LibreJS updated to 6.0.9

8 files changed, 175 insertions, 52 deletions

diff --git a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/html_script_finder/web_labels/script_hash_worker.js b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/html_script_finder/web_labels/script_hash_worker.js
index a8b2fdb..92ffb82 100644
--- a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/html_script_finder/web_labels/script_hash_worker.js
+++ b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/html_script_finder/web_labels/script_hash_worker.js
@@ -49,15 +49,28 @@ exports.addToCache = function (lic, delay, jsWebLabelsURL, callback) {
                     lic.fileUrl
                 );
                 console.debug('returning xhr from', lic.fileUrl);
-                callback(lic.fileUrl);
+                if (typeof callback === 'function') {
+                    callback(lic.fileUrl);
+                } else {
+                    console.debug('callback is not a function:', callback);
+                }
             } catch (e) {
-                callback(lic.fileUrl);
+                if (typeof callback === 'function') {
+                    callback(lic.fileUrl);
+                } else {
+                    console.debug('callback is not a function:', callback);
+                }
             }
         };
         // just callback after 5 seconds if we don't get the answer yet.
         timers.setTimeout(function() {
             cb = function() {};
-            callback(lic.fileUrl); }, 20000);
+            if (typeof callback === 'function') {
+                callback(lic.fileUrl);
+            } else {
+                console.debug('callback is not a function:', callback);
+            }
+        }, 20000);
 
         xhr({'url': lic.fileUrl}, cb);
     }, delay);
diff --git a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/http_observer/process_response.js b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/http_observer/process_response.js
index c1f5e88..924636b 100644
--- a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/http_observer/process_response.js
+++ b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/http_observer/process_response.js
@@ -81,17 +81,6 @@ var processResponseObject = {
     },
 
     /**
-     * genBinaryOutput
-     * Set or reset binaryOutputStream and storageStream.
-     */
-    genBinaryOutput: function () {
-        this.storageStream = Cc["@mozilla.org/storagestream;1"]
-            .createInstance(Ci.nsIStorageStream);
-        this.binaryOutputStream = Cc["@mozilla.org/binaryoutputstream;1"]
-            .createInstance(Ci.nsIBinaryOutputStream);
-    },
-
-    /**
      * Gather the data gathered from onDataAvailable.
      */
     setData: function () {
@@ -203,7 +192,6 @@ var processResponseObject = {
     processJS: function() {
         var checker, check, jsCheckString,
             that = this;
-        //var start = Date.now(), end;
 
         try {
             // make sure script isn't already listed as free
@@ -212,7 +200,6 @@ var processResponseObject = {
                 // this is free. we are done.
                 this.jsListenerCallback();
                 return;
-
             }
 
             // analyze javascript in response.
@@ -222,8 +209,6 @@ var processResponseObject = {
                 that.processJsCallback(checker);
             }, that.url);
 
-
-
         } catch(e) {
 
             // any error is considered nontrivial.
@@ -273,7 +258,6 @@ var processResponseObject = {
                     return true;
 
                 }
-
             }
         }
     },
@@ -393,21 +377,26 @@ var processResponseObject = {
 
     jsListenerCallback: function () {
 
-        var len = this.data.length;
+        var converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+            .createInstance(Ci.nsIScriptableUnicodeConverter);
 
-        this.genBinaryOutput();
+        if (typeof this.req.contentCharset !== 'undefined' &&
+            this.req.contentCharset !== '' &&
+            this.req.contentCharset !== null
+           ) {
+            converter.charset = this.req.contentCharset;
+        } else {
+            converter.charset = "UTF-8";
+        }
 
-        this.storageStream.init(8192, len, null);
-        this.binaryOutputStream.setOutputStream(
-            this.storageStream.getOutputStream(0));
-        this.binaryOutputStream.writeBytes(this.data, len);
+        var stream = converter.convertToInputStream(this.data);
 
         try {
             this.listener.onDataAvailable(
                 this.req,
                 this.resInfo.context,
-                this.storageStream.newInputStream(0),
-                0, len);
+                stream,
+                0, stream.available());
         } catch (e) {
             this.req.cancel(this.req.NS_BINDING_ABORTED);
         }
diff --git a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/http_observer/stream_loader.js b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/http_observer/stream_loader.js
index 42187c8..ec21cb7 100644
--- a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/http_observer/stream_loader.js
+++ b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/http_observer/stream_loader.js
@@ -22,6 +22,49 @@
 var {Cc, Ci, Cu, Cm, Cr} = require("chrome");
 
 const processResponse = require('http_observer/process_response');
+const CHARSETS = [
+    '866', 'ansi_x3.4-1968', 'arabic', 'ascii', 'asmo-708', 'big5',
+    'big5-hkscs', 'chinese', 'cn-big5', 'cp1250', 'cp1251', 'cp1252',
+    'cp1253', 'cp1254', 'cp1255', 'cp1256', 'cp1257', 'cp1258',
+    'cp819', 'cp866', 'csbig5', 'cseuckr', 'cseucpkdfmtjapanese',
+    'csgb2312', 'csibm866', 'csiso2022jp', 'csiso2022kr', 'csiso58gb231280',
+    'csiso88596e', 'csiso88596i', 'csiso88598e', 'csiso88598i', 'csisolatin1',
+    'csisolatin2', 'csisolatin3', 'csisolatin4', 'csisolatin5', 'csisolatin6',
+    'csisolatin9', 'csisolatinarabic', 'csisolatincyrillic',
+    'csisolatingreek', 'csisolatinhebrew', 'cskoi8r', 'csksc56011987',
+    'csmacintosh', 'csshiftjis', 'cyrillic', 'dos-874', 'ecma-114',
+    'ecma-118', 'elot_928', 'euc-jp', 'euc-kr', 'gb18030', 'gb2312',
+    'gb_2312', 'gb_2312-80', 'gbk', 'greek', 'greek8', 'hebrew',
+    'hz-gb-2312', 'ibm819', 'ibm866', 'iso-2022-cn', 'iso-2022-cn-ext',
+    'iso-2022-jp', 'iso-2022-kr', 'iso88591', 'iso_8859-1', 'iso-8859-1',
+    'iso8859-1', 'iso885910', 'iso-8859-10', 'iso8859-10', 'iso885911',
+    'iso-8859-11', 'iso8859-11', 'iso_8859-1:1987', 'iso885913', 'iso-8859-13',
+    'iso8859-13', 'iso885914', 'iso-8859-14', 'iso8859-14', 'iso885915',
+    'iso-8859-15', 'iso8859-15', 'iso-8859-16', 'iso88592', 'iso_8859-2',
+    'iso-8859-2', 'iso8859-2', 'iso_8859-2:1987', 'iso88593', 'iso_8859-3',
+    'iso-8859-3', 'iso8859-3', 'iso_8859-3:1988', 'iso88594', 'iso_8859-4',
+    'iso-8859-4', 'iso8859-4', 'iso_8859-4:1988', 'iso88595', 'iso_8859-5',
+    'iso-8859-5', 'iso_8859-5:1988', 'iso88596', 'iso_8859-6', 'iso-8859-6',
+    'iso8859-6', 'iso_8859-6:1987', 'iso-8859-6-e', 'iso-8859-6-i', 'iso88597',
+    'iso_8859-7', 'iso-8859-7', 'iso8859-7', 'iso_8859-7:1987', 'iso88598',
+    'iso_8859-8', 'iso-8859-8', 'iso8859-8', 'iso_8859-8:1988', 'iso-8859-8-e',
+    'iso-8859-8i', 'iso-8859-8-i', 'iso88599', 'iso_8859-9', 'iso-8859-9',
+    'iso8859-9', 'iso_8859-9:1989', 'iso-ir-100', 'iso-ir-101', 'iso-ir-109',
+    'iso-ir-110', 'iso-ir-126', 'iso-ir-127', 'iso-ir-138', 'iso-ir-144',
+    'iso-ir-148', 'iso-ir-149', 'iso-ir-157', 'iso-ir-58', 'koi', 'koi8',
+    'koi8_r', 'koi8-r', 'koi8-u', 'korean', 'ksc5601', 'ksc_5601',
+    'ks_c_5601-1987', 'ks_c_5601-1989', 'l1', 'l2', 'l3', 'l4', 'l5', 'l6',
+    'l9', 'latin1', 'latin2', 'latin3', 'latin4', 'latin5', 'latin6', 'latin9',
+    'logical', 'mac', 'macintosh', 'ms_kanji', 'replacement', 'shift_jis',
+    'shift-jis', 'sjis', 'sun_eu_greek', 'tis-620', 'unicode-1-1-utf-8',
+    'us-ascii', 'utf-16', 'utf-16be', 'utf-16le', 'utf8', 'utf-8', 'visual',
+    'windows-1250', 'windows-1251', 'windows-1252', 'windows-1253',
+    'windows-1254', 'windows-1255', 'windows-1256', 'windows-1257',
+    'windows-1258', 'windows-31j', 'windows-874', 'windows-949', 'x-cp1250',
+    'x-cp1251', 'x-cp1252', 'x-cp1253', 'x-cp1254', 'x-cp1255', 'x-cp1256',
+    'x-cp1257', 'x-cp1258', 'x-euc-jp', 'x-gbk', 'x-mac-cyrillic',
+    'x-mac-roman', 'x-mac-ukrainian', 'x-sjis', 'x-user-defined', 'x-x-big5'
+];
 
 var StreamLoader = function() {
     this.loader = null;
@@ -97,7 +140,11 @@ StreamListener.prototype.onDetermineCharset = function onDetermineCharset(
         return loader.channel.contentCharset;
     } else {
         match = getRegexForContentType(loader.channel.contentType).exec(data);
-        if (match) {
+        if (typeof match !== 'undefined' &&
+            match !== null &&
+            match.length > 0 &&
+            CHARSETS.indexOf(match[1].toLowerCase()) >= 0
+           ) {
             loader.channel.contentCharset = match[1];
             return match[1];
         } else {
diff --git a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/js_checker/js_checker.js b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/js_checker/js_checker.js
index 3b03084..da3ebe6 100644
--- a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/js_checker/js_checker.js
+++ b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/js_checker/js_checker.js
@@ -3,6 +3,7 @@
  * *
  * Copyright (C) 2011, 2012, 2013, 2014 Loic J. Duros
  * Copyright (C) 2014, 2015 Nik Nyby
+ * Copyright (C) 2015 Ruben Rodriguez
  *
  * This file is part of GNU LibreJS.
  *
@@ -115,12 +116,51 @@ var JsChecker = function() {
 };
 
 /**
+ * isFreeLicensed
+ *
+ * This function returns true if the input script is licensed under
+ * a free license. Otherwise, it returns false.
+ */
+JsChecker.prototype.isFreeLicensed = function(script) {
+    var magnets = '(' +
+        'magnet:\\?xt=urn:btih:90dc5c0be029de84e523b9b3922520e79e0e6f08&dn=cc0.txt|' +
+        'magnet:\\?xt=urn:btih:cf05388f2679ee054f2beb29a391d25f4e673ac3&dn=gpl-2.0.txt|' +
+        'magnet:\\?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt|' +
+        'magnet:\\?xt=urn:btih:8e4f440f4c65981c5bf93c76d35135ba5064d8b7&dn=apache-2.0.txt|' +
+        'magnet:\\?xt=urn:btih:5de60da917303dbfad4f93fb1b985ced5a89eac2&dn=lgpl-2.1.txt|' +
+        'magnet:\\?xt=urn:btih:0ef1b8170b3b615170ff270def6427c317705f85&dn=lgpl-3.0.txt|' +
+        'magnet:\\?xt=urn:btih:0b31508aeb0634b347b8270c7bee4d411b5d4109&dn=agpl-3.0.txt|' +
+        'magnet:\\?xt=urn:btih:c80d50af7d3db9be66a4d0a86db0286e4fd33292&dn=bsd-3-clause.txt|' +
+        'magnet:\\?xt=urn:btih:3877d6d54b3accd4bc32f8a48bf32ebc0901502a&dn=mpl-2.0.txt|' +
+        'magnet:\\?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt|' +
+        'magnet:\\?xt=urn:btih:5305d91886084f776adcf57509a648432709a7c7&dn=x11.txt|' +
+        'magnet:\\?xt=urn:btih:12f2ec9e8de2a3b0002a33d518d6010cc8ab2ae9&dn=xfree86.txt|' +
+        'magnet:\\?xt=urn:btih:87f119ba0b429ba17a44b4bffcab33165ebdacc0&dn=freebsd.txt|' +
+        'magnet:\\?xt=urn:btih:b8999bbaf509c08d127678643c515b9ab0836bae&dn=ISC.txt|' +
+        'magnet:\\?xt=urn:btih:54fd2283f9dbdf29466d2df1a98bf8f65cafe314&dn=artistic-2.0.txt|' +
+        'magnet:\\?xt=urn:btih:e95b018ef3580986a04669f1b5879592219e2a7a&dn=public-domain.txt' +
+        ')';
+
+    // Remove licensed parts
+    var re = new RegExp(
+        '@license +' + magnets + '.*([\n\r].*?)*@license-end', 'g');
+    script = script.replace(re, '');
+
+    // Remove comments and empty lines
+    re = new RegExp('\/\/.*|/\\*.*?\\*/|^\s*[\n\r]*', 'gm');
+
+    script = script.replace(re, "");
+
+    // If only spaces remain, the file has a free license
+    return (script.match(/\S/) === null);
+};
+
+/**
  * searchJs
  *
  * Takes in some javascript code (as string).
  * Uses Narcissus parser to build an abstract syntax tree.
  * Checks for trivialness.
- *
  */
 JsChecker.prototype.searchJs = function(jsCode, resultReady, url) {
     var that = this;
@@ -163,20 +203,39 @@ JsChecker.prototype.searchJs = function(jsCode, resultReady, url) {
         this.freeToken = types.emptyTypeObj();
         this.nontrivialness = types.emptyTypeObj();
 
-        // use this.hash to keep track of comments made by the nontrivial
-        // checker code about why/how the code is found to be nontrivial.
-        this.nonTrivialChecker =
-            nonTrivialModule.nonTrivialChecker(this.hash);
-
-        // register callback and hash. So that result
-        // can be passed.
-        setHashCallback(
-            this.hash, this.handleTree.bind(this), this.notification);
+        if (typeof url === 'undefined' || url === null) {
+            // use this.hash to keep track of comments made by the nontrivial
+            // checker code about why/how the code is found to be nontrivial.
+            this.nonTrivialChecker =
+                nonTrivialModule.nonTrivialChecker(this.hash);
+
+            // register callback and hash. So that result
+            // can be passed.
+            setHashCallback(
+                this.hash, this.handleTree.bind(this), this.notification);
+
+            // parse using ChromeWorker.
+            console.debug(
+                'JsChecker.searchJs(): starting narcissusWorker.parse()');
+            narcissusWorker.parse(this.jsCode, this.hash);
+        } else {
+            // Handle external scripts
+            console.debug('Analyzing external script: ' + url);
+            this.parseTree = {};
+            if (this.isFreeLicensed(this.jsCode)) {
+                this.parseTree.freeTrivialCheck = type.trivialWithComment(
+                    'This script is under free software licenses: ' + url);
+            } else {
+                this.parseTree.freeTrivialCheck = type.nontrivialWithComment(
+                    'This external script is not under free software ' +
+                        'licenses: ' + url);
+            }
 
-        // parse using ChromeWorker.
-        console.debug(
-            'JsChecker.searchJs(): starting narcissusWorker.parse()');
-        narcissusWorker.parse(this.jsCode, this.hash);
+            // Cache result with hash of script for future checks
+            scriptsCached.addEntry(
+                this.jsCode, this.parseTree.freeTrivialCheck,
+                this.relationChecker, true, this.url);
+        }
     } catch (x) {
         console.debug('error', x);
         this.handleTree(false, x);
diff --git a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/js_checker/license_definitions.js b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/js_checker/license_definitions.js
index 41cfb7b..d5fcd83 100644
--- a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/js_checker/license_definitions.js
+++ b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/js_checker/license_definitions.js
@@ -247,5 +247,23 @@ exports.licenses = {
                 type: type.SHORT
             }
         ]
+    },
+    WTFPL: {
+        licenseName: 'Do What The F*ck You Want To Public License (WTFPL)',
+        canonicalUrl: [
+            'http://www.wtfpl.net/txt/copying/',
+            'magnet:?xt=urn:btih:723febf9f6185544f57f0660a41489c7d6b4931b&dn=wtfpl.txt'
+        ],
+        identifier: 'WTFPL',
+        licenseFragments: [
+            {
+                text: 'DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE',
+                type: type.SHORT
+            },
+            {
+                text: '0. You just DO WHAT THE FUCK YOU WANT TO.',
+                type: type.SHORT
+            }
+        ]
     }
 };
diff --git a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/crypto.js b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/crypto.js
index 5fb920f..d8d9c0a 100644
--- a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/crypto.js
+++ b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/crypto.js
@@ -34,7 +34,6 @@ CryptoString.prototype.init = function(hashAlgorithm, charset) {
     this.converter.charset = charset;
     this.hashAlgorithm = hashAlgorithm;
     this.cryptoHash.init(this.cryptoHash[this.hashAlgorithm]);
-
 };
 
 CryptoString.prototype.encryptString = function(str) {
diff --git a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/free_libraries.js b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/free_libraries.js
index af2cf61..f3354f6 100644
--- a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/free_libraries.js
+++ b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/free_libraries.js
@@ -24,7 +24,7 @@
 
 var relationChecker = require("js_checker/relation_checker").relationChecker;
 var checkTypes = require("js_checker/constant_types").checkTypes;
-var scriptsCached = require("./scripts_cache").scriptsCached;
+var scriptsCached = require("script_entries/scripts_cache").scriptsCached;
 
 
 // find the json database path.
@@ -40,7 +40,6 @@ var freeLibraries = JSON.parse(dbContents); /* a database of the free libraries
  */
 
 var init = function () {
-
     // relationChecker, which roughly checks if variables are window
     //  variables or not, is useless in this case.  Use the same
     //  object for all entries.
diff --git a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/scripts_cache.js b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/scripts_cache.js
index 47ab069..dfc5e6a 100644
--- a/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/scripts_cache.js
+++ b/data/extensions/jid1-KtlZuoiikVfFew@jetpack/resources/librejs/lib/script_entries/scripts_cache.js
@@ -19,11 +19,7 @@
  * You should have received a copy of the GNU General Public License
  * along with GNU LibreJS.  If not, see <http://www.gnu.org/licenses/>.
  */
-var relationCheckerObj = require("js_checker/relation_checker")
-    .relationChecker;
-
-// import free_libraries to populate the cache hash map.
-var free_libraries = require("script_entries/free_libraries");
+var relationChecker = require("js_checker/relation_checker").relationChecker;
 
 var crypto = require('script_entries/crypto');
 const checkTypes = require("js_checker/constant_types").checkTypes;
@@ -52,6 +48,8 @@ ScriptsCached.prototype.getHash = function(scriptText) {
  */
 ScriptsCached.prototype.resetCache = function () {
     cachedResults = {};
+    // import free_libraries to populate the cache hash map.
+    var free_libraries = require("script_entries/free_libraries");
     free_libraries.init();
 };
   
@@ -64,11 +62,12 @@ ScriptsCached.prototype.resetCache = function () {
  * 
  */
 ScriptsCached.prototype.addEntry = function(
-        scriptText, result, relationChecker, allowTrivial, url) {
+    scriptText, result, relationCheckerObj, allowTrivial, url
+) {
     console.debug("result addEntry is", JSON.stringify(result));
     cachedResults[this.getHash(scriptText)] = {
         'result': result,
-        'relationChecker': relationCheckerObj(),
+        'relationChecker': relationChecker(),
         'allowTrivial': allowTrivial,
         'url': url
     };