Update HTTPS Everywhere to 2020.3.16.

1 files changed, 222 insertions, 164 deletions

diff --git a/data/extensions/https-everywhere@eff.org/background-scripts/rules.js b/data/extensions/https-everywhere@eff.org/background-scripts/rules.js
index c4ac18d..51da9b7 100644
--- a/data/extensions/https-everywhere@eff.org/background-scripts/rules.js
+++ b/data/extensions/https-everywhere@eff.org/background-scripts/rules.js
@@ -2,7 +2,8 @@
 
 (function(exports) {
 
-const util = require('./util');
+const util = require('./util'),
+  wasm = require('./wasm');
 
 let settings = {
   enableMixedRulesets: false,
@@ -12,19 +13,6 @@ let settings = {
 // To reduce memory usage for the numerous rules/cookies with trivial rules
 const trivial_cookie_rule_c = /.+/;
 
-// Empty iterable singleton to reduce memory usage
-const nullIterable = Object.create(null, {
-  [Symbol.iterator]: {
-    value: function* () {
-      // do nothing
-    }
-  },
-
-  size: {
-    value: 0
-  },
-});
-
 /* A map of all scope RegExp objects */
 const scopes = new Map();
 
@@ -220,33 +208,46 @@ RuleSets.prototype = {
   loadFromBrowserStorage: async function(store, applyStoredFunc) {
     this.store = store;
     this.ruleActiveStates = await this.store.get_promise('ruleActiveStates', {});
+    try {
+      this.wasm_rs = wasm.RuleSets.new();
+    } catch(e) {
+      util.log(util.WARN, 'Falling back to pure JS implementation: ' + e);
+    }
     await applyStoredFunc(this);
     await this.loadStoredUserRules();
     await this.addStoredCustomRulesets();
   },
 
   /**
-   * Iterate through data XML and load rulesets
+   * Convert XML to JS and load rulesets
    */
   addFromXml: function(ruleXml, scope) {
-    const scope_obj = getScope(scope);
-    const rulesets = ruleXml.getElementsByTagName("ruleset");
-    for (let ruleset of rulesets) {
-      try {
-        this.parseOneXmlRuleset(ruleset, scope_obj);
-      } catch (e) {
-        util.log(util.WARN, 'Error processing ruleset:' + e);
-      }
+    const rulesets_xml = ruleXml.getElementsByTagName("ruleset");
+
+    let rulesets = [];
+    for (let ruleset_xml of rulesets_xml) {
+      rulesets.push(this.convertOneXmlToJs(ruleset_xml));
     }
+
+    this.addFromJson(rulesets, scope);
   },
 
   addFromJson: function(ruleJson, scope) {
     const scope_obj = getScope(scope);
-    for (let ruleset of ruleJson) {
-      try {
-        this.parseOneJsonRuleset(ruleset, scope_obj);
-      } catch(e) {
-        util.log(util.WARN, 'Error processing ruleset:' + e);
+
+    if (this.wasm_rs) {
+      this.wasm_rs.add_all_from_js_array(
+        ruleJson,
+        settings.enableMixedRulesets,
+        this.ruleActiveStates,
+        scope);
+    } else {
+      for (let ruleset of ruleJson) {
+        try {
+          this.parseOneJsonRuleset(ruleset, scope_obj);
+        } catch(e) {
+          util.log(util.WARN, 'Error processing ruleset:' + e);
+        }
       }
     }
   },
@@ -323,7 +324,15 @@ RuleSets.prototype = {
    */
   addUserRule : function(params, scope) {
     util.log(util.INFO, 'adding new user rule for ' + JSON.stringify(params));
-    this.parseOneJsonRuleset(params, scope);
+    if (this.wasm_rs) {
+      this.wasm_rs.add_all_from_js_array(
+        [params],
+        settings.enableMixedRulesets,
+        this.ruleActiveStates,
+        scope);
+    } else {
+      this.parseOneJsonRuleset(params, scope);
+    }
 
     // clear cache so new rule take effect immediately
     for (const target of params.target) {
@@ -352,11 +361,16 @@ RuleSets.prototype = {
     this.ruleCache.delete(ruleset.name);
 
     if (src === 'popup') {
-      const tmp = this.targets.get(ruleset.name).filter(r => !r.isEquivalentTo(ruleset))
-      this.targets.set(ruleset.name, tmp);
 
-      if (this.targets.get(ruleset.name).length == 0) {
-        this.targets.delete(ruleset.name);
+      if (this.wasm_rs) {
+        this.wasm_rs.remove_ruleset(ruleset);
+      } else {
+        const tmp = this.targets.get(ruleset.name).filter(r => !r.isEquivalentTo(ruleset))
+        this.targets.set(ruleset.name, tmp);
+
+        if (this.targets.get(ruleset.name).length == 0) {
+          this.targets.delete(ruleset.name);
+        }
       }
     }
 
@@ -465,73 +479,70 @@ RuleSets.prototype = {
   },
 
   /**
-   * Does the loading of a ruleset.
+   * Converts an XML ruleset to a JS ruleset for parsing
    * @param ruletag The whole <ruleset> tag to parse
    */
-  parseOneXmlRuleset: function(ruletag, scope) {
-    var default_state = true;
-    var note = "";
-    var default_off = ruletag.getAttribute("default_off");
-    if (default_off) {
-      default_state = false;
-      if (default_off === "user rule") {
-        default_state = true;
-      }
-      note += default_off + "\n";
-    }
+  convertOneXmlToJs: function(ruletag) {
+    try {
+      let ruleset = {};
 
-    // If a ruleset declares a platform, and we don't match it, treat it as
-    // off-by-default. In practice, this excludes "mixedcontent" rules.
-    var platform = ruletag.getAttribute("platform");
-    if (platform) {
-      default_state = false;
-      if (platform == "mixedcontent" && settings.enableMixedRulesets) {
-        default_state = true;
+      let default_off = ruletag.getAttribute("default_off");
+      if (default_off) {
+        ruleset["default_off"] = platform;
       }
-      note += "Platform(s): " + platform + "\n";
-    }
 
-    var rule_set = new RuleSet(ruletag.getAttribute("name"),
-      default_state,
-      scope,
-      note.trim());
+      let platform = ruletag.getAttribute("platform");
+      if (platform) {
+        ruleset["platform"] = platform;
+      }
 
-    // Read user prefs
-    if (rule_set.name in this.ruleActiveStates) {
-      rule_set.active = (this.ruleActiveStates[rule_set.name] == "true");
-    }
+      let name = ruletag.getAttribute("name");
+      if (name) {
+        ruleset["name"] = name;
+      }
 
-    var rules = ruletag.getElementsByTagName("rule");
-    for (let rule of rules) {
-      rule_set.rules.push(getRule(rule.getAttribute("from"),
-        rule.getAttribute("to")));
-    }
+      let rules = [];
+      for (let rule of ruletag.getElementsByTagName("rule")) {
+        rules.push({
+          from: rule.getAttribute("from"),
+          to: rule.getAttribute("to")
+        });
+      }
+      if (rules.length > 0) {
+        ruleset["rule"] = rules;
+      }
 
-    var exclusions = Array();
-    for (let exclusion of ruletag.getElementsByTagName("exclusion")) {
-      exclusions.push(exclusion.getAttribute("pattern"));
-    }
-    if (exclusions.length > 0) {
-      rule_set.exclusions = new RegExp(exclusions.join("|"));
-    }
+      let exclusions = [];
+      for (let exclusion of ruletag.getElementsByTagName("exclusion")) {
+        exclusions.push(exclusion.getAttribute("pattern"));
+      }
+      if (exclusions.length > 0) {
+        ruleset["exclusion"] = exclusions;
+      }
 
-    var cookierules = ruletag.getElementsByTagName("securecookie");
-    if (cookierules.length > 0) {
-      rule_set.cookierules = [];
-      for (let cookierule of cookierules) {
-        rule_set.cookierules.push(
-          new CookieRule(cookierule.getAttribute("host"),
-            cookierule.getAttribute("name")));
+      let cookierules = [];
+      for (let cookierule of ruletag.getElementsByTagName("securecookie")) {
+        cookierules.push({
+          host: cookierule.getAttribute("host"),
+          name: cookierule.getAttribute("name")
+        });
+      }
+      if (cookierules.length > 0) {
+        ruleset["securecookie"] = cookierules;
       }
-    }
 
-    var targets = ruletag.getElementsByTagName("target");
-    for (let target of targets) {
-      var host = target.getAttribute("host");
-      if (!this.targets.has(host)) {
-        this.targets.set(host, []);
+      let targets = [];
+      for (let target of ruletag.getElementsByTagName("target")) {
+        targets.push(target.getAttribute("host"));
+      }
+      if (targets.length > 0) {
+        ruleset["target"] = targets;
       }
-      this.targets.get(host).push(rule_set);
+
+      return ruleset;
+    } catch (e) {
+      util.log(util.WARN, 'Error converting ruleset to JS:' + e);
+      return {};
     }
   },
 
@@ -550,51 +561,56 @@ RuleSets.prototype = {
       util.log(util.DBUG, "Ruleset cache miss for " + host);
     }
 
-    // Let's begin search
-    // Copy the host targets so we don't modify them.
-    let results = (this.targets.has(host) ?
-      new Set([...this.targets.get(host)]) :
-      new Set());
-
-    // Ensure host is well-formed (RFC 1035)
-    if (host.length <= 0 || host.length > 255 || host.indexOf("..") != -1) {
-      util.log(util.WARN, "Malformed host passed to potentiallyApplicableRulesets: " + host);
-      return nullIterable;
-    }
-
-    // Replace www.example.com with www.example.*
-    // eat away from the right for once and only once
-    let segmented = host.split(".");
-    if (segmented.length > 1) {
-      const tmp = segmented[segmented.length - 1];
-      segmented[segmented.length - 1] = "*";
-
-      results = (this.targets.has(segmented.join(".")) ?
-        new Set([...results, ...this.targets.get(segmented.join("."))]) :
-        results);
+    let results;
+    if (this.wasm_rs) {
+      let pa = this.wasm_rs.potentially_applicable(host);
+      results = new Set([...pa].map(ruleset => {
+        let rs = new RuleSet(ruleset.name, ruleset.default_state, getScope(ruleset.scope), ruleset.note);
 
-      segmented[segmented.length - 1] = tmp;
-    }
+        if (ruleset.cookierules) {
+          let cookierules = ruleset.cookierules.map(cookierule => {
+            return new CookieRule(cookierule.host, cookierule.name);
+          });
+          rs.cookierules = cookierules;
+        } else {
+          rs.cookierules = null;
+        }
 
-    // now eat away from the left, with *, so that for x.y.z.google.com we
-    // check *.y.z.google.com, *.z.google.com and *.google.com
-    for (let i = 1; i <= segmented.length - 2; i++) {
-      let t = "*." + segmented.slice(i, segmented.length).join(".");
+        let rules = ruleset.rules.map(rule => {
+          return getRule(rule.from, rule.to);
+        });
+        rs.rules = rules;
 
-      results = (this.targets.has(t) ?
-        new Set([...results, ...this.targets.get(t)]) :
-        results);
-    }
+        if (ruleset.exclusions) {
+          rs.exclusions = new RegExp(ruleset.exclusions);
+        } else {
+          rs.exclusions = null;
+        }
+        return rs;
+      }));
+    } else {
+      // Let's begin search
+      results = (this.targets.has(host) ?
+        new Set([...this.targets.get(host)]) :
+        new Set());
+
+      let expressions = util.getWildcardExpressions(host);
+      for (const expression of expressions) {
+        results = (this.targets.has(expression) ?
+          new Set([...results, ...this.targets.get(expression)]) :
+          results);
+      }
 
-    // Clean the results list, which may contain duplicates or undefined entries
-    results.delete(undefined);
+      // Clean the results list, which may contain duplicates or undefined entries
+      results.delete(undefined);
 
-    util.log(util.DBUG,"Applicable rules for " + host + ":");
-    if (results.size == 0) {
-      util.log(util.DBUG, "  None");
-      results = nullIterable;
-    } else {
-      results.forEach(result => util.log(util.DBUG, "  " + result.name));
+      util.log(util.DBUG,"Applicable rules for " + host + ":");
+      if (results.size == 0) {
+        util.log(util.DBUG, "  None");
+        results = util.nullIterable;
+      } else {
+        results.forEach(result => util.log(util.DBUG, "  " + result.name));
+      }
     }
 
     // Insert results into the ruleset cache
@@ -612,38 +628,15 @@ RuleSets.prototype = {
   /**
    * Check to see if the Cookie object c meets any of our cookierule criteria for being marked as secure.
    * @param cookie The cookie to test
-   * @returns {*} ruleset or null
+   * @returns {*} true or false
    */
   shouldSecureCookie: function(cookie) {
-    var hostname = cookie.domain;
+    let hostname = cookie.domain;
     // cookie domain scopes can start with .
     while (hostname.charAt(0) == ".") {
       hostname = hostname.slice(1);
     }
 
-    if (!this.safeToSecureCookie(hostname)) {
-      return null;
-    }
-
-    var potentiallyApplicable = this.potentiallyApplicableRulesets(hostname);
-    for (const ruleset of potentiallyApplicable) {
-      if (ruleset.cookierules !== null && ruleset.active) {
-        for (const cookierule of ruleset.cookierules) {
-          if (cookierule.host_c.test(cookie.domain) && cookierule.name_c.test(cookie.name)) {
-            return ruleset;
-          }
-        }
-      }
-    }
-    return null;
-  },
-
-  /**
-   * Check if it is secure to secure the cookie (=patch the secure flag in).
-   * @param domain The domain of the cookie
-   * @returns {*} true or false
-   */
-  safeToSecureCookie: function(domain) {
     // Check if the domain might be being served over HTTP.  If so, it isn't
     // safe to secure a cookie!  We can't always know this for sure because
     // observing cookie-changed doesn't give us enough context to know the
@@ -656,20 +649,57 @@ RuleSets.prototype = {
     // observed and the domain blacklisted, a cookie might already have been
     // flagged as secure.
 
-    if (settings.domainBlacklist.has(domain)) {
-      util.log(util.INFO, "cookies for " + domain + "blacklisted");
+    if (settings.domainBlacklist.has(hostname)) {
+      util.log(util.INFO, "cookies for " + hostname + "blacklisted");
       return false;
     }
-    var cached_item = this.cookieHostCache.get(domain);
-    if (cached_item !== undefined) {
-      util.log(util.DBUG, "Cookie host cache hit for " + domain);
-      return cached_item;
+
+    // Second, we need a cookie pass two tests before patching it
+    //   (1) it is safe to secure the cookie, as per safeToSecureCookie()
+    //   (2) it matches with the CookieRule
+    //
+    // We kept a cache of the results for (1), if we have a cached result which
+    //   (a) is false, we should not secure the cookie for sure
+    //   (b) is true, we need to perform test (2)
+    //
+    // Otherwise,
+    //   (c) We need to perform (1) and (2) in place
+
+    let safe = false;
+    if (this.cookieHostCache.has(hostname)) {
+      util.log(util.DBUG, "Cookie host cache hit for " + hostname);
+      safe = this.cookieHostCache.get(hostname); // true only if it is case (b)
+      if (!safe) {
+        return false; // (a)
+      }
+    } else {
+      util.log(util.DBUG, "Cookie host cache miss for " + hostname);
     }
-    util.log(util.DBUG, "Cookie host cache miss for " + domain);
 
-    // If we passed that test, make up a random URL on the domain, and see if
-    // we would HTTPSify that.
+    const potentiallyApplicable = this.potentiallyApplicableRulesets(hostname);
+    for (const ruleset of potentiallyApplicable) {
+      if (ruleset.cookierules !== null && ruleset.active) {
+        // safe is false only indicate the lack of a cached result
+        // we cannot use it to avoid looping here
+        for (const cookierule of ruleset.cookierules) {
+          // if safe is true, it is case (b); otherwise it is case (c)
+          if (cookierule.host_c.test(cookie.domain) && cookierule.name_c.test(cookie.name)) {
+            return safe || this.safeToSecureCookie(hostname, potentiallyApplicable);
+          }
+        }
+      }
+    }
+    return false;
+  },
 
+  /**
+   * Check if it is secure to secure the cookie (=patch the secure flag in).
+   * @param domain The domain of the cookie
+   * @param potentiallyApplicable
+   * @returns {*} true or false
+   */
+  safeToSecureCookie: function(domain, potentiallyApplicable) {
+    // Make up a random URL on the domain, and see if we would HTTPSify that.
     var nonce_path = "/" + Math.random().toString();
     var test_uri = "http://" + domain + nonce_path + nonce_path;
 
@@ -680,7 +710,6 @@ RuleSets.prototype = {
     }
 
     util.log(util.INFO, "Testing securecookie applicability with " + test_uri);
-    var potentiallyApplicable = this.potentiallyApplicableRulesets(domain);
     for (let ruleset of potentiallyApplicable) {
       if (ruleset.active && ruleset.apply(test_uri)) {
         util.log(util.INFO, "Cookie domain could be secured.");
@@ -694,6 +723,36 @@ RuleSets.prototype = {
   },
 
   /**
+   * Get a list of simple rules (active, with no exclusions) for all hosts that
+   * are in a single ruleset, and end in the specified ending.
+   * @param ending Target ending to search for
+   * @returns A list of { host, from_regex, to, scope_regex }
+   */
+  getSimpleRulesEndingWith: function(ending) {
+    let results;
+
+    if (this.wasm_rs) {
+      results = this.wasm_rs.get_simple_rules_ending_with(ending);
+    } else {
+      results = [];
+      for(let [host, rulesets] of this.targets) {
+        if (host.endsWith(ending) &&
+            rulesets.length == 1 &&
+            rulesets[0].active === true &&
+            rulesets[0].exclusions == null
+        ) {
+          for (let rule of rulesets[0].rules) {
+            if (rule.from_c.test("http://" + host + "/")) {
+              results.push({ host, from_regex: rule.from_c.toString(), to: rule.to, scope_regex: rulesets[0].scope.toString() });
+            }
+          }
+        }
+      }
+    }
+    return results;
+  },
+
+  /**
    * Rewrite an URI
    * @param urispec The uri to rewrite
    * @param host The host of this uri
@@ -712,7 +771,6 @@ RuleSets.prototype = {
 };
 
 Object.assign(exports, {
-  nullIterable,
   settings,
   trivial_rule,
   Rule,