diff options
Diffstat (limited to 'data/extensions/https-everywhere@eff.org/background-scripts')
7 files changed, 579 insertions, 456 deletions
diff --git a/data/extensions/https-everywhere@eff.org/background-scripts/background.js b/data/extensions/https-everywhere@eff.org/background-scripts/background.js index c432d74..7d999f7 100644 --- a/data/extensions/https-everywhere@eff.org/background-scripts/background.js +++ b/data/extensions/https-everywhere@eff.org/background-scripts/background.js @@ -7,12 +7,15 @@ const rules = require('./rules'), incognito = require('./incognito'), util = require('./util'), update = require('./update'), - { update_channels } = require('./update_channels'); + { update_channels } = require('./update_channels'), + wasm = require('./wasm'), + ipUtils = require('./ip_utils'); let all_rules = new rules.RuleSets(); async function initialize() { + await wasm.initialize(); await store.initialize(); await store.performMigrations(); await initializeStoredGlobals(); @@ -39,6 +42,33 @@ async function initializeAllRules() { var httpNowhereOn = false; var isExtensionEnabled = true; let disabledList = new Set(); +let httpOnceList = new Set(); + +/** + * Check if HTTPS Everywhere should be ON for host + */ +function isExtensionDisabledOnSite(host) { + // make sure the host is not matched in the httpOnceList + if (httpOnceList.has(host)) { + return true; + } + + // make sure the host is not matched in the disabledList + if (disabledList.has(host)) { + return true; + } + + // make sure the host is matched by any wildcard expressions in the disabledList + const experessions = util.getWildcardExpressions(host); + for (const expression of experessions) { + if (disabledList.has(expression)) { + return true; + } + } + + // otherwise return false + return false; +} function initializeStoredGlobals() { return new Promise(resolve => { @@ -112,20 +142,23 @@ if (chrome.windows) { chrome.windows.onFocusChanged.addListener(function() { updateState(); }); + + // Grant access to HTTP site only during session, clear once window is closed + chrome.windows.onRemoved.addListener(function() { + chrome.windows.getAll({}, function(windows) { + if(windows.length > 0) { + return; + } else { + httpOnceList.clear(); + } + }); + }); + } chrome.webNavigation.onCompleted.addListener(function() { updateState(); }); -// Records which tabId's are active in the HTTPS Switch Planner (see -// pages/devtools/panel-ux.js). -var switchPlannerEnabledFor = {}; -// Detailed information recorded when the HTTPS Switch Planner is active. -// Structure is: -// switchPlannerInfo[tabId]["rw"/"nrw"][resource_host][active_content][url]; -// rw / nrw stand for "rewritten" versus "not rewritten" -var switchPlannerInfo = {}; - /** * Set the icon color correctly * active: extension is enabled. @@ -153,8 +186,9 @@ function updateState () { return; } const tabUrl = new URL(tabs[0].url); + const hostname = util.getNormalisedHostname(tabUrl.hostname); - if (disabledList.has(tabUrl.host) || iconState == "disabled") { + if (isExtensionDisabledOnSite(hostname) || iconState == "disabled") { if ('setIcon' in chrome.browserAction) { chrome.browserAction.setIcon({ path: { @@ -184,38 +218,57 @@ chrome.browserAction.onClicked.addListener(e => { }); }); - - /** - * Add a listener for removed tabs + * A centralized storage for browsing data within the browser session. */ -function AppliedRulesets() { - this.active_tab_rules = new Map(); - this.active_tab_main_frames = new Map(); +function BrowserSession() { + this.tabs = new Map(); + this.requests = new Map(); - let that = this; if (chrome.tabs) { - chrome.tabs.onRemoved.addListener(function(tabId) { - that.removeTab(tabId); + chrome.tabs.onRemoved.addListener(tabId => { + this.deleteTab(tabId); }); } } -AppliedRulesets.prototype = { - addRulesetToTab: function(tabId, type, ruleset) { - if (!this.active_tab_main_frames.has(tabId)) { - this.active_tab_main_frames.set(tabId, false); +BrowserSession.prototype = { + putTab: function(tabId, key, value, overwrite) { + if (!this.tabs.has(tabId)) { + this.tabs.set(tabId, {}); + } + + if (!(key in this.tabs.get(tabId)) || overwrite) { + this.tabs.get(tabId)[key] = value; } + }, + + getTab: function(tabId, key, defaultValue) { + if (this.tabs.has(tabId) && key in this.tabs.get(tabId)) { + return this.tabs.get(tabId)[key]; + } + return defaultValue; + }, + + deleteTab: function(tabId) { + if (this.tabs.has(tabId)) { + this.tabs.delete(tabId); + } + }, + + putTabAppliedRulesets: function(tabId, type, ruleset) { + this.putTab(tabId, "main_frame", false, false); // always show main_frame ruleset on the top if (type == "main_frame") { - this.active_tab_main_frames.set(tabId, true); - this.active_tab_rules.set(tabId, [ruleset,]); + this.putTab(tabId, "main_frame", true, true); + this.putTab(tabId, "applied_rulesets", [ruleset,], true); return ; } - if (this.active_tab_rules.has(tabId)) { - let rulesets = this.active_tab_rules.get(tabId); + // sort by ruleset names alphabetically, case-insensitive + if (this.getTab(tabId, "applied_rulesets", null)) { + let rulesets = this.getTab(tabId, "applied_rulesets"); let insertIndex = 0; const ruleset_name = ruleset.name.toLowerCase(); @@ -225,7 +278,7 @@ AppliedRulesets.prototype = { if (item_name == ruleset_name) { return ; - } else if (insertIndex == 0 && this.active_tab_main_frames.get(tabId)) { + } else if (insertIndex == 0 && this.getTab(tabId, "main_frame", false)) { insertIndex = 1; } else if (item_name < ruleset_name) { insertIndex++; @@ -233,59 +286,46 @@ AppliedRulesets.prototype = { } rulesets.splice(insertIndex, 0, ruleset); } else { - this.active_tab_rules.set(tabId, [ruleset,]); + this.putTab(tabId, "applied_rulesets", [ruleset,], true); } }, - getRulesets: function(tabId) { - if (this.active_tab_rules.has(tabId)) { - return this.active_tab_rules.get(tabId); - } else { - return null; - } + getTabAppliedRulesets: function(tabId) { + return this.getTab(tabId, "applied_rulesets", null); }, - removeTab: function(tabId) { - this.active_tab_rules.delete(tabId); - this.active_tab_main_frames.delete(tabId); + putRequest: function(requestId, key, value) { + if (!this.requests.has(requestId)) { + this.requests.set(requestId, {}); + } + this.requests.get(requestId)[key] = value; }, - getActiveRulesetCount: function (tabId) { - let activeCount = 0; + getRequest: function(requestId, key, defaultValue) { + if (this.requests.has(requestId) && key in this.requests.get(requestId)) { + return this.requests.get(requestId)[key]; + } + return defaultValue; + }, - const rulesets = this.getRulesets(tabId); - if (rulesets) { - for (const ruleset of rulesets) { - if (ruleset.active) { - activeCount++; - } - } + deleteRequest: function(requestId) { + if (this.requests.has(requestId)) { + this.requests.delete(requestId); } - return activeCount; } -}; +} -var appliedRulesets = new AppliedRulesets(); +let browserSession = new BrowserSession(); var urlBlacklist = new Set(); -// redirect counter workaround -// TODO: Remove this code if they ever give us a real counter -var redirectCounter = new Map(); - -// Create a map to indicate whether a given request has been subject to a simple -// HTTP Nowhere redirect. -let simpleHTTPNowhereRedirect = new Map(); - const cancelUrl = chrome.runtime.getURL("/pages/cancel/index.html"); function redirectOnCancel(shouldCancel, originURL) { return shouldCancel ? {redirectUrl: newCancelUrl(originURL)} : {cancel: false}; } -function newCancelUrl(originURL) { - return cancelUrl + "?originURL=" + encodeURI(originURL); -} +const newCancelUrl = originURL => `${cancelUrl}?originURL=${encodeURIComponent(originURL)}`; /** * Called before a HTTP(s) request. Does the heavy lifting @@ -298,40 +338,33 @@ function onBeforeRequest(details) { return; } - // Clear the content shown in the extension popup. - // This needed to be done before this listener returns, - // otherwise, the extension page might include rulesets - // from previous page. - if (details.type == "main_frame") { - appliedRulesets.removeTab(details.tabId); - } - let uri = new URL(details.url); - // Check if a user has disabled HTTPS Everywhere on this site. We should - // ensure that all subresources are not run through HTTPS Everywhere as well. - let firstPartyHost; - if (details.type == "main_frame") { - firstPartyHost = uri.host; - } else { - // In Firefox, documentUrl is preferable here, since it will always be the - // URL in the URL bar, but it was only introduced in FF 54. We should get - // rid of `originUrl` at some point. - if ('documentUrl' in details) { // Firefox 54+ - firstPartyHost = new URL(details.documentUrl).host; - } else if ('originUrl' in details) { // Firefox < 54 - firstPartyHost = new URL(details.originUrl).host; - } else if('initiator' in details) { // Chrome - firstPartyHost = new URL(details.initiator).host; - } + // Normalise hosts with tailing dots, e.g. "www.example.com." + uri.hostname = util.getNormalisedHostname(uri.hostname); + + let ip = ipUtils.parseIp(uri.hostname); + + let isLocalIp = false; + + if (ip !== -1) { + isLocalIp = ipUtils.isLocalIp(ip); } - if (disabledList.has(firstPartyHost)) { - return; + + if (details.type == "main_frame") { + // Clear the content from previous browser session. + // This needed to be done before this listener returns, + // otherwise, the extension popup might include rulesets + // from previous page. + browserSession.deleteTab(details.tabId); + + // Check if an user has disabled HTTPS Everywhere on this site. We should + // ensure that all subresources are not run through HTTPS Everywhere as well. + browserSession.putTab(details.tabId, 'first_party_host', uri.hostname, true); } - // Normalise hosts with tailing dots, e.g. "www.example.com." - while (uri.hostname[uri.hostname.length - 1] === '.' && uri.hostname !== '.') { - uri.hostname = uri.hostname.slice(0, -1); + if (isExtensionDisabledOnSite(browserSession.getTab(details.tabId, 'first_party_host', null))) { + return; } // Should the request be canceled? @@ -341,9 +374,8 @@ function onBeforeRequest(details) { (uri.protocol === 'http:' || uri.protocol === 'ftp:') && uri.hostname.slice(-6) !== '.onion' && uri.hostname !== 'localhost' && - !/^127(\.[0-9]{1,3}){3}$/.test(uri.hostname) && - uri.hostname !== '0.0.0.0' && - uri.hostname !== '[::1]'; + uri.hostname !== '[::1]' && + !isLocalIp; // If there is a username / password, put them aside during the ruleset // analysis process @@ -366,7 +398,7 @@ function onBeforeRequest(details) { return redirectOnCancel(shouldCancel, details.url); } - if (redirectCounter.get(details.requestId) >= 8) { + if (browserSession.getRequest(details.requestId, "redirect_count") >= 8) { util.log(util.NOTE, "Redirect counter hit for " + uri.href); urlBlacklist.add(uri.href); rules.settings.domainBlacklist.add(uri.hostname); @@ -382,7 +414,7 @@ function onBeforeRequest(details) { for (let ruleset of potentiallyApplicable) { if (details.url.match(ruleset.scope)) { - appliedRulesets.addRulesetToTab(details.tabId, details.type, ruleset); + browserSession.putTabAppliedRulesets(details.tabId, details.type, ruleset); if (ruleset.active && !newuristr) { newuristr = ruleset.apply(uri.href); } @@ -411,23 +443,13 @@ function onBeforeRequest(details) { } } - // In Switch Planner Mode, record any non-rewriteable - // HTTP URIs by parent hostname, along with the resource type. - if (switchPlannerEnabledFor[details.tabId] && uri.protocol !== "https:") { - writeToSwitchPlanner(details.type, - details.tabId, - uri.hostname, - details.url, - newuristr); - } - if (httpNowhereOn) { // If loading a main frame, try the HTTPS version as an alternative to // failing. if (shouldCancel) { if (!newuristr) { newuristr = uri.href.replace(/^http:/, "https:"); - simpleHTTPNowhereRedirect.set(details.requestId, true); + browserSession.putRequest(details.requestId, "simple_http_nowhere_redirect", true); upgradeToSecure = true; } else { newuristr = newuristr.replace(/^http:/, "https:"); @@ -457,86 +479,6 @@ function onBeforeRequest(details) { } } - -// Map of which values for the `type' enum denote active vs passive content. -// https://developer.chrome.com/extensions/webRequest.html#event-onBeforeRequest -const mixedContentTypes = { - object: 1, other: 1, script: 1, stylesheet: 1, sub_frame: 1, xmlhttprequest: 1, - image: 0, main_frame: 0 -}; - -/** - * Record a non-HTTPS URL loaded by a given hostname in the Switch Planner, for - * use in determining which resources need to be ported to HTTPS. - * (Reminder: Switch planner is the pro-tool enabled by switching into debug-mode) - * - * @param type: type of the resource (see activeTypes and passiveTypes arrays) - * @param tab_id: The id of the tab - * @param resource_host: The host of the original url - * @param resource_url: the original url - * @param rewritten_url: The url rewritten to - * */ -function writeToSwitchPlanner(type, tab_id, resource_host, resource_url, rewritten_url) { - let rw = rewritten_url ? "rw" : "nrw"; - - let active_content = 1; - if (mixedContentTypes.hasOwnProperty(type)) { - active_content = mixedContentTypes[type]; - } else { - util.log(util.WARN, "Unknown type from onBeforeRequest details: `" + type + "', assuming active"); - } - - if (!switchPlannerInfo[tab_id]) { - switchPlannerInfo[tab_id] = {}; - switchPlannerInfo[tab_id]["rw"] = {}; - switchPlannerInfo[tab_id]["nrw"] = {}; - } - if (!switchPlannerInfo[tab_id][rw][resource_host]) - switchPlannerInfo[tab_id][rw][resource_host] = {}; - if (!switchPlannerInfo[tab_id][rw][resource_host][active_content]) - switchPlannerInfo[tab_id][rw][resource_host][active_content] = {}; - - switchPlannerInfo[tab_id][rw][resource_host][active_content][resource_url] = 1; -} - -/** - * Return the number of properties in an object. For associative maps, this is - * their size. - * @param obj: object to calc the size for - * */ -function objSize(obj) { - if (typeof obj == 'undefined') return 0; - var size = 0, key; - for (key in obj) { - if (obj.hasOwnProperty(key)) size++; - } - return size; -} - -/** - * Make an array of asset hosts by score so we can sort them, - * presenting the most important ones first. - * */ -function sortSwitchPlanner(tab_id, rewritten) { - var asset_host_list = []; - if (typeof switchPlannerInfo[tab_id] === 'undefined' || - typeof switchPlannerInfo[tab_id][rewritten] === 'undefined') { - return []; - } - var tabInfo = switchPlannerInfo[tab_id][rewritten]; - for (var asset_host in tabInfo) { - var ah = tabInfo[asset_host]; - var activeCount = objSize(ah[1]); - var passiveCount = objSize(ah[0]); - var score = activeCount * 100 + passiveCount; - asset_host_list.push([score, activeCount, passiveCount, asset_host]); - } - asset_host_list.sort(function(a,b) { - return a[0]-b[0]; - }); - return asset_host_list; -} - /** * monitor cookie changes. Automatically convert them to secure cookies * @param changeInfo Cookie changed info, see Chrome doc @@ -592,13 +534,12 @@ function onBeforeRedirect(details) { // Catch redirect loops (ignoring about:blank, etc. caused by other extensions) let prefix = details.redirectUrl.substring(0, 5); if (prefix === "http:" || prefix === "https") { - let count = redirectCounter.get(details.requestId); + let count = browserSession.getRequest(details.requestId, "redirect_count", 0); if (count) { - redirectCounter.set(details.requestId, count + 1); - util.log(util.DBUG, "Got redirect id "+details.requestId+ - ": "+count); + browserSession.putRequest(details.requestId, "redirect_count", count + 1); + util.log(util.DBUG, "Got redirect id " + details.requestId + ": "+count); } else { - redirectCounter.set(details.requestId, 1); + browserSession.putRequest(details.requestId, "redirect_count", 1); } } } @@ -608,12 +549,7 @@ function onBeforeRedirect(details) { * @param details details for the chrome.webRequest (see chrome doc) */ function onCompleted(details) { - if (redirectCounter.has(details.requestId)) { - redirectCounter.delete(details.requestId); - } - if (simpleHTTPNowhereRedirect.has(details.requestId)) { - simpleHTTPNowhereRedirect.delete(details.requestId); - } + browserSession.deleteRequest(details.requestId); } /** @@ -623,7 +559,7 @@ function onCompleted(details) { function onErrorOccurred(details) { if (httpNowhereOn && details.type == "main_frame" && - simpleHTTPNowhereRedirect.get(details.requestId) && + browserSession.getRequest(details.requestId, "simple_http_nowhere_redirect", false) && ( // Enumerate a class of errors that are likely due to HTTPS misconfigurations details.error.indexOf("net::ERR_SSL_") == 0 || details.error.indexOf("net::ERR_CERT_") == 0 || @@ -633,7 +569,6 @@ function onErrorOccurred(details) { details.error.indexOf("NS_ERROR_CONNECTION_REFUSED") == 0 || details.error.indexOf("NS_ERROR_NET_TIMEOUT") == 0 || details.error.indexOf("NS_ERROR_NET_ON_TLS_HANDSHAKE_ENDED") == 0 || - details.error.indexOf("NS_BINDING_ABORTED") == 0 || details.error.indexOf("SSL received a record that exceeded the maximum permissible length.") == 0 || details.error.indexOf("Peer’s Certificate has expired.") == 0 || details.error.indexOf("Unable to communicate securely with peer: requested domain name does not match the server’s certificate.") == 0 || @@ -654,12 +589,7 @@ function onErrorOccurred(details) { chrome.tabs.update(details.tabId, {url: newCancelUrl(url.toString())}); } - if (redirectCounter.has(details.requestId)) { - redirectCounter.delete(details.requestId); - } - if (simpleHTTPNowhereRedirect.has(details.requestId)) { - simpleHTTPNowhereRedirect.delete(details.requestId); - } + browserSession.deleteRequest(details.requestId); } /** @@ -672,28 +602,14 @@ function onHeadersReceived(details) { // Do not upgrade the .onion requests in EASE mode, // See https://github.com/EFForg/https-everywhere/pull/14600#discussion_r168072480 const uri = new URL(details.url); - if (uri.hostname.slice(-6) == '.onion') { + const hostname = util.getNormalisedHostname(uri.hostname); + if (hostname.slice(-6) == '.onion') { return {}; } // Do not upgrade resources if the first-party domain disbled EASE mode // This is needed for HTTPS sites serve mixed content and is broken - let firstPartyHost; - if (details.type == "main_frame") { - firstPartyHost = uri.host; - } else { - // In Firefox, documentUrl is preferable here, since it will always be the - // URL in the URL bar, but it was only introduced in FF 54. We should get - // rid of `originUrl` at some point. - if ('documentUrl' in details) { // Firefox 54+ - firstPartyHost = new URL(details.documentUrl).host; - } else if ('originUrl' in details) { // Firefox < 54 - firstPartyHost = new URL(details.originUrl).host; - } else if('initiator' in details) { // Chrome - firstPartyHost = new URL(details.initiator).host; - } - } - if (disabledList.has(firstPartyHost)) { + if (isExtensionDisabledOnSite(browserSession.getTab(details.tabId, 'first_party_host', null))) { return {}; } @@ -761,59 +677,6 @@ chrome.webRequest.onHeadersReceived.addListener(onHeadersReceived, {urls: ["http // Listen for cookies set/updated and secure them if applicable. This function is async/nonblocking. chrome.cookies.onChanged.addListener(onCookieChanged); - -/** - * disable switch Planner - * @param tabId the Tab to disable for - */ -function disableSwitchPlannerFor(tabId) { - delete switchPlannerEnabledFor[tabId]; - // Clear stored URL info. - delete switchPlannerInfo[tabId]; -} - -/** - * Enable switch planner for specific tab - * @param tabId the tab to enable it for - */ -function enableSwitchPlannerFor(tabId) { - switchPlannerEnabledFor[tabId] = true; -} - -// Listen for connection from the DevTools panel so we can set up communication. -chrome.runtime.onConnect.addListener(function (port) { - if (port.name == "devtools-page") { - chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) { - var tabId = message.tabId; - - var disableOnCloseCallback = function() { - util.log(util.DBUG, "Devtools window for tab " + tabId + " closed, clearing data."); - disableSwitchPlannerFor(tabId); - }; - - const responses = { - enable: () => { - enableSwitchPlannerFor(tabId); - port.onDisconnect.addListener(disableOnCloseCallback); - }, - disable: () => { - disableSwitchPlannerFor(tabId); - }, - getHosts: () => { - sendResponse({ - nrw: sortSwitchPlanner(tabId, "nrw"), - rw: sortSwitchPlanner(tabId, "rw") - }); - return true; - } - }; - if (message.type in responses) { - return responses[message.type](); - } - }); - } -}); - // This is necessary for communication with the popup in Firefox Private // Browsing Mode, see https://bugzilla.mozilla.org/show_bug.cgi?id=1329304 chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) { @@ -836,11 +699,21 @@ chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) { }); } - function storeDisabledList() { + function storeDisabledList(message) { + const disabledListArray = Array.from(disabledList); - store.set({disabledList: disabledListArray}, () => { - sendResponse(true); - }); + const httpOnceListArray = Array.from(httpOnceList); + + if (message === 'once') { + store.set({httpOnceList: httpOnceListArray}, () => { + sendResponse(true); + }); + } else { + store.set({disabledList: disabledListArray}, () => { + sendResponse(true); + }); + } + return true; } @@ -859,12 +732,12 @@ chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) { delete_from_ruleset_cache: () => { all_rules.ruleCache.delete(message.object); }, - get_active_rulesets: () => { - sendResponse(appliedRulesets.getRulesets(message.object)); + get_applied_rulesets: () => { + sendResponse(browserSession.getTabAppliedRulesets(message.object)); return true; }, set_ruleset_active_status: () => { - let rulesets = appliedRulesets.getRulesets(message.object.tab_id); + let rulesets = browserSession.getTabAppliedRulesets(message.object.tab_id); for (let ruleset of rulesets) { if (ruleset.name == message.object.name) { @@ -1011,23 +884,34 @@ chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) { }); return true; }, + get_simple_rules_ending_with: () => { + return sendResponse(all_rules.getSimpleRulesEndingWith(message.object)); + }, get_last_checked: () => { store.local.get({'last-checked': false}, item => { sendResponse(item['last-checked']); }); return true; }, + disable_on_site_once: () => { + httpOnceList.add(message.object); + return storeDisabledList('once'); + }, disable_on_site: () => { - disabledList.add(message.object); - return storeDisabledList(); + const host = util.getNormalisedHostname(message.object); + // always validate hostname before adding it to the disabled list + if (util.isValidHostname(host)) { + disabledList.add(host); + return storeDisabledList('disable'); + } + return sendResponse(false); }, enable_on_site: () => { - disabledList.delete(message.object); - return storeDisabledList(); + disabledList.delete(util.getNormalisedHostname(message.object)); + return storeDisabledList('enable'); }, check_if_site_disabled: () => { - sendResponse(disabledList.has(message.object)); - return true; + return sendResponse(isExtensionDisabledOnSite(util.getNormalisedHostname(message.object))); }, is_firefox: () => { if(typeof(browser) != "undefined") { @@ -1050,6 +934,21 @@ chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) { }); /** + * @description Upboarding event for visual changelog + */ +chrome.runtime.onInstalled.addListener(async ({reason, temporary}) => { + if (temporary) return; + switch (reason) { + case "update": + { + const url = chrome.runtime.getURL("pages/onboarding/updated.html"); + await chrome.tabs.create({ url }); + } + break; + } +}); + +/** * Clear any cache/ blacklist we have. */ function destroy_caches() { @@ -1058,13 +957,12 @@ function destroy_caches() { all_rules.ruleCache.clear(); rules.settings.domainBlacklist.clear(); urlBlacklist.clear(); + httpOnceList.clear(); } Object.assign(exports, { all_rules, - urlBlacklist, - sortSwitchPlanner, - switchPlannerInfo + urlBlacklist }); })(typeof exports == 'undefined' ? require.scopes.background = {} : exports); diff --git a/data/extensions/https-everywhere@eff.org/background-scripts/ip_utils.js b/data/extensions/https-everywhere@eff.org/background-scripts/ip_utils.js new file mode 100644 index 0000000..be7c1c8 --- /dev/null +++ b/data/extensions/https-everywhere@eff.org/background-scripts/ip_utils.js @@ -0,0 +1,53 @@ +'use strict'; + +(function (exports) { + +const parseIp = ip => { + if (!/^[0-9.]+$/.test(ip)) { + return -1; + } + + const octets = ip.split('.'); + + if (octets.length !== 4) { + return -1; + } + + let ipN = 0; + + for (const octet of octets) { + if (octet === '') { + return -1; + } + + const octetN = parseInt(octet); + + if (octetN < 0 || octetN > 255) { + return -1; + } + + ipN = (ipN << 8) | octet; + } + + return ipN >>> 0; +}; + +const isIpInRange = (ip, [rangeIp, mask]) => (ip & mask) >>> 0 === rangeIp; + +const localRanges = [ + [/* 0.0.0.0 */ 0x00000000, /* 255.255.255.255 */ 0xffffffff], + [/* 127.0.0.0 */ 0x7f000000, /* 255.0.0.0 */ 0xff000000], + [/* 10.0.0.0 */ 0x0a000000, /* 255.0.0.0 */ 0xff000000], + [/* 172.16.0.0 */ 0xac100000, /* 255.240.0.0 */ 0xfff00000], + [/* 192.168.0.0 */ 0xc0a80000, /* 255.255.0.0 */ 0xffff0000], +]; + +const isLocalIp = ip => localRanges.some(range => isIpInRange(ip, range)); + +Object.assign(exports, { + parseIp, + isIpInRange, + isLocalIp +}); + +})(typeof exports !== 'undefined' ? exports : require.scopes.ip_utils = {}); diff --git a/data/extensions/https-everywhere@eff.org/background-scripts/modules/on_before.js b/data/extensions/https-everywhere@eff.org/background-scripts/modules/on_before.js new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/data/extensions/https-everywhere@eff.org/background-scripts/modules/on_before.js diff --git a/data/extensions/https-everywhere@eff.org/background-scripts/rules.js b/data/extensions/https-everywhere@eff.org/background-scripts/rules.js index c4ac18d..51da9b7 100644 --- a/data/extensions/https-everywhere@eff.org/background-scripts/rules.js +++ b/data/extensions/https-everywhere@eff.org/background-scripts/rules.js @@ -2,7 +2,8 @@ (function(exports) { -const util = require('./util'); +const util = require('./util'), + wasm = require('./wasm'); let settings = { enableMixedRulesets: false, @@ -12,19 +13,6 @@ let settings = { // To reduce memory usage for the numerous rules/cookies with trivial rules const trivial_cookie_rule_c = /.+/; -// Empty iterable singleton to reduce memory usage -const nullIterable = Object.create(null, { - [Symbol.iterator]: { - value: function* () { - // do nothing - } - }, - - size: { - value: 0 - }, -}); - /* A map of all scope RegExp objects */ const scopes = new Map(); @@ -220,33 +208,46 @@ RuleSets.prototype = { loadFromBrowserStorage: async function(store, applyStoredFunc) { this.store = store; this.ruleActiveStates = await this.store.get_promise('ruleActiveStates', {}); + try { + this.wasm_rs = wasm.RuleSets.new(); + } catch(e) { + util.log(util.WARN, 'Falling back to pure JS implementation: ' + e); + } await applyStoredFunc(this); await this.loadStoredUserRules(); await this.addStoredCustomRulesets(); }, /** - * Iterate through data XML and load rulesets + * Convert XML to JS and load rulesets */ addFromXml: function(ruleXml, scope) { - const scope_obj = getScope(scope); - const rulesets = ruleXml.getElementsByTagName("ruleset"); - for (let ruleset of rulesets) { - try { - this.parseOneXmlRuleset(ruleset, scope_obj); - } catch (e) { - util.log(util.WARN, 'Error processing ruleset:' + e); - } + const rulesets_xml = ruleXml.getElementsByTagName("ruleset"); + + let rulesets = []; + for (let ruleset_xml of rulesets_xml) { + rulesets.push(this.convertOneXmlToJs(ruleset_xml)); } + + this.addFromJson(rulesets, scope); }, addFromJson: function(ruleJson, scope) { const scope_obj = getScope(scope); - for (let ruleset of ruleJson) { - try { - this.parseOneJsonRuleset(ruleset, scope_obj); - } catch(e) { - util.log(util.WARN, 'Error processing ruleset:' + e); + + if (this.wasm_rs) { + this.wasm_rs.add_all_from_js_array( + ruleJson, + settings.enableMixedRulesets, + this.ruleActiveStates, + scope); + } else { + for (let ruleset of ruleJson) { + try { + this.parseOneJsonRuleset(ruleset, scope_obj); + } catch(e) { + util.log(util.WARN, 'Error processing ruleset:' + e); + } } } }, @@ -323,7 +324,15 @@ RuleSets.prototype = { */ addUserRule : function(params, scope) { util.log(util.INFO, 'adding new user rule for ' + JSON.stringify(params)); - this.parseOneJsonRuleset(params, scope); + if (this.wasm_rs) { + this.wasm_rs.add_all_from_js_array( + [params], + settings.enableMixedRulesets, + this.ruleActiveStates, + scope); + } else { + this.parseOneJsonRuleset(params, scope); + } // clear cache so new rule take effect immediately for (const target of params.target) { @@ -352,11 +361,16 @@ RuleSets.prototype = { this.ruleCache.delete(ruleset.name); if (src === 'popup') { - const tmp = this.targets.get(ruleset.name).filter(r => !r.isEquivalentTo(ruleset)) - this.targets.set(ruleset.name, tmp); - if (this.targets.get(ruleset.name).length == 0) { - this.targets.delete(ruleset.name); + if (this.wasm_rs) { + this.wasm_rs.remove_ruleset(ruleset); + } else { + const tmp = this.targets.get(ruleset.name).filter(r => !r.isEquivalentTo(ruleset)) + this.targets.set(ruleset.name, tmp); + + if (this.targets.get(ruleset.name).length == 0) { + this.targets.delete(ruleset.name); + } } } @@ -465,73 +479,70 @@ RuleSets.prototype = { }, /** - * Does the loading of a ruleset. + * Converts an XML ruleset to a JS ruleset for parsing * @param ruletag The whole <ruleset> tag to parse */ - parseOneXmlRuleset: function(ruletag, scope) { - var default_state = true; - var note = ""; - var default_off = ruletag.getAttribute("default_off"); - if (default_off) { - default_state = false; - if (default_off === "user rule") { - default_state = true; - } - note += default_off + "\n"; - } + convertOneXmlToJs: function(ruletag) { + try { + let ruleset = {}; - // If a ruleset declares a platform, and we don't match it, treat it as - // off-by-default. In practice, this excludes "mixedcontent" rules. - var platform = ruletag.getAttribute("platform"); - if (platform) { - default_state = false; - if (platform == "mixedcontent" && settings.enableMixedRulesets) { - default_state = true; + let default_off = ruletag.getAttribute("default_off"); + if (default_off) { + ruleset["default_off"] = platform; } - note += "Platform(s): " + platform + "\n"; - } - var rule_set = new RuleSet(ruletag.getAttribute("name"), - default_state, - scope, - note.trim()); + let platform = ruletag.getAttribute("platform"); + if (platform) { + ruleset["platform"] = platform; + } - // Read user prefs - if (rule_set.name in this.ruleActiveStates) { - rule_set.active = (this.ruleActiveStates[rule_set.name] == "true"); - } + let name = ruletag.getAttribute("name"); + if (name) { + ruleset["name"] = name; + } - var rules = ruletag.getElementsByTagName("rule"); - for (let rule of rules) { - rule_set.rules.push(getRule(rule.getAttribute("from"), - rule.getAttribute("to"))); - } + let rules = []; + for (let rule of ruletag.getElementsByTagName("rule")) { + rules.push({ + from: rule.getAttribute("from"), + to: rule.getAttribute("to") + }); + } + if (rules.length > 0) { + ruleset["rule"] = rules; + } - var exclusions = Array(); - for (let exclusion of ruletag.getElementsByTagName("exclusion")) { - exclusions.push(exclusion.getAttribute("pattern")); - } - if (exclusions.length > 0) { - rule_set.exclusions = new RegExp(exclusions.join("|")); - } + let exclusions = []; + for (let exclusion of ruletag.getElementsByTagName("exclusion")) { + exclusions.push(exclusion.getAttribute("pattern")); + } + if (exclusions.length > 0) { + ruleset["exclusion"] = exclusions; + } - var cookierules = ruletag.getElementsByTagName("securecookie"); - if (cookierules.length > 0) { - rule_set.cookierules = []; - for (let cookierule of cookierules) { - rule_set.cookierules.push( - new CookieRule(cookierule.getAttribute("host"), - cookierule.getAttribute("name"))); + let cookierules = []; + for (let cookierule of ruletag.getElementsByTagName("securecookie")) { + cookierules.push({ + host: cookierule.getAttribute("host"), + name: cookierule.getAttribute("name") + }); + } + if (cookierules.length > 0) { + ruleset["securecookie"] = cookierules; } - } - var targets = ruletag.getElementsByTagName("target"); - for (let target of targets) { - var host = target.getAttribute("host"); - if (!this.targets.has(host)) { - this.targets.set(host, []); + let targets = []; + for (let target of ruletag.getElementsByTagName("target")) { + targets.push(target.getAttribute("host")); + } + if (targets.length > 0) { + ruleset["target"] = targets; } - this.targets.get(host).push(rule_set); + + return ruleset; + } catch (e) { + util.log(util.WARN, 'Error converting ruleset to JS:' + e); + return {}; } }, @@ -550,51 +561,56 @@ RuleSets.prototype = { util.log(util.DBUG, "Ruleset cache miss for " + host); } - // Let's begin search - // Copy the host targets so we don't modify them. - let results = (this.targets.has(host) ? - new Set([...this.targets.get(host)]) : - new Set()); - - // Ensure host is well-formed (RFC 1035) - if (host.length <= 0 || host.length > 255 || host.indexOf("..") != -1) { - util.log(util.WARN, "Malformed host passed to potentiallyApplicableRulesets: " + host); - return nullIterable; - } - - // Replace www.example.com with www.example.* - // eat away from the right for once and only once - let segmented = host.split("."); - if (segmented.length > 1) { - const tmp = segmented[segmented.length - 1]; - segmented[segmented.length - 1] = "*"; - - results = (this.targets.has(segmented.join(".")) ? - new Set([...results, ...this.targets.get(segmented.join("."))]) : - results); + let results; + if (this.wasm_rs) { + let pa = this.wasm_rs.potentially_applicable(host); + results = new Set([...pa].map(ruleset => { + let rs = new RuleSet(ruleset.name, ruleset.default_state, getScope(ruleset.scope), ruleset.note); - segmented[segmented.length - 1] = tmp; - } + if (ruleset.cookierules) { + let cookierules = ruleset.cookierules.map(cookierule => { + return new CookieRule(cookierule.host, cookierule.name); + }); + rs.cookierules = cookierules; + } else { + rs.cookierules = null; + } - // now eat away from the left, with *, so that for x.y.z.google.com we - // check *.y.z.google.com, *.z.google.com and *.google.com - for (let i = 1; i <= segmented.length - 2; i++) { - let t = "*." + segmented.slice(i, segmented.length).join("."); + let rules = ruleset.rules.map(rule => { + return getRule(rule.from, rule.to); + }); + rs.rules = rules; - results = (this.targets.has(t) ? - new Set([...results, ...this.targets.get(t)]) : - results); - } + if (ruleset.exclusions) { + rs.exclusions = new RegExp(ruleset.exclusions); + } else { + rs.exclusions = null; + } + return rs; + })); + } else { + // Let's begin search + results = (this.targets.has(host) ? + new Set([...this.targets.get(host)]) : + new Set()); + + let expressions = util.getWildcardExpressions(host); + for (const expression of expressions) { + results = (this.targets.has(expression) ? + new Set([...results, ...this.targets.get(expression)]) : + results); + } - // Clean the results list, which may contain duplicates or undefined entries - results.delete(undefined); + // Clean the results list, which may contain duplicates or undefined entries + results.delete(undefined); - util.log(util.DBUG,"Applicable rules for " + host + ":"); - if (results.size == 0) { - util.log(util.DBUG, " None"); - results = nullIterable; - } else { - results.forEach(result => util.log(util.DBUG, " " + result.name)); + util.log(util.DBUG,"Applicable rules for " + host + ":"); + if (results.size == 0) { + util.log(util.DBUG, " None"); + results = util.nullIterable; + } else { + results.forEach(result => util.log(util.DBUG, " " + result.name)); + } } // Insert results into the ruleset cache @@ -612,38 +628,15 @@ RuleSets.prototype = { /** * Check to see if the Cookie object c meets any of our cookierule criteria for being marked as secure. * @param cookie The cookie to test - * @returns {*} ruleset or null + * @returns {*} true or false */ shouldSecureCookie: function(cookie) { - var hostname = cookie.domain; + let hostname = cookie.domain; // cookie domain scopes can start with . while (hostname.charAt(0) == ".") { hostname = hostname.slice(1); } - if (!this.safeToSecureCookie(hostname)) { - return null; - } - - var potentiallyApplicable = this.potentiallyApplicableRulesets(hostname); - for (const ruleset of potentiallyApplicable) { - if (ruleset.cookierules !== null && ruleset.active) { - for (const cookierule of ruleset.cookierules) { - if (cookierule.host_c.test(cookie.domain) && cookierule.name_c.test(cookie.name)) { - return ruleset; - } - } - } - } - return null; - }, - - /** - * Check if it is secure to secure the cookie (=patch the secure flag in). - * @param domain The domain of the cookie - * @returns {*} true or false - */ - safeToSecureCookie: function(domain) { // Check if the domain might be being served over HTTP. If so, it isn't // safe to secure a cookie! We can't always know this for sure because // observing cookie-changed doesn't give us enough context to know the @@ -656,20 +649,57 @@ RuleSets.prototype = { // observed and the domain blacklisted, a cookie might already have been // flagged as secure. - if (settings.domainBlacklist.has(domain)) { - util.log(util.INFO, "cookies for " + domain + "blacklisted"); + if (settings.domainBlacklist.has(hostname)) { + util.log(util.INFO, "cookies for " + hostname + "blacklisted"); return false; } - var cached_item = this.cookieHostCache.get(domain); - if (cached_item !== undefined) { - util.log(util.DBUG, "Cookie host cache hit for " + domain); - return cached_item; + + // Second, we need a cookie pass two tests before patching it + // (1) it is safe to secure the cookie, as per safeToSecureCookie() + // (2) it matches with the CookieRule + // + // We kept a cache of the results for (1), if we have a cached result which + // (a) is false, we should not secure the cookie for sure + // (b) is true, we need to perform test (2) + // + // Otherwise, + // (c) We need to perform (1) and (2) in place + + let safe = false; + if (this.cookieHostCache.has(hostname)) { + util.log(util.DBUG, "Cookie host cache hit for " + hostname); + safe = this.cookieHostCache.get(hostname); // true only if it is case (b) + if (!safe) { + return false; // (a) + } + } else { + util.log(util.DBUG, "Cookie host cache miss for " + hostname); } - util.log(util.DBUG, "Cookie host cache miss for " + domain); - // If we passed that test, make up a random URL on the domain, and see if - // we would HTTPSify that. + const potentiallyApplicable = this.potentiallyApplicableRulesets(hostname); + for (const ruleset of potentiallyApplicable) { + if (ruleset.cookierules !== null && ruleset.active) { + // safe is false only indicate the lack of a cached result + // we cannot use it to avoid looping here + for (const cookierule of ruleset.cookierules) { + // if safe is true, it is case (b); otherwise it is case (c) + if (cookierule.host_c.test(cookie.domain) && cookierule.name_c.test(cookie.name)) { + return safe || this.safeToSecureCookie(hostname, potentiallyApplicable); + } + } + } + } + return false; + }, + /** + * Check if it is secure to secure the cookie (=patch the secure flag in). + * @param domain The domain of the cookie + * @param potentiallyApplicable + * @returns {*} true or false + */ + safeToSecureCookie: function(domain, potentiallyApplicable) { + // Make up a random URL on the domain, and see if we would HTTPSify that. var nonce_path = "/" + Math.random().toString(); var test_uri = "http://" + domain + nonce_path + nonce_path; @@ -680,7 +710,6 @@ RuleSets.prototype = { } util.log(util.INFO, "Testing securecookie applicability with " + test_uri); - var potentiallyApplicable = this.potentiallyApplicableRulesets(domain); for (let ruleset of potentiallyApplicable) { if (ruleset.active && ruleset.apply(test_uri)) { util.log(util.INFO, "Cookie domain could be secured."); @@ -694,6 +723,36 @@ RuleSets.prototype = { }, /** + * Get a list of simple rules (active, with no exclusions) for all hosts that + * are in a single ruleset, and end in the specified ending. + * @param ending Target ending to search for + * @returns A list of { host, from_regex, to, scope_regex } + */ + getSimpleRulesEndingWith: function(ending) { + let results; + + if (this.wasm_rs) { + results = this.wasm_rs.get_simple_rules_ending_with(ending); + } else { + results = []; + for(let [host, rulesets] of this.targets) { + if (host.endsWith(ending) && + rulesets.length == 1 && + rulesets[0].active === true && + rulesets[0].exclusions == null + ) { + for (let rule of rulesets[0].rules) { + if (rule.from_c.test("http://" + host + "/")) { + results.push({ host, from_regex: rule.from_c.toString(), to: rule.to, scope_regex: rulesets[0].scope.toString() }); + } + } + } + } + } + return results; + }, + + /** * Rewrite an URI * @param urispec The uri to rewrite * @param host The host of this uri @@ -712,7 +771,6 @@ RuleSets.prototype = { }; Object.assign(exports, { - nullIterable, settings, trivial_rule, Rule, diff --git a/data/extensions/https-everywhere@eff.org/background-scripts/store.js b/data/extensions/https-everywhere@eff.org/background-scripts/store.js index 9698c18..3f32d03 100644 --- a/data/extensions/https-everywhere@eff.org/background-scripts/store.js +++ b/data/extensions/https-everywhere@eff.org/background-scripts/store.js @@ -3,6 +3,7 @@ (function(exports) { const rules = require('./rules'); +const util = require("./util"); function initialize() { return new Promise(resolve => { @@ -96,6 +97,22 @@ async function performMigrations() { migration_version = 2; await set_promise('migration_version', migration_version); } + + if (migration_version <= 2) { + await get_promise('disabledList', []) + .then(disabledList => { + disabledList = disabledList.map(item => { + return util.getNormalisedHostname(item); + }) + return disabledList; + }) + .then(disabledList => { + return set_promise('disabledList', disabledList); + }) + + migration_version = 3; + await set_promise('migration_version', migration_version); + } } const local = { diff --git a/data/extensions/https-everywhere@eff.org/background-scripts/util.js b/data/extensions/https-everywhere@eff.org/background-scripts/util.js index 1946e14..5a4097c 100644 --- a/data/extensions/https-everywhere@eff.org/background-scripts/util.js +++ b/data/extensions/https-everywhere@eff.org/background-scripts/util.js @@ -61,6 +61,73 @@ function loadExtensionFile(url, returnType) { } /** + * Remove tailing dots from hostname, e.g. "www.example.com." + */ +function getNormalisedHostname(hostname) { + while (hostname && hostname[hostname.length - 1] === '.' && hostname !== '.') { + hostname = hostname.slice(0, -1); + } + return hostname; +} + +// Empty iterable singleton to reduce memory usage +const nullIterable = Object.create(null, { + [Symbol.iterator]: { + value: function* () { + // do nothing + } + }, + + size: { + value: 0 + }, +}); + +/** + * Return true if host is well-formed (RFC 1035) + */ +function isValidHostname(host) { + if (host && host.length > 0 && host.length <= 255 && host.indexOf("..") === -1) { + return true; + } + return false; +} + +/** + * Return a list of wildcard expressions which support + * the host under HTTPS Everywhere's implementation + */ +function getWildcardExpressions(host) { + // Ensure host is well-formed (RFC 1035) + if (!isValidHostname(host)) { + return nullIterable; + } + + // Ensure host does not contain a wildcard itself + if (host.indexOf("*") != -1) { + return nullIterable; + } + + let results = []; + + // Replace www.example.com with www.example.* + // eat away from the right for once and only once + let segmented = host.split("."); + if (segmented.length > 1) { + const tmp = [...segmented.slice(0, segmented.length - 1), "*"].join("."); + results.push(tmp); + } + + // now eat away from the left, with *, so that for x.y.z.google.com we + // check *.y.z.google.com, *.z.google.com and *.google.com + for (let i = 1; i < segmented.length - 1; i++) { + const tmp = ["*", ...segmented.slice(i, segmented.length)].join("."); + results.push(tmp); + } + return results; +} + +/** * Convert an ArrayBuffer to string * * @param array: an ArrayBuffer to convert @@ -84,6 +151,10 @@ Object.assign(exports, { NOTE, WARN, log, + nullIterable, + isValidHostname, + getNormalisedHostname, + getWildcardExpressions, setDefaultLogLevel, getDefaultLogLevel, loadExtensionFile, diff --git a/data/extensions/https-everywhere@eff.org/background-scripts/wasm.js b/data/extensions/https-everywhere@eff.org/background-scripts/wasm.js new file mode 100644 index 0000000..3385bb7 --- /dev/null +++ b/data/extensions/https-everywhere@eff.org/background-scripts/wasm.js @@ -0,0 +1,26 @@ +"use strict"; + +(function(exports) { + +const util = require('./util'), + { RuleSets } = wasm_bindgen; + +async function initialize() { + try { + await wasm_bindgen(chrome.runtime.getURL('wasm/https_everywhere_lib_wasm_bg.wasm')); + } catch(e) { + util.log(util.WARN, 'The wasm library has not loaded correctly: ' + e); + } +} + +function is_enabled() { + return true; +} + +Object.assign(exports, { + initialize, + RuleSets, + is_enabled, +}); + +})(typeof exports == 'undefined' ? require.scopes.wasm = {} : exports); |