/*
* This file is part of Adblock Plus ,
* Copyright (C) 2006-2015 Eyeo GmbH
*
* Adblock Plus is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 3 as
* published by the Free Software Foundation.
*
* Adblock Plus is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Adblock Plus. If not, see .
*/
/**
* @fileOverview Downloads a set of URLs in regular time intervals.
*/
let {Utils} = require("utils");
let MILLIS_IN_SECOND = exports.MILLIS_IN_SECOND = 1000;
let MILLIS_IN_MINUTE = exports.MILLIS_IN_MINUTE = 60 * MILLIS_IN_SECOND;
let MILLIS_IN_HOUR = exports.MILLIS_IN_HOUR = 60 * MILLIS_IN_MINUTE;
let MILLIS_IN_DAY = exports.MILLIS_IN_DAY = 24 * MILLIS_IN_HOUR;
/**
* Creates a new downloader instance.
* @param {Function} dataSource Function that will yield downloadable objects on each check
* @param {Integer} initialDelay Number of milliseconds to wait before the first check
* @param {Integer} checkInterval Interval between the checks
* @constructor
*/
let Downloader = exports.Downloader = function Downloader(dataSource, initialDelay, checkInterval)
{
this.dataSource = dataSource;
this._timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
this._timer.initWithCallback(function()
{
this._timer.delay = checkInterval;
this._doCheck();
}.bind(this), initialDelay, Ci.nsITimer.TYPE_REPEATING_SLACK);
this._downloading = Object.create(null);
}
Downloader.prototype =
{
/**
* Timer triggering the downloads.
* @type nsITimer
*/
_timer: null,
/**
* Map containing the URLs of objects currently being downloaded as its keys.
*/
_downloading: null,
/**
* Function that will yield downloadable objects on each check.
* @type Function
*/
dataSource: null,
/**
* Maximal time interval that the checks can be left out until the soft
* expiration interval increases.
* @type Integer
*/
maxAbsenceInterval: 1 * MILLIS_IN_DAY,
/**
* Minimal time interval before retrying a download after an error.
* @type Integer
*/
minRetryInterval: 1 * MILLIS_IN_DAY,
/**
* Maximal allowed expiration interval, larger expiration intervals will be
* corrected.
* @type Integer
*/
maxExpirationInterval: 14 * MILLIS_IN_DAY,
/**
* Maximal number of redirects before the download is considered as failed.
* @type Integer
*/
maxRedirects: 5,
/**
* Called whenever expiration intervals for an object need to be adapted.
* @type Function
*/
onExpirationChange: null,
/**
* Callback to be triggered whenever a download starts.
* @type Function
*/
onDownloadStarted: null,
/**
* Callback to be triggered whenever a download finishes successfully. The
* callback can return an error code to indicate that the data is wrong.
* @type Function
*/
onDownloadSuccess: null,
/**
* Callback to be triggered whenever a download fails.
* @type Function
*/
onDownloadError: null,
/**
* Checks whether anything needs downloading.
*/
_doCheck: function()
{
let now = Date.now();
for (let downloadable of this.dataSource())
{
if (downloadable.lastCheck && now - downloadable.lastCheck > this.maxAbsenceInterval)
{
// No checks for a long time interval - user must have been offline, e.g.
// during a weekend. Increase soft expiration to prevent load peaks on the
// server.
downloadable.softExpiration += now - downloadable.lastCheck;
}
downloadable.lastCheck = now;
// Sanity check: do expiration times make sense? Make sure people changing
// system clock don't get stuck with outdated subscriptions.
if (downloadable.hardExpiration - now > this.maxExpirationInterval)
downloadable.hardExpiration = now + this.maxExpirationInterval;
if (downloadable.softExpiration - now > this.maxExpirationInterval)
downloadable.softExpiration = now + this.maxExpirationInterval;
// Notify the caller about changes to expiration parameters
if (this.onExpirationChange)
this.onExpirationChange(downloadable);
// Does that object need downloading?
if (downloadable.softExpiration > now && downloadable.hardExpiration > now)
continue;
// Do not retry downloads too often
if (downloadable.lastError && now - downloadable.lastError < this.minRetryInterval)
continue;
this._download(downloadable, 0);
}
},
/**
* Stops the periodic checks.
*/
cancel: function()
{
this._timer.cancel();
},
/**
* Checks whether an address is currently being downloaded.
*/
isDownloading: function(/**String*/ url) /**Boolean*/
{
return url in this._downloading;
},
/**
* Starts downloading for an object.
*/
download: function(/**Downloadable*/ downloadable)
{
// Make sure to detach download from the current execution context
Utils.runAsync(this._download.bind(this, downloadable, 0));
},
/**
* Generates the real download URL for an object by appending various
* parameters.
*/
getDownloadUrl: function(/**Downloadable*/ downloadable) /** String*/
{
let {addonName, addonVersion, application, applicationVersion, platform, platformVersion} = require("info");
let url = downloadable.redirectURL || downloadable.url;
if (url.indexOf("?") >= 0)
url += "&";
else
url += "?";
// We limit the download count to 4+ to keep the request anonymized
let downloadCount = downloadable.downloadCount;
if (downloadCount > 4)
downloadCount = "4+";
url += "addonName=" + encodeURIComponent(addonName) +
"&addonVersion=" + encodeURIComponent(addonVersion) +
"&application=" + encodeURIComponent(application) +
"&applicationVersion=" + encodeURIComponent(applicationVersion) +
"&platform=" + encodeURIComponent(platform) +
"&platformVersion=" + encodeURIComponent(platformVersion) +
"&lastVersion=" + encodeURIComponent(downloadable.lastVersion) +
"&downloadCount=" + encodeURIComponent(downloadCount);
return url;
},
_download: function(downloadable, redirects)
{
if (this.isDownloading(downloadable.url))
return;
let downloadUrl = this.getDownloadUrl(downloadable);
let request = null;
let errorCallback = function errorCallback(error)
{
let channelStatus = -1;
try
{
channelStatus = request.channel.status;
} catch (e) {}
let responseStatus = request.status;
Cu.reportError("Adblock Plus: Downloading URL " + downloadable.url + " failed (" + error + ")\n" +
"Download address: " + downloadUrl + "\n" +
"Channel status: " + channelStatus + "\n" +
"Server response: " + responseStatus);
if (this.onDownloadError)
{
// Allow one extra redirect if the error handler gives us a redirect URL
let redirectCallback = null;
if (redirects <= this.maxRedirects)
{
redirectCallback = function redirectCallback(url)
{
downloadable.redirectURL = url;
this._download(downloadable, redirects + 1);
}.bind(this);
}
this.onDownloadError(downloadable, downloadUrl, error, channelStatus, responseStatus, redirectCallback);
}
}.bind(this);
try
{
request = new XMLHttpRequest();
request.mozBackgroundRequest = true;
request.open("GET", downloadUrl);
}
catch (e)
{
errorCallback("synchronize_invalid_url");
return;
}
try {
request.overrideMimeType("text/plain");
request.channel.loadFlags = request.channel.loadFlags |
request.channel.INHIBIT_CACHING |
request.channel.VALIDATE_ALWAYS;
// Override redirect limit from preferences, user might have set it to 1
if (request.channel instanceof Ci.nsIHttpChannel)
request.channel.redirectionLimit = this.maxRedirects;
}
catch (e)
{
Cu.reportError(e)
}
request.addEventListener("error", function(event)
{
if (onShutdown.done)
return;
delete this._downloading[downloadable.url];
errorCallback("synchronize_connection_error");
}.bind(this), false);
request.addEventListener("load", function(event)
{
if (onShutdown.done)
return;
delete this._downloading[downloadable.url];
// Status will be 0 for non-HTTP requests
if (request.status && request.status != 200)
{
errorCallback("synchronize_connection_error");
return;
}
downloadable.downloadCount++;
this.onDownloadSuccess(downloadable, request.responseText, errorCallback, function redirectCallback(url)
{
if (redirects >= this.maxRedirects)
errorCallback("synchronize_connection_error");
else
{
downloadable.redirectURL = url;
this._download(downloadable, redirects + 1);
}
}.bind(this));
}.bind(this), false);
request.send(null);
this._downloading[downloadable.url] = true;
if (this.onDownloadStarted)
this.onDownloadStarted(downloadable);
},
/**
* Produces a soft and a hard expiration interval for a given supplied
* expiration interval.
* @return {Array} soft and hard expiration interval
*/
processExpirationInterval: function(/**Integer*/ interval)
{
interval = Math.min(Math.max(interval, 0), this.maxExpirationInterval);
let soft = Math.round(interval * (Math.random() * 0.4 + 0.8));
let hard = interval * 2;
let now = Date.now();
return [now + soft, now + hard];
}
};
/**
* An object that can be downloaded by the downloadable
* @param {String} url URL that has to be requested for the object
* @constructor
*/
let Downloadable = exports.Downloadable = function Downloadable(url)
{
this.url = url;
}
Downloadable.prototype =
{
/**
* URL that has to be requested for the object.
* @type String
*/
url: null,
/**
* URL that the download was redirected to if any.
* @type String
*/
redirectURL: null,
/**
* Time of last download error or 0 if the last download was successful.
* @type Integer
*/
lastError: 0,
/**
* Time of last check whether the object needs downloading.
* @type Integer
*/
lastCheck: 0,
/**
* Object version corresponding to the last successful download.
* @type Integer
*/
lastVersion: 0,
/**
* Soft expiration interval, will increase if no checks are performed for a
* while.
* @type Integer
*/
softExpiration: 0,
/**
* Hard expiration interval, this is fixed.
* @type Integer
*/
hardExpiration: 0,
/**
* Number indicating how often the object was downloaded.
* @type Integer
*/
downloadCount: 0,
};