Initial release

.travis.yml

@@ -4,7 +4,6 @@ script:
   - "npm run travis"
 
 node_js:
-  - "4"
   - "6"
   - "8"
 
@@ -15,7 +14,7 @@ branches:
 matrix:
   fast_finish: true
   include:
-    - node_js: "4"
+    - node_js: "6"
       env: COVERAGE=true
   allow_failures:
     - env: COVERAGE=true

README.md

@@ -12,10 +12,150 @@
 
 ## Usage
 
+Use [got](https://www.npmjs.com/package/got) to make an HTTP request with automatic retries for network errors.
+
+Designed for downloading large files. If transfer fails part-way through, will retry, resuming from point where previous attempt finished, using HTTP range headers.
+
+### gotResume( [url], [options] ) -> Stream
+
+```js
+const stream = gotResume('http://google.com/');
+const writeStream = fs.createWriteStream('foo.html');
+stream.pipe(writeStream);
+```
+
+### Options
+
+#### url
+
+Alternative way to provide URL.
+
+```js
+const stream = gotResume( {url: 'http://google.com/'} );
+```
+
+#### attempts
+
+Max number of attempts in a row yielding no data (i.e. failed connection, empty response) before aborting.
+
+Set to `0` for no limit. Default `10`.
+
+```js
+const stream = gotResume( 'http://google.com/', {attempts: 0} );
+```
+
+#### attemptsTotal
+
+Max number of total attempts before aborting.
+
+Set to `0` for no limit. Default `0`.
+
+#### backoff
+
+Function to determine wait in milliseconds before retry. Called with arguments `(attempt, transfer)`.
+
+`attempt` is what attempt number for current chunk (reset to zero when a new chunk is successfully received).
+
+`transfer` is the internal `Transfer` object (see below).
+
+If function returns `false`, the transfer is aborted. If using this mechanism, `options.attempts` should be set to `0` so it does not interfere.
+
+If not provided, default backoff function starts with 1000ms and doubles each time:
+
+```js
+function backoff(attempt) {
+	return Math.pow(2, attempt - 1) * 1000;
+};
+```
+
+#### length
+
+Length of response expected in bytes. If undefined, `length` will be determined from HTTP `content-length` header.
+
+If server does not provide `content-length` header, and `options.length` is not set, transfer will be considered complete when first successful request complete.
+
+If `options.length` is set, only that number of bytes will be fetched.
+
+#### offset
+
+Number of bytes at start of resource to skip. Default `0`.
+
+NB Number of bytes to be streamed is `length - offset`. i.e. `length` is actually not length of response, but end of range.
+
+e.g. `{offset: 5, length: 10}` will stream 5 bytes.
+
+#### pre
+
+An async function that is run before each chunk request. Must return a `Promise`. Request will commence once promise resolves.
+
+Useful where some authentication requires being set up before transfer HTTP request, or where resource has a different URL each time (e.g. Amazon EC2).
+
+`pre` function is called with `Transfer` object (see below). To set URL for next chunk, `pre` should set `transfer.url`. To alter `got` options, should set `transfer.gotOptions`.
+
+```js
+function pre(transfer) {
+	transfer.gotOptions.headers['user-agent'] = 'Stealth 2.0';
+	return Promise.resolve();
+}
+```
+
+#### log
+
+Function to receive logging information e.g. HTTP responses
+
+```js
+const stream = gotResume( 'http://google.com/', {log: console.log} );
+```
+
+#### got
+
+Options to pass to `got`. See [got documentation](https://www.npmjs.com/package/got) for details.
+
+```js
+const stream = gotResume( 'http://google.com/', {got: {method: 'POST'} } );
+```
+
+### Events
+
+#### error
+
+Emitted with a `gotResume.TransferError` on stream when transfer fails and has exhausted retries.
+
+#### end
+
+Emitted when transfer completes.
+
+NB Is also emitted after `error` event if transfer fails.
+
+#### request
+
+Emitted with HTTP request object when first HTTP request made to server.
+
+NB Not emitted again for each retry HTTP request. You cannot abort the transfer with `request.abort()` as the request may be finished if a retry has happened.
+
+#### response
+
+Emitted when first successful HTTP response is received. NB Not emitted again for each retry HTTP request.
+
+Useful for e.g. determining length of transfer:
+
+```js
+const stream = gotResume('http://google.com/');
+stream.on( 'response', res => console.log('Length: ', stream.transfer.length) );
+```
+
+### Transfer object
+
+A transfer in progress is represented internally as an instance of `gotResume.Transfer` class.
+
+Transfer object is stored as `stream.transfer` and also passed to `options.pre` function.
+
 ## Tests
 
 Use `npm test` to run the tests. Use `npm run cover` to check coverage.
 
+No tests yet but seems to work fine!
+
 ## Changelog
 
 See [changelog.md](https://github.com/overlookmotel/got-resume/blob/master/changelog.md)

changelog.md

@@ -1,3 +1,5 @@
 # Changelog
 
 ## Next
+
+* Initial release

lib/backoff.js

@@ -0,0 +1,19 @@
+/* --------------------
+ * got-resume module
+ * Default backoff function
+ * ------------------*/
+
+'use strict';
+
+// Exports
+
+/**
+ * Default backoff function
+ * 1 sec on first attempt, increasing exponentially, doubling each time.
+ * On 10th attempt is 512 secs = 8.5 mins.
+ * @param {number} attempt - Attempt number (starting at 1)
+ * @returns {number} - Wait (in milliseconds)
+ */
+module.exports = function(attempt) {
+	return Math.pow(2, attempt - 1) * 1000;
+};

lib/errors.js

@@ -0,0 +1,34 @@
+/* --------------------
+ * got-resume module
+ * Errors
+ * ------------------*/
+
+'use strict';
+
+// Exports
+class BaseError extends Error {
+	constructor(message) {
+		super(message);
+		this.name = 'GotResumeError';
+	}
+}
+
+class OptionsError extends BaseError {
+	constructor(message) {
+		super(message);
+		this.name = 'GotResumeOptionsError';
+	}
+}
+
+class TransferError extends BaseError {
+	constructor(message) {
+		super(message);
+		this.name = 'GotResumeTransferError';
+	}
+}
+
+module.exports = {
+	Error: BaseError,
+	OptionsError,
+	TransferError
+};

lib/index.js

@@ -4,6 +4,82 @@
 
 'use strict';
 
+// Imports
+const errors = require('./errors'),
+	Transfer = require('./transfer');
+
 // Exports
-module.exports = function() {
+/**
+ * Fetch URL with retries if failure
+ * @param {string} [url] - URL (optional)
+ * @param {Object} [options] - Options object
+ * @param {number} [options.attempts=10] - Number of attempts to make before failing (0 for no limit)
+ * @param {number} [options.attemptsTotal=0] - Total number of attempts to make before failing (0 for no limit)
+ * @param {Object} [options.got] - Options to pass to `got` module
+ * @param {number} [options.length] - Length of transfer (NB is actually range end - does not take into account options.offset)
+ * @param {number} [options.offset=0] - Number of bytes at start of file to skip
+ * @param {Function} [options.backoff] - Function called with `backoff(attempt, transfer)` and should return milliseconds to wait before next attempt
+ * @param {Function} [options.pre] - Function to call before HTTP requests. Is passed `transfer` object, should set `transfer.url` and `transfer.gotOptions` and return a promise.
+ * @param {Function} [options.log] - Function to call with logging information
+ * @returns {Stream}
+ */
+module.exports = function(url, options) {
+	// Conform params
+	if (url && typeof url == 'object') {
+		options = url;
+		url = undefined;
+	} else if (url == null) {
+		if (!options) throw new errors.OptionsError('url or options must be provided');
+		if (typeof options != 'object') throw new errors.OptionsError('options must be an object');
+	} else if (typeof url != 'string') {
+		throw new errors.OptionsError('url must be a string');
+	} else if (!options) {
+		options = {};
+	} else if (typeof options != 'object') {
+		throw new errors.OptionsError('options must be an object');
+	}
+
+	// Set default options
+	options = Object.assign({
+		attempts: 10,
+		attemptsTotal: 0,
+		backoff: undefined,
+		length: undefined,
+		offset: 0,
+		pre: undefined,
+		log: undefined
+	}, options);
+
+	if (url) options.url = url;
+
+	if (!options.url && !options.pre) throw new errors.OptionsError('url or pre function must be provided');
+
+	// Set default got options
+	if (options.got && typeof options.got != 'object') throw new errors.OptionsError('options.got must be an object');
+	options.got = Object.assign({retries: 0}, options.got || {});
+
+	const {got} = options;
+	if (got.headers && typeof got.headers != 'object') throw new errors.OptionsError('options.got.headers must be an object');
+	got.headers = Object.assign({}, got.headers || {});
+
+	if (typeof got.timeout == 'number') {
+		got.timeout = {request: got.timeout};
+	} else if (got.timeout && typeof got.timeout != 'object') {
+		throw new errors.OptionsError('options.got.timeout must be an object or a number');
+	}
+	got.timeout = Object.assign({connect: 5000, socket: 5000, request: 5000}, got.timeout || {});
+
+	// Start transfer
+	const transfer = new Transfer(options);
+	transfer.start();
+
+	// Return stream
+	return transfer.stream;
 };
+
+/*
+ * Add Transfer and errors to export
+ */
+
+module.exports.Transfer = Transfer;
+Object.assign(module.exports, errors);