diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 0000000..5888d8a --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1 @@ +repo_token: mStRafz0Xc5kShTIDOInJXhjgq0QtCoJh diff --git a/.gitignore b/.gitignore index 552f221..65975a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ node_modules/ +.nyc_output/ *.log + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..3753e9d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: node_js +sudo: false +node_js: + - 6 + - 8 + - 9 +script: + - npm run test-travis +after_script: + - npm run report-coverage diff --git a/index.js b/index.js index 9601214..8efc0bd 100644 --- a/index.js +++ b/index.js @@ -1,69 +1,88 @@ var axios = require('axios') -var Abstract = require('abstract-random-access') -var inherits = require('inherits') -var http = require('http') -var https = require('https') +var randomAccess = require('random-access-storage') +var logger = require('./lib/logger') +var isNode = require('./lib/is-node') +var validUrl = require('./lib/valid-url') -var Store = function (filename, options) { - if (!(this instanceof Store)) return new Store(filename, options) - Abstract.call(this) - this.axios = axios.create({ - baseURL: options.url, - responseType: 'arraybuffer', - timeout: 60000, - // keepAlive pools and reuses TCP connections, so it's faster - httpAgent: new http.Agent({ keepAlive: true }), - httpsAgent: new https.Agent({ keepAlive: true }), - // follow up to 10 HTTP 3xx redirects - maxRedirects: 10, - maxContentLength: 50 * 1000 * 1000 // cap at 50MB - }) - this.url = options.url - this.file = filename - this.verbose = !!options.verbose - inherits(Store, Abstract) +var defaultOptions = { + responseType: 'arraybuffer', + timeout: 60000, + // follow up to 10 HTTP 3xx redirects + maxRedirects: 10, + maxContentLength: 50 * 1000 * 1000 // cap at 50MB, } -Store.prototype._open = function (callback) { - if (this.verbose) console.log('Testing to see if server accepts range requests', this.url, this.file) - this.axios.head(this.file) - .then((response) => { - if (this.verbose) console.log('Received headers from server') - const accepts = response.headers['accept-ranges'] - if (accepts && accepts.toLowerCase().indexOf('bytes') !== -1) { - return callback(null) - } - return callback(new Error('Accept-Ranges does not include "bytes"')) - }) - .catch((err) => { - if (this.verbose) console.log('Error opening', this.file, '-', err) - callback(err) - }) +if (isNode) { + var http = require('http') + var https = require('https') + // keepAlive pools and reuses TCP connections, so it's faster + defaultOptions.httpAgent = new http.Agent({ keepAlive: true }) + defaultOptions.httpsAgent = new https.Agent({ keepAlive: true }) } -Store.prototype._read = function (offset, length, callback) { - var headers = { - range: `bytes=${offset}-${offset + length - 1}` +var randomAccessHttp = function (filename, options) { + var url = options && options.url + if (!filename || (!validUrl(filename) && !validUrl(url))) { + throw new Error('Expect first argument to be a valid URL or a relative path, with url set in options') } - if (this.verbose) console.log('Trying to read', this.file, headers.Range) - this.axios.get(this.file, { headers: headers }) - .then((response) => { - if (this.verbose) console.log('read', JSON.stringify(response.headers, null, 2)) - callback(null, Buffer.from(response.data)) - }) - .catch((err) => { - if (this.verbose) { - console.log('error', this.file, headers.Range) - console.log(err, err.stack) - } - callback(err) - }) -} + var axiosConfig = Object.assign({}, defaultOptions) + if (options) { + if (url) axiosConfig.baseURL = url + if (options.timeout) axiosConfig.timeout = options.timeout + if (options.maxRedirects) axiosConfig.maxRedirects = options.maxRedirects + if (options.maxContentLength) axiosConfig.maxContentLength = options.maxContentLength + } + var _axios = axios.create(axiosConfig) + var file = filename + var verbose = !!(options && options.verbose) -// This is a dummy write function - does not write, but fails silently -Store.prototype._write = function (offset, buffer, callback) { - if (this.verbose) console.log('trying to write', this.file, offset, buffer) - callback() + return randomAccess({ + open: (req) => { + if (verbose) logger.log('Testing to see if server accepts range requests', url, file) + // should cache this + _axios.head(file) + .then((response) => { + if (verbose) logger.log('Received headers from server') + const accepts = response.headers['accept-ranges'] + if (accepts && accepts.toLowerCase().indexOf('bytes') !== -1) { + return req.callback(null) + } + return req.callback(new Error('Accept-Ranges does not include "bytes"')) + }) + .catch((err) => { + if (verbose) logger.log('Error opening', file, '-', err) + req.callback(err) + }) + }, + read: (req) => { + var headers = { + range: `bytes=${req.offset}-${req.offset + req.size - 1}` + } + if (verbose) logger.log('Trying to read', file, headers.Range) + _axios.get(file, { headers: headers }) + .then((response) => { + if (verbose) logger.log('read', JSON.stringify(response.headers, null, 2)) + req.callback(null, Buffer.from(response.data)) + }) + .catch((err) => { + if (verbose) { + logger.log('error', file, headers.Range) + logger.log(err, err.stack) + } + req.callback(err) + }) + }, + write: (req) => { + // This is a dummy write function - does not write, but fails silently + if (verbose) logger.log('trying to write', file, req.offset, req.data) + req.callback() + }, + del: (req) => { + // This is a dummy del function - does not del, but fails silently + if (verbose) logger.log('trying to del', file, req.offset, req.size) + req.callback() + } + }) } -module.exports = Store +module.exports = randomAccessHttp diff --git a/lib/is-node.js b/lib/is-node.js new file mode 100644 index 0000000..8310c84 --- /dev/null +++ b/lib/is-node.js @@ -0,0 +1 @@ +module.exports = !!(process && process.release && process.release.name === 'node') diff --git a/lib/logger.js b/lib/logger.js new file mode 100644 index 0000000..67dd2ff --- /dev/null +++ b/lib/logger.js @@ -0,0 +1,4 @@ +// This is simply for testing +module.exports = { + log: console.log +} diff --git a/lib/valid-url.js b/lib/valid-url.js new file mode 100644 index 0000000..f15479a --- /dev/null +++ b/lib/valid-url.js @@ -0,0 +1,7 @@ +var url = require('url') + +module.exports = function (str) { + if (typeof str !== 'string') return false + var parsed = url.parse(str) + return ['http:', 'https:'].includes(parsed.protocol) +} diff --git a/package.json b/package.json index 0159233..689b1ee 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,10 @@ "description": "A random access interface for files served over http", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "standard && tape tests/**.test.js", + "test-travis": "nyc tape tests/**.test.js | tap-spec", + "tdd": "tape-watch tests/**.test.js", + "report-coverage": "nyc report --reporter=text-lcov | coveralls" }, "keywords": [ "http", @@ -17,11 +20,19 @@ "url": "git+https://github.com/e-e-e/http-random-access.git" }, "dependencies": { - "abstract-random-access": "^1.1.2", "axios": "^0.17.0", - "inherits": "^2.0.3" + "random-access-storage": "^1.1.1", + "url": "^0.11.0" }, "devDependencies": { - "standard": "^10.0.3" + "coveralls": "^3.0.0", + "nyc": "^11.4.1", + "proxyquire": "^1.8.0", + "sinon": "^4.2.2", + "standard": "^10.0.3", + "stoppable": "^1.0.5", + "tap-spec": "^4.1.1", + "tape": "^4.8.0", + "tape-watch": "^2.3.0" } } diff --git a/readme.md b/readme.md index c24cceb..86d145b 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,8 @@ # http-random-access -An implementation of [abstract-random-access](https://www.npmjs.com/package/abstract-random-access) to access content via http/s. +[![Build Status](https://travis-ci.org/e-e-e/http-random-access.svg?branch=master)](https://travis-ci.org/e-e-e/http-random-access) [![Coverage Status](https://coveralls.io/repos/github/e-e-e/http-random-access/badge.svg?branch=master)](https://coveralls.io/github/e-e-e/http-random-access?branch=master) + +An implementation of [random-access-storage](https://www.npmjs.com/package/random-access-storage) to access content via http/s. Providing the same interface as [random-access-file](https://www.npmjs.com/package/random-access-file) and [random-access-memory](https://www.npmjs.com/package/random-access-memory). This implementation is intended as a drop-in replacement for random-access-file or random-access-memory in the dat-storage configuration. You might want to look at [random-access-http](https://www.npmjs.com/package/random-access-http) for an alternative implementation to see if it better suits your needs. diff --git a/tests/random-access-http.test.js b/tests/random-access-http.test.js new file mode 100644 index 0000000..5c74380 --- /dev/null +++ b/tests/random-access-http.test.js @@ -0,0 +1,181 @@ +var test = require('tape') +var proxyquire = require('proxyquire').noPreserveCache() +var sinon = require('sinon') +var stoppable = require('stoppable') +var http = require('http') +var port = 3000 + +var server + +var standardHandler = (req, res) => { + res.setHeader('Content-Type', 'plain/text') + res.setHeader('Accept-Ranges', 'Bytes') + if (req.method === 'HEAD') { + return res.end() + } + var range = req.headers.range + if (range) { + var match = range.match(/bytes=(\d+)-(\d+)/) + if (match) { + var a = parseInt(match[1], 10) + var b = parseInt(match[2], 10) + var str = '' + for (var i = a; i <= b; i++) { + str += (a % 10) + } + res.write(str) + } + } + res.end() +} + +function startServer (fn, cb) { + if (server && server.listening) { + return server.stop(() => startServer(fn, cb)) + } + server = stoppable(http.createServer(fn)) + server.listen(port, cb) +} + +function stopServer (t) { + if (!server.listening) return t.end() + server.stop(() => { + t.end() + }) +} + +test('it uses node http/s agent setting with keepAlive when run in node', (t) => { + var httpStub = sinon.stub() + var httpsStub = sinon.stub() + proxyquire('../index.js', { + 'http': { + Agent: httpStub + }, + 'https': { + Agent: httpsStub + } + }) + t.ok(httpStub.calledWithNew()) + t.ok(httpStub.calledWith({ keepAlive: true })) + t.ok(httpsStub.calledWithNew()) + t.ok(httpsStub.calledWith({ keepAlive: true })) + t.end() +}) + +test('it does not use node http/s when in browser', (t) => { + var httpStub = sinon.stub() + var httpsStub = sinon.stub() + proxyquire('../index.js', { + './lib/is-node': false, + 'http': { + Agent: httpStub + }, + 'https': { + Agent: httpsStub + } + }) + t.ok(httpStub.notCalled) + t.ok(httpsStub.notCalled) + t.end() +}) + +test('raHttp.open() callback returns error if server does not support range requests', (t) => { + var raHttp = require('../index.js') + var withoutRangeSupportHandler = (req, res) => { + res.end() + } + startServer(withoutRangeSupportHandler, (err) => { + t.error(err) + var ra = raHttp('without-range-support', { url: 'http://localhost:3000' }) + ra.read(0, 10, (err, res) => { + t.ok(err.message.search(/Not opened/) !== -1) + stopServer(t) + }) + }) +}) + +test('raHttp.open() callback returns error if call to axios.head() fails', (t) => { + var raHttp = require('../index.js') + var notFoundHandler = (req, res) => { + res.statusCode = 404 + res.end() + } + startServer(notFoundHandler, (err) => { + t.error(err) + var ra = raHttp('not-found', { url: 'http://localhost:3000' }) + ra.read(0, 10, (err, res) => { + t.ok(err.message.search(/Not opened/) !== -1) + stopServer(t) + }) + }) +}) + +test('raHttp.read() returns a buffer of length requested', (t) => { + var raHttp = require('../index.js') + startServer(standardHandler, (err) => { + t.error(err) + var ra = raHttp('test', { url: 'http://localhost:3000' }) + ra.read(10, 20, (err, data) => { + t.error(err) + t.ok(data instanceof Buffer) + t.same(data.length, 20) + stopServer(t) + }) + }) +}) + +test('raHttp.write does not throw error', (t) => { + var raHttp = require('../index.js') + startServer(standardHandler, (err) => { + t.error(err) + var ra = raHttp('test-write', { url: 'http://localhost:3000' }) + t.doesNotThrow(ra.write.bind(ra, 10, 'some-data')) + stopServer(t) + }) +}) + +test('raHttp.write logs with options.verbose === true', (t) => { + var stub = sinon.stub() + var proxyRaHttp = proxyquire('../index', { + './lib/logger': { + log: stub + } + }) + startServer(standardHandler, (err) => { + t.error(err) + var ra = proxyRaHttp('test-write', { url: 'http://localhost:3000', verbose: true }) + ra.write(10, 'some-data', (err, res) => { + t.error(err) + t.ok(stub.calledWith('trying to write', 'test-write', 10, 'some-data')) + stopServer(t) + }) + }) +}) + +test('raHttp.del does not throw error', (t) => { + var raHttp = require('../index.js') + startServer(standardHandler, (err) => { + t.error(err) + var ra = raHttp('test-del', { url: 'http://localhost:3000' }) + t.doesNotThrow(ra.del.bind(ra, 10, 100)) + stopServer(t) + }) +}) + +test('raHttp.del logs with options.verbose === true', (t) => { + var stub = sinon.stub() + var proxyRaHttp = proxyquire('../index', { + './lib/logger': { + log: stub + } + }) + startServer(standardHandler, (err) => { + t.error(err) + var ra = proxyRaHttp('test-del', { url: 'http://localhost:3000', verbose: true }) + ra.del(10, 100, (err, res) => { + t.error(err) + t.ok(stub.calledWith('trying to del', 'test-del', 10, 100)) + stopServer(t) + }) + }) +}) diff --git a/tests/simple.test.js b/tests/simple.test.js new file mode 100644 index 0000000..93831bd --- /dev/null +++ b/tests/simple.test.js @@ -0,0 +1,73 @@ +var rahttp = require('../') +var tape = require('tape') + +var testUrl = 'https://ia800309.us.archive.org/2/items/Popeye_Nearlyweds/Popeye_Nearlyweds_512kb.mp4' + +tape('open and close', function (t) { + t.plan(4) + var popeye = rahttp(testUrl) + popeye.on('close', function () { + t.pass('close event fired') + }) + popeye.on('open', function () { + t.pass('open event fired') + }) + popeye.open(function (err) { + t.error(err, 'url opened without error') + popeye.close(function (err) { + t.error(err, 'url closed without error') + }) + }) +}) + +tape('read 10 bytes', function (t) { + t.plan(3) + var popeye = rahttp(testUrl) + var length = 10 + popeye.read(0, 10, function (err, buf) { + t.error(err, 'url read without error') + t.equal(buf.length, length) + popeye.close(function (err) { + t.error(err, 'url closed without error') + }) + }) +}) + +tape('read 100 bytes at an offset of 2000', function (t) { + t.plan(3) + var popeye = rahttp(testUrl) + var length = 100 + popeye.read(2000, length, function (err, buf) { + t.error(err, 'url read without error') + t.equal(buf.length, length) + popeye.close(function (err) { + t.error(err, 'url closed without error') + }) + }) +}) + +tape('read from https flickr', function (t) { + t.plan(3) + var popeye = rahttp('https://c1.staticflickr.com/3/2892/12196828014_eb4ffac150_o.jpg') + var length = 10 + popeye.read(0, 10, function (err, buf) { + t.error(err, 'url read without error') + t.equal(buf.length, length) + popeye.close(function (err) { + t.error(err, 'url closed without error') + }) + }) +}) + +tape('read from http flickr', function (t) { + t.plan(3) + var popeye = rahttp('http://c1.staticflickr.com/3/2892/12196828014_eb4ffac150_o.jpg') + var length = 10 + popeye.read(30, 10, function (err, buf) { + t.error(err, 'url read without error') + t.equal(buf.length, length) + popeye.close(function (err) { + t.error(err, 'url closed without error') + }) + }) +}) diff --git a/tests/valid-url.test.js b/tests/valid-url.test.js new file mode 100644 index 0000000..0b86c98 --- /dev/null +++ b/tests/valid-url.test.js @@ -0,0 +1,30 @@ +var test = require('tape') +var validUrl = require('../lib/valid-url') + +test('validUrl returns false if url is not a string', (t) => { + t.notOk(validUrl()) + t.notOk(validUrl(null)) + t.notOk(validUrl({})) + t.notOk(validUrl(['foo'])) + t.end() +}) + +test('validUrl returns false for rubbish strings', (t) => { + t.notOk(validUrl('f234324 ff43 f43f4 f43 ')) + t.notOk(validUrl('company tax cuts will increase wages...')) + t.end() +}) + +test('validUrl returns false if url is not http/s', (t) => { + t.notOk(validUrl('ftp://ok.com')) + t.notOk(validUrl('ssh://this:not@ok.net')) + t.notOk(validUrl('mailto:not@ok.net')) + t.end() +}) + +test('validUrl returns true if url is http/s', (t) => { + t.ok(validUrl('http://theanarchistlibrary.org/')) + t.ok(validUrl('http://127.0.0.1:4000')) + t.ok(validUrl('https://en.wikipedia.org/wiki/S._R._Ranganathan')) + t.end() +})