Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .coveralls.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
repo_token: mStRafz0Xc5kShTIDOInJXhjgq0QtCoJh
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
node_modules/
.nyc_output/
*.log

10 changes: 10 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
language: node_js
sudo: false
node_js:
- 6
- 8
- 9
script:
- npm run test-travis
after_script:
- npm run report-coverage
137 changes: 78 additions & 59 deletions index.js
Original file line number Diff line number Diff line change
@@ -1,69 +1,88 @@
var axios = require('axios')
var Abstract = require('abstract-random-access')
var inherits = require('inherits')
var http = require('http')
var https = require('https')
var randomAccess = require('random-access-storage')
var logger = require('./lib/logger')
var isNode = require('./lib/is-node')
var validUrl = require('./lib/valid-url')

var Store = function (filename, options) {
if (!(this instanceof Store)) return new Store(filename, options)
Abstract.call(this)
this.axios = axios.create({
baseURL: options.url,
responseType: 'arraybuffer',
timeout: 60000,
// keepAlive pools and reuses TCP connections, so it's faster
httpAgent: new http.Agent({ keepAlive: true }),
httpsAgent: new https.Agent({ keepAlive: true }),
// follow up to 10 HTTP 3xx redirects
maxRedirects: 10,
maxContentLength: 50 * 1000 * 1000 // cap at 50MB
})
this.url = options.url
this.file = filename
this.verbose = !!options.verbose
inherits(Store, Abstract)
var defaultOptions = {
responseType: 'arraybuffer',
timeout: 60000,
// follow up to 10 HTTP 3xx redirects
maxRedirects: 10,
maxContentLength: 50 * 1000 * 1000 // cap at 50MB,
}

Store.prototype._open = function (callback) {
if (this.verbose) console.log('Testing to see if server accepts range requests', this.url, this.file)
this.axios.head(this.file)
.then((response) => {
if (this.verbose) console.log('Received headers from server')
const accepts = response.headers['accept-ranges']
if (accepts && accepts.toLowerCase().indexOf('bytes') !== -1) {
return callback(null)
}
return callback(new Error('Accept-Ranges does not include "bytes"'))
})
.catch((err) => {
if (this.verbose) console.log('Error opening', this.file, '-', err)
callback(err)
})
if (isNode) {
var http = require('http')
var https = require('https')
// keepAlive pools and reuses TCP connections, so it's faster
defaultOptions.httpAgent = new http.Agent({ keepAlive: true })
defaultOptions.httpsAgent = new https.Agent({ keepAlive: true })
}

Store.prototype._read = function (offset, length, callback) {
var headers = {
range: `bytes=${offset}-${offset + length - 1}`
var randomAccessHttp = function (filename, options) {
var url = options && options.url
if (!filename || (!validUrl(filename) && !validUrl(url))) {
throw new Error('Expect first argument to be a valid URL or a relative path, with url set in options')
}
if (this.verbose) console.log('Trying to read', this.file, headers.Range)
this.axios.get(this.file, { headers: headers })
.then((response) => {
if (this.verbose) console.log('read', JSON.stringify(response.headers, null, 2))
callback(null, Buffer.from(response.data))
})
.catch((err) => {
if (this.verbose) {
console.log('error', this.file, headers.Range)
console.log(err, err.stack)
}
callback(err)
})
}
var axiosConfig = Object.assign({}, defaultOptions)
if (options) {
if (url) axiosConfig.baseURL = url
if (options.timeout) axiosConfig.timeout = options.timeout
if (options.maxRedirects) axiosConfig.maxRedirects = options.maxRedirects
if (options.maxContentLength) axiosConfig.maxContentLength = options.maxContentLength
}
var _axios = axios.create(axiosConfig)
var file = filename
var verbose = !!(options && options.verbose)

// This is a dummy write function - does not write, but fails silently
Store.prototype._write = function (offset, buffer, callback) {
if (this.verbose) console.log('trying to write', this.file, offset, buffer)
callback()
return randomAccess({
open: (req) => {
if (verbose) logger.log('Testing to see if server accepts range requests', url, file)
// should cache this
_axios.head(file)
.then((response) => {
if (verbose) logger.log('Received headers from server')
const accepts = response.headers['accept-ranges']
if (accepts && accepts.toLowerCase().indexOf('bytes') !== -1) {
return req.callback(null)
}
return req.callback(new Error('Accept-Ranges does not include "bytes"'))
})
.catch((err) => {
if (verbose) logger.log('Error opening', file, '-', err)
req.callback(err)
})
},
read: (req) => {
var headers = {
range: `bytes=${req.offset}-${req.offset + req.size - 1}`
}
if (verbose) logger.log('Trying to read', file, headers.Range)
_axios.get(file, { headers: headers })
.then((response) => {
if (verbose) logger.log('read', JSON.stringify(response.headers, null, 2))
req.callback(null, Buffer.from(response.data))
})
.catch((err) => {
if (verbose) {
logger.log('error', file, headers.Range)
logger.log(err, err.stack)
}
req.callback(err)
})
},
write: (req) => {
// This is a dummy write function - does not write, but fails silently
if (verbose) logger.log('trying to write', file, req.offset, req.data)
req.callback()
},
del: (req) => {
// This is a dummy del function - does not del, but fails silently
if (verbose) logger.log('trying to del', file, req.offset, req.size)
req.callback()
}
})
}

module.exports = Store
module.exports = randomAccessHttp
1 change: 1 addition & 0 deletions lib/is-node.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
module.exports = !!(process && process.release && process.release.name === 'node')
4 changes: 4 additions & 0 deletions lib/logger.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
// This is simply for testing
module.exports = {
log: console.log
}
7 changes: 7 additions & 0 deletions lib/valid-url.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
var url = require('url')

module.exports = function (str) {
if (typeof str !== 'string') return false
var parsed = url.parse(str)
return ['http:', 'https:'].includes(parsed.protocol)
}
19 changes: 15 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
"description": "A random access interface for files served over http",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
"test": "standard && tape tests/**.test.js",
"test-travis": "nyc tape tests/**.test.js | tap-spec",
"tdd": "tape-watch tests/**.test.js",
"report-coverage": "nyc report --reporter=text-lcov | coveralls"
},
"keywords": [
"http",
Expand All @@ -17,11 +20,19 @@
"url": "git+https://github.com/e-e-e/http-random-access.git"
},
"dependencies": {
"abstract-random-access": "^1.1.2",
"axios": "^0.17.0",
"inherits": "^2.0.3"
"random-access-storage": "^1.1.1",
"url": "^0.11.0"
},
"devDependencies": {
"standard": "^10.0.3"
"coveralls": "^3.0.0",
"nyc": "^11.4.1",
"proxyquire": "^1.8.0",
"sinon": "^4.2.2",
"standard": "^10.0.3",
"stoppable": "^1.0.5",
"tap-spec": "^4.1.1",
"tape": "^4.8.0",
"tape-watch": "^2.3.0"
}
}
4 changes: 3 additions & 1 deletion readme.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
# http-random-access

An implementation of [abstract-random-access](https://www.npmjs.com/package/abstract-random-access) to access content via http/s.
[![Build Status](https://travis-ci.org/e-e-e/http-random-access.svg?branch=master)](https://travis-ci.org/e-e-e/http-random-access) [![Coverage Status](https://coveralls.io/repos/github/e-e-e/http-random-access/badge.svg?branch=master)](https://coveralls.io/github/e-e-e/http-random-access?branch=master)

An implementation of [random-access-storage](https://www.npmjs.com/package/random-access-storage) to access content via http/s.
Providing the same interface as [random-access-file](https://www.npmjs.com/package/random-access-file) and [random-access-memory](https://www.npmjs.com/package/random-access-memory).

This implementation is intended as a drop-in replacement for random-access-file or random-access-memory in the dat-storage configuration. You might want to look at [random-access-http](https://www.npmjs.com/package/random-access-http) for an alternative implementation to see if it better suits your needs.
Expand Down
Loading