From f1ab34cce4d98b00db2fbc252622deed5f0a9c53 Mon Sep 17 00:00:00 2001 From: Benjamin Forster Date: Mon, 5 Feb 2018 11:05:13 +0100 Subject: [PATCH 1/6] update readme and package --- package.json | 6 +++--- readme.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 1ee6cb7..de61521 100644 --- a/package.json +++ b/package.json @@ -15,11 +15,11 @@ "license": "MIT", "repository": { "type": "git", - "url": "git+https://github.com/e-e-e/random-access-s3.git" + "url": "git+https://github.com/random-access-storage/random-access-s3.git" }, "dependencies": { - "abstract-random-access": "^1.1.2", - "inherits": "^2.0.3" + "inherits": "^2.0.3", + "random-access-storage": "^1.1.0" }, "devDependencies": { "standard": "^10.0.3" diff --git a/readme.md b/readme.md index 3ded36a..1d0e776 100644 --- a/readme.md +++ b/readme.md @@ -1,11 +1,11 @@ # random-access-s3 -An implementation of [abstract-random-access](https://www.npmjs.com/package/abstract-random-access) on top of an AWS S3 bucket. +An implementation of [random-access-storage](https://www.npmjs.com/package/random-access-storage) on top of an AWS S3 bucket. Providing the same interface as [random-access-file](https://www.npmjs.com/package/random-access-file) and [random-access-memory](https://www.npmjs.com/package/random-access-memory). ## Why? -This is an experiment to see if we can serve [dat](http://datproject.org) data over aws s3. It is possible. +This is an experiment to see if we can serve [dat](http://datproject.org) data over aws s3. It is possible. **TLDR;** Latency is a killer. ## Installation From 85e39278ce009e93fca7216005ec9c0ba5ada9cb Mon Sep 17 00:00:00 2001 From: Benjamin Forster Date: Mon, 5 Feb 2018 11:18:01 +0100 Subject: [PATCH 2/6] update s3 to use random-access-storage --- index.js | 67 ++++++++++++++++++++++++++-------------------------- package.json | 2 +- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/index.js b/index.js index 4fd4327..e681366 100644 --- a/index.js +++ b/index.js @@ -1,47 +1,46 @@ var S3 = require('aws-sdk/clients/s3') -var Abstract = require('abstract-random-access') -var inherits = require('inherits') +var randomAccess = require('random-access-storage') function sanitizeKeyForS3 (key) { if (typeof key === 'string' && key.length && key[0] === '/') return key.slice(1) return key } -var Store = function (filename, options) { - if (!(this instanceof Store)) return new Store(filename, options) - Abstract.call(this) - this.s3 = options.s3 || new S3() - this.key = sanitizeKeyForS3(filename) - this.bucket = options.bucket - this.verbose = !!options.verbose - inherits(Store, Abstract) -} - -Store.prototype._read = function (offset, length, callback) { - var params = { - Bucket: this.bucket, - Key: this.key, - Range: `bytes=${offset}-${offset + length - 1}` - } - if (this.verbose) console.log('Trying to read', this.key, params.Range) - this.s3.getObject(params, (err, data) => { - if (err) { - if (this.verbose) { - console.log('error', this.key, params.Range) - console.log(err, err.stack) +function s3 (filename, options) { + const s3 = options.s3 || new S3() + const key = sanitizeKeyForS3(filename) + const bucket = options.bucket + const verbose = !!options.verbose + return randomAccess({ + read: function (req) { + var params = { + Bucket: bucket, + Key: key, + Range: `bytes=${req.offset}-${req.offset + req.size - 1}` } - return callback(err) + if (verbose) console.log('Trying to read', key, params.Range) + s3.getObject(params, (err, data) => { + if (err) { + if (verbose) { + console.log('error', key, params.Range) + console.log(err, err.stack) + } + return req.callback(err) + } + if (verbose) console.log('read', data.Body) + req.callback(null, data.Body) + }) + }, + write: function (req) { + if (verbose) console.log('trying to write', key, req.offset, req.data) + req.callback() + }, + del: function (req) { + if (verbose) console.log('trying to del', key, req.offset, req.size) + req.callback() } - if (this.verbose) console.log('read', data.Body) - callback(null, data.Body) }) } -// This is a dummy write function - does not write, but fails silently -Store.prototype._write = function (offset, buffer, callback) { - if (this.verbose) console.log('trying to write', this.key, offset, buffer) - callback() -} - -module.exports = Store +module.exports = s3 diff --git a/package.json b/package.json index de61521..c4e1f1e 100644 --- a/package.json +++ b/package.json @@ -18,10 +18,10 @@ "url": "git+https://github.com/random-access-storage/random-access-s3.git" }, "dependencies": { - "inherits": "^2.0.3", "random-access-storage": "^1.1.0" }, "devDependencies": { + "aws-sdk": "^2.188.0", "standard": "^10.0.3" } } From 7e6c052d7fc56ad4838a40cfea67d15d203fd2fb Mon Sep 17 00:00:00 2001 From: Benjamin Forster Date: Mon, 5 Feb 2018 11:37:28 +0100 Subject: [PATCH 3/6] move aws-sdk into deps --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index c4e1f1e..6abfc4b 100644 --- a/package.json +++ b/package.json @@ -18,10 +18,9 @@ "url": "git+https://github.com/random-access-storage/random-access-s3.git" }, "dependencies": { + "aws-sdk": "^2.188.0", "random-access-storage": "^1.1.0" }, "devDependencies": { - "aws-sdk": "^2.188.0", - "standard": "^10.0.3" } } From 1dfeb2422c080cdc7300695707ed40ad401ea9ae Mon Sep 17 00:00:00 2001 From: Benjamin Forster Date: Thu, 8 Feb 2018 11:45:31 +0100 Subject: [PATCH 4/6] add tests --- .travis.yml | 6 +++ index.js | 23 +++++---- lib/logger.js | 3 ++ package.json | 8 +++- tests/ras3.test.js | 116 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 145 insertions(+), 11 deletions(-) create mode 100644 .travis.yml create mode 100644 lib/logger.js create mode 100644 tests/ras3.test.js diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..ddb052e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +language: node_js +sudo: false +node_js: + - 6 + - 8 + - 9 diff --git a/index.js b/index.js index e681366..18fc26f 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,6 @@ var S3 = require('aws-sdk/clients/s3') - var randomAccess = require('random-access-storage') +var logger = require('./lib/logger') function sanitizeKeyForS3 (key) { if (typeof key === 'string' && key.length && key[0] === '/') return key.slice(1) @@ -8,10 +8,13 @@ function sanitizeKeyForS3 (key) { } function s3 (filename, options) { - const s3 = options.s3 || new S3() + if (!filename) throw new Error('Random Access S3 requires a filename!') + if (!options) throw new Error('Random Access S3 requires configuration options to be set!') + if (!options.bucket) throw new Error('Random Access S3 requires options.bucket!') + const s3 = (options && options.s3) || new S3() const key = sanitizeKeyForS3(filename) - const bucket = options.bucket - const verbose = !!options.verbose + const bucket = options && options.bucket + const verbose = !!(options && options.verbose) return randomAccess({ read: function (req) { var params = { @@ -19,25 +22,25 @@ function s3 (filename, options) { Key: key, Range: `bytes=${req.offset}-${req.offset + req.size - 1}` } - if (verbose) console.log('Trying to read', key, params.Range) + if (verbose) logger.log('Trying to read', key, params.Range) s3.getObject(params, (err, data) => { if (err) { if (verbose) { - console.log('error', key, params.Range) - console.log(err, err.stack) + logger.log('error', key, params.Range) + logger.log(err, err.stack) } return req.callback(err) } - if (verbose) console.log('read', data.Body) + if (verbose) logger.log('read', data.Body) req.callback(null, data.Body) }) }, write: function (req) { - if (verbose) console.log('trying to write', key, req.offset, req.data) + if (verbose) logger.log('trying to write', key, req.offset, req.data) req.callback() }, del: function (req) { - if (verbose) console.log('trying to del', key, req.offset, req.size) + if (verbose) logger.log('trying to del', key, req.offset, req.size) req.callback() } }) diff --git a/lib/logger.js b/lib/logger.js new file mode 100644 index 0000000..e6c91b6 --- /dev/null +++ b/lib/logger.js @@ -0,0 +1,3 @@ +module.exports = { + log: console.log +} diff --git a/package.json b/package.json index 6abfc4b..4e3cf23 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "A read-only random access interface for aws s3 buckets", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "standard && tape tests/**.test.js", + "tdd": "tape-watch tests/**.test.js" }, "keywords": [ "aws", @@ -22,5 +23,10 @@ "random-access-storage": "^1.1.0" }, "devDependencies": { + "proxyquire": "^1.8.0", + "sinon": "^4.2.2", + "standard": "^10.0.3", + "tape": "^4.8.0", + "tape-watch": "^2.3.0" } } diff --git a/tests/ras3.test.js b/tests/ras3.test.js new file mode 100644 index 0000000..2ef68f0 --- /dev/null +++ b/tests/ras3.test.js @@ -0,0 +1,116 @@ +const test = require('tape') +const sinon = require('sinon') +const proxyquire = require('proxyquire') +const ras3 = require('../index') + +test('it throws error if not provided a filename', (t) => { + t.throws(ras3.bind({}, undefined, {}), /Random Access S3 requires a filename!/) + t.end() +}) + +test('it throws error if not provided options', (t) => { + t.throws(ras3.bind({}, 'filename', undefined), /Random Access S3 requires configuration options to be set!/) + t.end() +}) + +test('it throws error if not provided options', (t) => { + t.throws(ras3.bind({}, 'filename', {}), /Random Access S3 requires options.bucket/) + t.end() +}) + +test('ras3 returns a object with expected interface', (t) => { + var s3 = ras3('filename', { bucket: 'fake-bucket' }) + t.ok(typeof s3.read === 'function') + t.ok(typeof s3.write === 'function') + t.ok(typeof s3.del === 'function') + t.end() +}) + +test('uses default s3 client if options.s3 is not set', (t) => { + var stub = sinon.stub().returns({}) + var proxyRas3 = proxyquire('../index', { + 'aws-sdk/clients/s3': stub + }) + proxyRas3('filename', { bucket: 'fake-bucket' }) + t.ok(stub.calledWithNew) + t.end() +}) + +test('ras3.read calls s3.getObject with offsets', (t) => { + t.comment('strips preceding forward slashes on filenames') + var stub = sinon.stub().callsFake((params, cb) => cb(null, { Body: 'somedata' })) + var proxyRas3 = proxyquire('../index', { + 'aws-sdk/clients/s3': function () { + this.getObject = stub + } + }) + var s3 = proxyRas3('/filename', { bucket: 'fake-bucket' }) + s3.read(10, 20, (err, res) => { + t.error(err) + t.ok(stub.calledOnce) + t.ok(stub.calledWith({ + Bucket: 'fake-bucket', + Key: 'filename', + Range: `bytes=10-29` + })) + t.end() + }) +}) + +test('ras3.read returns error in callback if s3.getObject errors', (t) => { + var stub = sinon.stub().callsFake((params, cb) => cb(new Error('BOOM!'))) + var proxyRas3 = proxyquire('../index', { + 'aws-sdk/clients/s3': function () { + this.getObject = stub + } + }) + var s3 = proxyRas3('/filename', { bucket: 'fake-bucket' }) + s3.read(10, 20, (err, res) => { + t.ok(err.message === 'BOOM!') + t.end() + }) +}) + +test('ras3.write does not throw error', (t) => { + var s3 = ras3('test-write', { bucket: 'fake-bucket' }) + t.doesNotThrow(s3.write.bind(s3, 10, 'some-data')) + t.end() +}) + +test('ras3.write logs with options.verbose === true', (t) => { + var stub = sinon.stub() + var proxyRas3 = proxyquire('../index', { + './lib/logger': { + log: stub + } + }) + var s3 = proxyRas3('test-write', { bucket: 'fake-bucket', verbose: true }) + s3.write(10, 'some-data', (err, res) => { + t.error(err) + t.ok(stub.calledOnce) + t.ok(stub.calledWith('trying to write', 'test-write', 10, 'some-data')) + t.end() + }) +}) + +test('ras3.del does not throw error', (t) => { + var s3 = ras3('test-del', { bucket: 'fake-bucket' }) + t.doesNotThrow(s3.del.bind(s3, 10, 100)) + t.end() +}) + +test('ras3.del logs with options.verbose === true', (t) => { + var stub = sinon.stub() + var proxyRas3 = proxyquire('../index', { + './lib/logger': { + log: stub + } + }) + var s3 = proxyRas3('test-del', { bucket: 'fake-bucket', verbose: true }) + s3.del(10, 100, (err, res) => { + t.error(err) + t.ok(stub.calledOnce) + t.ok(stub.calledWith('trying to del', 'test-del', 10, 100)) + t.end() + }) +}) From b90c9f224577f43083b3cd4074a7b84cc0b9fb35 Mon Sep 17 00:00:00 2001 From: Benjamin Forster Date: Thu, 8 Feb 2018 11:54:27 +0100 Subject: [PATCH 5/6] add test coverage --- .coveralls.yml | 1 + .gitignore | 3 ++- .travis.yml | 4 ++++ package.json | 7 ++++++- 4 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 0000000..06d85d4 --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1 @@ +repo_token: XnMFMbdXR7zMIwLNKvdLzx6oFQ80GJeIG diff --git a/.gitignore b/.gitignore index 19d68ad..96326be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ +.nyc_output/ *.log -config.json +coverage.lcov diff --git a/.travis.yml b/.travis.yml index ddb052e..3753e9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,3 +4,7 @@ node_js: - 6 - 8 - 9 +script: + - npm run test-travis +after_script: + - npm run report-coverage diff --git a/package.json b/package.json index 4e3cf23..5e7c7bb 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,9 @@ "main": "index.js", "scripts": { "test": "standard && tape tests/**.test.js", - "tdd": "tape-watch tests/**.test.js" + "test-travis": "nyc tape tests/**.test.js | tap-spec", + "tdd": "tape-watch tests/**.test.js", + "report-coverage": "nyc report --reporter=text-lcov | coveralls" }, "keywords": [ "aws", @@ -23,9 +25,12 @@ "random-access-storage": "^1.1.0" }, "devDependencies": { + "coveralls": "^3.0.0", + "nyc": "^11.4.1", "proxyquire": "^1.8.0", "sinon": "^4.2.2", "standard": "^10.0.3", + "tap-spec": "^4.1.1", "tape": "^4.8.0", "tape-watch": "^2.3.0" } From 445fd51b4aca170b0b83ca9f6b312b6c68ffbb61 Mon Sep 17 00:00:00 2001 From: Benjamin Forster Date: Thu, 8 Feb 2018 11:55:58 +0100 Subject: [PATCH 6/6] add ci badges to readme --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 1d0e776..9e31413 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,5 @@ # random-access-s3 +[![Build Status](https://travis-ci.org/random-access-storage/random-access-s3.svg?branch=master)](https://travis-ci.org/random-access-storage/random-access-s3) [![Coverage Status](https://coveralls.io/repos/github/random-access-storage/random-access-s3/badge.svg?branch=master)](https://coveralls.io/github/random-access-storage/random-access-s3?branch=master) An implementation of [random-access-storage](https://www.npmjs.com/package/random-access-storage) on top of an AWS S3 bucket. Providing the same interface as [random-access-file](https://www.npmjs.com/package/random-access-file) and [random-access-memory](https://www.npmjs.com/package/random-access-memory).