Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .coveralls.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
repo_token: XnMFMbdXR7zMIwLNKvdLzx6oFQ80GJeIG
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
node_modules/
.nyc_output/
*.log
config.json
coverage.lcov
10 changes: 10 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
language: node_js
sudo: false
node_js:
- 6
- 8
- 9
script:
- npm run test-travis
after_script:
- npm run report-coverage
72 changes: 37 additions & 35 deletions index.js
Original file line number Diff line number Diff line change
@@ -1,47 +1,49 @@
var S3 = require('aws-sdk/clients/s3')

var Abstract = require('abstract-random-access')
var inherits = require('inherits')
var randomAccess = require('random-access-storage')
var logger = require('./lib/logger')

function sanitizeKeyForS3 (key) {
if (typeof key === 'string' && key.length && key[0] === '/') return key.slice(1)
return key
}

var Store = function (filename, options) {
if (!(this instanceof Store)) return new Store(filename, options)
Abstract.call(this)
this.s3 = options.s3 || new S3()
this.key = sanitizeKeyForS3(filename)
this.bucket = options.bucket
this.verbose = !!options.verbose
inherits(Store, Abstract)
}

Store.prototype._read = function (offset, length, callback) {
var params = {
Bucket: this.bucket,
Key: this.key,
Range: `bytes=${offset}-${offset + length - 1}`
}
if (this.verbose) console.log('Trying to read', this.key, params.Range)
this.s3.getObject(params, (err, data) => {
if (err) {
if (this.verbose) {
console.log('error', this.key, params.Range)
console.log(err, err.stack)
function s3 (filename, options) {
if (!filename) throw new Error('Random Access S3 requires a filename!')
if (!options) throw new Error('Random Access S3 requires configuration options to be set!')
if (!options.bucket) throw new Error('Random Access S3 requires options.bucket!')
const s3 = (options && options.s3) || new S3()
const key = sanitizeKeyForS3(filename)
const bucket = options && options.bucket
const verbose = !!(options && options.verbose)
return randomAccess({
read: function (req) {
var params = {
Bucket: bucket,
Key: key,
Range: `bytes=${req.offset}-${req.offset + req.size - 1}`
}
return callback(err)
if (verbose) logger.log('Trying to read', key, params.Range)
s3.getObject(params, (err, data) => {
if (err) {
if (verbose) {
logger.log('error', key, params.Range)
logger.log(err, err.stack)
}
return req.callback(err)
}
if (verbose) logger.log('read', data.Body)
req.callback(null, data.Body)
})
},
write: function (req) {
if (verbose) logger.log('trying to write', key, req.offset, req.data)
req.callback()
},
del: function (req) {
if (verbose) logger.log('trying to del', key, req.offset, req.size)
req.callback()
}
if (this.verbose) console.log('read', data.Body)
callback(null, data.Body)
})
}

// This is a dummy write function - does not write, but fails silently
Store.prototype._write = function (offset, buffer, callback) {
if (this.verbose) console.log('trying to write', this.key, offset, buffer)
callback()
}

module.exports = Store
module.exports = s3
3 changes: 3 additions & 0 deletions lib/logger.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
module.exports = {
log: console.log
}
20 changes: 15 additions & 5 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
"description": "A read-only random access interface for aws s3 buckets",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
"test": "standard && tape tests/**.test.js",
"test-travis": "nyc tape tests/**.test.js | tap-spec",
"tdd": "tape-watch tests/**.test.js",
"report-coverage": "nyc report --reporter=text-lcov | coveralls"
},
"keywords": [
"aws",
Expand All @@ -15,13 +18,20 @@
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/e-e-e/random-access-s3.git"
"url": "git+https://github.com/random-access-storage/random-access-s3.git"
},
"dependencies": {
"abstract-random-access": "^1.1.2",
"inherits": "^2.0.3"
"aws-sdk": "^2.188.0",
"random-access-storage": "^1.1.0"
},
"devDependencies": {
"standard": "^10.0.3"
"coveralls": "^3.0.0",
"nyc": "^11.4.1",
"proxyquire": "^1.8.0",
"sinon": "^4.2.2",
"standard": "^10.0.3",
"tap-spec": "^4.1.1",
"tape": "^4.8.0",
"tape-watch": "^2.3.0"
}
}
5 changes: 3 additions & 2 deletions readme.md
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
# random-access-s3
[![Build Status](https://travis-ci.org/random-access-storage/random-access-s3.svg?branch=master)](https://travis-ci.org/random-access-storage/random-access-s3) [![Coverage Status](https://coveralls.io/repos/github/random-access-storage/random-access-s3/badge.svg?branch=master)](https://coveralls.io/github/random-access-storage/random-access-s3?branch=master)

An implementation of [abstract-random-access](https://www.npmjs.com/package/abstract-random-access) on top of an AWS S3 bucket.
An implementation of [random-access-storage](https://www.npmjs.com/package/random-access-storage) on top of an AWS S3 bucket.
Providing the same interface as [random-access-file](https://www.npmjs.com/package/random-access-file) and [random-access-memory](https://www.npmjs.com/package/random-access-memory).

## Why?
This is an experiment to see if we can serve [dat](http://datproject.org) data over aws s3. It is possible.

This is an experiment to see if we can serve [dat](http://datproject.org) data over aws s3. It is possible.
**TLDR;** Latency is a killer.

## Installation
Expand Down
116 changes: 116 additions & 0 deletions tests/ras3.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
const test = require('tape')
const sinon = require('sinon')
const proxyquire = require('proxyquire')
const ras3 = require('../index')

test('it throws error if not provided a filename', (t) => {
t.throws(ras3.bind({}, undefined, {}), /Random Access S3 requires a filename!/)
t.end()
})

test('it throws error if not provided options', (t) => {
t.throws(ras3.bind({}, 'filename', undefined), /Random Access S3 requires configuration options to be set!/)
t.end()
})

test('it throws error if not provided options', (t) => {
t.throws(ras3.bind({}, 'filename', {}), /Random Access S3 requires options.bucket/)
t.end()
})

test('ras3 returns a object with expected interface', (t) => {
var s3 = ras3('filename', { bucket: 'fake-bucket' })
t.ok(typeof s3.read === 'function')
t.ok(typeof s3.write === 'function')
t.ok(typeof s3.del === 'function')
t.end()
})

test('uses default s3 client if options.s3 is not set', (t) => {
var stub = sinon.stub().returns({})
var proxyRas3 = proxyquire('../index', {
'aws-sdk/clients/s3': stub
})
proxyRas3('filename', { bucket: 'fake-bucket' })
t.ok(stub.calledWithNew)
t.end()
})

test('ras3.read calls s3.getObject with offsets', (t) => {
t.comment('strips preceding forward slashes on filenames')
var stub = sinon.stub().callsFake((params, cb) => cb(null, { Body: 'somedata' }))
var proxyRas3 = proxyquire('../index', {
'aws-sdk/clients/s3': function () {
this.getObject = stub
}
})
var s3 = proxyRas3('/filename', { bucket: 'fake-bucket' })
s3.read(10, 20, (err, res) => {
t.error(err)
t.ok(stub.calledOnce)
t.ok(stub.calledWith({
Bucket: 'fake-bucket',
Key: 'filename',
Range: `bytes=10-29`
}))
t.end()
})
})

test('ras3.read returns error in callback if s3.getObject errors', (t) => {
var stub = sinon.stub().callsFake((params, cb) => cb(new Error('BOOM!')))
var proxyRas3 = proxyquire('../index', {
'aws-sdk/clients/s3': function () {
this.getObject = stub
}
})
var s3 = proxyRas3('/filename', { bucket: 'fake-bucket' })
s3.read(10, 20, (err, res) => {
t.ok(err.message === 'BOOM!')
t.end()
})
})

test('ras3.write does not throw error', (t) => {
var s3 = ras3('test-write', { bucket: 'fake-bucket' })
t.doesNotThrow(s3.write.bind(s3, 10, 'some-data'))
t.end()
})

test('ras3.write logs with options.verbose === true', (t) => {
var stub = sinon.stub()
var proxyRas3 = proxyquire('../index', {
'./lib/logger': {
log: stub
}
})
var s3 = proxyRas3('test-write', { bucket: 'fake-bucket', verbose: true })
s3.write(10, 'some-data', (err, res) => {
t.error(err)
t.ok(stub.calledOnce)
t.ok(stub.calledWith('trying to write', 'test-write', 10, 'some-data'))
t.end()
})
})

test('ras3.del does not throw error', (t) => {
var s3 = ras3('test-del', { bucket: 'fake-bucket' })
t.doesNotThrow(s3.del.bind(s3, 10, 100))
t.end()
})

test('ras3.del logs with options.verbose === true', (t) => {
var stub = sinon.stub()
var proxyRas3 = proxyquire('../index', {
'./lib/logger': {
log: stub
}
})
var s3 = proxyRas3('test-del', { bucket: 'fake-bucket', verbose: true })
s3.del(10, 100, (err, res) => {
t.error(err)
t.ok(stub.calledOnce)
t.ok(stub.calledWith('trying to del', 'test-del', 10, 100))
t.end()
})
})