diff --git a/API.md b/API.md new file mode 100644 index 0000000..f8de02c --- /dev/null +++ b/API.md @@ -0,0 +1,76 @@ +# API + +```js +const BlockService = require('ipfs-block-service') +``` + +### `new BlockService(repo)` + +Creates a new block service backed by [IPFS Repo][repo] `repo` for storage. + +### `goOnline(bitswap)` + +Add a bitswap instance that communicates with the network to retreive blocks +that are not in the local store. + +If the node is online all requests for blocks first check locally and +afterwards ask the network for the blocks. + +### `goOffline()` + +Remove the bitswap instance and fall back to offline mode. + +### `isOnline()` + +Returns a `Boolean` indicating if the block service is online or not. + +### `addBlock(block, callback(err))` + +Asynchronously adds a block instance to the underlying repo. + +### `addBlocks(blocks, callback(err))` + +Asynchronously adds an array of block instances to the underlying repo. + +*Does not guarantee atomicity.* + +### `getBlock(multihash, callback(err, block))` + +Asynchronously returns the block whose content multihash matches `multihash`. +Returns an error (`err.code === 'ENOENT'`) if the block does not exist. + +If the block could not be found, expect `err.code` to be `'ENOENT'`. + +### `getBlocks(multihashes, callback(err, blocks))` + +Asynchronously returns the blocks whose content multihashes match the array +`multihashes`. + +`blocks` is an object that maps each `multihash` to an object of the form + +```js +{ + err: Error + block: Block +} +``` + +Expect `blocks[multihash].err.code === 'ENOENT'` and `blocks[multihash].block +=== null` if a block did not exist. + +*Does not guarantee atomicity.* + +### `deleteBlock(multihash, callback(err))` + +Asynchronously deletes the block from the store with content multihash matching +`multihash`, if it exists. + +### `bs.deleteBlocks(multihashes, callback(err))` + +Asynchronously deletes all blocks from the store with content multihashes matching +from the array `multihashes`. + +*Does not guarantee atomicity.* + +[multihash]: https://github.com/jbenet/js-multihash +[repo]: https://github.com/ipfs/specs/tree/master/repo diff --git a/README.md b/README.md index 8905480..2600b8b 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,7 @@ IPFS Block Service JavaScript Implementation **BlockService** - A BlockService is a content-addressable store for blocks, providing an API for adding, deleting, and retrieving blocks. A BlockService is -backed by an [IPFS Repo][repo] as its datastore for blocks, and uses an [IPFS -Exchange][bitswap] implementation to fetch blocks from the network. +backed by an [IPFS Repo][repo] as its datastore for blocks, and uses [Bitswap][bitswap] to fetch blocks from the network. ```markdown ┌────────────────────┐ @@ -27,9 +26,9 @@ Exchange][bitswap] implementation to fetch blocks from the network. │ ┌─────┴─────┐ ▼ ▼ -┌─────────┐ ┌────────┐ -│IPFS Repo│ │Exchange│ -└─────────┘ └────────┘ +┌─────────┐ ┌───────┐ +│IPFS Repo│ |Bitswap│ +└─────────┘ └───────┘ ``` ## Example @@ -99,7 +98,7 @@ var BlockService = require('ipfs-block-service') ### Browser: ` ``` -## API - -```js -const BlockService = require('ipfs-block-service') -``` - -### var bs = new BlockService(repo[, exchange]) - -Creates a new block service backed by [IPFS Repo][repo] `repo` for storage, and -[IPFS Exchange][bitswap] for retrieving blocks from the network. Providing an -`exchange` is optional. - -#### bs.addBlock(block, callback(err)) - -Asynchronously adds a block instance to the underlying repo. - -#### bs.addBlocks(blocks, callback(err)) - -Asynchronously adds an array of block instances to the underlying repo. - -*Does not guarantee atomicity.* - -#### bs.getBlock(multihash, callback(err, block)) - -Asynchronously returns the block whose content multihash matches `multihash`. -Returns an error (`err.code === 'ENOENT'`) if the block does not exist. - -If the block could not be found, expect `err.code` to be `'ENOENT'`. - -#### bs.getBlocks(multihashes, callback(err, blocks)) - -Asynchronously returns the blocks whose content multihashes match the array -`multihashes`. - -`blocks` is an object that maps each `multihash` to an object of the form - -```js -{ - err: Error - block: Block -} -``` - -Expect `blocks[multihash].err.code === 'ENOENT'` and `blocks[multihash].block -=== null` if a block did not exist. - -*Does not guarantee atomicity.* - -#### bs.deleteBlock(multihash, callback(err)) - -Asynchronously deletes the block from the store with content multihash matching -`multihash`, if it exists. - -#### bs.deleteBlocks(multihashes, callback(err)) - -Asynchronously deletes all blocks from the store with content multihashes matching -from the array `multihashes`. - -*Does not guarantee atomicity.* +You can find the [API documentation here](API.md) ## License MIT [ipfs]: https://ipfs.io -[repo]: https://github.com/ipfs/specs/tree/master/repo [bitswap]: https://github.com/ipfs/specs/tree/master/bitswap -[multihash]: https://github.com/jbenet/js-multihash diff --git a/src/block-service.js b/src/block-service.js deleted file mode 100644 index d3de445..0000000 --- a/src/block-service.js +++ /dev/null @@ -1,68 +0,0 @@ -'use strict' - -const async = require('async') - -// BlockService is a hybrid block datastore. It stores data in a local -// datastore and may retrieve data from a remote Exchange. -// It uses an internal `datastore.Datastore` instance to store values. -function BlockService (ipfsRepo, exchange) { - this.addBlock = ipfsRepo.datastore.put.bind(ipfsRepo.datastore) - - this.addBlocks = (blocks, callback) => { - if (!Array.isArray(blocks)) { - return callback(new Error('expects an array of Blocks')) - } - - async.eachLimit(blocks, 100, (block, next) => { - this.addBlock(block, next) - }, callback) - } - - this.getBlock = ipfsRepo.datastore.get.bind(ipfsRepo.datastore) - - this.getBlocks = (multihashes, extension, callback) => { - if (typeof extension === 'function') { - callback = extension - extension = undefined - } - - if (!Array.isArray(multihashes)) { - return callback(new Error('Invalid batch of multihashes')) - } - - var results = {} - - async.eachLimit(multihashes, 100, (multihash, next) => { - this.getBlock(multihash, extension, (err, block) => { - results[multihash] = { - err: err, - block: block - } - next() - }) - }, (err) => { - callback(err, results) - }) - } - - this.deleteBlock = ipfsRepo.datastore.delete.bind(ipfsRepo.datastore) - - this.deleteBlocks = (multihashes, extension, callback) => { - if (typeof extension === 'function') { - callback = extension - extension = undefined - } - - if (!Array.isArray(multihashes)) { - return callback(new Error('Invalid batch of multihashes')) - } - - async.eachLimit(multihashes, 100, (multihash, next) => { - this.deleteBlock(multihash, extension, next) - }, (err) => { - callback(err) - }) - } -} - -module.exports = BlockService diff --git a/src/index.js b/src/index.js index 16a596c..f2d6ca5 100644 --- a/src/index.js +++ b/src/index.js @@ -1,3 +1,107 @@ 'use strict' -exports = module.exports = require('./block-service.js') +const async = require('async') + +// BlockService is a hybrid block datastore. It stores data in a local +// datastore and may retrieve data from a remote Exchange. +// It uses an internal `datastore.Datastore` instance to store values. +module.exports = class BlockService { + constructor (ipfsRepo) { + this._repo = ipfsRepo + this._bitswap = null + } + + goOnline (bitswap) { + this._bitswap = bitswap + } + + goOffline () { + this._bitswap = null + } + + isOnline () { + return this._bitswap != null + } + + addBlock (block, extension, callback) { + if (this.isOnline()) { + if (typeof extension === 'function') { + callback = extension + extension = undefined + } + + this._bitswap.hasBlock(block, callback) + } else { + this._repo.datastore.put(block, extension, callback) + } + } + + addBlocks (blocks, callback) { + if (!Array.isArray(blocks)) { + return callback(new Error('expects an array of Blocks')) + } + + async.eachLimit(blocks, 100, (block, next) => { + this.addBlock(block, next) + }, callback) + } + + getBlock (key, extension, callback) { + if (this.isOnline()) { + if (typeof extension === 'function') { + callback = extension + extension = undefined + } + + this._bitswap.getBlock(key, callback) + } else { + this._repo.datastore.get(key, extension, callback) + } + } + + getBlocks (multihashes, extension, callback) { + if (typeof extension === 'function') { + callback = extension + extension = undefined + } + + if (!Array.isArray(multihashes)) { + return callback(new Error('Invalid batch of multihashes')) + } + + var results = {} + + async.eachLimit(multihashes, 100, (multihash, next) => { + this.getBlock(multihash, extension, (err, block) => { + results[multihash] = { + err: err, + block: block + } + next() + }) + }, (err) => { + callback(err, results) + }) + } + + deleteBlock (key, extension, callback) { + this._repo.datastore.delete(key, extension, callback) + } + + deleteBlocks (multihashes, extension, callback) { + if (typeof extension === 'function') { + callback = extension + extension = undefined + } + + if (!Array.isArray(multihashes)) { + return callback(new Error('Invalid batch of multihashes')) + } + + async.eachLimit(multihashes, 100, (multihash, next) => { + this.deleteBlock(multihash, extension, next) + }, (err) => { + callback(err) + }) + } +} diff --git a/test/block-service-test.js b/test/block-service-test.js index 28bd9c7..63ea619 100644 --- a/test/block-service-test.js +++ b/test/block-service-test.js @@ -9,210 +9,254 @@ module.exports = (repo) => { describe('block-service', () => { let bs - before((done) => { + before(() => { bs = new BlockService(repo) - expect(bs).to.exist - done() }) - it('store and get a block', (done) => { - const b = new Block('A random data block') - bs.addBlock(b, (err) => { - expect(err).to.not.exist - bs.getBlock(b.key, (err, block) => { + describe('offline', () => { + it('store and get a block', (done) => { + const b = new Block('A random data block') + bs.addBlock(b, (err) => { expect(err).to.not.exist - expect(b.data.equals(block.data)).to.equal(true) - expect(b.key.equals(block.key)).to.equal(true) - done() + bs.getBlock(b.key, (err, block) => { + expect(err).to.not.exist + expect(b.data.equals(block.data)).to.equal(true) + expect(b.key.equals(block.key)).to.equal(true) + done() + }) }) }) - }) - it('store and get a block, with custom extension', (done) => { - const b = new Block('A random data block 2', 'ext') - bs.addBlock(b, (err) => { - expect(err).to.not.exist - bs.getBlock(b.key, 'ext', (err, block) => { + it('store and get a block, with custom extension', (done) => { + const b = new Block('A random data block 2', 'ext') + bs.addBlock(b, (err) => { expect(err).to.not.exist - expect(b.data.equals(block.data)).to.equal(true) - expect(b.key.equals(block.key)).to.equal(true) - done() + bs.getBlock(b.key, 'ext', (err, block) => { + expect(err).to.not.exist + expect(b.data.equals(block.data)).to.equal(true) + expect(b.key.equals(block.key)).to.equal(true) + done() + }) }) }) - }) - it('get a non existent block', (done) => { - const b = new Block('Not stored') - bs.getBlock(b.key, (err, block) => { - expect(err).to.exist - done() + it('get a non existent block', (done) => { + const b = new Block('Not stored') + bs.getBlock(b.key, (err, block) => { + expect(err).to.exist + done() + }) }) - }) - it('store many blocks', (done) => { - const b1 = new Block('1') - const b2 = new Block('2') - const b3 = new Block('3') + it('store many blocks', (done) => { + const b1 = new Block('1') + const b2 = new Block('2') + const b3 = new Block('3') - bs.addBlocks([b1, b2, b3], (err) => { - expect(err).to.not.exist - done() + bs.addBlocks([b1, b2, b3], (err) => { + expect(err).to.not.exist + done() + }) }) - }) - it('addBlocks: bad invocation', (done) => { - const b1 = new Block('1') + it('addBlocks: bad invocation', (done) => { + const b1 = new Block('1') - bs.addBlocks(b1, (err) => { - expect(err).to.be.an('error') - done() + bs.addBlocks(b1, (err) => { + expect(err).to.be.an('error') + done() + }) }) - }) - it('getBlock: bad invocation', (done) => { - bs.getBlock(null, (err) => { - expect(err).to.be.an('error') - done() + it('getBlock: bad invocation', (done) => { + bs.getBlock(null, (err) => { + expect(err).to.be.an('error') + done() + }) }) - }) - it('getBlocks: bad invocation', (done) => { - bs.getBlocks(null, 'protobuf', (err) => { - expect(err).to.be.an('error') - done() + it('getBlocks: bad invocation', (done) => { + bs.getBlocks(null, 'protobuf', (err) => { + expect(err).to.be.an('error') + done() + }) }) - }) - - it('get many blocks', (done) => { - const b1 = new Block('1') - const b2 = new Block('2') - const b3 = new Block('3') - bs.addBlocks([b1, b2, b3], (err) => { - expect(err).to.not.exist + it('get many blocks', (done) => { + const b1 = new Block('1') + const b2 = new Block('2') + const b3 = new Block('3') - bs.getBlocks([b1.key, b2.key, b3.key], (err, blocks) => { + bs.addBlocks([b1, b2, b3], (err) => { expect(err).to.not.exist - expect(Object.keys(blocks)).to.have.lengthOf(3) - expect(blocks[b1.key]).to.exist - expect(blocks[b1.key].err).to.not.exist - expect(blocks[b1.key].block.data).to.deep.equal(b1.data) - expect(blocks[b2.key]).to.exist - expect(blocks[b2.key].err).to.not.exist - expect(blocks[b2.key].block.data).to.deep.equal(b2.data) - expect(blocks[b3.key]).to.exist - expect(blocks[b3.key].err).to.not.exist - expect(blocks[b3.key].block.data).to.deep.equal(b3.data) - done() + + bs.getBlocks([b1.key, b2.key, b3.key], (err, blocks) => { + expect(err).to.not.exist + expect(Object.keys(blocks)).to.have.lengthOf(3) + expect(blocks[b1.key]).to.exist + expect(blocks[b1.key].err).to.not.exist + expect(blocks[b1.key].block.data).to.deep.equal(b1.data) + expect(blocks[b2.key]).to.exist + expect(blocks[b2.key].err).to.not.exist + expect(blocks[b2.key].block.data).to.deep.equal(b2.data) + expect(blocks[b3.key]).to.exist + expect(blocks[b3.key].err).to.not.exist + expect(blocks[b3.key].block.data).to.deep.equal(b3.data) + done() + }) }) }) - }) - it('get many blocks: partial success', (done) => { - const b1 = new Block('a1') - const b2 = new Block('a2') - const b3 = new Block('a3') + it('get many blocks: partial success', (done) => { + const b1 = new Block('a1') + const b2 = new Block('a2') + const b3 = new Block('a3') - bs.addBlocks([b1, b3], (err) => { - expect(err).to.not.exist + bs.addBlocks([b1, b3], (err) => { + expect(err).to.not.exist + + bs.getBlocks([b1.key, b2.key, b3.key], (err, blocks) => { + expect(err).to.not.exist + expect(Object.keys(blocks)).to.have.lengthOf(3) + expect(blocks[b1.key]).to.exist + expect(blocks[b1.key].err).to.not.exist + expect(blocks[b1.key].block.data).to.deep.equal(b1.data) + expect(blocks[b2.key]).to.exist + expect(blocks[b2.key].err).to.exist + expect(blocks[b2.key].block).to.not.exist + expect(blocks[b3.key]).to.exist + expect(blocks[b3.key].err).to.not.exist + expect(blocks[b3.key].block.data).to.deep.equal(b3.data) + done() + }) + }) + }) - bs.getBlocks([b1.key, b2.key, b3.key], (err, blocks) => { + it('delete a block', (done) => { + const b = new Block('Will not live that much') + bs.addBlock(b, (err) => { expect(err).to.not.exist - expect(Object.keys(blocks)).to.have.lengthOf(3) - expect(blocks[b1.key]).to.exist - expect(blocks[b1.key].err).to.not.exist - expect(blocks[b1.key].block.data).to.deep.equal(b1.data) - expect(blocks[b2.key]).to.exist - expect(blocks[b2.key].err).to.exist - expect(blocks[b2.key].block).to.not.exist - expect(blocks[b3.key]).to.exist - expect(blocks[b3.key].err).to.not.exist - expect(blocks[b3.key].block.data).to.deep.equal(b3.data) + bs.deleteBlock(b.key, (err) => { + expect(err).to.not.exist + bs.getBlock(b.key, (err, block) => { + expect(err).to.exist + done() + }) + }) + }) + }) + + it('deleteBlock: bad invocation', (done) => { + bs.deleteBlock(null, (err) => { + expect(err).to.be.an('error') done() }) }) - }) - it('delete a block', (done) => { - const b = new Block('Will not live that much') - bs.addBlock(b, (err) => { - expect(err).to.not.exist - bs.deleteBlock(b.key, (err) => { + it('delete a block, with custom extension', (done) => { + const b = new Block('Will not live that much', 'ext') + bs.addBlock(b, (err) => { expect(err).to.not.exist - bs.getBlock(b.key, (err, block) => { - expect(err).to.exist - done() + bs.deleteBlock(b.key, 'ext', (err) => { + expect(err).to.not.exist + bs.getBlock(b.key, 'ext', (err, block) => { + expect(err).to.exist + done() + }) }) }) }) - }) - it('deleteBlock: bad invocation', (done) => { - bs.deleteBlock(null, (err) => { - expect(err).to.be.an('error') - done() + it('delete a non existent block', (done) => { + const b = new Block('I do not exist') + bs.deleteBlock(b.key, (err) => { + expect(err).to.not.exist + done() + }) }) - }) - it('delete a block, with custom extension', (done) => { - const b = new Block('Will not live that much', 'ext') - bs.addBlock(b, (err) => { - expect(err).to.not.exist - bs.deleteBlock(b.key, 'ext', (err) => { + it('delete many blocks', (done) => { + const b1 = new Block('1') + const b2 = new Block('2') + const b3 = new Block('3') + + bs.deleteBlocks([b1, b2, b3], 'data', (err) => { expect(err).to.not.exist - bs.getBlock(b.key, 'ext', (err, block) => { - expect(err).to.exist - done() - }) + done() }) }) - }) - it('delete a non existent block', (done) => { - const b = new Block('I do not exist') - bs.deleteBlock(b.key, (err) => { - expect(err).to.not.exist - done() + it('deleteBlocks: bad invocation', (done) => { + bs.deleteBlocks(null, (err) => { + expect(err).to.be.an('error') + done() + }) }) - }) - it('delete many blocks', (done) => { - const b1 = new Block('1') - const b2 = new Block('2') - const b3 = new Block('3') + it('stores and gets lots of blocks', function (done) { + this.timeout(60 * 1000) + + const blocks = [] + const count = 1000 + while (blocks.length < count) { + blocks.push(new Block('hello-' + Math.random())) + } + + bs.addBlocks(blocks, (err) => { + expect(err).to.not.exist + + bs.getBlocks(blocks.map((b) => b.key), (err, res) => { + expect(err).to.not.exist + expect(Object.keys(res)).to.have.length(count) - bs.deleteBlocks([b1, b2, b3], 'data', (err) => { - expect(err).to.not.exist - done() + done() + }) + }) }) - }) - it('deleteBlocks: bad invocation', (done) => { - bs.deleteBlocks(null, (err) => { - expect(err).to.be.an('error') - done() + it('goes offline', () => { + bs = new BlockService(repo) + bs.goOnline({}) + expect(bs.isOnline()).to.be.eql(true) + bs.goOffline() + expect(bs.isOnline()).to.be.eql(false) }) }) - it('stores and gets lots of blocks', function (done) { - this.timeout(60 * 1000) + describe('online', () => { + beforeEach(() => { + bs = new BlockService(repo) + }) - const blocks = [] - const count = 1000 - while (blocks.length < count) { - blocks.push(new Block('hello-' + Math.random())) - } + it('isOnline returns true when online', () => { + bs.goOnline({}) + expect(bs.isOnline()).to.be.eql(true) + }) - bs.addBlocks(blocks, (err) => { - expect(err).to.not.exist + it('retrieves a block through bitswap', (done) => { + const bitswap = { + getBlock (key, cb) { + cb(null, new Block(key)) + } + } + bs.goOnline(bitswap) - bs.getBlocks(blocks.map((b) => b.key), (err, res) => { + bs.getBlock('secret', (err, res) => { expect(err).to.not.exist - expect(Object.keys(res)).to.have.length(count) - + expect(res).to.be.eql(new Block('secret')) done() }) }) + + it('puts the block through bitswap', (done) => { + const bitswap = { + hasBlock (block, cb) { + cb() + } + } + bs.goOnline(bitswap) + bs.addBlock(new Block('secret sauce'), done) + }) }) }) }