Skip to content
This repository has been archived by the owner on Aug 11, 2021. It is now read-only.

Commit

Permalink
feat: implementation of the new get() function
Browse files Browse the repository at this point in the history 
BREAKING CHANGE: `get()` is replacing the `getMany()` function.

The API docs for it:

> Retrieve several IPLD Nodes at once.

 - `cids` (`Iterable<CID>`): the CIDs of the IPLD Nodes that should be retrieved.

Returns an async iterator with the IPLD Nodes that correspond to the given `cids`.

Throws an error if a IPLD Node can’t be retrieved.
  • Loading branch information
@vmx
vmx committed Mar 21, 2019
1 parent 8b737b1 commit 743e679
Show file tree
Hide file tree
Showing 9 changed files with 277 additions and 334 deletions.
90 changes: 70 additions & 20 deletions src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,29 +146,56 @@ class IPLDResolver {
/**
* Get multiple nodes back from an array of CIDs.
*
* @param {Array<CID>} cids
* @param {function(Error, Array)} callback
* @returns {void}
* @param {Iterable.<CID>} cids - The CIDs of the IPLD Nodes that should be retrieved.
* @returns {Iterable.<Promise.<Object>>} - Returns an async iterator with the IPLD Nodes that correspond to the given `cids`.
*/
getMany (cids, callback) {
if (!Array.isArray(cids)) {
return callback(new Error('Argument must be an array of CIDs'))
get (cids) {
if (!typical.isIterable(cids) || typical.isString(cids) ||
Buffer.isBuffer(cids)) {
throw new Error('`cids` must be an iterable of CIDs')
}
this.bs.getMany(cids, (err, blocks) => {
if (err) {
return callback(err)

let blocks
const next = () => {
// End of iteration if there aren't any blocks left to return
if (cids.length === 0 ||
(blocks !== undefined && blocks.length === 0)
) {
return Promise.resolve({ done: true })
}
map(blocks, (block, mapCallback) => {
// TODO vmx 2018-12-07: Make this one async/await once
// `util.serialize()` is a Promise
this._getFormat(block.cid.codec).then((format) => {
format.util.deserialize(block.data, mapCallback)
}).catch((err) => {
mapCallback(err)
})
},
callback)
})

return new Promise(async (resolve, reject) => {
// Lazy load block.
// Currntly the BlockService return all nodes as an array. In the
// future this will also be an iterator
if (blocks === undefined) {
const cidsArray = Array.from(cids)
this.bs.getMany(cidsArray, async (err, returnedBlocks) => {
if (err) {
return reject(err)
}
blocks = returnedBlocks
const block = blocks.shift()
try {
const node = await this._deserialize(block)
return resolve({ done: false, value: node })
} catch (err) {
return reject(err)
}
})
} else {
const block = blocks.shift()
try {
const node = await this._deserialize(block)
return resolve({ done: false, value: node })
} catch (err) {
return reject(err)
}
}
})
}

return fancyIterator(next)
}

/**
Expand Down Expand Up @@ -410,6 +437,29 @@ class IPLDResolver {
})
}

/**
* Deserialize a given block
*
* @param {Object} block - The block to deserialize
* @return {Object} = Returns the deserialized node
*/
async _deserialize (block) {
return new Promise((resolve, reject) => {
this._getFormat(block.cid.codec).then((format) => {
// TODO vmx 2018-12-11: Make this one async/await once
// `util.serialize()` is a Promise
format.util.deserialize(block.data, (err, deserialized) => {
if (err) {
return reject(err)
}
return resolve(deserialized)
})
}).catch((err) => {
return reject(err)
})
})
}

/**
* Return a CID instance if it is a link.
*
Expand Down
13 changes: 0 additions & 13 deletions test/basics.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,19 +50,6 @@ module.exports = (repo) => {
'No resolver found for codec "blake2b-8"')
})

// TODO vmx 2018-11-29 Change this test to use `get()`.
// it('_get - errors on unknown resolver', (done) => {
// const bs = new BlockService(repo)
// const r = new IPLDResolver({ blockService: bs })
// // choosing a format that is not supported
// const cid = new CID(1, 'base1', multihash.encode(Buffer.from('abcd', 'hex'), 'sha1'))
// r.get(cid, (err, result) => {
// expect(err).to.exist()
// expect(err.message).to.eql('No resolver found for codec "base1"')
// done()
// })
// }

it('put - errors on unknown resolver', async () => {
const bs = new BlockService(repo)
const r = new IPLDResolver({ blockService: bs })
Expand Down
69 changes: 36 additions & 33 deletions test/ipld-all.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@
*/

const chai = require('chai')
const chaiAsProised = require('chai-as-promised')
const dirtyChai = require('dirty-chai')
const expect = chai.expect
chai.use(chaiAsProised)
chai.use(dirtyChai)
const dagPB = require('ipld-dag-pb')
const dagCBOR = require('ipld-dag-cbor')
Expand Down Expand Up @@ -104,49 +106,50 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => {
})
})

describe('getMany', () => {
it('should return nodes correctly', (done) => {
resolver.getMany([cidCbor, cidPb], (err, result) => {
expect(err).to.not.exist()
expect(result.length).to.equal(2)
expect(result).to.deep.equal([nodeCbor, nodePb])
done()
})
describe('get', () => {
it('should return nodes correctly', async () => {
const result = resolver.get([cidCbor, cidPb])
const node1 = await result.first()
expect(node1).to.eql(nodeCbor)

const node2 = await result.first()
expect(node2).to.eql(nodePb)
})

it('should return nodes in input order', (done) => {
resolver.getMany([cidPb, cidCbor], (err, result) => {
expect(err).to.not.exist()
expect(result.length).to.equal(2)
expect(result).to.deep.equal([nodePb, nodeCbor])
done()
})
it('should return nodes in input order', async () => {
const result = resolver.get([cidPb, cidCbor])
const node1 = await result.first()
expect(node1).to.eql(nodePb)

const node2 = await result.first()
expect(node2).to.eql(nodeCbor)
})

it('should return error on invalid CID', (done) => {
resolver.getMany([cidCbor, 'invalidcid'], (err, result) => {
expect(err.message).to.equal('Not a valid cid')
expect(result).to.be.undefined()
done()
})
it('should return error on invalid CID', async () => {
const result = resolver.get([cidCbor, 'invalidcid'])
// TODO vmx 2018-12-11: This should really fail on the second node
// we get, as the first one is valid. This is only possible once
// the `getmany()` call of the BlockService takes and returns an
// iterator and not an array.
await expect(result.next()).to.be.rejectedWith(
'Not a valid cid')
})

it('should return error on non-existent CID', (done) => {
it('should return error on non-existent CID', async () => {
const nonExistentCid = new CID(
'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP')
resolver.getMany([cidCbor, nonExistentCid], (err, result) => {
expect(err.message).to.equal('Not Found')
expect(result).to.be.undefined()
done()
})
const result = resolver.get([cidCbor, nonExistentCid])
// TODO vmx 2018-12-11: This should really fail on the second node
// we get, as the first one is valid. This is only possible once
// the `getmany()` call of the BlockService takes and returns an
// iterator and not an array.
await expect(result.next()).to.be.rejectedWith(
'Not Found')
})

it('should return error on invalid input', (done) => {
resolver.getMany('astring', (err, result) => {
expect(err.message).to.equal('Argument must be an array of CIDs')
expect(result).to.be.undefined()
done()
})
it('should return error on invalid input', () => {
expect(() => resolver.get('astring')).to.throw(
'`cids` must be an iterable of CIDs')
})
})
})
74 changes: 28 additions & 46 deletions test/ipld-bitcoin.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,18 +102,6 @@ module.exports = (repo) => {
resolver._put(nc.cid, nc.node, cb)
}, done)
})

// TODO vmx 2018-11-30 Change this test to use `get()`.
// it('resolver._get', (done) => {
// resolver.put(node1, { cid: cid1 }, (err) => {
// expect(err).to.not.exist()
// resolver.get(cid1, (err, result) => {
// expect(err).to.not.exist()
// expect(node1.version).to.eql(result.value.version)
// done()
// })
// })
// })
})

describe('public api', () => {
Expand All @@ -139,19 +127,6 @@ module.exports = (repo) => {
expect(mh.name).to.equal('sha3-512')
})

// TODO vmx 2018-11-30: Implement getting the whole object properly
// it('root path (same as get)', (done) => {
// resolver.get(cid1, '/', (err, result) => {
// expect(err).to.not.exist()
//
// ipldBitcoin.util.cid(result.value, (err, cid) => {
// expect(err).to.not.exist()
// expect(cid).to.eql(cid1)
// done()
// })
// })
// })

it('resolves value within 1st node scope', async () => {
const result = resolver.resolve(cid1, 'version')
const node = await result.first()
Expand Down Expand Up @@ -187,27 +162,34 @@ module.exports = (repo) => {
expect(node3.value).to.eql(1)
})

// // TODO vmx 2018-11-30: remove this `get()` call with the new `get()`
// it('resolver.remove', (done) => {
// resolver.put(node1, { cid: cid1 }, (err) => {
// expect(err).to.not.exist()
// resolver.get(cid1, (err, result) => {
// expect(err).to.not.exist()
// expect(result.value.version).to.eql(1)
// remove()
// })
// })
//
// function remove () {
// resolver.remove(cid1, (err) => {
// expect(err).to.not.exist()
// resolver.get(cid1, (err) => {
// expect(err).to.exist()
// done()
// })
// })
// }
// })
it('resolver.get round-trip', async () => {
const resultPut = resolver.put([node1], multicodec.BITCOIN_BLOCK)
const cid = await resultPut.first()
const resultGet = resolver.get([cid])
const node = await resultGet.first()
expect(node).to.deep.equal(node1)
})

it('resolver.remove', async () => {
const resultPut = resolver.put([node1], multicodec.BITCOIN_BLOCK)
const cid = await resultPut.first()
const resultGet = resolver.get([cid])
const sameAsNode1 = await resultGet.first()
expect(sameAsNode1).to.deep.equal(node1)
return remove()

function remove () {
return new Promise((resolve, reject) => {
resolver.remove(cid, (err) => {
expect(err).to.not.exist()
const resultGet = resolver.get([cid])
expect(resultGet.next()).to.eventually.be.rejected()
.then(() => resolve())
.catch((err) => reject(err))
})
})
}
})
})
})
}
33 changes: 8 additions & 25 deletions test/ipld-dag-cbor.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,18 +89,6 @@ module.exports = (repo) => {
resolver._put(nc.cid, nc.node, cb)
}, done)
})

// TODO vmx 2018-11-30 Change this test to use `get()`.
// it('resolver._get', (done) => {
// resolver.put(node1, { cid: cid1 }, (err) => {
// expect(err).to.not.exist()
// resolver._get(cid1, (err, node) => {
// expect(err).to.not.exist()
// expect(node1).to.eql(node)
// done()
// })
// })
// })
})

describe('public api', () => {
Expand All @@ -127,19 +115,6 @@ module.exports = (repo) => {
expect(mh.name).to.equal('sha3-512')
})

// TODO vmx 2018-11-30: Implement getting the whole object properly
// it('resolver.get root path', (done) => {
// resolver.get(cid1, '/', (err, result) => {
// expect(err).to.not.exist()
//
// dagCBOR.util.cid(result.value, (err, cid) => {
// expect(err).to.not.exist()
// expect(cid).to.eql(cid1)
// done()
// })
// })
// })

it('resolves value within 1st node scope', async () => {
const result = resolver.resolve(cid1, 'someData')
const node = await result.first()
Expand Down Expand Up @@ -193,6 +168,14 @@ module.exports = (repo) => {
'path not available at root')
})

it('resolver.get round-trip', async () => {
const resultPut = resolver.put([node1], multicodec.DAG_CBOR)
const cid = await resultPut.first()
const resultGet = resolver.get([cid])
const node = await resultGet.first()
expect(node).to.deep.equal(node1)
})

it('resolver.tree', (done) => {
pull(
resolver.treeStream(cid3),
Expand Down

0 comments on commit 743e679

Please sign in to comment.