From 98017654b993cf8c9e2a8e7f09a602e52b4c4b32 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 13 Dec 2018 00:26:26 +0100 Subject: [PATCH] feat: implementation of the new `tree()` function BREAKING CHANGE: This replaces the `treeStream()` function. The API docs for it: > Returns all the paths that can be resolved into. - `cid` (`CID`, required): the CID to get the paths from. - `path` (`IPLD Path`, default: ''): the path to start to retrieve the other paths from. - `options`: - `recursive` (`bool`, default: false): whether to get the paths recursively or not. `false` resolves only the paths of the given CID. Returns an async iterator of all the paths (as Strings) you could resolve into. --- package.json | 3 - src/index.js | 244 +++++++++++++++++++++--------------------- test/basics.js | 14 +-- test/ipld-dag-cbor.js | 98 +++++++---------- 4 files changed, 164 insertions(+), 195 deletions(-) diff --git a/package.json b/package.json index 7c7306f..34445eb 100644 --- a/package.json +++ b/package.json @@ -62,9 +62,6 @@ "ipld-raw": "^2.0.1", "merge-options": "^1.0.1", "multicodec": "~0.5.0", - "pull-defer": "~0.2.3", - "pull-stream": "^3.6.9", - "pull-traverse": "^1.0.3", "typical": "^3.0.0" }, "contributors": [ diff --git a/src/index.js b/src/index.js index 0b9f39f..2fcf8b6 100644 --- a/src/index.js +++ b/src/index.js @@ -1,11 +1,7 @@ 'use strict' const Block = require('ipfs-block') -const pull = require('pull-stream') const CID = require('cids') -const pullDeferSource = require('pull-defer').source -const pullTraverse = require('pull-traverse') -const map = require('async/map') const waterfall = require('async/waterfall') const mergeOptions = require('merge-options') const ipldDagCbor = require('ipld-dag-cbor') @@ -283,122 +279,6 @@ class IPLDResolver { return fancyIterator(next) } - treeStream (cid, path, options) { - if (typeof path === 'object') { - options = path - path = undefined - } - - options = options || {} - - let p - - if (!options.recursive) { - p = pullDeferSource() - - waterfall([ - async () => { - return this._getFormat(cid.codec) - }, - (format, cb) => this.bs.get(cid, (err, block) => { - if (err) return cb(err) - cb(null, format, block) - }), - (format, block, cb) => format.resolver.tree(block.data, cb) - ], (err, paths) => { - if (err) { - p.abort(err) - return p - } - p.resolve(pull.values(paths)) - }) - } - - // recursive - if (options.recursive) { - p = pull( - pullTraverse.widthFirst({ - basePath: null, - cid: cid - }, (el) => { - // pass the paths through the pushable pull stream - // continue traversing the graph by returning - // the next cids with deferred - - if (typeof el === 'string') { - return pull.empty() - } - - const deferred = pullDeferSource() - const cid = el.cid - - waterfall([ - async () => { - return this._getFormat(cid.codec) - }, - (format, cb) => this.bs.get(cid, (err, block) => { - if (err) return cb(err) - cb(null, format, block) - }), - (format, block, cb) => format.resolver.tree(block.data, (err, paths) => { - if (err) { - return cb(err) - } - map(paths, (p, cb) => { - format.resolver.isLink(block.data, p, (err, link) => { - if (err) { - return cb(err) - } - cb(null, { path: p, link: link }) - }) - }, cb) - }) - ], (err, paths) => { - if (err) { - deferred.abort(err) - return deferred - } - - deferred.resolve(pull.values(paths.map((p) => { - const base = el.basePath ? el.basePath + '/' + p.path : p.path - if (p.link) { - return { - basePath: base, - cid: IPLDResolver._maybeCID(p.link) - } - } - return base - }))) - }) - return deferred - }), - pull.map((e) => { - if (typeof e === 'string') { - return e - } - return e.basePath - }), - pull.filter(Boolean) - ) - } - - // filter out by path - if (path) { - return pull( - p, - pull.map((el) => { - if (el.indexOf(path) === 0) { - el = el.slice(path.length + 1) - return el - } - }), - pull.filter(Boolean) - ) - } - - return p - } - /** * Remove IPLD Nodes by the given CIDs. * @@ -434,6 +314,130 @@ class IPLDResolver { return fancyIterator(next) } + /** + * Returns all the paths that can be resolved into. + * + * @param {Object} cid - The ID to get the paths from + * @param {string} [offsetPath=''] - the path to start to retrieve the other paths from. + * @param {Object} [userOptions] + * @param {number} [userOptions.recursive=false] - whether to get the paths recursively or not. `false` resolves only the paths of the given CID. + * @returns {Iterable.>} - Returns an async iterator with paths that can be resolved into + */ + tree (cid, offsetPath, userOptions) { + if (typeof offsetPath === 'object') { + userOptions = offsetPath + offsetPath = undefined + } + offsetPath = offsetPath || '' + + const defaultOptions = { + recursive: false + } + const options = mergeOptions(defaultOptions, userOptions) + + // Get available paths from a block + const getPaths = (cid) => { + return new Promise(async (resolve, reject) => { + let format + try { + format = await this._getFormat(cid.codec) + } catch (error) { + return reject(error) + } + this.bs.get(cid, (err, block) => { + if (err) { + return reject(err) + } + format.resolver.tree(block.data, (err, paths) => { + if (err) { + return reject(err) + } + return resolve({ paths, block }) + }) + }) + }) + } + + // If a path is a link then follow it and return its CID + const maybeRecurse = (block, treePath) => { + return new Promise(async (resolve, reject) => { + // A treepath we might want to follow recursively + const format = await this._getFormat(block.cid.codec) + format.resolver.isLink(block.data, treePath, (err, link) => { + if (err) { + return reject(err) + } + // Something to follow recusively, hence push it into the queue + if (link) { + const cid = IPLDResolver._maybeCID(link) + resolve(cid) + } else { + resolve(null) + } + }) + }) + } + + // The list of paths that will get returned + let treePaths = [] + // The current block, needed to call `isLink()` on every interation + let block + // The list of items we want to follow recursively. The items are + // an object consisting of the CID and the currently already resolved + // path + const queue = [{ cid, basePath: '' }] + // The path that was already traversed + let basePath + + const next = () => { + // End of iteration if there aren't any paths left to return or + // if we don't want to traverse recursively and have already + // returne the first level + if (treePaths.length === 0 && queue.length === 0) { + return { done: true } + } + + return new Promise(async (resolve, reject) => { + // There aren't any paths left, get them from the given CID + if (treePaths.length === 0 && queue.length > 0) { + ({ cid, basePath } = queue.shift()) + + let paths + try { + ({ block, paths } = await getPaths(cid)) + } catch (error) { + return reject(error) + } + treePaths.push(...paths) + } + const treePath = treePaths.shift() + let fullPath = basePath + treePath + + // Only follow links if recursion is intended + if (options.recursive) { + cid = await maybeRecurse(block, treePath) + if (cid !== null) { + queue.push({ cid, basePath: fullPath + '/' }) + } + } + + // Return it if it matches the given offset path, but is not the + // offset path itself + if (fullPath.startsWith(offsetPath) && + fullPath.length > offsetPath.length) { + if (offsetPath.length > 0) { + fullPath = fullPath.slice(offsetPath.length + 1) + } + return resolve({ done: false, value: fullPath }) + } else { // Else move on to the next iteration before returning + return resolve(next()) + } + }) + } + + return fancyIterator(next) + } + /* */ /* internals */ /* */ diff --git a/test/basics.js b/test/basics.js index a6d0eaa..f0fd58f 100644 --- a/test/basics.js +++ b/test/basics.js @@ -11,7 +11,6 @@ const BlockService = require('ipfs-block-service') const CID = require('cids') const multihash = require('multihashes') const multicodec = require('multicodec') -const pull = require('pull-stream') const inMemory = require('ipld-in-memory') const IPLDResolver = require('../src') @@ -81,7 +80,7 @@ module.exports = (repo) => { }) }) - it('treeStream - errors on unknown resolver', (done) => { + it('tree - errors on unknown resolver', async () => { const bs = new BlockService(repo) const r = new IPLDResolver({ blockService: bs }) // choosing a format that is not supported @@ -90,14 +89,9 @@ module.exports = (repo) => { 'blake2b-8', multihash.encode(Buffer.from('abcd', 'hex'), 'sha1') ) - pull( - r.treeStream(cid, '/', {}), - pull.collect(function (err) { - expect(err).to.exist() - expect(err.message).to.eql('No resolver found for codec "blake2b-8"') - done() - }) - ) + const result = r.tree(cid) + await expect(result.next()).to.be.rejectedWith( + 'No resolver found for codec "blake2b-8"') }) }) } diff --git a/test/ipld-dag-cbor.js b/test/ipld-dag-cbor.js index 13f80b4..11e2766 100644 --- a/test/ipld-dag-cbor.js +++ b/test/ipld-dag-cbor.js @@ -11,7 +11,6 @@ const BlockService = require('ipfs-block-service') const dagCBOR = require('ipld-dag-cbor') const series = require('async/series') const each = require('async/each') -const pull = require('pull-stream') const multicodec = require('multicodec') const multihash = require('multihashes') @@ -176,75 +175,50 @@ module.exports = (repo) => { expect(node).to.deep.equal(node1) }) - it('resolver.tree', (done) => { - pull( - resolver.treeStream(cid3), - pull.collect((err, values) => { - expect(err).to.not.exist() - expect(values).to.eql([ - 'one', - 'two', - 'someData' - ]) - done() - }) - ) + it('resolver.tree', async () => { + const result = resolver.tree(cid3) + const paths = await result.all() + expect(paths).to.eql([ + 'one', + 'two', + 'someData' + ]) }) - it('resolver.tree with exist()ent path', (done) => { - pull( - resolver.treeStream(cid3, 'one'), - pull.collect((err, values) => { - expect(err).to.not.exist() - expect(values).to.eql([]) - done() - }) - ) + it('resolver.tree with exist()ent path', async () => { + const result = resolver.tree(cid3, 'one') + const paths = await result.all() + expect(paths).to.eql([]) }) - it('resolver.tree with non exist()ent path', (done) => { - pull( - resolver.treeStream(cid3, 'bananas'), - pull.collect((err, values) => { - expect(err).to.not.exist() - expect(values).to.eql([]) - done() - }) - ) + it('resolver.tree with non exist()ent path', async () => { + const result = resolver.tree(cid3, 'bananas') + const paths = await result.all() + expect(paths).to.eql([]) }) - it('resolver.tree recursive', (done) => { - pull( - resolver.treeStream(cid3, { recursive: true }), - pull.collect((err, values) => { - expect(err).to.not.exist() - expect(values).to.eql([ - 'one', - 'two', - 'someData', - 'one/someData', - 'two/one', - 'two/someData', - 'two/one/someData' - ]) - done() - }) - ) + it('resolver.tree recursive', async () => { + const result = resolver.tree(cid3, { recursive: true }) + const paths = await result.all() + expect(paths).to.eql([ + 'one', + 'two', + 'someData', + 'one/someData', + 'two/one', + 'two/someData', + 'two/one/someData' + ]) }) - it('resolver.tree with exist()ent path recursive', (done) => { - pull( - resolver.treeStream(cid3, 'two', { recursive: true }), - pull.collect((err, values) => { - expect(err).to.not.exist() - expect(values).to.eql([ - 'one', - 'someData', - 'one/someData' - ]) - done() - }) - ) + it('resolver.tree with exist()ent path recursive', async () => { + const result = resolver.tree(cid3, 'two', { recursive: true }) + const paths = await result.all() + expect(paths).to.eql([ + 'one', + 'someData', + 'one/someData' + ]) }) it('resolver.remove', async () => {