From e343b3195ea74d6528f9479bf70385d2e8fe377c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 20 Sep 2019 10:44:04 +0100 Subject: [PATCH 1/4] feat: accept async iterators into blockstore.putMany Allows streaming into the blockstore. The blockstore currently uses the batch functionality of the underlying datatore, so we might need to change `interface-datastore` next to add streaming primitives but this at least allows streaming this far down the stack. BREAKING CHANGE: you must pass an iterable or async iterable to putMany - this should be relatively painless as the current API is to pass an array which is iterable, but it does change the API. --- src/blockstore.js | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/blockstore.js b/src/blockstore.js index 27fc5415..04e1fab7 100644 --- a/src/blockstore.js +++ b/src/blockstore.js @@ -85,26 +85,21 @@ function createBaseStore (store) { /** * Like put, but for more. * - * @param {Array} blocks + * @param {AsyncIterable} blocks * @returns {Promise} */ async putMany (blocks) { - const keys = blocks.map((b) => ({ - key: cidToKey(b.cid), - block: b - })) - const batch = store.batch() - await Promise.all( - keys.map(async k => { - if (await store.has(k.key)) { - return - } + for await (const block of blocks) { + const key = cidToKey(block.cid) - batch.put(k.key, k.block.data) - }) - ) + if (await store.has(key)) { + continue + } + + batch.put(key, block.data) + } return batch.commit() }, From 41921a0080a492c6fa1d37a81e12ead999e5e3bb Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 20 Sep 2019 11:32:37 +0100 Subject: [PATCH 2/4] docs: Update src/blockstore.js Co-Authored-By: Hugo Dias --- src/blockstore.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blockstore.js b/src/blockstore.js index 04e1fab7..7d3a4bc0 100644 --- a/src/blockstore.js +++ b/src/blockstore.js @@ -85,7 +85,7 @@ function createBaseStore (store) { /** * Like put, but for more. * - * @param {AsyncIterable} blocks + * @param {AsyncIterable|Iterable} blocks * @returns {Promise} */ async putMany (blocks) { From f7e2e8ed6b0c47cc318a9fddb95074a5da9a3c70 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Fri, 20 Sep 2019 12:36:24 +0200 Subject: [PATCH 3/4] chore: remove CI commitlint chore: add node 12 to CI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d2e373b0..8507920f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ stages: node_js: - '10' + - '12' os: - linux @@ -20,7 +21,6 @@ jobs: include: - stage: check script: - - npx aegir commitlint --travis - npx aegir dep-check - npm run lint From efd4360614c2f1eb24d73647fa06e4814a9c7fe4 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 20 Sep 2019 12:05:34 +0100 Subject: [PATCH 4/4] docs: update docs with new api --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f5e46902..c48600eb 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ Get a value at the root of the repo. Put many blocks. -* `block` should be an array of type [Block](https://github.com/ipfs/js-ipfs-block#readme). +* `block` should be an Iterable or AsyncIterable that yields entries of type [Block](https://github.com/ipfs/js-ipfs-block#readme). #### `Promise repo.blocks.get (cid)`