From 0c0780cf35124713c92ed547b31057ee945069f8 Mon Sep 17 00:00:00 2001 From: Norton Wang Date: Thu, 5 Apr 2018 10:32:51 -0700 Subject: [PATCH 1/2] add remote datastore --- .gitignore | 1 + package-lock.json | 57 ++++++++++++++++++++++++++++++++++++---------- package.json | 2 ++ remoteDatastore.js | 52 ++++++++++++++++++++++++++++++++++++++++++ tests/index.js | 34 +++++++++++++++++++++++++++ tests/remote.js | 26 +++++++++++++++++++++ 6 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 remoteDatastore.js create mode 100644 tests/remote.js diff --git a/.gitignore b/.gitignore index c250503..822031f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules testdb +localdb diff --git a/package-lock.json b/package-lock.json index 9442358..10b80c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1332,7 +1332,7 @@ "requires": { "bignumber.js": "6.0.0", "commander": "2.15.1", - "ieee754": "1.1.10", + "ieee754": "1.1.11", "json-text-sequence": "0.1.1" } }, @@ -3026,6 +3026,17 @@ "map-cache": "0.2.2" } }, + "fs-extra": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-5.0.0.tgz", + "integrity": "sha512-66Pm4RYbjzdyeuqudYqhFiNBbCIuI9kgRqLPSHIlXHidW8NIQtVdkM1yeZ4lXwuhbTETv3EUGMNHAAw6hiundQ==", + "dev": true, + "requires": { + "graceful-fs": "4.1.11", + "jsonfile": "4.0.0", + "universalify": "0.1.1" + } + }, "fs-mkdirp-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-mkdirp-stream/-/fs-mkdirp-stream-1.0.0.tgz", @@ -4340,9 +4351,9 @@ "dev": true }, "ieee754": { - "version": "1.1.10", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.10.tgz", - "integrity": "sha512-byWFX8OyW/qeVxcY21r6Ncxl0ZYHgnf0cPup2h34eHXrCJbOp7IuqnJ4Q0omfyWl6Z++BTI6bByf31pZt7iRLg==" + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.11.tgz", + "integrity": "sha512-VhDzCKN7K8ufStx/CLj5/PDTMgph+qwN5Pkd5i0sGnVwk56zJ0lkT8Qzi1xqWLS0Wp29DgDtNeS7v8/wMoZeHg==" }, "ignore": { "version": "3.3.7", @@ -4834,6 +4845,18 @@ "requires": { "node-fetch": "1.7.3", "whatwg-fetch": "2.0.3" + }, + "dependencies": { + "node-fetch": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", + "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", + "dev": true, + "requires": { + "encoding": "0.1.12", + "is-stream": "1.1.0" + } + } } }, "isstream": { @@ -4924,6 +4947,15 @@ "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=", "dev": true }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "dev": true, + "requires": { + "graceful-fs": "4.1.11" + } + }, "jsonify": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz", @@ -5593,14 +5625,9 @@ } }, "node-fetch": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", - "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", - "dev": true, - "requires": { - "encoding": "0.1.12", - "is-stream": "1.1.0" - } + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz", + "integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U=" }, "noop-logger": { "version": "0.1.1", @@ -10648,6 +10675,12 @@ "unist-util-is": "2.1.1" } }, + "universalify": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz", + "integrity": "sha1-+nG63UQ3r0wUiEHjs7Fl+enlkLc=", + "dev": true + }, "unset-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", diff --git a/package.json b/package.json index 083b5d7..6d3b079 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "devDependencies": { "coveralls": "^3.0.0", "documentation": "^6.1.0", + "fs-extra": "^5.0.0", "level-browserify": "^1.1.2", "nyc": "^11.6.0", "standard": "^11.0.1", @@ -29,6 +30,7 @@ "dependencies": { "borc": "git+https://github.com/dignifiedquire/borc.git#fix/nested-array", "ipld-graph-builder": "^1.3.8", + "node-fetch": "^2.1.2", "text-encoding": "^0.6.4", "uint1array": "^1.0.5" }, diff --git a/remoteDatastore.js b/remoteDatastore.js new file mode 100644 index 0000000..c984c40 --- /dev/null +++ b/remoteDatastore.js @@ -0,0 +1,52 @@ +const Buffer = require('safe-buffer').Buffer +const TreeDAG = require('./datastore.js') +const fetch = typeof window !== 'undefined' ? window.fetch : require('node-fetch') + +module.exports = class RemoteTreeDAG extends TreeDAG { + /** + * @param dag {object} a level db instance + * @param remoteOpts + * @param remoteOpts.uri {string} the HTTP uri which has an interface: GET /:key -> value + * @param remoteOpts.encoding {string} the encoding of the reponse + * @param opts.decoder {object} a cbor decoder + */ + constructor (dag, remoteOpts, decoder) { + super(dag, decoder) + this.remoteOpts = Object.assign({ + uri: null, + encoding: 'base64' + }, remoteOpts) + } + + async get (link) { + try { + return await super.get(link) + } catch (e) {} + + if (this.remoteOpts.uri) { + await this.fetchRemote(link) + return super.get(link) + } + } + + fetchRemote (key) { + if (!Buffer.isBuffer(key)) { + key = Buffer.from(key.buffer) + } + + const route = `${this.remoteOpts.uri}/${key.toString('hex')}` + return fetch(route) + .then(res => res.text()) + .then(text => { + const encoded = Buffer.from(text, this.remoteOpts.encoding) + return new Promise((resolve, reject) => { + this._dag.put(key, encoded.toString('hex'), () => { + resolve(key) + }) + }) + }) + .catch(err => { + console.warn(`error fetching ${route}:`, err.message) + }) + } +} diff --git a/tests/index.js b/tests/index.js index 444e630..2749b63 100644 --- a/tests/index.js +++ b/tests/index.js @@ -1,7 +1,10 @@ +const fs = require('fs-extra') const tape = require('tape') const crypto = require('crypto') const level = require('level-browserify') const RadixTree = require('../') +const RemoteDataStore = require('../remoteDatastore') +const remote = require('./remote') const db = level('./testdb') tape('root existance', async t => { @@ -203,3 +206,34 @@ tape('random', async t => { t.end() }) + +tape('remote', async t => { + // remote + const remoteTree = new RadixTree({ + db: db + }) + const server = remote.listen(db) + + const entries = 100 + for (let i = 0; i < entries; i++) { + const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) + remoteTree.set(key, Buffer.from([i])) + } + const stateRoot = await remoteTree.flush() + + // local + fs.removeSync('./localdb') + const localTree = new RadixTree({ + dag: new RemoteDataStore(level('./localdb'), {uri: 'http://localhost:3000'}) + }) + localTree.root = stateRoot + + for (let i = 0; i < entries; i++) { + const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) + const value = await localTree.get(key) + t.equals(value.value[0], i) + } + + server.close() + t.end() +}) diff --git a/tests/remote.js b/tests/remote.js new file mode 100644 index 0000000..003c7df --- /dev/null +++ b/tests/remote.js @@ -0,0 +1,26 @@ +const RadixTree = require('../') + +const cbor = require('borc') +const http = require('http') + +let tree + +const server = http.createServer(async (req, res) => { + const key = Buffer.from(req.url.slice(1), 'hex') + const value = await tree.graph._dag.get(key) + res.end(cbor.encode(value).toString('base64')) +}) + +module.exports = { + listen: (db, port = 3000) => { + server.listen(port, (err) => { + if (err) { return console.error(err) } + + tree = new RadixTree({db}) + + console.log(`server is listening on ${port}`) + }) + + return server + } +} From eb2b91e814ad8589fb6c3b7c0c1d9714a040b688 Mon Sep 17 00:00:00 2001 From: Norton Wang Date: Thu, 5 Apr 2018 11:57:13 -0700 Subject: [PATCH 2/2] minor fixes --- fetch.js | 1 + package.json | 3 +++ remoteDatastore.js | 12 ++++++------ 3 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 fetch.js diff --git a/fetch.js b/fetch.js new file mode 100644 index 0000000..4db9e3e --- /dev/null +++ b/fetch.js @@ -0,0 +1 @@ +module.exports = window.fetch diff --git a/package.json b/package.json index 6d3b079..a4dab8a 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,9 @@ "text-encoding": "^0.6.4", "uint1array": "^1.0.5" }, + "browser": { + "node-fetch": "./fetch.js" + }, "standard": { "ignore": [ "/benchmark/" diff --git a/remoteDatastore.js b/remoteDatastore.js index c984c40..ed42e86 100644 --- a/remoteDatastore.js +++ b/remoteDatastore.js @@ -1,6 +1,6 @@ const Buffer = require('safe-buffer').Buffer const TreeDAG = require('./datastore.js') -const fetch = typeof window !== 'undefined' ? window.fetch : require('node-fetch') +const fetch = require('node-fetch') module.exports = class RemoteTreeDAG extends TreeDAG { /** @@ -21,11 +21,11 @@ module.exports = class RemoteTreeDAG extends TreeDAG { async get (link) { try { return await super.get(link) - } catch (e) {} - - if (this.remoteOpts.uri) { - await this.fetchRemote(link) - return super.get(link) + } catch (e) { + if (this.remoteOpts.uri) { + await this.fetchRemote(link) + return super.get(link) + } } }