diff --git a/.gitignore b/.gitignore index c250503..822031f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules testdb +localdb diff --git a/fetch.js b/fetch.js new file mode 100644 index 0000000..4db9e3e --- /dev/null +++ b/fetch.js @@ -0,0 +1 @@ +module.exports = window.fetch diff --git a/package-lock.json b/package-lock.json index 9442358..10b80c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1332,7 +1332,7 @@ "requires": { "bignumber.js": "6.0.0", "commander": "2.15.1", - "ieee754": "1.1.10", + "ieee754": "1.1.11", "json-text-sequence": "0.1.1" } }, @@ -3026,6 +3026,17 @@ "map-cache": "0.2.2" } }, + "fs-extra": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-5.0.0.tgz", + "integrity": "sha512-66Pm4RYbjzdyeuqudYqhFiNBbCIuI9kgRqLPSHIlXHidW8NIQtVdkM1yeZ4lXwuhbTETv3EUGMNHAAw6hiundQ==", + "dev": true, + "requires": { + "graceful-fs": "4.1.11", + "jsonfile": "4.0.0", + "universalify": "0.1.1" + } + }, "fs-mkdirp-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-mkdirp-stream/-/fs-mkdirp-stream-1.0.0.tgz", @@ -4340,9 +4351,9 @@ "dev": true }, "ieee754": { - "version": "1.1.10", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.10.tgz", - "integrity": "sha512-byWFX8OyW/qeVxcY21r6Ncxl0ZYHgnf0cPup2h34eHXrCJbOp7IuqnJ4Q0omfyWl6Z++BTI6bByf31pZt7iRLg==" + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.11.tgz", + "integrity": "sha512-VhDzCKN7K8ufStx/CLj5/PDTMgph+qwN5Pkd5i0sGnVwk56zJ0lkT8Qzi1xqWLS0Wp29DgDtNeS7v8/wMoZeHg==" }, "ignore": { "version": "3.3.7", @@ -4834,6 +4845,18 @@ "requires": { "node-fetch": "1.7.3", "whatwg-fetch": "2.0.3" + }, + "dependencies": { + "node-fetch": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", + "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", + "dev": true, + "requires": { + "encoding": "0.1.12", + "is-stream": "1.1.0" + } + } } }, "isstream": { @@ -4924,6 +4947,15 @@ "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=", "dev": true }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "dev": true, + "requires": { + "graceful-fs": "4.1.11" + } + }, "jsonify": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz", @@ -5593,14 +5625,9 @@ } }, "node-fetch": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", - "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", - "dev": true, - "requires": { - "encoding": "0.1.12", - "is-stream": "1.1.0" - } + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz", + "integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U=" }, "noop-logger": { "version": "0.1.1", @@ -10648,6 +10675,12 @@ "unist-util-is": "2.1.1" } }, + "universalify": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz", + "integrity": "sha1-+nG63UQ3r0wUiEHjs7Fl+enlkLc=", + "dev": true + }, "unset-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", diff --git a/package.json b/package.json index 083b5d7..a4dab8a 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "devDependencies": { "coveralls": "^3.0.0", "documentation": "^6.1.0", + "fs-extra": "^5.0.0", "level-browserify": "^1.1.2", "nyc": "^11.6.0", "standard": "^11.0.1", @@ -29,9 +30,13 @@ "dependencies": { "borc": "git+https://github.com/dignifiedquire/borc.git#fix/nested-array", "ipld-graph-builder": "^1.3.8", + "node-fetch": "^2.1.2", "text-encoding": "^0.6.4", "uint1array": "^1.0.5" }, + "browser": { + "node-fetch": "./fetch.js" + }, "standard": { "ignore": [ "/benchmark/" diff --git a/remoteDatastore.js b/remoteDatastore.js new file mode 100644 index 0000000..ed42e86 --- /dev/null +++ b/remoteDatastore.js @@ -0,0 +1,52 @@ +const Buffer = require('safe-buffer').Buffer +const TreeDAG = require('./datastore.js') +const fetch = require('node-fetch') + +module.exports = class RemoteTreeDAG extends TreeDAG { + /** + * @param dag {object} a level db instance + * @param remoteOpts + * @param remoteOpts.uri {string} the HTTP uri which has an interface: GET /:key -> value + * @param remoteOpts.encoding {string} the encoding of the reponse + * @param opts.decoder {object} a cbor decoder + */ + constructor (dag, remoteOpts, decoder) { + super(dag, decoder) + this.remoteOpts = Object.assign({ + uri: null, + encoding: 'base64' + }, remoteOpts) + } + + async get (link) { + try { + return await super.get(link) + } catch (e) { + if (this.remoteOpts.uri) { + await this.fetchRemote(link) + return super.get(link) + } + } + } + + fetchRemote (key) { + if (!Buffer.isBuffer(key)) { + key = Buffer.from(key.buffer) + } + + const route = `${this.remoteOpts.uri}/${key.toString('hex')}` + return fetch(route) + .then(res => res.text()) + .then(text => { + const encoded = Buffer.from(text, this.remoteOpts.encoding) + return new Promise((resolve, reject) => { + this._dag.put(key, encoded.toString('hex'), () => { + resolve(key) + }) + }) + }) + .catch(err => { + console.warn(`error fetching ${route}:`, err.message) + }) + } +} diff --git a/tests/index.js b/tests/index.js index 444e630..2749b63 100644 --- a/tests/index.js +++ b/tests/index.js @@ -1,7 +1,10 @@ +const fs = require('fs-extra') const tape = require('tape') const crypto = require('crypto') const level = require('level-browserify') const RadixTree = require('../') +const RemoteDataStore = require('../remoteDatastore') +const remote = require('./remote') const db = level('./testdb') tape('root existance', async t => { @@ -203,3 +206,34 @@ tape('random', async t => { t.end() }) + +tape('remote', async t => { + // remote + const remoteTree = new RadixTree({ + db: db + }) + const server = remote.listen(db) + + const entries = 100 + for (let i = 0; i < entries; i++) { + const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) + remoteTree.set(key, Buffer.from([i])) + } + const stateRoot = await remoteTree.flush() + + // local + fs.removeSync('./localdb') + const localTree = new RadixTree({ + dag: new RemoteDataStore(level('./localdb'), {uri: 'http://localhost:3000'}) + }) + localTree.root = stateRoot + + for (let i = 0; i < entries; i++) { + const key = crypto.createHash('sha256').update(i.toString()).digest().slice(0, 20) + const value = await localTree.get(key) + t.equals(value.value[0], i) + } + + server.close() + t.end() +}) diff --git a/tests/remote.js b/tests/remote.js new file mode 100644 index 0000000..003c7df --- /dev/null +++ b/tests/remote.js @@ -0,0 +1,26 @@ +const RadixTree = require('../') + +const cbor = require('borc') +const http = require('http') + +let tree + +const server = http.createServer(async (req, res) => { + const key = Buffer.from(req.url.slice(1), 'hex') + const value = await tree.graph._dag.get(key) + res.end(cbor.encode(value).toString('base64')) +}) + +module.exports = { + listen: (db, port = 3000) => { + server.listen(port, (err) => { + if (err) { return console.error(err) } + + tree = new RadixTree({db}) + + console.log(`server is listening on ${port}`) + }) + + return server + } +}