From dca374938c2e51a86820815a6511825d3ce52055 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 24 Jul 2018 14:51:43 +0100 Subject: [PATCH 01/21] feat: preload content This PR adds a new config property `preload`: ```js new IPFS({ preload: { enabled: false, addresses: [{ bootstrap: '/multiaddr/bootstrap/address/tcp/4001/ipfs/peerId', gateway: '/multiaddr/gateway/address/tcp/9090' }] } }) ``` * `preload.enabled` (default `false`) enables/disabled preloading * `preload.addresses` array of node addresses to preload content on * `preload.addresses.0.bootstrap` the swarm address of the preload node. This is added to the node's bootstrap addresses * `preload.addresses.0.gateway` the gateway address of the preload node. This is the address we make a HTTP HEAD request to, to initiate the preload After adding content with `ipfs.files.add*` or `ipfs.dag.put`, we make HTTP HEAD requests to _all_ preload gateway addresses (providing `preload.enabled` is true). License: MIT Signed-off-by: Alan Shaw --- README.md | 5 ++++ package.json | 2 ++ src/core/components/libp2p.js | 13 ++++++++++ src/core/config.js | 15 +++++++++++ src/core/preload.js | 40 +++++++++++++++++++++++++++++ src/core/runtime/preload-browser.js | 27 +++++++++++++++++++ src/core/runtime/preload-nodejs.js | 33 ++++++++++++++++++++++++ 7 files changed, 135 insertions(+) create mode 100644 src/core/preload.js create mode 100644 src/core/runtime/preload-browser.js create mode 100644 src/core/runtime/preload-nodejs.js diff --git a/README.md b/README.md index 0c20f35cdb..0f45d25fcf 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,11 @@ Creates and returns an instance of an IPFS node. Use the `options` argument to s - `enabled` (boolean): Make this node a relay (other nodes can connect *through* it). (Default: `false`) - `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`) +- `preload` (object): Configure external nodes that will preload any content added to this node + - `enabled` (boolean): Enable content preloading (Default: `false`) + - `addresses` (array): Bootstrap and gateway addresses for the preload nodes + - `bootstrap` (string): Multiaddr swarm addresses of a node that should preload content + - `gateway` (string): Multiaddr gateway addresses of a node that should preload content - `EXPERIMENTAL` (object): Enable and configure experimental features. - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`) - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`) diff --git a/package.json b/package.json index 2cd7bcb477..e98cd5a278 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "./src/core/components/init-assets.js": false, "./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js", "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js", + "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js", "./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js", "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js", "./test/utils/create-repo-nodejs.js": "./test/utils/create-repo-browser.js", @@ -140,6 +141,7 @@ "mime-types": "^2.1.18", "mkdirp": "~0.5.1", "multiaddr": "^5.0.0", + "multiaddr-to-uri": "^4.0.0", "multibase": "~0.4.0", "multihashes": "~0.4.13", "once": "^1.4.0", diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index 1ad0d1ecac..4ee772d19d 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -58,6 +58,19 @@ module.exports = function libp2p (self) { libp2pDefaults ) + // Add the addresses for the preload nodes to the bootstrap addresses + if (get(self._options, 'preload.enabled')) { + let bootstrapList = libp2pOptions.config.peerDiscovery.bootstrap.list + + const preloadBootstrap = get(self._options, 'preload.addresses', []) + .map(address => address.bootstrap) + .filter(Boolean) // A preload node doesn't _have_ to be added to the boostrap + .filter(address => !bootstrapList.includes(address)) // De-dupe + + bootstrapList = bootstrapList.concat(preloadBootstrap) + libp2pOptions.config.peerDiscovery.bootstrap.list = bootstrapList + } + self._libp2pNode = new Node(libp2pOptions) self._libp2pNode.on('peer:discovery', (peerInfo) => { diff --git a/src/core/config.js b/src/core/config.js index 1b04d10a2f..0c8342efad 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -8,6 +8,21 @@ const schema = Joi.object().keys({ Joi.string() ).allow(null), repoOwner: Joi.boolean().default(true), + preload: Joi.object().keys({ + enabled: Joi.boolean().default(false), + addresses: Joi.array() + .items(Joi.object().keys({ + bootstrap: Joi.multiaddr().options({ convert: false }), + gateway: Joi.multiaddr().options({ convert: false }) + })) + .default([{ + bootstrap: '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + gateway: '/dns4/wss0.bootstrap.libp2p.io/tcp/443' + }, { + bootstrap: '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + gateway: '/dns4/wss1.bootstrap.libp2p.io/tcp/443' + }]) + }).allow(null), init: Joi.alternatives().try( Joi.boolean(), Joi.object().keys({ bits: Joi.number().integer() }) diff --git a/src/core/preload.js b/src/core/preload.js new file mode 100644 index 0000000000..4293c856a5 --- /dev/null +++ b/src/core/preload.js @@ -0,0 +1,40 @@ +const get = require('lodash/get') +const setImmediate = require('async/setImmediate') +const each = require('async/each') +const toUri = require('multiaddr-to-uri') +const preload = require('./runtime/preload-nodejs') + +// Tools like IPFS Companion redirect requests to IPFS gateways to your local +// gateway. This is a hint to those tools that they shouldn't redirect these +// requests as they will effectively disable the preloading. +const redirectOptOutHint = 'x-ipfs-preload' + +module.exports = self => { + const enabled = get(self._options, 'preload.enabled') + const gateways = get(self._options, 'preload.addresses', []) + .map(address => address.gateway) + .filter(Boolean) + + if (!enabled || !gateways.length) { + return (_, callback) => { + if (!callback) return + setImmediate(() => callback()) + } + } + + return (cid, callback) => { + each(gateways, (gatewayAddr, cb) => { + let gatewayUri + + try { + gatewayUri = toUri(gatewayAddr) + } catch (err) { + return cb(err) + } + + const preloadUrl = `${gatewayUri}/ipfs/${cid.toBaseEncodedString()}#${redirectOptOutHint}` + + preload(preloadUrl, cb) + }, callback) + } +} diff --git a/src/core/runtime/preload-browser.js b/src/core/runtime/preload-browser.js new file mode 100644 index 0000000000..dd765395c2 --- /dev/null +++ b/src/core/runtime/preload-browser.js @@ -0,0 +1,27 @@ +const debug = require('debug') + +const log = debug('jsipfs:preload') +log.error = debug('jsipfs:preload:error') + +module.exports = function preload (url, callback) { + log(url) + + const req = new window.XMLHttpRequest() + + req.open('HEAD', url) + + req.onreadystatechange = function () { + if (this.readyState !== this.DONE) { + return + } + + if (this.status !== 200) { + log.error('failed to preload', url, this.status, this.statusText) + return callback(new Error(`failed to preload ${url}`)) + } + + callback() + } + + req.send() +} diff --git a/src/core/runtime/preload-nodejs.js b/src/core/runtime/preload-nodejs.js new file mode 100644 index 0000000000..8f09caca5e --- /dev/null +++ b/src/core/runtime/preload-nodejs.js @@ -0,0 +1,33 @@ +const http = require('http') +const URL = require('url') +const debug = require('debug') + +const log = debug('jsipfs:preload') +log.error = debug('jsipfs:preload:error') + +module.exports = function preload (url, callback) { + log(url) + + url = new URL(url) + + const req = http.request({ + protocol: url.protocol, + hostname: url.hostname, + port: url.port, + path: url.pathname, + method: 'HEAD' + }, (res) => { + if (res.statusCode !== 200) { + log.error('failed to preload', url, res.statusCode, res.statusMessage) + return callback(new Error(`failed to preload ${url}`)) + } + callback() + }) + + req.on('error', err => { + log.error('error preloading', url, err) + callback(err) + }) + + req.end() +} From f5b585454f3446892e24e75c0f001330310188de Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 25 Jul 2018 16:30:57 +0100 Subject: [PATCH 02/21] test: add preload tests for ipfs.files.add and make fixes License: MIT Signed-off-by: Alan Shaw --- .aegir.js | 31 ++++++- README.md | 6 +- src/core/components/files.js | 17 ++++ src/core/components/libp2p.js | 13 --- src/core/config.js | 18 ++--- src/core/index.js | 2 + src/core/preload.js | 37 ++++++--- src/core/runtime/preload-browser.js | 7 +- src/core/runtime/preload-nodejs.js | 16 +++- test/core/preload.spec.js | 120 ++++++++++++++++++++++++++++ test/utils/mock-preload-node.js | 120 ++++++++++++++++++++++++++++ 11 files changed, 336 insertions(+), 51 deletions(-) create mode 100644 test/core/preload.spec.js create mode 100644 test/utils/mock-preload-node.js diff --git a/.aegir.js b/.aegir.js index 979ffde3df..5d87aa66d7 100644 --- a/.aegir.js +++ b/.aegir.js @@ -1,8 +1,11 @@ 'use strict' -const createServer = require('ipfsd-ctl').createServer +const IPFSFactory = require('ipfsd-ctl') +const parallel = require('async/parallel') +const MockPreloadNode = require('./test/utils/mock-preload-node') -const server = createServer() +const ipfsdServer = IPFSFactory.createServer() +const preloadNode = MockPreloadNode.createNode() module.exports = { webpack: { @@ -21,9 +24,29 @@ module.exports = { singleRun: true }, hooks: { + node: { + pre: (cb) => preloadNode.start(cb), + post: (cb) => preloadNode.stop(cb) + }, browser: { - pre: server.start.bind(server), - post: server.stop.bind(server) + pre: (cb) => { + parallel([ + (cb) => { + ipfsdServer.start() + cb() + }, + (cb) => preloadNode.start(cb) + ], cb) + }, + post: (cb) => { + parallel([ + (cb) => { + ipfsdServer.stop() + cb() + }, + (cb) => preloadNode.stop(cb) + ], cb) + } } } } diff --git a/README.md b/README.md index 0f45d25fcf..e312940788 100644 --- a/README.md +++ b/README.md @@ -231,11 +231,9 @@ Creates and returns an instance of an IPFS node. Use the `options` argument to s - `enabled` (boolean): Make this node a relay (other nodes can connect *through* it). (Default: `false`) - `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`) -- `preload` (object): Configure external nodes that will preload any content added to this node +- `preload` (object): Configure external nodes that will preload content added to this node - `enabled` (boolean): Enable content preloading (Default: `false`) - - `addresses` (array): Bootstrap and gateway addresses for the preload nodes - - `bootstrap` (string): Multiaddr swarm addresses of a node that should preload content - - `gateway` (string): Multiaddr gateway addresses of a node that should preload content + - `gateways` (array): Multiaddr gateway addresses of nodes that should preload content. NOTE: nodes specified here should also be added to your node's bootstrap address list at `config.Boostrap` - `EXPERIMENTAL` (object): Enable and configure experimental features. - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`) - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`) diff --git a/src/core/components/files.js b/src/core/components/files.js index f69868bd77..632da9e154 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -89,6 +89,22 @@ function normalizeContent (opts, content) { }) } +function preloadFile (self, opts, file, cb) { + const isRootFile = opts.wrapWithDirectory + ? file.path === '' + : !file.path.includes('/') + + const shouldPreload = isRootFile && !opts.onlyHash && opts.preload !== false + + if (!shouldPreload) return cb(null, file) + + self._preload(new CID(file.hash), (err) => { + // Preload error is not fatal + if (err) console.error(err) + cb(null, file) + }) +} + function pinFile (self, opts, file, cb) { // Pin a file if it is the root dir of a recursive add or the single file // of a direct add. @@ -158,6 +174,7 @@ module.exports = function files (self) { pull.flatten(), importer(self._ipld, opts), pull.asyncMap(prepareFile.bind(null, self, opts)), + pull.asyncMap(preloadFile.bind(null, self, opts)), pull.asyncMap(pinFile.bind(null, self, opts)) ) } diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index 4ee772d19d..1ad0d1ecac 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -58,19 +58,6 @@ module.exports = function libp2p (self) { libp2pDefaults ) - // Add the addresses for the preload nodes to the bootstrap addresses - if (get(self._options, 'preload.enabled')) { - let bootstrapList = libp2pOptions.config.peerDiscovery.bootstrap.list - - const preloadBootstrap = get(self._options, 'preload.addresses', []) - .map(address => address.bootstrap) - .filter(Boolean) // A preload node doesn't _have_ to be added to the boostrap - .filter(address => !bootstrapList.includes(address)) // De-dupe - - bootstrapList = bootstrapList.concat(preloadBootstrap) - libp2pOptions.config.peerDiscovery.bootstrap.list = bootstrapList - } - self._libp2pNode = new Node(libp2pOptions) self._libp2pNode.on('peer:discovery', (peerInfo) => { diff --git a/src/core/config.js b/src/core/config.js index 0c8342efad..7f2b82f925 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -10,18 +10,12 @@ const schema = Joi.object().keys({ repoOwner: Joi.boolean().default(true), preload: Joi.object().keys({ enabled: Joi.boolean().default(false), - addresses: Joi.array() - .items(Joi.object().keys({ - bootstrap: Joi.multiaddr().options({ convert: false }), - gateway: Joi.multiaddr().options({ convert: false }) - })) - .default([{ - bootstrap: '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - gateway: '/dns4/wss0.bootstrap.libp2p.io/tcp/443' - }, { - bootstrap: '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - gateway: '/dns4/wss1.bootstrap.libp2p.io/tcp/443' - }]) + gateways: Joi.array() + .items(Joi.multiaddr().options({ convert: false })) + .default([ + '/dns4/wss0.bootstrap.libp2p.io/https', + '/dns4/wss1.bootstrap.libp2p.io/https' + ]) }).allow(null), init: Joi.alternatives().try( Joi.boolean(), diff --git a/src/core/index.js b/src/core/index.js index 36f7c3a118..ad9c9af6ba 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -22,6 +22,7 @@ const boot = require('./boot') const components = require('./components') // replaced by repo-browser when running in the browser const defaultRepo = require('./runtime/repo-nodejs') +const preload = require('./preload') class IPFS extends EventEmitter { constructor (options) { @@ -78,6 +79,7 @@ class IPFS extends EventEmitter { this._blockService = new BlockService(this._repo) this._ipld = new Ipld(this._blockService) this._pubsub = undefined + this._preload = preload(this._options.preload) // IPFS Core exposed components // - for booting up a node diff --git a/src/core/preload.js b/src/core/preload.js index 4293c856a5..dcefc52150 100644 --- a/src/core/preload.js +++ b/src/core/preload.js @@ -1,40 +1,51 @@ -const get = require('lodash/get') +'use strict' + const setImmediate = require('async/setImmediate') const each = require('async/each') const toUri = require('multiaddr-to-uri') +const debug = require('debug') const preload = require('./runtime/preload-nodejs') +const log = debug('jsipfs:preload') +log.error = debug('jsipfs:preload:error') + // Tools like IPFS Companion redirect requests to IPFS gateways to your local // gateway. This is a hint to those tools that they shouldn't redirect these // requests as they will effectively disable the preloading. const redirectOptOutHint = 'x-ipfs-preload' -module.exports = self => { - const enabled = get(self._options, 'preload.enabled') - const gateways = get(self._options, 'preload.addresses', []) - .map(address => address.gateway) - .filter(Boolean) +module.exports = (options) => { + options = options || {} + options.enabled = !!options.enabled + options.gateways = options.gateways || [] - if (!enabled || !gateways.length) { + if (!options.enabled || !options.gateways.length) { return (_, callback) => { - if (!callback) return - setImmediate(() => callback()) + if (callback) { + setImmediate(() => callback()) + } } } + const noop = (err) => { + if (err) log.error(err) + } + return (cid, callback) => { - each(gateways, (gatewayAddr, cb) => { + callback = callback || noop + + each(options.gateways, (gatewayAddr, cb) => { let gatewayUri try { gatewayUri = toUri(gatewayAddr) + gatewayUri = gatewayUri.startsWith('http') ? gatewayUri : `http://${gatewayUri}` } catch (err) { return cb(err) } - const preloadUrl = `${gatewayUri}/ipfs/${cid.toBaseEncodedString()}#${redirectOptOutHint}` - - preload(preloadUrl, cb) + const url = `${gatewayUri}/ipfs/${cid.toBaseEncodedString()}#${redirectOptOutHint}` + preload(url, cb) }, callback) } } diff --git a/src/core/runtime/preload-browser.js b/src/core/runtime/preload-browser.js index dd765395c2..b80c8901a5 100644 --- a/src/core/runtime/preload-browser.js +++ b/src/core/runtime/preload-browser.js @@ -1,3 +1,6 @@ +/* eslint-env browser */ +'use strict' + const debug = require('debug') const log = debug('jsipfs:preload') @@ -6,7 +9,7 @@ log.error = debug('jsipfs:preload:error') module.exports = function preload (url, callback) { log(url) - const req = new window.XMLHttpRequest() + const req = new self.XMLHttpRequest() req.open('HEAD', url) @@ -15,7 +18,7 @@ module.exports = function preload (url, callback) { return } - if (this.status !== 200) { + if (this.status < 200 || this.status >= 300) { log.error('failed to preload', url, this.status, this.statusText) return callback(new Error(`failed to preload ${url}`)) } diff --git a/src/core/runtime/preload-nodejs.js b/src/core/runtime/preload-nodejs.js index 8f09caca5e..7c20df4d3a 100644 --- a/src/core/runtime/preload-nodejs.js +++ b/src/core/runtime/preload-nodejs.js @@ -1,6 +1,9 @@ +'use strict' + const http = require('http') -const URL = require('url') +const { URL } = require('url') const debug = require('debug') +const setImmediate = require('async/setImmediate') const log = debug('jsipfs:preload') log.error = debug('jsipfs:preload:error') @@ -8,7 +11,11 @@ log.error = debug('jsipfs:preload:error') module.exports = function preload (url, callback) { log(url) - url = new URL(url) + try { + url = new URL(url) + } catch (err) { + return setImmediate(() => callback(err)) + } const req = http.request({ protocol: url.protocol, @@ -17,10 +24,13 @@ module.exports = function preload (url, callback) { path: url.pathname, method: 'HEAD' }, (res) => { - if (res.statusCode !== 200) { + res.resume() + + if (res.statusCode < 200 || res.statusCode >= 300) { log.error('failed to preload', url, res.statusCode, res.statusMessage) return callback(new Error(`failed to preload ${url}`)) } + callback() }) diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js new file mode 100644 index 0000000000..1068dba05f --- /dev/null +++ b/test/core/preload.spec.js @@ -0,0 +1,120 @@ +/* eslint-env mocha */ +'use strict' + +const hat = require('hat') +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) + +const MockPreloadNode = require('../utils/mock-preload-node') +const IPFS = require('../../src') + +describe('preload', () => { + let ipfs + + before((done) => { + ipfs = new IPFS({ + config: { + Addresses: { + Swarm: [] + } + }, + preload: { + enabled: true, + gateways: [MockPreloadNode.defaultAddr] + } + }) + + ipfs.on('ready', done) + }) + + afterEach((done) => MockPreloadNode.clearPreloadUrls(done)) + + after((done) => ipfs.stop(done)) + + it('should preload content added with ipfs.files.add', (done) => { + ipfs.files.add(Buffer.from(hat()), (err, res) => { + expect(err).to.not.exist() + + MockPreloadNode.getPreloadUrls((err, urls) => { + expect(err).to.not.exist() + expect(urls.length).to.equal(1) + expect(urls[0]).to.equal(`/ipfs/${res[0].hash}`) + done() + }) + }) + }) + + it('should preload multiple content added with ipfs.files.add', (done) => { + ipfs.files.add([{ + content: Buffer.from(hat()) + }, { + content: Buffer.from(hat()) + }, { + content: Buffer.from(hat()) + }], (err, res) => { + expect(err).to.not.exist() + + MockPreloadNode.getPreloadUrls((err, urls) => { + expect(err).to.not.exist() + expect(urls.length).to.equal(res.length) + res.forEach(file => { + const url = urls.find(url => url === `/ipfs/${file.hash}`) + expect(url).to.exist() + }) + done() + }) + }) + }) + + it('should preload root dir for multiple content added with ipfs.files.add', (done) => { + ipfs.files.add([{ + path: 'dir0/dir1/file0', + content: Buffer.from(hat()) + }, { + path: 'dir0/dir1/file1', + content: Buffer.from(hat()) + }, { + path: 'dir0/file2', + content: Buffer.from(hat()) + }], (err, res) => { + expect(err).to.not.exist() + + const rootDir = res.find(file => file.path === 'dir0') + expect(rootDir).to.exist() + + MockPreloadNode.getPreloadUrls((err, urls) => { + expect(err).to.not.exist() + expect(urls.length).to.equal(1) + expect(urls[0]).to.equal(`/ipfs/${rootDir.hash}`) + done() + }) + }) + }) + + it('should preload wrapping dir for content added with ipfs.files.add and wrapWithDirectory option', (done) => { + ipfs.files.add([{ + path: 'dir0/dir1/file0', + content: Buffer.from(hat()) + }, { + path: 'dir0/dir1/file1', + content: Buffer.from(hat()) + }, { + path: 'dir0/file2', + content: Buffer.from(hat()) + }], { wrapWithDirectory: true }, (err, res) => { + expect(err).to.not.exist() + + const wrappingDir = res.find(file => file.path === '') + expect(wrappingDir).to.exist() + + MockPreloadNode.getPreloadUrls((err, urls) => { + expect(err).to.not.exist() + expect(urls.length).to.equal(1) + expect(urls[0]).to.equal(`/ipfs/${wrappingDir.hash}`) + done() + }) + }) + }) +}) diff --git a/test/utils/mock-preload-node.js b/test/utils/mock-preload-node.js new file mode 100644 index 0000000000..2ca78c33a3 --- /dev/null +++ b/test/utils/mock-preload-node.js @@ -0,0 +1,120 @@ +/* eslint-env browser */ +'use strict' + +const http = require('http') +const toUri = require('multiaddr-to-uri') +const URL = require('url').URL || self.URL + +const defaultPort = 1138 +const defaultAddr = `/dnsaddr/localhost/tcp/${defaultPort}/http` + +module.exports.defaultAddr = defaultAddr + +// Create a mock preload IPFS node with a gateway that'll respond 204 to a HEAD +// request. It also remembers the preload URLs it has been called with, and you +// can ask it for them and also clear them by issuing a GET/DELETE request. +module.exports.createNode = () => { + let urls = [] + + const server = http.createServer((req, res) => { + switch (req.method) { + case 'HEAD': + res.statusCode = 204 + urls = urls.concat(req.url) + break + case 'DELETE': + res.statusCode = 204 + urls = [] + break + case 'GET': + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + res.write(JSON.stringify(urls)) + break + default: + res.statusCode = 500 + } + + res.end() + }) + + server.start = (opts, cb) => { + if (typeof opts === 'function') { + cb = opts + opts = {} + } + return server.listen(Object.assign({ port: defaultPort }, opts), cb) + } + + server.stop = (cb) => server.close(cb) + + return server +} + +function parseMultiaddr (addr) { + let url = toUri(addr) + url = url.startsWith('http://') ? url : `http://${url}` + return new URL(url) +} + +// Get the stored preload URLs for the server at `addr` +module.exports.getPreloadUrls = (addr, cb) => { + if (typeof addr === 'function') { + cb = addr + addr = defaultAddr + } + + const { protocol, hostname, port } = parseMultiaddr(addr) + + const req = http.get({ protocol, hostname, port }, (res) => { + if (res.statusCode !== 200) { + res.resume() + return cb(new Error('failed to get preloaded URLs from mock preload node')) + } + + let data = '' + + res.on('error', cb) + res.on('data', chunk => { data += chunk }) + + res.on('end', () => { + let obj + try { + obj = JSON.parse(data) + } catch (err) { + return cb(err) + } + cb(null, obj) + }) + }) + + req.on('error', cb) +} + +// Clear the stored preload URLs for the server at `addr` +module.exports.clearPreloadUrls = (addr, cb) => { + if (typeof addr === 'function') { + cb = addr + addr = defaultAddr + } + + const { protocol, hostname, port } = parseMultiaddr(addr) + + const req = http.request({ + method: 'DELETE', + protocol, + hostname, + port + }, (res) => { + res.resume() + + if (res.statusCode !== 204) { + return cb(new Error('failed to reset mock preload node')) + } + + cb() + }) + + req.on('error', cb) + req.end() +} From 332ef98eba9a5260cd8afb6fda4d52ee2b502db9 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 25 Jul 2018 16:47:32 +0100 Subject: [PATCH 03/21] refactor: only create new CID if not already stringy License: MIT Signed-off-by: Alan Shaw --- src/core/components/files.js | 2 +- src/core/preload.js | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/core/components/files.js b/src/core/components/files.js index 632da9e154..ecb4c05b0d 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -98,7 +98,7 @@ function preloadFile (self, opts, file, cb) { if (!shouldPreload) return cb(null, file) - self._preload(new CID(file.hash), (err) => { + self._preload(file.hash, (err) => { // Preload error is not fatal if (err) console.error(err) cb(null, file) diff --git a/src/core/preload.js b/src/core/preload.js index dcefc52150..e17ed27dab 100644 --- a/src/core/preload.js +++ b/src/core/preload.js @@ -4,6 +4,7 @@ const setImmediate = require('async/setImmediate') const each = require('async/each') const toUri = require('multiaddr-to-uri') const debug = require('debug') +const CID = require('cids') const preload = require('./runtime/preload-nodejs') const log = debug('jsipfs:preload') @@ -44,7 +45,15 @@ module.exports = (options) => { return cb(err) } - const url = `${gatewayUri}/ipfs/${cid.toBaseEncodedString()}#${redirectOptOutHint}` + if (typeof cid !== 'string') { + try { + cid = new CID(cid).toBaseEncodedString() + } catch (err) { + return cb(err) + } + } + + const url = `${gatewayUri}/ipfs/${cid}#${redirectOptOutHint}` preload(url, cb) }, callback) } From e6d22d1afdd0b05abd3df583f8005e1367605a37 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 25 Jul 2018 16:58:13 +0100 Subject: [PATCH 04/21] chore: appease the linters License: MIT Signed-off-by: Alan Shaw --- test/core/preload.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js index 1068dba05f..7ff54682d0 100644 --- a/test/core/preload.spec.js +++ b/test/core/preload.spec.js @@ -1,3 +1,4 @@ +/* eslint max-nested-callbacks: ["error", 8] */ /* eslint-env mocha */ 'use strict' From 2f958b92dd06d4ff7f01e7c98eee636c86c5f9ba Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 25 Jul 2018 17:54:48 +0100 Subject: [PATCH 05/21] chore: more linter changes License: MIT Signed-off-by: Alan Shaw --- src/core/preload.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/preload.js b/src/core/preload.js index e17ed27dab..7b3265a47b 100644 --- a/src/core/preload.js +++ b/src/core/preload.js @@ -17,7 +17,7 @@ const redirectOptOutHint = 'x-ipfs-preload' module.exports = (options) => { options = options || {} - options.enabled = !!options.enabled + options.enabled = Boolean(options.enabled) options.gateways = options.gateways || [] if (!options.enabled || !options.gateways.length) { From 557c8da03abc61e5d72155a48a073b13e762e68b Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 26 Jul 2018 13:43:45 +0100 Subject: [PATCH 06/21] refactor: switch to /api/v0/refs License: MIT Signed-off-by: Alan Shaw --- README.md | 4 +- src/core/components/files.js | 14 +++-- src/core/config.js | 8 +-- src/core/index.js | 2 +- src/core/preload.js | 79 +++++++++++++++++++---------- src/core/runtime/config-browser.js | 4 +- src/core/runtime/config-nodejs.js | 4 +- src/core/runtime/preload-browser.js | 7 +++ src/core/runtime/preload-nodejs.js | 39 ++++++++++---- test/core/preload.spec.js | 69 ++++++++++++++----------- test/utils/mock-preload-node.js | 60 +++++++++++----------- 11 files changed, 176 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index e312940788..c2dbefa5b3 100644 --- a/README.md +++ b/README.md @@ -232,8 +232,8 @@ Creates and returns an instance of an IPFS node. Use the `options` argument to s - `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`) - `preload` (object): Configure external nodes that will preload content added to this node - - `enabled` (boolean): Enable content preloading (Default: `false`) - - `gateways` (array): Multiaddr gateway addresses of nodes that should preload content. NOTE: nodes specified here should also be added to your node's bootstrap address list at `config.Boostrap` + - `enabled` (boolean): Enable content preloading (Default: `true`) + - `addresses` (array): Multiaddr API addresses of nodes that should preload content. NOTE: nodes specified here should also be added to your node's bootstrap address list at `config.Boostrap` - `EXPERIMENTAL` (object): Enable and configure experimental features. - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`) - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`) diff --git a/src/core/components/files.js b/src/core/components/files.js index ecb4c05b0d..12a1fcdce0 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -89,20 +89,18 @@ function normalizeContent (opts, content) { }) } -function preloadFile (self, opts, file, cb) { +function preloadFile (self, opts, file) { const isRootFile = opts.wrapWithDirectory ? file.path === '' : !file.path.includes('/') const shouldPreload = isRootFile && !opts.onlyHash && opts.preload !== false - if (!shouldPreload) return cb(null, file) + if (shouldPreload) { + self._preload(file.hash) + } - self._preload(file.hash, (err) => { - // Preload error is not fatal - if (err) console.error(err) - cb(null, file) - }) + return file } function pinFile (self, opts, file, cb) { @@ -174,7 +172,7 @@ module.exports = function files (self) { pull.flatten(), importer(self._ipld, opts), pull.asyncMap(prepareFile.bind(null, self, opts)), - pull.asyncMap(preloadFile.bind(null, self, opts)), + pull.map(preloadFile.bind(null, self, opts)), pull.asyncMap(pinFile.bind(null, self, opts)) ) } diff --git a/src/core/config.js b/src/core/config.js index 7f2b82f925..5b2a5d5e42 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -9,12 +9,12 @@ const schema = Joi.object().keys({ ).allow(null), repoOwner: Joi.boolean().default(true), preload: Joi.object().keys({ - enabled: Joi.boolean().default(false), - gateways: Joi.array() + enabled: Joi.boolean().default(true), + addresses: Joi.array() .items(Joi.multiaddr().options({ convert: false })) .default([ - '/dns4/wss0.bootstrap.libp2p.io/https', - '/dns4/wss1.bootstrap.libp2p.io/https' + '/dnsaddr/node0.preload.ipfs.io', + '/dnsaddr/node1.preload.ipfs.io' ]) }).allow(null), init: Joi.alternatives().try( diff --git a/src/core/index.js b/src/core/index.js index ad9c9af6ba..122f45d797 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -79,7 +79,7 @@ class IPFS extends EventEmitter { this._blockService = new BlockService(this._repo) this._ipld = new Ipld(this._blockService) this._pubsub = undefined - this._preload = preload(this._options.preload) + this._preload = preload(this) // IPFS Core exposed components // - for booting up a node diff --git a/src/core/preload.js b/src/core/preload.js index 7b3265a47b..58ffab29ed 100644 --- a/src/core/preload.js +++ b/src/core/preload.js @@ -1,26 +1,24 @@ 'use strict' const setImmediate = require('async/setImmediate') -const each = require('async/each') +const retry = require('async/retry') const toUri = require('multiaddr-to-uri') const debug = require('debug') const CID = require('cids') +const shuffle = require('lodash/shuffle') const preload = require('./runtime/preload-nodejs') const log = debug('jsipfs:preload') log.error = debug('jsipfs:preload:error') -// Tools like IPFS Companion redirect requests to IPFS gateways to your local -// gateway. This is a hint to those tools that they shouldn't redirect these -// requests as they will effectively disable the preloading. -const redirectOptOutHint = 'x-ipfs-preload' +const noop = (err) => { if (err) log.error(err) } -module.exports = (options) => { - options = options || {} +module.exports = self => { + const options = self._options.preload || {} options.enabled = Boolean(options.enabled) - options.gateways = options.gateways || [] + options.addresses = options.addresses || [] - if (!options.enabled || !options.gateways.length) { + if (!options.enabled || !options.addresses.length) { return (_, callback) => { if (callback) { setImmediate(() => callback()) @@ -28,33 +26,60 @@ module.exports = (options) => { } } - const noop = (err) => { - if (err) log.error(err) - } + let stopped = true + let requests = [] + + self.on('start', () => { + stopped = false + }) + + self.on('stop', () => { + stopped = true + requests.forEach(r => r.cancel()) + requests = [] + }) + + const apiUris = options.addresses.map(apiAddrToUri) return (cid, callback) => { callback = callback || noop - each(options.gateways, (gatewayAddr, cb) => { - let gatewayUri - + if (typeof cid !== 'string') { try { - gatewayUri = toUri(gatewayAddr) - gatewayUri = gatewayUri.startsWith('http') ? gatewayUri : `http://${gatewayUri}` + cid = new CID(cid).toBaseEncodedString() } catch (err) { - return cb(err) + return setImmediate(() => callback(err)) } + } + + const shuffledApiUris = shuffle(apiUris) + let request - if (typeof cid !== 'string') { - try { - cid = new CID(cid).toBaseEncodedString() - } catch (err) { - return cb(err) - } + retry({ times: shuffledApiUris.length }, (cb) => { + if (stopped) return cb() + + // Remove failed request from a previous attempt + requests = requests.filter(r => r === request) + + const apiUri = shuffledApiUris.pop() + + request = preload(`${apiUri}/api/v0/refs?r=true&arg=${cid}`, cb) + requests = requests.concat(request) + }, (err) => { + requests = requests.filter(r => r === request) + + if (err) { + return callback(err) } - const url = `${gatewayUri}/ipfs/${cid}#${redirectOptOutHint}` - preload(url, cb) - }, callback) + callback() + }) + } +} + +function apiAddrToUri (addr) { + if (!(addr.endsWith('http') || addr.endsWith('https'))) { + addr = addr + '/http' } + return toUri(addr) } diff --git a/src/core/runtime/config-browser.js b/src/core/runtime/config-browser.js index 9819c04aaa..f7662420cb 100644 --- a/src/core/runtime/config-browser.js +++ b/src/core/runtime/config-browser.js @@ -24,6 +24,8 @@ module.exports = () => ({ '/dns4/nyc-1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm', '/dns4/nyc-2.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' + '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' ] }) diff --git a/src/core/runtime/config-nodejs.js b/src/core/runtime/config-nodejs.js index 995f66261d..d56550e181 100644 --- a/src/core/runtime/config-nodejs.js +++ b/src/core/runtime/config-nodejs.js @@ -37,6 +37,8 @@ module.exports = () => ({ '/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3', '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' + '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' ] }) diff --git a/src/core/runtime/preload-browser.js b/src/core/runtime/preload-browser.js index b80c8901a5..8d123a12be 100644 --- a/src/core/runtime/preload-browser.js +++ b/src/core/runtime/preload-browser.js @@ -27,4 +27,11 @@ module.exports = function preload (url, callback) { } req.send() + + return { + cancel: () => { + req.abort() + callback(new Error('request aborted')) + } + } } diff --git a/src/core/runtime/preload-nodejs.js b/src/core/runtime/preload-nodejs.js index 7c20df4d3a..96a14d0f96 100644 --- a/src/core/runtime/preload-nodejs.js +++ b/src/core/runtime/preload-nodejs.js @@ -1,6 +1,7 @@ 'use strict' const http = require('http') +const https = require('https') const { URL } = require('url') const debug = require('debug') const setImmediate = require('async/setImmediate') @@ -17,27 +18,47 @@ module.exports = function preload (url, callback) { return setImmediate(() => callback(err)) } - const req = http.request({ - protocol: url.protocol, + const transport = url.protocol === 'https:' ? https : http + + const req = transport.get({ hostname: url.hostname, port: url.port, - path: url.pathname, - method: 'HEAD' + path: url.pathname + url.search }, (res) => { - res.resume() - if (res.statusCode < 200 || res.statusCode >= 300) { + res.resume() log.error('failed to preload', url, res.statusCode, res.statusMessage) return callback(new Error(`failed to preload ${url}`)) } - callback() + res.on('data', chunk => log(chunk)) + + res.on('abort', () => { + callback(new Error('request aborted')) + }) + + res.on('error', err => { + log.error('response error preloading', url, err) + callback(err) + }) + + res.on('end', () => { + // If aborted, callback is called in the abort handler + if (!res.aborted) callback() + }) }) req.on('error', err => { - log.error('error preloading', url, err) + log.error('request error preloading', url, err) callback(err) }) - req.end() + return { + cancel: () => { + // No need to call callback here + // before repsonse - called in req error handler + // after response - called in res abort hander + req.abort() + } + } } diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js index 7ff54682d0..21d39bc20d 100644 --- a/test/core/preload.spec.js +++ b/test/core/preload.spec.js @@ -11,7 +11,7 @@ chai.use(dirtyChai) const MockPreloadNode = require('../utils/mock-preload-node') const IPFS = require('../../src') -describe('preload', () => { +describe.only('preload', () => { let ipfs before((done) => { @@ -23,14 +23,14 @@ describe('preload', () => { }, preload: { enabled: true, - gateways: [MockPreloadNode.defaultAddr] + addresses: [MockPreloadNode.defaultAddr] } }) ipfs.on('ready', done) }) - afterEach((done) => MockPreloadNode.clearPreloadUrls(done)) + afterEach((done) => MockPreloadNode.clearPreloadCids(done)) after((done) => ipfs.stop(done)) @@ -38,12 +38,15 @@ describe('preload', () => { ipfs.files.add(Buffer.from(hat()), (err, res) => { expect(err).to.not.exist() - MockPreloadNode.getPreloadUrls((err, urls) => { - expect(err).to.not.exist() - expect(urls.length).to.equal(1) - expect(urls[0]).to.equal(`/ipfs/${res[0].hash}`) - done() - }) + // Wait for preloading to finish + setTimeout(() => { + MockPreloadNode.getPreloadCids((err, cids) => { + expect(err).to.not.exist() + expect(cids.length).to.equal(1) + expect(cids[0]).to.equal(res[0].hash) + done() + }) + }, 100) }) }) @@ -57,19 +60,19 @@ describe('preload', () => { }], (err, res) => { expect(err).to.not.exist() - MockPreloadNode.getPreloadUrls((err, urls) => { - expect(err).to.not.exist() - expect(urls.length).to.equal(res.length) - res.forEach(file => { - const url = urls.find(url => url === `/ipfs/${file.hash}`) - expect(url).to.exist() + // Wait for preloading to finish + setTimeout(() => { + MockPreloadNode.getPreloadCids((err, cids) => { + expect(err).to.not.exist() + expect(cids.length).to.equal(res.length) + res.forEach(file => expect(cids).to.include(file.hash)) + done() }) - done() - }) + }, 100) }) }) - it('should preload root dir for multiple content added with ipfs.files.add', (done) => { + it('should preload multiple content and intermediate dirs added with ipfs.files.add', (done) => { ipfs.files.add([{ path: 'dir0/dir1/file0', content: Buffer.from(hat()) @@ -85,16 +88,19 @@ describe('preload', () => { const rootDir = res.find(file => file.path === 'dir0') expect(rootDir).to.exist() - MockPreloadNode.getPreloadUrls((err, urls) => { - expect(err).to.not.exist() - expect(urls.length).to.equal(1) - expect(urls[0]).to.equal(`/ipfs/${rootDir.hash}`) - done() - }) + // Wait for preloading to finish + setTimeout(() => { + MockPreloadNode.getPreloadCids((err, cids) => { + expect(err).to.not.exist() + expect(cids.length).to.equal(1) + expect(cids[0]).to.equal(rootDir.hash) + done() + }) + }, 100) }) }) - it('should preload wrapping dir for content added with ipfs.files.add and wrapWithDirectory option', (done) => { + it('should preload multiple content and wrapping dir for content added with ipfs.files.add and wrapWithDirectory option', (done) => { ipfs.files.add([{ path: 'dir0/dir1/file0', content: Buffer.from(hat()) @@ -110,11 +116,14 @@ describe('preload', () => { const wrappingDir = res.find(file => file.path === '') expect(wrappingDir).to.exist() - MockPreloadNode.getPreloadUrls((err, urls) => { - expect(err).to.not.exist() - expect(urls.length).to.equal(1) - expect(urls[0]).to.equal(`/ipfs/${wrappingDir.hash}`) - done() + // Wait for preloading to finish + setTimeout(() => { + MockPreloadNode.getPreloadCids((err, cids) => { + expect(err).to.not.exist() + expect(cids.length).to.equal(1) + expect(cids[0]).to.equal(wrappingDir.hash) + done() + }) }) }) }) diff --git a/test/utils/mock-preload-node.js b/test/utils/mock-preload-node.js index 2ca78c33a3..0f08c8d61d 100644 --- a/test/utils/mock-preload-node.js +++ b/test/utils/mock-preload-node.js @@ -6,33 +6,29 @@ const toUri = require('multiaddr-to-uri') const URL = require('url').URL || self.URL const defaultPort = 1138 -const defaultAddr = `/dnsaddr/localhost/tcp/${defaultPort}/http` +const defaultAddr = `/dnsaddr/localhost/tcp/${defaultPort}` module.exports.defaultAddr = defaultAddr -// Create a mock preload IPFS node with a gateway that'll respond 204 to a HEAD -// request. It also remembers the preload URLs it has been called with, and you -// can ask it for them and also clear them by issuing a GET/DELETE request. +// Create a mock preload IPFS node with a gateway that'll respond 200 to a +// request for /api/v0/refs?arg=*. It remembers the preload CIDs it has been +// called with, and you can ask it for them and also clear them by issuing a +// GET/DELETE request to /cids. module.exports.createNode = () => { - let urls = [] + let cids = [] const server = http.createServer((req, res) => { - switch (req.method) { - case 'HEAD': - res.statusCode = 204 - urls = urls.concat(req.url) - break - case 'DELETE': - res.statusCode = 204 - urls = [] - break - case 'GET': - res.statusCode = 200 - res.setHeader('Content-Type', 'application/json') - res.write(JSON.stringify(urls)) - break - default: - res.statusCode = 500 + if (req.url.startsWith('/api/v0/refs')) { + const arg = new URL(`https://ipfs.io${req.url}`).searchParams.get('arg') + cids = cids.concat(arg) + } else if (req.method === 'DELETE' && req.url === '/cids') { + res.statusCode = 204 + cids = [] + } else if (req.method === 'GET' && req.url === '/cids') { + res.setHeader('Content-Type', 'application/json') + res.write(JSON.stringify(cids)) + } else { + res.statusCode = 500 } res.end() @@ -52,13 +48,14 @@ module.exports.createNode = () => { } function parseMultiaddr (addr) { - let url = toUri(addr) - url = url.startsWith('http://') ? url : `http://${url}` - return new URL(url) + if (!(addr.endsWith('http') || addr.endsWith('https'))) { + addr = addr + '/http' + } + return new URL(toUri(addr)) } -// Get the stored preload URLs for the server at `addr` -module.exports.getPreloadUrls = (addr, cb) => { +// Get the stored preload CIDs for the server at `addr` +module.exports.getPreloadCids = (addr, cb) => { if (typeof addr === 'function') { cb = addr addr = defaultAddr @@ -66,10 +63,10 @@ module.exports.getPreloadUrls = (addr, cb) => { const { protocol, hostname, port } = parseMultiaddr(addr) - const req = http.get({ protocol, hostname, port }, (res) => { + const req = http.get({ protocol, hostname, port, path: '/cids' }, (res) => { if (res.statusCode !== 200) { res.resume() - return cb(new Error('failed to get preloaded URLs from mock preload node')) + return cb(new Error('failed to get preloaded CIDs from mock preload node')) } let data = '' @@ -92,7 +89,7 @@ module.exports.getPreloadUrls = (addr, cb) => { } // Clear the stored preload URLs for the server at `addr` -module.exports.clearPreloadUrls = (addr, cb) => { +module.exports.clearPreloadCids = (addr, cb) => { if (typeof addr === 'function') { cb = addr addr = defaultAddr @@ -104,12 +101,13 @@ module.exports.clearPreloadUrls = (addr, cb) => { method: 'DELETE', protocol, hostname, - port + port, + path: '/cids' }, (res) => { res.resume() if (res.statusCode !== 204) { - return cb(new Error('failed to reset mock preload node')) + return cb(new Error('failed to clear CIDs from mock preload node')) } cb() From 1cdd7240a27bbad986f645be2fad3196ec3271c2 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 26 Jul 2018 14:23:01 +0100 Subject: [PATCH 07/21] fix: many fixes License: MIT Signed-off-by: Alan Shaw --- src/core/components/start.js | 2 ++ src/core/components/stop.js | 1 + src/core/config.js | 7 +------ src/core/index.js | 9 ++++++++- src/core/preload.js | 30 ++++++++++++++---------------- src/core/runtime/preload-nodejs.js | 6 +++--- 6 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/core/components/start.js b/src/core/components/start.js index fd4832e35a..ebcff2f627 100644 --- a/src/core/components/start.js +++ b/src/core/components/start.js @@ -34,6 +34,8 @@ module.exports = (self) => { }, (cb) => self.libp2p.start(cb), (cb) => { + self._preload.start() + self._bitswap = new Bitswap( self._libp2pNode, self._repo.blocks, diff --git a/src/core/components/stop.js b/src/core/components/stop.js index 4d35190d21..d38a3719be 100644 --- a/src/core/components/stop.js +++ b/src/core/components/stop.js @@ -30,6 +30,7 @@ module.exports = (self) => { self.state.stop() self._blockService.unsetExchange() self._bitswap.stop() + self._preload.stop() series([ (cb) => self.libp2p.stop(cb), diff --git a/src/core/config.js b/src/core/config.js index 5b2a5d5e42..7b16c17d06 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -10,12 +10,7 @@ const schema = Joi.object().keys({ repoOwner: Joi.boolean().default(true), preload: Joi.object().keys({ enabled: Joi.boolean().default(true), - addresses: Joi.array() - .items(Joi.multiaddr().options({ convert: false })) - .default([ - '/dnsaddr/node0.preload.ipfs.io', - '/dnsaddr/node1.preload.ipfs.io' - ]) + addresses: Joi.array().items(Joi.multiaddr().options({ convert: false })) }).allow(null), init: Joi.alternatives().try( Joi.boolean(), diff --git a/src/core/index.js b/src/core/index.js index 122f45d797..7184942612 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -31,7 +31,14 @@ class IPFS extends EventEmitter { this._options = { init: true, start: true, - EXPERIMENTAL: {} + EXPERIMENTAL: {}, + preload: { + enabled: true, + addresses: [ + '/dnsaddr/node0.preload.ipfs.io/https', + '/dnsaddr/node1.preload.ipfs.io/https' + ] + } } options = config.validate(options || {}) diff --git a/src/core/preload.js b/src/core/preload.js index 58ffab29ed..08a4364a83 100644 --- a/src/core/preload.js +++ b/src/core/preload.js @@ -26,22 +26,10 @@ module.exports = self => { } } - let stopped = true let requests = [] - - self.on('start', () => { - stopped = false - }) - - self.on('stop', () => { - stopped = true - requests.forEach(r => r.cancel()) - requests = [] - }) - const apiUris = options.addresses.map(apiAddrToUri) - return (cid, callback) => { + const api = (cid, callback) => { callback = callback || noop if (typeof cid !== 'string') { @@ -56,17 +44,17 @@ module.exports = self => { let request retry({ times: shuffledApiUris.length }, (cb) => { - if (stopped) return cb() + if (self.state.state() === 'stopped') return cb() // Remove failed request from a previous attempt - requests = requests.filter(r => r === request) + requests = requests.filter(r => r !== request) const apiUri = shuffledApiUris.pop() request = preload(`${apiUri}/api/v0/refs?r=true&arg=${cid}`, cb) requests = requests.concat(request) }, (err) => { - requests = requests.filter(r => r === request) + requests = requests.filter(r => r !== request) if (err) { return callback(err) @@ -75,6 +63,16 @@ module.exports = self => { callback() }) } + + api.start = () => {} + + api.stop = () => { + log(`canceling ${requests.length} pending preload requests`) + requests.forEach(r => r.cancel()) + requests = [] + } + + return api } function apiAddrToUri (addr) { diff --git a/src/core/runtime/preload-nodejs.js b/src/core/runtime/preload-nodejs.js index 96a14d0f96..bc4c4fc657 100644 --- a/src/core/runtime/preload-nodejs.js +++ b/src/core/runtime/preload-nodejs.js @@ -27,7 +27,7 @@ module.exports = function preload (url, callback) { }, (res) => { if (res.statusCode < 200 || res.statusCode >= 300) { res.resume() - log.error('failed to preload', url, res.statusCode, res.statusMessage) + log.error('failed to preload', url.href, res.statusCode, res.statusMessage) return callback(new Error(`failed to preload ${url}`)) } @@ -38,7 +38,7 @@ module.exports = function preload (url, callback) { }) res.on('error', err => { - log.error('response error preloading', url, err) + log.error('response error preloading', url.href, err) callback(err) }) @@ -49,7 +49,7 @@ module.exports = function preload (url, callback) { }) req.on('error', err => { - log.error('request error preloading', url, err) + log.error('request error preloading', url.href, err) callback(err) }) From f62bae1061e6b587dfa54fbd0b7ee296a73b58a2 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 26 Jul 2018 14:39:49 +0100 Subject: [PATCH 08/21] fix: remove .only License: MIT Signed-off-by: Alan Shaw --- test/core/preload.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js index 21d39bc20d..54fbed45cf 100644 --- a/test/core/preload.spec.js +++ b/test/core/preload.spec.js @@ -11,7 +11,7 @@ chai.use(dirtyChai) const MockPreloadNode = require('../utils/mock-preload-node') const IPFS = require('../../src') -describe.only('preload', () => { +describe('preload', () => { let ipfs before((done) => { From ac9d246ceadcd934b86086bdc9153b6f99dbd553 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 26 Jul 2018 14:43:18 +0100 Subject: [PATCH 09/21] fix: fallback in order License: MIT Signed-off-by: Alan Shaw --- src/core/preload.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/preload.js b/src/core/preload.js index 08a4364a83..4e087ea339 100644 --- a/src/core/preload.js +++ b/src/core/preload.js @@ -5,7 +5,6 @@ const retry = require('async/retry') const toUri = require('multiaddr-to-uri') const debug = require('debug') const CID = require('cids') -const shuffle = require('lodash/shuffle') const preload = require('./runtime/preload-nodejs') const log = debug('jsipfs:preload') @@ -40,16 +39,16 @@ module.exports = self => { } } - const shuffledApiUris = shuffle(apiUris) + const fallbackApiUris = Array.from(apiUris) let request - retry({ times: shuffledApiUris.length }, (cb) => { + retry({ times: fallbackApiUris.length }, (cb) => { if (self.state.state() === 'stopped') return cb() // Remove failed request from a previous attempt requests = requests.filter(r => r !== request) - const apiUri = shuffledApiUris.pop() + const apiUri = fallbackApiUris.shift() request = preload(`${apiUri}/api/v0/refs?r=true&arg=${cid}`, cb) requests = requests.concat(request) From 89502aaef76d12b06250df847e6724bf006f7bb5 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 26 Jul 2018 15:09:00 +0100 Subject: [PATCH 10/21] fix: start/stop fix and add preload request time logging License: MIT Signed-off-by: Alan Shaw --- src/core/preload.js | 12 +++++++++--- src/core/runtime/preload-nodejs.js | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/core/preload.js b/src/core/preload.js index 4e087ea339..d99a9d8f20 100644 --- a/src/core/preload.js +++ b/src/core/preload.js @@ -25,6 +25,7 @@ module.exports = self => { } } + let stopped = true let requests = [] const apiUris = options.addresses.map(apiAddrToUri) @@ -41,9 +42,10 @@ module.exports = self => { const fallbackApiUris = Array.from(apiUris) let request + const now = Date.now() retry({ times: fallbackApiUris.length }, (cb) => { - if (self.state.state() === 'stopped') return cb() + if (stopped) return cb(new Error(`preload aborted for ${cid}`)) // Remove failed request from a previous attempt requests = requests.filter(r => r !== request) @@ -59,14 +61,18 @@ module.exports = self => { return callback(err) } + log(`preloaded ${cid} in ${Date.now() - now}ms`) callback() }) } - api.start = () => {} + api.start = () => { + stopped = false + } api.stop = () => { - log(`canceling ${requests.length} pending preload requests`) + stopped = true + log(`canceling ${requests.length} pending preload request(s)`) requests.forEach(r => r.cancel()) requests = [] } diff --git a/src/core/runtime/preload-nodejs.js b/src/core/runtime/preload-nodejs.js index bc4c4fc657..405798ca34 100644 --- a/src/core/runtime/preload-nodejs.js +++ b/src/core/runtime/preload-nodejs.js @@ -31,7 +31,7 @@ module.exports = function preload (url, callback) { return callback(new Error(`failed to preload ${url}`)) } - res.on('data', chunk => log(chunk)) + res.on('data', chunk => log(`data ${chunk}`)) res.on('abort', () => { callback(new Error('request aborted')) From 7c731423acbb293560825df968311898d6347466 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 26 Jul 2018 16:48:52 +0100 Subject: [PATCH 11/21] fix: bootstrap default list in tests License: MIT Signed-off-by: Alan Shaw --- test/core/bootstrap.spec.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/core/bootstrap.spec.js b/test/core/bootstrap.spec.js index f092765354..d95d622f18 100644 --- a/test/core/bootstrap.spec.js +++ b/test/core/bootstrap.spec.js @@ -59,7 +59,9 @@ describe('bootstrap', () => { '/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3', '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' + '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' ] const updatedList = [ @@ -82,6 +84,8 @@ describe('bootstrap', () => { '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', '/ip4/111.111.111.111/tcp/1001/ipfs/QmXFX2P5ammdmXQgfqGkfswtEVFsZUJ5KeHRXQYCTdiTAb' ] From 3aa2a1de1bd347c7056470498c70a0ec31ad33e4 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 26 Jul 2018 16:57:19 +0100 Subject: [PATCH 12/21] fix: more fix bootstrap tests License: MIT Signed-off-by: Alan Shaw --- test/cli/bootstrap.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/cli/bootstrap.js b/test/cli/bootstrap.js index 8807a12411..a301bca3f2 100644 --- a/test/cli/bootstrap.js +++ b/test/cli/bootstrap.js @@ -31,7 +31,9 @@ describe('bootstrap', () => runOnAndOff((thing) => { '/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3', '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' + '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' ] const updatedList = [ @@ -54,6 +56,8 @@ describe('bootstrap', () => runOnAndOff((thing) => { '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', '/ip4/111.111.111.111/tcp/1001/ipfs/QmcyFFKfLDGJKwufn2GeitxvhricsBQyNKTkrD14psikoD' ] From ecf1cc3d4be34023e4b0f53e99181bfe3327940a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 26 Jul 2018 22:50:53 +0100 Subject: [PATCH 13/21] fix: more config fix License: MIT Signed-off-by: Alan Shaw --- test/fixtures/go-ipfs-repo/config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/fixtures/go-ipfs-repo/config b/test/fixtures/go-ipfs-repo/config index 00f467f95f..9843d866a8 100644 --- a/test/fixtures/go-ipfs-repo/config +++ b/test/fixtures/go-ipfs-repo/config @@ -65,7 +65,9 @@ "/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3", "/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx", "/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic", - "/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6" + "/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6", + "/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic", + "/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6" ], "Tour": { "Last": "" @@ -106,4 +108,4 @@ "hash": "sha2-512" } } -} \ No newline at end of file +} From 977a0740f1c0f6163e5c5768af21c4e8be75236c Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 26 Jul 2018 16:38:09 +0100 Subject: [PATCH 14/21] feat: preload mfs root preiodically and preload dag.put License: MIT Signed-off-by: Alan Shaw --- src/core/components/dag.js | 6 ++++- src/core/components/index.js | 2 +- src/core/components/mfs.js | 24 +++++++++++++++++ src/core/components/pin-set.js | 5 ++-- src/core/components/pin.js | 4 +-- src/core/components/start.js | 1 + src/core/components/stop.js | 1 + src/core/index.js | 4 ++- src/core/mfs-preload.js | 47 ++++++++++++++++++++++++++++++++++ 9 files changed, 87 insertions(+), 7 deletions(-) create mode 100644 src/core/components/mfs.js create mode 100644 src/core/mfs-preload.js diff --git a/src/core/components/dag.js b/src/core/components/dag.js index 88a80bdcce..105985df68 100644 --- a/src/core/components/dag.js +++ b/src/core/components/dag.js @@ -24,7 +24,11 @@ module.exports = function dag (self) { options = options.cid ? options : Object.assign({}, optionDefaults, options) - self._ipld.put(dagNode, options, callback) + self._ipld.put(dagNode, options, (err, cid) => { + if (err) return callback(err) + if (options.preload !== false) self._preload(cid) + callback(null, cid) + }) }), get: promisify((cid, path, options, callback) => { diff --git a/src/core/components/index.js b/src/core/components/index.js index 9eb36ad4c3..1f6f084dee 100644 --- a/src/core/components/index.js +++ b/src/core/components/index.js @@ -26,4 +26,4 @@ exports.dht = require('./dht') exports.dns = require('./dns') exports.key = require('./key') exports.stats = require('./stats') -exports.mfs = require('ipfs-mfs/core') +exports.mfs = require('./mfs') diff --git a/src/core/components/mfs.js b/src/core/components/mfs.js new file mode 100644 index 0000000000..9f033545b4 --- /dev/null +++ b/src/core/components/mfs.js @@ -0,0 +1,24 @@ +'use strict' + +const promisify = require('promisify-es6') +const mfs = require('ipfs-mfs/core') + +module.exports = self => { + const mfsSelf = Object.assign({}, self) + + // A patched dag API to ensure preload doesn't happen for MFS operations + mfsSelf.dag = Object.assign({}, self.dag, { + put: promisify((node, opts, cb) => { + if (typeof opts === 'function') { + cb = opts + opts = {} + } + + opts = Object.assign({}, opts, { preload: false }) + + return self.dag.put(node, opts, cb) + }) + }) + + return mfs(mfsSelf, mfsSelf._options) +} diff --git a/src/core/components/pin-set.js b/src/core/components/pin-set.js index f18a248604..806df5f05f 100644 --- a/src/core/components/pin-set.js +++ b/src/core/components/pin-set.js @@ -90,7 +90,7 @@ exports = module.exports = function (dag) { pinSet.storeItems(pins, (err, rootNode) => { if (err) { return callback(err) } - const opts = { cid: new CID(rootNode.multihash) } + const opts = { cid: new CID(rootNode.multihash), preload: false } dag.put(rootNode, opts, (err, cid) => { if (err) { return callback(err) } callback(null, rootNode) @@ -168,7 +168,8 @@ exports = module.exports = function (dag) { function storeChild (err, child, binIdx, cb) { if (err) { return cb(err) } - dag.put(child, { cid: new CID(child._multihash) }, err => { + const opts = { cid: new CID(child._multihash), preload: false } + dag.put(child, opts, err => { if (err) { return cb(err) } fanoutLinks[binIdx] = new DAGLink('', child.size, child.multihash) cb(null) diff --git a/src/core/components/pin.js b/src/core/components/pin.js index d2bd670e14..ce0cd72d84 100644 --- a/src/core/components/pin.js +++ b/src/core/components/pin.js @@ -80,14 +80,14 @@ module.exports = (self) => { // the pin-set nodes link to a special 'empty' node, so make sure it exists cb => DAGNode.create(Buffer.alloc(0), (err, empty) => { if (err) { return cb(err) } - dag.put(empty, { cid: new CID(empty.multihash) }, cb) + dag.put(empty, { cid: new CID(empty.multihash), preload: false }, cb) }), // create a root node with DAGLinks to the direct and recursive DAGs cb => DAGNode.create(Buffer.alloc(0), [dLink, rLink], (err, node) => { if (err) { return cb(err) } root = node - dag.put(root, { cid: new CID(root.multihash) }, cb) + dag.put(root, { cid: new CID(root.multihash), preload: false }, cb) }), // hack for CLI tests diff --git a/src/core/components/start.js b/src/core/components/start.js index ebcff2f627..d66a084803 100644 --- a/src/core/components/start.js +++ b/src/core/components/start.js @@ -35,6 +35,7 @@ module.exports = (self) => { (cb) => self.libp2p.start(cb), (cb) => { self._preload.start() + self._mfsPreload.start() self._bitswap = new Bitswap( self._libp2pNode, diff --git a/src/core/components/stop.js b/src/core/components/stop.js index d38a3719be..321bf03cbe 100644 --- a/src/core/components/stop.js +++ b/src/core/components/stop.js @@ -31,6 +31,7 @@ module.exports = (self) => { self._blockService.unsetExchange() self._bitswap.stop() self._preload.stop() + self._mfsPreload.stop() series([ (cb) => self.libp2p.stop(cb), diff --git a/src/core/index.js b/src/core/index.js index 7184942612..52266b7b41 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -23,6 +23,7 @@ const components = require('./components') // replaced by repo-browser when running in the browser const defaultRepo = require('./runtime/repo-nodejs') const preload = require('./preload') +const mfsPreload = require('./mfs-preload') class IPFS extends EventEmitter { constructor (options) { @@ -87,6 +88,7 @@ class IPFS extends EventEmitter { this._ipld = new Ipld(this._blockService) this._pubsub = undefined this._preload = preload(this) + this._mfsPreload = mfsPreload(this) // IPFS Core exposed components // - for booting up a node @@ -143,7 +145,7 @@ class IPFS extends EventEmitter { } // ipfs.files - const mfs = components.mfs(this, this._options) + const mfs = components.mfs(this) Object.keys(mfs).forEach(key => { this.files[key] = mfs[key] diff --git a/src/core/mfs-preload.js b/src/core/mfs-preload.js new file mode 100644 index 0000000000..c779323f0f --- /dev/null +++ b/src/core/mfs-preload.js @@ -0,0 +1,47 @@ +'use strict' + +const debug = require('debug') + +const log = debug('jsipfs:mfs-preload') +log.error = debug('jsipfs:mfs-preload:error') + +module.exports = (self, options) => { + options = options || {} + options.interval = options.interval || 30 * 1000 + + let rootCid + let timeoutId + + const preloadMfs = () => { + self.files.stat('/', (err, stats) => { + if (err) { + timeoutId = setTimeout(preloadMfs, options.interval) + return log.error('failed to stat MFS root for preload', err) + } + + if (rootCid !== stats.hash) { + log(`preloading updated MFS root ${rootCid} -> ${stats.hash}`) + + self._preload(stats.hash, (err) => { + timeoutId = setTimeout(preloadMfs, options.interval) + if (err) return log.error(`failed to preload MFS root ${stats.hash}`, err) + rootCid = stats.hash + }) + } + }) + } + + return { + start () { + self.files.stat('/', (err, stats) => { + if (err) return log.error('failed to stat MFS root for preload', err) + rootCid = stats.hash + log(`monitoring MFS root ${rootCid}`) + timeoutId = setTimeout(preloadMfs, options.interval) + }) + }, + stop () { + clearTimeout(timeoutId) + } + } +} From ef4616a94c21656880a8cacb6bab71417108de08 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 27 Jul 2018 07:02:00 +0100 Subject: [PATCH 15/21] fix: do not monitor if stopped while starting License: MIT Signed-off-by: Alan Shaw --- src/core/mfs-preload.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/mfs-preload.js b/src/core/mfs-preload.js index c779323f0f..9efb2e082b 100644 --- a/src/core/mfs-preload.js +++ b/src/core/mfs-preload.js @@ -31,16 +31,21 @@ module.exports = (self, options) => { }) } + let stopped = true + return { start () { + stopped = false self.files.stat('/', (err, stats) => { if (err) return log.error('failed to stat MFS root for preload', err) + if (stopped) return // Stopped inbetween! rootCid = stats.hash log(`monitoring MFS root ${rootCid}`) timeoutId = setTimeout(preloadMfs, options.interval) }) }, stop () { + stopped = true clearTimeout(timeoutId) } } From 9a4b5e09435895213898e50e6408f5f1e833f15f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 27 Jul 2018 08:52:17 +0100 Subject: [PATCH 16/21] feat: async mfs-preload start and stop License: MIT Signed-off-by: Alan Shaw --- src/core/components/start.js | 7 +++---- src/core/components/stop.js | 2 +- src/core/mfs-preload.js | 13 +++++-------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/core/components/start.js b/src/core/components/start.js index d66a084803..3a7a5716ce 100644 --- a/src/core/components/start.js +++ b/src/core/components/start.js @@ -34,9 +34,6 @@ module.exports = (self) => { }, (cb) => self.libp2p.start(cb), (cb) => { - self._preload.start() - self._mfsPreload.start() - self._bitswap = new Bitswap( self._libp2pNode, self._repo.blocks, @@ -45,7 +42,9 @@ module.exports = (self) => { self._bitswap.start() self._blockService.setExchange(self._bitswap) - cb() + + self._preload.start() + self._mfsPreload.start(cb) } ], done) }) diff --git a/src/core/components/stop.js b/src/core/components/stop.js index 321bf03cbe..cf97b6ec6a 100644 --- a/src/core/components/stop.js +++ b/src/core/components/stop.js @@ -31,9 +31,9 @@ module.exports = (self) => { self._blockService.unsetExchange() self._bitswap.stop() self._preload.stop() - self._mfsPreload.stop() series([ + (cb) => self._mfsPreload.stop(cb), (cb) => self.libp2p.stop(cb), (cb) => self._repo.close(cb) ], done) diff --git a/src/core/mfs-preload.js b/src/core/mfs-preload.js index 9efb2e082b..fc75f25a0a 100644 --- a/src/core/mfs-preload.js +++ b/src/core/mfs-preload.js @@ -31,22 +31,19 @@ module.exports = (self, options) => { }) } - let stopped = true - return { - start () { - stopped = false + start (cb) { self.files.stat('/', (err, stats) => { - if (err) return log.error('failed to stat MFS root for preload', err) - if (stopped) return // Stopped inbetween! + if (err) return cb(err) rootCid = stats.hash log(`monitoring MFS root ${rootCid}`) timeoutId = setTimeout(preloadMfs, options.interval) + cb() }) }, - stop () { - stopped = true + stop (cb) { clearTimeout(timeoutId) + cb() } } } From eb7e01f9c9b5678cec306980a2ef3b3b5f2a5a93 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 27 Jul 2018 10:13:39 +0100 Subject: [PATCH 17/21] feat: preload object.* API License: MIT Signed-off-by: Alan Shaw --- src/core/components/object.js | 48 ++++++++--- test/core/preload.spec.js | 142 ++++++++++++++++++++++---------- test/utils/mock-preload-node.js | 41 ++++++++- 3 files changed, 177 insertions(+), 54 deletions(-) diff --git a/src/core/components/object.js b/src/core/components/object.js index 5e8be00e4a..e7ca331056 100644 --- a/src/core/components/object.js +++ b/src/core/components/object.js @@ -80,10 +80,17 @@ module.exports = function object (self) { if (err) { return cb(err) } - self._ipld.put(node, { - cid: new CID(node.multihash) - }, (err) => { - cb(err, node) + + const cid = new CID(node.multihash) + + self._ipld.put(node, { cid }, (err) => { + if (err) return cb(err) + + if (options.preload !== false) { + self._preload(cid) + } + + cb(null, node) }) }) } @@ -92,12 +99,20 @@ module.exports = function object (self) { } return { - new: promisify((template, callback) => { + new: promisify((template, options, callback) => { if (typeof template === 'function') { callback = template template = undefined + options = {} + } + + if (typeof options === 'function') { + callback = options + options = {} } + options = options || {} + let data if (template) { @@ -111,13 +126,18 @@ module.exports = function object (self) { if (err) { return callback(err) } - self._ipld.put(node, { - cid: new CID(node.multihash) - }, (err) => { + + const cid = new CID(node.multihash) + + self._ipld.put(node, { cid }, (err) => { if (err) { return callback(err) } + if (options.preload !== false) { + self._preload(cid) + } + callback(null, node) }) }) @@ -166,13 +186,17 @@ module.exports = function object (self) { } function next () { - self._ipld.put(node, { - cid: new CID(node.multihash) - }, (err) => { + const cid = new CID(node.multihash) + + self._ipld.put(node, { cid }, (err) => { if (err) { return callback(err) } + if (options.preload !== false) { + self._preload(cid) + } + self.object.get(node.multihash, callback) }) } @@ -282,6 +306,8 @@ module.exports = function object (self) { editAndSave((node, cb) => { if (DAGLink.isDAGLink(linkRef)) { linkRef = linkRef._name + } else if (linkRef && linkRef.name) { + linkRef = linkRef.name } DAGNode.rmLink(node, linkRef, cb) })(multihash, options, callback) diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js index 54fbed45cf..284231fc01 100644 --- a/test/core/preload.spec.js +++ b/test/core/preload.spec.js @@ -3,6 +3,9 @@ 'use strict' const hat = require('hat') +const CID = require('cids') +const parallel = require('async/parallel') +const waterfall = require('async/waterfall') const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect @@ -11,7 +14,7 @@ chai.use(dirtyChai) const MockPreloadNode = require('../utils/mock-preload-node') const IPFS = require('../../src') -describe('preload', () => { +describe.only('preload', () => { let ipfs before((done) => { @@ -34,23 +37,14 @@ describe('preload', () => { after((done) => ipfs.stop(done)) - it('should preload content added with ipfs.files.add', (done) => { + it('should preload content added with files.add', (done) => { ipfs.files.add(Buffer.from(hat()), (err, res) => { expect(err).to.not.exist() - - // Wait for preloading to finish - setTimeout(() => { - MockPreloadNode.getPreloadCids((err, cids) => { - expect(err).to.not.exist() - expect(cids.length).to.equal(1) - expect(cids[0]).to.equal(res[0].hash) - done() - }) - }, 100) + MockPreloadNode.waitForCids(res[0].hash, done) }) }) - it('should preload multiple content added with ipfs.files.add', (done) => { + it('should preload multiple content added with files.add', (done) => { ipfs.files.add([{ content: Buffer.from(hat()) }, { @@ -59,20 +53,11 @@ describe('preload', () => { content: Buffer.from(hat()) }], (err, res) => { expect(err).to.not.exist() - - // Wait for preloading to finish - setTimeout(() => { - MockPreloadNode.getPreloadCids((err, cids) => { - expect(err).to.not.exist() - expect(cids.length).to.equal(res.length) - res.forEach(file => expect(cids).to.include(file.hash)) - done() - }) - }, 100) + MockPreloadNode.waitForCids(res.map(file => file.hash), done) }) }) - it('should preload multiple content and intermediate dirs added with ipfs.files.add', (done) => { + it('should preload multiple content and intermediate dirs added with files.add', (done) => { ipfs.files.add([{ path: 'dir0/dir1/file0', content: Buffer.from(hat()) @@ -88,19 +73,11 @@ describe('preload', () => { const rootDir = res.find(file => file.path === 'dir0') expect(rootDir).to.exist() - // Wait for preloading to finish - setTimeout(() => { - MockPreloadNode.getPreloadCids((err, cids) => { - expect(err).to.not.exist() - expect(cids.length).to.equal(1) - expect(cids[0]).to.equal(rootDir.hash) - done() - }) - }, 100) + MockPreloadNode.waitForCids(rootDir.hash, done) }) }) - it('should preload multiple content and wrapping dir for content added with ipfs.files.add and wrapWithDirectory option', (done) => { + it('should preload multiple content and wrapping dir for content added with files.add and wrapWithDirectory option', (done) => { ipfs.files.add([{ path: 'dir0/dir1/file0', content: Buffer.from(hat()) @@ -116,14 +93,95 @@ describe('preload', () => { const wrappingDir = res.find(file => file.path === '') expect(wrappingDir).to.exist() - // Wait for preloading to finish - setTimeout(() => { - MockPreloadNode.getPreloadCids((err, cids) => { - expect(err).to.not.exist() - expect(cids.length).to.equal(1) - expect(cids[0]).to.equal(wrappingDir.hash) - done() - }) + MockPreloadNode.waitForCids(wrappingDir.hash, done) + }) + }) + + it('should preload content added with object.new', (done) => { + ipfs.object.new((err, node) => { + expect(err).to.not.exist() + + const cid = new CID(node.multihash) + MockPreloadNode.waitForCids(cid.toBaseEncodedString(), done) + }) + }) + + it('should preload content added with object.put', (done) => { + ipfs.object.put({ Data: Buffer.from(hat()), Links: [] }, (err, node) => { + expect(err).to.not.exist() + + const cid = new CID(node.multihash) + MockPreloadNode.waitForCids(cid.toBaseEncodedString(), done) + }) + }) + + it('should preload content added with object.patch.addLink', (done) => { + parallel({ + parent: (cb) => ipfs.object.put({ Data: Buffer.from(hat()), Links: [] }, cb), + link: (cb) => ipfs.object.put({ Data: Buffer.from(hat()), Links: [] }, cb) + }, (err, nodes) => { + expect(err).to.not.exist() + + ipfs.object.patch.addLink(nodes.parent.multihash, { + name: 'link', + multihash: nodes.link.multihash, + size: nodes.link.size + }, (err, node) => { + expect(err).to.not.exist() + + const cid = new CID(node.multihash) + MockPreloadNode.waitForCids(cid.toBaseEncodedString(), done) + }) + }) + }) + + it('should preload content added with object.patch.rmLink', (done) => { + waterfall([ + (cb) => ipfs.object.put({ Data: Buffer.from(hat()), Links: [] }, cb), + (link, cb) => { + ipfs.object.put({ + Data: Buffer.from(hat()), + Links: [{ + name: 'link', + multihash: link.multihash, + size: link.size + }] + }, cb) + } + ], (err, parent) => { + expect(err).to.not.exist() + + ipfs.object.patch.rmLink(parent.multihash, { name: 'link' }, (err, node) => { + expect(err).to.not.exist() + + const cid = new CID(node.multihash) + MockPreloadNode.waitForCids(cid.toBaseEncodedString(), done) + }) + }) + }) + + it('should preload content added with object.patch.setData', (done) => { + ipfs.object.put({ Data: Buffer.from(hat()), Links: [] }, (err, node) => { + expect(err).to.not.exist() + + ipfs.object.patch.setData(node.multihash, Buffer.from(hat()), (err, node) => { + expect(err).to.not.exist() + + const cid = new CID(node.multihash) + MockPreloadNode.waitForCids(cid.toBaseEncodedString(), done) + }) + }) + }) + + it('should preload content added with object.patch.appendData', (done) => { + ipfs.object.put({ Data: Buffer.from(hat()), Links: [] }, (err, node) => { + expect(err).to.not.exist() + + ipfs.object.patch.appendData(node.multihash, Buffer.from(hat()), (err, node) => { + expect(err).to.not.exist() + + const cid = new CID(node.multihash) + MockPreloadNode.waitForCids(cid.toBaseEncodedString(), done) }) }) }) diff --git a/test/utils/mock-preload-node.js b/test/utils/mock-preload-node.js index 0f08c8d61d..370847bf72 100644 --- a/test/utils/mock-preload-node.js +++ b/test/utils/mock-preload-node.js @@ -55,12 +55,14 @@ function parseMultiaddr (addr) { } // Get the stored preload CIDs for the server at `addr` -module.exports.getPreloadCids = (addr, cb) => { +const getPreloadCids = (addr, cb) => { if (typeof addr === 'function') { cb = addr addr = defaultAddr } + addr = addr || defaultAddr + const { protocol, hostname, port } = parseMultiaddr(addr) const req = http.get({ protocol, hostname, port, path: '/cids' }, (res) => { @@ -88,6 +90,8 @@ module.exports.getPreloadCids = (addr, cb) => { req.on('error', cb) } +module.exports.getPreloadCids = getPreloadCids + // Clear the stored preload URLs for the server at `addr` module.exports.clearPreloadCids = (addr, cb) => { if (typeof addr === 'function') { @@ -95,6 +99,8 @@ module.exports.clearPreloadCids = (addr, cb) => { addr = defaultAddr } + addr = addr || defaultAddr + const { protocol, hostname, port } = parseMultiaddr(addr) const req = http.request({ @@ -116,3 +122,36 @@ module.exports.clearPreloadCids = (addr, cb) => { req.on('error', cb) req.end() } + +// Wait for the passed CIDs to appear in the CID list from the preload node +module.exports.waitForCids = (cids, opts, cb) => { + if (typeof opts === 'function') { + cb = opts + opts = {} + } + + opts = opts || {} + opts.timeout = opts.timeout || 1000 + + cids = Array.isArray(cids) ? cids : [cids] + + const start = Date.now() + + const checkForCid = () => { + getPreloadCids(opts.addr, (err, preloadCids) => { + if (err) return cb(err) + + if (cids.every(cid => preloadCids.includes(cid))) { + return cb() + } + + if (Date.now() > start + opts.timeout) { + return cb(new Error('Timed out waiting for CIDs to be preloaded')) + } + + setTimeout(checkForCid, 10) + }) + } + + checkForCid() +} From fa1c9f330d3bfbf4167856aeb9c1a05dc945e8e1 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 27 Jul 2018 10:30:16 +0100 Subject: [PATCH 18/21] feat: preload block.put License: MIT Signed-off-by: Alan Shaw --- src/core/components/block.js | 5 +++++ test/core/preload.spec.js | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/src/core/components/block.js b/src/core/components/block.js index 89b18db7d3..b68140af36 100644 --- a/src/core/components/block.js +++ b/src/core/components/block.js @@ -52,6 +52,11 @@ module.exports = function block (self) { if (err) { return cb(err) } + + if (options.preload !== false) { + self._preload(block.cid) + } + cb(null, block) }) ], callback) diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js index 284231fc01..dbbf029642 100644 --- a/test/core/preload.spec.js +++ b/test/core/preload.spec.js @@ -185,4 +185,11 @@ describe.only('preload', () => { }) }) }) + + it('should preload content added with block.put', (done) => { + ipfs.block.put(Buffer.from(hat()), (err, block) => { + expect(err).to.not.exist() + MockPreloadNode.waitForCids(block.cid.toBaseEncodedString(), done) + }) + }) }) From 787d1dd97d852fc151c397b719a6db8a6d449d2f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 27 Jul 2018 10:43:54 +0100 Subject: [PATCH 19/21] chore: remove .only License: MIT Signed-off-by: Alan Shaw --- src/core/components/dag.js | 6 +++++- test/core/preload.spec.js | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/components/dag.js b/src/core/components/dag.js index 105985df68..3aa67d1e88 100644 --- a/src/core/components/dag.js +++ b/src/core/components/dag.js @@ -26,7 +26,11 @@ module.exports = function dag (self) { self._ipld.put(dagNode, options, (err, cid) => { if (err) return callback(err) - if (options.preload !== false) self._preload(cid) + + if (options.preload !== false) { + self._preload(cid) + } + callback(null, cid) }) }), diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js index dbbf029642..8c7b03f313 100644 --- a/test/core/preload.spec.js +++ b/test/core/preload.spec.js @@ -14,7 +14,7 @@ chai.use(dirtyChai) const MockPreloadNode = require('../utils/mock-preload-node') const IPFS = require('../../src') -describe.only('preload', () => { +describe('preload', () => { let ipfs before((done) => { From d49f17efe18b6ecdbc7aee146f285b61a17cb3d7 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 27 Jul 2018 10:50:02 +0100 Subject: [PATCH 20/21] test: add dag.put preload test License: MIT Signed-off-by: Alan Shaw --- test/core/preload.spec.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js index 8c7b03f313..38db0f3412 100644 --- a/test/core/preload.spec.js +++ b/test/core/preload.spec.js @@ -192,4 +192,12 @@ describe('preload', () => { MockPreloadNode.waitForCids(block.cid.toBaseEncodedString(), done) }) }) + + it('should preload content added with dag.put', (done) => { + const obj = { test: hat() } + ipfs.dag.put(obj, { format: 'dag-cbor', hashAlg: 'sha2-256' }, (err, cid) => { + expect(err).to.not.exist() + MockPreloadNode.waitForCids(cid.toBaseEncodedString(), done) + }) + }) }) From 9df1012b9f3b984a58eb72df747dc65c470ecd06 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 27 Jul 2018 13:18:34 +0100 Subject: [PATCH 21/21] fix: ensure MFS is continually preloaded License: MIT Signed-off-by: Alan Shaw --- src/core/mfs-preload.js | 4 ++- test/core/mfs-preload.spec.js | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 test/core/mfs-preload.spec.js diff --git a/src/core/mfs-preload.js b/src/core/mfs-preload.js index fc75f25a0a..4f0cfd16a5 100644 --- a/src/core/mfs-preload.js +++ b/src/core/mfs-preload.js @@ -22,12 +22,14 @@ module.exports = (self, options) => { if (rootCid !== stats.hash) { log(`preloading updated MFS root ${rootCid} -> ${stats.hash}`) - self._preload(stats.hash, (err) => { + return self._preload(stats.hash, (err) => { timeoutId = setTimeout(preloadMfs, options.interval) if (err) return log.error(`failed to preload MFS root ${stats.hash}`, err) rootCid = stats.hash }) } + + timeoutId = setTimeout(preloadMfs, options.interval) }) } diff --git a/test/core/mfs-preload.spec.js b/test/core/mfs-preload.spec.js new file mode 100644 index 0000000000..98d1fb70ed --- /dev/null +++ b/test/core/mfs-preload.spec.js @@ -0,0 +1,54 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) + +const mfsPreload = require('../../src/core/mfs-preload') + +const createMockFilesStat = (cids = []) => { + let n = 0 + return (path, cb) => cb(null, { hash: cids[n++] || 'QmHash' }) +} + +const createMockPreload = () => { + return function preload (cid, cb) { + preload.cids = preload.cids || [] + preload.cids.push(cid) + cb() + } +} + +describe('MFS preload', () => { + it('should preload MFS root periodically', (done) => { + // CIDs returned from our mock files.stat function + const statCids = ['QmInitial', 'QmSame', 'QmSame', 'QmUpdated'] + // The CIDs we expect to have been preloaded + const expectedPreloadCids = ['QmSame', 'QmUpdated'] + + const mockPreload = createMockPreload() + const mockFilesStat = createMockFilesStat(statCids) + const mockIpfs = { files: { stat: mockFilesStat }, _preload: mockPreload } + + const interval = 10 + const preloader = mfsPreload(mockIpfs, { interval }) + + preloader.start((err) => { + expect(err).to.not.exist() + + setTimeout(() => { + preloader.stop((err) => { + expect(err).to.not.exist() + expect( + // Slice off any extra CIDs it processed + mockPreload.cids.slice(0, expectedPreloadCids.length) + ).to.deep.equal(expectedPreloadCids) + done() + }) + }, statCids.length * (interval + 5)) + }) + }) +})