diff --git a/.aegir.js b/.aegir.js index 979ffde3df..5d87aa66d7 100644 --- a/.aegir.js +++ b/.aegir.js @@ -1,8 +1,11 @@ 'use strict' -const createServer = require('ipfsd-ctl').createServer +const IPFSFactory = require('ipfsd-ctl') +const parallel = require('async/parallel') +const MockPreloadNode = require('./test/utils/mock-preload-node') -const server = createServer() +const ipfsdServer = IPFSFactory.createServer() +const preloadNode = MockPreloadNode.createNode() module.exports = { webpack: { @@ -21,9 +24,29 @@ module.exports = { singleRun: true }, hooks: { + node: { + pre: (cb) => preloadNode.start(cb), + post: (cb) => preloadNode.stop(cb) + }, browser: { - pre: server.start.bind(server), - post: server.stop.bind(server) + pre: (cb) => { + parallel([ + (cb) => { + ipfsdServer.start() + cb() + }, + (cb) => preloadNode.start(cb) + ], cb) + }, + post: (cb) => { + parallel([ + (cb) => { + ipfsdServer.stop() + cb() + }, + (cb) => preloadNode.stop(cb) + ], cb) + } } } } diff --git a/README.md b/README.md index 0c20f35cdb..c2dbefa5b3 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,9 @@ Creates and returns an instance of an IPFS node. Use the `options` argument to s - `enabled` (boolean): Make this node a relay (other nodes can connect *through* it). (Default: `false`) - `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`) +- `preload` (object): Configure external nodes that will preload content added to this node + - `enabled` (boolean): Enable content preloading (Default: `true`) + - `addresses` (array): Multiaddr API addresses of nodes that should preload content. NOTE: nodes specified here should also be added to your node's bootstrap address list at `config.Boostrap` - `EXPERIMENTAL` (object): Enable and configure experimental features. - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`) - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`) diff --git a/package.json b/package.json index 2cd7bcb477..e98cd5a278 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "./src/core/components/init-assets.js": false, "./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js", "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js", + "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js", "./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js", "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js", "./test/utils/create-repo-nodejs.js": "./test/utils/create-repo-browser.js", @@ -140,6 +141,7 @@ "mime-types": "^2.1.18", "mkdirp": "~0.5.1", "multiaddr": "^5.0.0", + "multiaddr-to-uri": "^4.0.0", "multibase": "~0.4.0", "multihashes": "~0.4.13", "once": "^1.4.0", diff --git a/src/core/components/dag.js b/src/core/components/dag.js index 88a80bdcce..105985df68 100644 --- a/src/core/components/dag.js +++ b/src/core/components/dag.js @@ -24,7 +24,11 @@ module.exports = function dag (self) { options = options.cid ? options : Object.assign({}, optionDefaults, options) - self._ipld.put(dagNode, options, callback) + self._ipld.put(dagNode, options, (err, cid) => { + if (err) return callback(err) + if (options.preload !== false) self._preload(cid) + callback(null, cid) + }) }), get: promisify((cid, path, options, callback) => { diff --git a/src/core/components/files.js b/src/core/components/files.js index f69868bd77..12a1fcdce0 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -89,6 +89,20 @@ function normalizeContent (opts, content) { }) } +function preloadFile (self, opts, file) { + const isRootFile = opts.wrapWithDirectory + ? file.path === '' + : !file.path.includes('/') + + const shouldPreload = isRootFile && !opts.onlyHash && opts.preload !== false + + if (shouldPreload) { + self._preload(file.hash) + } + + return file +} + function pinFile (self, opts, file, cb) { // Pin a file if it is the root dir of a recursive add or the single file // of a direct add. @@ -158,6 +172,7 @@ module.exports = function files (self) { pull.flatten(), importer(self._ipld, opts), pull.asyncMap(prepareFile.bind(null, self, opts)), + pull.map(preloadFile.bind(null, self, opts)), pull.asyncMap(pinFile.bind(null, self, opts)) ) } diff --git a/src/core/components/index.js b/src/core/components/index.js index 9eb36ad4c3..1f6f084dee 100644 --- a/src/core/components/index.js +++ b/src/core/components/index.js @@ -26,4 +26,4 @@ exports.dht = require('./dht') exports.dns = require('./dns') exports.key = require('./key') exports.stats = require('./stats') -exports.mfs = require('ipfs-mfs/core') +exports.mfs = require('./mfs') diff --git a/src/core/components/mfs.js b/src/core/components/mfs.js new file mode 100644 index 0000000000..9f033545b4 --- /dev/null +++ b/src/core/components/mfs.js @@ -0,0 +1,24 @@ +'use strict' + +const promisify = require('promisify-es6') +const mfs = require('ipfs-mfs/core') + +module.exports = self => { + const mfsSelf = Object.assign({}, self) + + // A patched dag API to ensure preload doesn't happen for MFS operations + mfsSelf.dag = Object.assign({}, self.dag, { + put: promisify((node, opts, cb) => { + if (typeof opts === 'function') { + cb = opts + opts = {} + } + + opts = Object.assign({}, opts, { preload: false }) + + return self.dag.put(node, opts, cb) + }) + }) + + return mfs(mfsSelf, mfsSelf._options) +} diff --git a/src/core/components/pin-set.js b/src/core/components/pin-set.js index f18a248604..806df5f05f 100644 --- a/src/core/components/pin-set.js +++ b/src/core/components/pin-set.js @@ -90,7 +90,7 @@ exports = module.exports = function (dag) { pinSet.storeItems(pins, (err, rootNode) => { if (err) { return callback(err) } - const opts = { cid: new CID(rootNode.multihash) } + const opts = { cid: new CID(rootNode.multihash), preload: false } dag.put(rootNode, opts, (err, cid) => { if (err) { return callback(err) } callback(null, rootNode) @@ -168,7 +168,8 @@ exports = module.exports = function (dag) { function storeChild (err, child, binIdx, cb) { if (err) { return cb(err) } - dag.put(child, { cid: new CID(child._multihash) }, err => { + const opts = { cid: new CID(child._multihash), preload: false } + dag.put(child, opts, err => { if (err) { return cb(err) } fanoutLinks[binIdx] = new DAGLink('', child.size, child.multihash) cb(null) diff --git a/src/core/components/pin.js b/src/core/components/pin.js index d2bd670e14..ce0cd72d84 100644 --- a/src/core/components/pin.js +++ b/src/core/components/pin.js @@ -80,14 +80,14 @@ module.exports = (self) => { // the pin-set nodes link to a special 'empty' node, so make sure it exists cb => DAGNode.create(Buffer.alloc(0), (err, empty) => { if (err) { return cb(err) } - dag.put(empty, { cid: new CID(empty.multihash) }, cb) + dag.put(empty, { cid: new CID(empty.multihash), preload: false }, cb) }), // create a root node with DAGLinks to the direct and recursive DAGs cb => DAGNode.create(Buffer.alloc(0), [dLink, rLink], (err, node) => { if (err) { return cb(err) } root = node - dag.put(root, { cid: new CID(root.multihash) }, cb) + dag.put(root, { cid: new CID(root.multihash), preload: false }, cb) }), // hack for CLI tests diff --git a/src/core/components/start.js b/src/core/components/start.js index fd4832e35a..3a7a5716ce 100644 --- a/src/core/components/start.js +++ b/src/core/components/start.js @@ -42,7 +42,9 @@ module.exports = (self) => { self._bitswap.start() self._blockService.setExchange(self._bitswap) - cb() + + self._preload.start() + self._mfsPreload.start(cb) } ], done) }) diff --git a/src/core/components/stop.js b/src/core/components/stop.js index 4d35190d21..cf97b6ec6a 100644 --- a/src/core/components/stop.js +++ b/src/core/components/stop.js @@ -30,8 +30,10 @@ module.exports = (self) => { self.state.stop() self._blockService.unsetExchange() self._bitswap.stop() + self._preload.stop() series([ + (cb) => self._mfsPreload.stop(cb), (cb) => self.libp2p.stop(cb), (cb) => self._repo.close(cb) ], done) diff --git a/src/core/config.js b/src/core/config.js index 1b04d10a2f..7b16c17d06 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -8,6 +8,10 @@ const schema = Joi.object().keys({ Joi.string() ).allow(null), repoOwner: Joi.boolean().default(true), + preload: Joi.object().keys({ + enabled: Joi.boolean().default(true), + addresses: Joi.array().items(Joi.multiaddr().options({ convert: false })) + }).allow(null), init: Joi.alternatives().try( Joi.boolean(), Joi.object().keys({ bits: Joi.number().integer() }) diff --git a/src/core/index.js b/src/core/index.js index 36f7c3a118..52266b7b41 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -22,6 +22,8 @@ const boot = require('./boot') const components = require('./components') // replaced by repo-browser when running in the browser const defaultRepo = require('./runtime/repo-nodejs') +const preload = require('./preload') +const mfsPreload = require('./mfs-preload') class IPFS extends EventEmitter { constructor (options) { @@ -30,7 +32,14 @@ class IPFS extends EventEmitter { this._options = { init: true, start: true, - EXPERIMENTAL: {} + EXPERIMENTAL: {}, + preload: { + enabled: true, + addresses: [ + '/dnsaddr/node0.preload.ipfs.io/https', + '/dnsaddr/node1.preload.ipfs.io/https' + ] + } } options = config.validate(options || {}) @@ -78,6 +87,8 @@ class IPFS extends EventEmitter { this._blockService = new BlockService(this._repo) this._ipld = new Ipld(this._blockService) this._pubsub = undefined + this._preload = preload(this) + this._mfsPreload = mfsPreload(this) // IPFS Core exposed components // - for booting up a node @@ -134,7 +145,7 @@ class IPFS extends EventEmitter { } // ipfs.files - const mfs = components.mfs(this, this._options) + const mfs = components.mfs(this) Object.keys(mfs).forEach(key => { this.files[key] = mfs[key] diff --git a/src/core/mfs-preload.js b/src/core/mfs-preload.js new file mode 100644 index 0000000000..fc75f25a0a --- /dev/null +++ b/src/core/mfs-preload.js @@ -0,0 +1,49 @@ +'use strict' + +const debug = require('debug') + +const log = debug('jsipfs:mfs-preload') +log.error = debug('jsipfs:mfs-preload:error') + +module.exports = (self, options) => { + options = options || {} + options.interval = options.interval || 30 * 1000 + + let rootCid + let timeoutId + + const preloadMfs = () => { + self.files.stat('/', (err, stats) => { + if (err) { + timeoutId = setTimeout(preloadMfs, options.interval) + return log.error('failed to stat MFS root for preload', err) + } + + if (rootCid !== stats.hash) { + log(`preloading updated MFS root ${rootCid} -> ${stats.hash}`) + + self._preload(stats.hash, (err) => { + timeoutId = setTimeout(preloadMfs, options.interval) + if (err) return log.error(`failed to preload MFS root ${stats.hash}`, err) + rootCid = stats.hash + }) + } + }) + } + + return { + start (cb) { + self.files.stat('/', (err, stats) => { + if (err) return cb(err) + rootCid = stats.hash + log(`monitoring MFS root ${rootCid}`) + timeoutId = setTimeout(preloadMfs, options.interval) + cb() + }) + }, + stop (cb) { + clearTimeout(timeoutId) + cb() + } + } +} diff --git a/src/core/preload.js b/src/core/preload.js new file mode 100644 index 0000000000..d99a9d8f20 --- /dev/null +++ b/src/core/preload.js @@ -0,0 +1,88 @@ +'use strict' + +const setImmediate = require('async/setImmediate') +const retry = require('async/retry') +const toUri = require('multiaddr-to-uri') +const debug = require('debug') +const CID = require('cids') +const preload = require('./runtime/preload-nodejs') + +const log = debug('jsipfs:preload') +log.error = debug('jsipfs:preload:error') + +const noop = (err) => { if (err) log.error(err) } + +module.exports = self => { + const options = self._options.preload || {} + options.enabled = Boolean(options.enabled) + options.addresses = options.addresses || [] + + if (!options.enabled || !options.addresses.length) { + return (_, callback) => { + if (callback) { + setImmediate(() => callback()) + } + } + } + + let stopped = true + let requests = [] + const apiUris = options.addresses.map(apiAddrToUri) + + const api = (cid, callback) => { + callback = callback || noop + + if (typeof cid !== 'string') { + try { + cid = new CID(cid).toBaseEncodedString() + } catch (err) { + return setImmediate(() => callback(err)) + } + } + + const fallbackApiUris = Array.from(apiUris) + let request + const now = Date.now() + + retry({ times: fallbackApiUris.length }, (cb) => { + if (stopped) return cb(new Error(`preload aborted for ${cid}`)) + + // Remove failed request from a previous attempt + requests = requests.filter(r => r !== request) + + const apiUri = fallbackApiUris.shift() + + request = preload(`${apiUri}/api/v0/refs?r=true&arg=${cid}`, cb) + requests = requests.concat(request) + }, (err) => { + requests = requests.filter(r => r !== request) + + if (err) { + return callback(err) + } + + log(`preloaded ${cid} in ${Date.now() - now}ms`) + callback() + }) + } + + api.start = () => { + stopped = false + } + + api.stop = () => { + stopped = true + log(`canceling ${requests.length} pending preload request(s)`) + requests.forEach(r => r.cancel()) + requests = [] + } + + return api +} + +function apiAddrToUri (addr) { + if (!(addr.endsWith('http') || addr.endsWith('https'))) { + addr = addr + '/http' + } + return toUri(addr) +} diff --git a/src/core/runtime/config-browser.js b/src/core/runtime/config-browser.js index 9819c04aaa..f7662420cb 100644 --- a/src/core/runtime/config-browser.js +++ b/src/core/runtime/config-browser.js @@ -24,6 +24,8 @@ module.exports = () => ({ '/dns4/nyc-1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm', '/dns4/nyc-2.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' + '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' ] }) diff --git a/src/core/runtime/config-nodejs.js b/src/core/runtime/config-nodejs.js index 995f66261d..d56550e181 100644 --- a/src/core/runtime/config-nodejs.js +++ b/src/core/runtime/config-nodejs.js @@ -37,6 +37,8 @@ module.exports = () => ({ '/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3', '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' + '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' ] }) diff --git a/src/core/runtime/preload-browser.js b/src/core/runtime/preload-browser.js new file mode 100644 index 0000000000..8d123a12be --- /dev/null +++ b/src/core/runtime/preload-browser.js @@ -0,0 +1,37 @@ +/* eslint-env browser */ +'use strict' + +const debug = require('debug') + +const log = debug('jsipfs:preload') +log.error = debug('jsipfs:preload:error') + +module.exports = function preload (url, callback) { + log(url) + + const req = new self.XMLHttpRequest() + + req.open('HEAD', url) + + req.onreadystatechange = function () { + if (this.readyState !== this.DONE) { + return + } + + if (this.status < 200 || this.status >= 300) { + log.error('failed to preload', url, this.status, this.statusText) + return callback(new Error(`failed to preload ${url}`)) + } + + callback() + } + + req.send() + + return { + cancel: () => { + req.abort() + callback(new Error('request aborted')) + } + } +} diff --git a/src/core/runtime/preload-nodejs.js b/src/core/runtime/preload-nodejs.js new file mode 100644 index 0000000000..405798ca34 --- /dev/null +++ b/src/core/runtime/preload-nodejs.js @@ -0,0 +1,64 @@ +'use strict' + +const http = require('http') +const https = require('https') +const { URL } = require('url') +const debug = require('debug') +const setImmediate = require('async/setImmediate') + +const log = debug('jsipfs:preload') +log.error = debug('jsipfs:preload:error') + +module.exports = function preload (url, callback) { + log(url) + + try { + url = new URL(url) + } catch (err) { + return setImmediate(() => callback(err)) + } + + const transport = url.protocol === 'https:' ? https : http + + const req = transport.get({ + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search + }, (res) => { + if (res.statusCode < 200 || res.statusCode >= 300) { + res.resume() + log.error('failed to preload', url.href, res.statusCode, res.statusMessage) + return callback(new Error(`failed to preload ${url}`)) + } + + res.on('data', chunk => log(`data ${chunk}`)) + + res.on('abort', () => { + callback(new Error('request aborted')) + }) + + res.on('error', err => { + log.error('response error preloading', url.href, err) + callback(err) + }) + + res.on('end', () => { + // If aborted, callback is called in the abort handler + if (!res.aborted) callback() + }) + }) + + req.on('error', err => { + log.error('request error preloading', url.href, err) + callback(err) + }) + + return { + cancel: () => { + // No need to call callback here + // before repsonse - called in req error handler + // after response - called in res abort hander + req.abort() + } + } +} diff --git a/test/cli/bootstrap.js b/test/cli/bootstrap.js index 8807a12411..a301bca3f2 100644 --- a/test/cli/bootstrap.js +++ b/test/cli/bootstrap.js @@ -31,7 +31,9 @@ describe('bootstrap', () => runOnAndOff((thing) => { '/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3', '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' + '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' ] const updatedList = [ @@ -54,6 +56,8 @@ describe('bootstrap', () => runOnAndOff((thing) => { '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', '/ip4/111.111.111.111/tcp/1001/ipfs/QmcyFFKfLDGJKwufn2GeitxvhricsBQyNKTkrD14psikoD' ] diff --git a/test/core/bootstrap.spec.js b/test/core/bootstrap.spec.js index f092765354..d95d622f18 100644 --- a/test/core/bootstrap.spec.js +++ b/test/core/bootstrap.spec.js @@ -59,7 +59,9 @@ describe('bootstrap', () => { '/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3', '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', - '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' + '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' ] const updatedList = [ @@ -82,6 +84,8 @@ describe('bootstrap', () => { '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx', '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', + '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', + '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6', '/ip4/111.111.111.111/tcp/1001/ipfs/QmXFX2P5ammdmXQgfqGkfswtEVFsZUJ5KeHRXQYCTdiTAb' ] diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js new file mode 100644 index 0000000000..54fbed45cf --- /dev/null +++ b/test/core/preload.spec.js @@ -0,0 +1,130 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const hat = require('hat') +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) + +const MockPreloadNode = require('../utils/mock-preload-node') +const IPFS = require('../../src') + +describe('preload', () => { + let ipfs + + before((done) => { + ipfs = new IPFS({ + config: { + Addresses: { + Swarm: [] + } + }, + preload: { + enabled: true, + addresses: [MockPreloadNode.defaultAddr] + } + }) + + ipfs.on('ready', done) + }) + + afterEach((done) => MockPreloadNode.clearPreloadCids(done)) + + after((done) => ipfs.stop(done)) + + it('should preload content added with ipfs.files.add', (done) => { + ipfs.files.add(Buffer.from(hat()), (err, res) => { + expect(err).to.not.exist() + + // Wait for preloading to finish + setTimeout(() => { + MockPreloadNode.getPreloadCids((err, cids) => { + expect(err).to.not.exist() + expect(cids.length).to.equal(1) + expect(cids[0]).to.equal(res[0].hash) + done() + }) + }, 100) + }) + }) + + it('should preload multiple content added with ipfs.files.add', (done) => { + ipfs.files.add([{ + content: Buffer.from(hat()) + }, { + content: Buffer.from(hat()) + }, { + content: Buffer.from(hat()) + }], (err, res) => { + expect(err).to.not.exist() + + // Wait for preloading to finish + setTimeout(() => { + MockPreloadNode.getPreloadCids((err, cids) => { + expect(err).to.not.exist() + expect(cids.length).to.equal(res.length) + res.forEach(file => expect(cids).to.include(file.hash)) + done() + }) + }, 100) + }) + }) + + it('should preload multiple content and intermediate dirs added with ipfs.files.add', (done) => { + ipfs.files.add([{ + path: 'dir0/dir1/file0', + content: Buffer.from(hat()) + }, { + path: 'dir0/dir1/file1', + content: Buffer.from(hat()) + }, { + path: 'dir0/file2', + content: Buffer.from(hat()) + }], (err, res) => { + expect(err).to.not.exist() + + const rootDir = res.find(file => file.path === 'dir0') + expect(rootDir).to.exist() + + // Wait for preloading to finish + setTimeout(() => { + MockPreloadNode.getPreloadCids((err, cids) => { + expect(err).to.not.exist() + expect(cids.length).to.equal(1) + expect(cids[0]).to.equal(rootDir.hash) + done() + }) + }, 100) + }) + }) + + it('should preload multiple content and wrapping dir for content added with ipfs.files.add and wrapWithDirectory option', (done) => { + ipfs.files.add([{ + path: 'dir0/dir1/file0', + content: Buffer.from(hat()) + }, { + path: 'dir0/dir1/file1', + content: Buffer.from(hat()) + }, { + path: 'dir0/file2', + content: Buffer.from(hat()) + }], { wrapWithDirectory: true }, (err, res) => { + expect(err).to.not.exist() + + const wrappingDir = res.find(file => file.path === '') + expect(wrappingDir).to.exist() + + // Wait for preloading to finish + setTimeout(() => { + MockPreloadNode.getPreloadCids((err, cids) => { + expect(err).to.not.exist() + expect(cids.length).to.equal(1) + expect(cids[0]).to.equal(wrappingDir.hash) + done() + }) + }) + }) + }) +}) diff --git a/test/fixtures/go-ipfs-repo/config b/test/fixtures/go-ipfs-repo/config index 00f467f95f..9843d866a8 100644 --- a/test/fixtures/go-ipfs-repo/config +++ b/test/fixtures/go-ipfs-repo/config @@ -65,7 +65,9 @@ "/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3", "/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx", "/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic", - "/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6" + "/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6", + "/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic", + "/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6" ], "Tour": { "Last": "" @@ -106,4 +108,4 @@ "hash": "sha2-512" } } -} \ No newline at end of file +} diff --git a/test/utils/mock-preload-node.js b/test/utils/mock-preload-node.js new file mode 100644 index 0000000000..0f08c8d61d --- /dev/null +++ b/test/utils/mock-preload-node.js @@ -0,0 +1,118 @@ +/* eslint-env browser */ +'use strict' + +const http = require('http') +const toUri = require('multiaddr-to-uri') +const URL = require('url').URL || self.URL + +const defaultPort = 1138 +const defaultAddr = `/dnsaddr/localhost/tcp/${defaultPort}` + +module.exports.defaultAddr = defaultAddr + +// Create a mock preload IPFS node with a gateway that'll respond 200 to a +// request for /api/v0/refs?arg=*. It remembers the preload CIDs it has been +// called with, and you can ask it for them and also clear them by issuing a +// GET/DELETE request to /cids. +module.exports.createNode = () => { + let cids = [] + + const server = http.createServer((req, res) => { + if (req.url.startsWith('/api/v0/refs')) { + const arg = new URL(`https://ipfs.io${req.url}`).searchParams.get('arg') + cids = cids.concat(arg) + } else if (req.method === 'DELETE' && req.url === '/cids') { + res.statusCode = 204 + cids = [] + } else if (req.method === 'GET' && req.url === '/cids') { + res.setHeader('Content-Type', 'application/json') + res.write(JSON.stringify(cids)) + } else { + res.statusCode = 500 + } + + res.end() + }) + + server.start = (opts, cb) => { + if (typeof opts === 'function') { + cb = opts + opts = {} + } + return server.listen(Object.assign({ port: defaultPort }, opts), cb) + } + + server.stop = (cb) => server.close(cb) + + return server +} + +function parseMultiaddr (addr) { + if (!(addr.endsWith('http') || addr.endsWith('https'))) { + addr = addr + '/http' + } + return new URL(toUri(addr)) +} + +// Get the stored preload CIDs for the server at `addr` +module.exports.getPreloadCids = (addr, cb) => { + if (typeof addr === 'function') { + cb = addr + addr = defaultAddr + } + + const { protocol, hostname, port } = parseMultiaddr(addr) + + const req = http.get({ protocol, hostname, port, path: '/cids' }, (res) => { + if (res.statusCode !== 200) { + res.resume() + return cb(new Error('failed to get preloaded CIDs from mock preload node')) + } + + let data = '' + + res.on('error', cb) + res.on('data', chunk => { data += chunk }) + + res.on('end', () => { + let obj + try { + obj = JSON.parse(data) + } catch (err) { + return cb(err) + } + cb(null, obj) + }) + }) + + req.on('error', cb) +} + +// Clear the stored preload URLs for the server at `addr` +module.exports.clearPreloadCids = (addr, cb) => { + if (typeof addr === 'function') { + cb = addr + addr = defaultAddr + } + + const { protocol, hostname, port } = parseMultiaddr(addr) + + const req = http.request({ + method: 'DELETE', + protocol, + hostname, + port, + path: '/cids' + }, (res) => { + res.resume() + + if (res.statusCode !== 204) { + return cb(new Error('failed to clear CIDs from mock preload node')) + } + + cb() + }) + + req.on('error', cb) + req.end() +}