feat: preload object.new object.put object.patch.*

.aegir.js

@@ -1,8 +1,11 @@
 'use strict'
 
-const createServer = require('ipfsd-ctl').createServer
+const IPFSFactory = require('ipfsd-ctl')
+const parallel = require('async/parallel')
+const MockPreloadNode = require('./test/utils/mock-preload-node')
 
-const server = createServer()
+const ipfsdServer = IPFSFactory.createServer()
+const preloadNode = MockPreloadNode.createNode()
 
 module.exports = {
   webpack: {
@@ -21,9 +24,29 @@ module.exports = {
     singleRun: true
   },
   hooks: {
+    node: {
+      pre: (cb) => preloadNode.start(cb),
+      post: (cb) => preloadNode.stop(cb)
+    },
     browser: {
-      pre: server.start.bind(server),
-      post: server.stop.bind(server)
+      pre: (cb) => {
+        parallel([
+          (cb) => {
+            ipfsdServer.start()
+            cb()
+          },
+          (cb) => preloadNode.start(cb)
+        ], cb)
+      },
+      post: (cb) => {
+        parallel([
+          (cb) => {
+            ipfsdServer.stop()
+            cb()
+          },
+          (cb) => preloadNode.stop(cb)
+        ], cb)
+      }
     }
   }
 }

README.md

@@ -231,6 +231,9 @@ Creates and returns an instance of an IPFS node. Use the `options` argument to s
         - `enabled` (boolean): Make this node a relay (other nodes can connect *through* it). (Default: `false`)
         - `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`)
 
+- `preload` (object): Configure external nodes that will preload content added to this node
+    - `enabled` (boolean): Enable content preloading (Default: `true`)
+    - `addresses` (array): Multiaddr API addresses of nodes that should preload content. NOTE: nodes specified here should also be added to your node's bootstrap address list at `config.Boostrap`
 - `EXPERIMENTAL` (object): Enable and configure experimental features.
     - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`)
     - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`)

package.json

@@ -11,6 +11,7 @@
     "./src/core/components/init-assets.js": false,
     "./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js",
     "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js",
+    "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js",
     "./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js",
     "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js",
     "./test/utils/create-repo-nodejs.js": "./test/utils/create-repo-browser.js",
@@ -140,6 +141,7 @@
     "mime-types": "^2.1.18",
     "mkdirp": "~0.5.1",
     "multiaddr": "^5.0.0",
+    "multiaddr-to-uri": "^4.0.0",
     "multibase": "~0.4.0",
     "multihashes": "~0.4.13",
     "once": "^1.4.0",

src/core/components/block.js

@@ -52,6 +52,11 @@ module.exports = function block (self) {
           if (err) {
             return cb(err)
           }
+
+          if (options.preload !== false) {
+            self._preload(block.cid)
+          }
+
           cb(null, block)
         })
       ], callback)

src/core/components/files.js

@@ -89,6 +89,20 @@ function normalizeContent (opts, content) {
   })
 }
 
+function preloadFile (self, opts, file) {
+  const isRootFile = opts.wrapWithDirectory
+    ? file.path === ''
+    : !file.path.includes('/')
+
+  const shouldPreload = isRootFile && !opts.onlyHash && opts.preload !== false
+
+  if (shouldPreload) {
+    self._preload(file.hash)
+  }
+
+  return file
+}
+
 function pinFile (self, opts, file, cb) {
   // Pin a file if it is the root dir of a recursive add or the single file
   // of a direct add.
@@ -158,6 +172,7 @@ module.exports = function files (self) {
       pull.flatten(),
       importer(self._ipld, opts),
       pull.asyncMap(prepareFile.bind(null, self, opts)),
+      pull.map(preloadFile.bind(null, self, opts)),
       pull.asyncMap(pinFile.bind(null, self, opts))
     )
   }

src/core/components/object.js

@@ -80,10 +80,17 @@ module.exports = function object (self) {
             if (err) {
               return cb(err)
             }
-            self._ipld.put(node, {
-              cid: new CID(node.multihash)
-            }, (err) => {
-              cb(err, node)
+
+            const cid = new CID(node.multihash)
+
+            self._ipld.put(node, { cid }, (err) => {
+              if (err) return cb(err)
+
+              if (options.preload !== false) {
+                self._preload(cid)
+              }
+
+              cb(null, node)
             })
           })
         }
@@ -92,12 +99,20 @@ module.exports = function object (self) {
   }
 
   return {
-    new: promisify((template, callback) => {
+    new: promisify((template, options, callback) => {
       if (typeof template === 'function') {
         callback = template
         template = undefined
+        options = {}
+      }
+
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
       }
 
+      options = options || {}
+
       let data
 
       if (template) {
@@ -111,13 +126,18 @@ module.exports = function object (self) {
         if (err) {
           return callback(err)
         }
-        self._ipld.put(node, {
-          cid: new CID(node.multihash)
-        }, (err) => {
+
+        const cid = new CID(node.multihash)
+
+        self._ipld.put(node, { cid }, (err) => {
           if (err) {
             return callback(err)
           }
 
+          if (options.preload !== false) {
+            self._preload(cid)
+          }
+
           callback(null, node)
         })
       })
@@ -166,13 +186,17 @@ module.exports = function object (self) {
       }
 
       function next () {
-        self._ipld.put(node, {
-          cid: new CID(node.multihash)
-        }, (err) => {
+        const cid = new CID(node.multihash)
+
+        self._ipld.put(node, { cid }, (err) => {
           if (err) {
             return callback(err)
           }
 
+          if (options.preload !== false) {
+            self._preload(cid)
+          }
+
           self.object.get(node.multihash, callback)
         })
       }
@@ -282,6 +306,8 @@ module.exports = function object (self) {
         editAndSave((node, cb) => {
           if (DAGLink.isDAGLink(linkRef)) {
             linkRef = linkRef._name
+          } else if (linkRef && linkRef.name) {
+            linkRef = linkRef.name
           }
           DAGNode.rmLink(node, linkRef, cb)
         })(multihash, options, callback)

src/core/components/start.js

@@ -34,6 +34,8 @@ module.exports = (self) => {
       },
       (cb) => self.libp2p.start(cb),
       (cb) => {
+        self._preload.start()
+
         self._bitswap = new Bitswap(
           self._libp2pNode,
           self._repo.blocks,

src/core/components/stop.js

@@ -30,6 +30,7 @@ module.exports = (self) => {
     self.state.stop()
     self._blockService.unsetExchange()
     self._bitswap.stop()
+    self._preload.stop()
 
     series([
       (cb) => self.libp2p.stop(cb),

src/core/config.js

@@ -8,6 +8,10 @@ const schema = Joi.object().keys({
     Joi.string()
   ).allow(null),
   repoOwner: Joi.boolean().default(true),
+  preload: Joi.object().keys({
+    enabled: Joi.boolean().default(true),
+    addresses: Joi.array().items(Joi.multiaddr().options({ convert: false }))
+  }).allow(null),
   init: Joi.alternatives().try(
     Joi.boolean(),
     Joi.object().keys({ bits: Joi.number().integer() })

src/core/index.js

@@ -22,6 +22,7 @@ const boot = require('./boot')
 const components = require('./components')
 // replaced by repo-browser when running in the browser
 const defaultRepo = require('./runtime/repo-nodejs')
+const preload = require('./preload')
 
 class IPFS extends EventEmitter {
   constructor (options) {
@@ -30,7 +31,14 @@ class IPFS extends EventEmitter {
     this._options = {
       init: true,
       start: true,
-      EXPERIMENTAL: {}
+      EXPERIMENTAL: {},
+      preload: {
+        enabled: true,
+        addresses: [
+          '/dnsaddr/node0.preload.ipfs.io/https',
+          '/dnsaddr/node1.preload.ipfs.io/https'
+        ]
+      }
     }
 
     options = config.validate(options || {})
@@ -78,6 +86,7 @@ class IPFS extends EventEmitter {
     this._blockService = new BlockService(this._repo)
     this._ipld = new Ipld(this._blockService)
     this._pubsub = undefined
+    this._preload = preload(this)
 
     // IPFS Core exposed components
     //   - for booting up a node

src/core/preload.js

@@ -0,0 +1,88 @@
+'use strict'
+
+const setImmediate = require('async/setImmediate')
+const retry = require('async/retry')
+const toUri = require('multiaddr-to-uri')
+const debug = require('debug')
+const CID = require('cids')
+const preload = require('./runtime/preload-nodejs')
+
+const log = debug('jsipfs:preload')
+log.error = debug('jsipfs:preload:error')
+
+const noop = (err) => { if (err) log.error(err) }
+
+module.exports = self => {
+  const options = self._options.preload || {}
+  options.enabled = Boolean(options.enabled)
+  options.addresses = options.addresses || []
+
+  if (!options.enabled || !options.addresses.length) {
+    return (_, callback) => {
+      if (callback) {
+        setImmediate(() => callback())
+      }
+    }
+  }
+
+  let stopped = true
+  let requests = []
+  const apiUris = options.addresses.map(apiAddrToUri)
+
+  const api = (cid, callback) => {
+    callback = callback || noop
+
+    if (typeof cid !== 'string') {
+      try {
+        cid = new CID(cid).toBaseEncodedString()
+      } catch (err) {
+        return setImmediate(() => callback(err))
+      }
+    }
+
+    const fallbackApiUris = Array.from(apiUris)
+    let request
+    const now = Date.now()
+
+    retry({ times: fallbackApiUris.length }, (cb) => {
+      if (stopped) return cb(new Error(`preload aborted for ${cid}`))
+
+      // Remove failed request from a previous attempt
+      requests = requests.filter(r => r !== request)
+
+      const apiUri = fallbackApiUris.shift()
+
+      request = preload(`${apiUri}/api/v0/refs?r=true&arg=${cid}`, cb)
+      requests = requests.concat(request)
+    }, (err) => {
+      requests = requests.filter(r => r !== request)
+
+      if (err) {
+        return callback(err)
+      }
+
+      log(`preloaded ${cid} in ${Date.now() - now}ms`)
+      callback()
+    })
+  }
+
+  api.start = () => {
+    stopped = false
+  }
+
+  api.stop = () => {
+    stopped = true
+    log(`canceling ${requests.length} pending preload request(s)`)
+    requests.forEach(r => r.cancel())
+    requests = []
+  }
+
+  return api
+}
+
+function apiAddrToUri (addr) {
+  if (!(addr.endsWith('http') || addr.endsWith('https'))) {
+    addr = addr + '/http'
+  }
+  return toUri(addr)
+}

src/core/runtime/config-browser.js

@@ -24,6 +24,8 @@ module.exports = () => ({
     '/dns4/nyc-1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm',
     '/dns4/nyc-2.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64',
     '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
-    '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
+    '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
+    '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
+    '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
   ]
 })

src/core/runtime/config-nodejs.js

@@ -37,6 +37,8 @@ module.exports = () => ({
     '/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
     '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
     '/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
-    '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
+    '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
+    '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
+    '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
   ]
 })