feat: support --raw-leaves

License: MIT
Signed-off-by: achingbrain <alex@achingbrain.net>

Author: achingbrain

package.json

@@ -39,6 +39,7 @@
   "devDependencies": {
     "aegir": "^15.0.0",
     "chai": "^4.1.2",
+    "detect-node": "^2.0.3",
     "detect-webworker": "^1.0.0",
     "dirty-chai": "^2.0.1",
     "ipfs": "~0.30.0",
@@ -48,15 +49,13 @@
   "dependencies": {
     "async": "^2.6.1",
     "blob": "~0.0.4",
-    "bs58": "^4.0.1",
     "cids": "~0.5.3",
     "debug": "^3.1.0",
-    "detect-node": "^2.0.3",
     "file-api": "~0.10.4",
     "filereader-stream": "^2.0.0",
     "interface-datastore": "~0.4.2",
     "ipfs-unixfs": "~0.1.15",
-    "ipfs-unixfs-engine": "~0.30.0",
+    "ipfs-unixfs-engine": "~0.31.1",
     "is-pull-stream": "~0.0.0",
     "is-stream": "^1.1.0",
     "joi": "^13.4.0",
@@ -67,7 +66,7 @@
     "pull-cat": "^1.1.11",
     "pull-paramap": "^1.2.2",
     "pull-pushable": "^2.2.0",
-    "pull-stream": "^3.6.7",
+    "pull-stream": "^3.6.8",
     "pull-stream-to-stream": "^1.3.4",
     "pull-traverse": "^1.0.3",
     "stream-to-pull-stream": "^1.7.2"

src/cli/write.js

@@ -16,7 +16,7 @@ module.exports = {
       describe: 'Create any non-existent intermediate directories'
     },
     create: {
-      alias: 'c',
+      alias: 'e',
       type: 'boolean',
       default: false,
       coerce: asBoolean,
@@ -42,7 +42,7 @@ module.exports = {
     rawLeaves: {
       alias: 'r',
       type: 'boolean',
-      default: true,
+      default: false,
       coerce: asBoolean,
       describe: 'Whether to write leaf nodes as raw UnixFS nodes'
     },
@@ -65,9 +65,9 @@ module.exports = {
       default: 'balanced'
     },
     cidVersion: {
-      alias: 'c',
+      alias: ['cid-ver', 'cid-version'],
       type: 'number',
-      default: 1
+      default: 0
     },
     hashAlg: {
       alias: 'h',
@@ -105,7 +105,7 @@ module.exports = {
         length,
         create,
         truncate,
-        rawLeafNodes: rawLeaves,
+        rawLeaves,
         reduceSingleLeafToSelf,
         cidVersion,
         hashAlg,

src/core/mkdir.js

@@ -12,7 +12,7 @@ const {
 const defaultOptions = {
   parents: false,
   hash: undefined,
-  cidVersion: undefined
+  cidVersion: 0
 }
 
 module.exports = (ipfs) => {

src/core/utils/load-node.js

@@ -3,21 +3,22 @@
 const waterfall = require('async/waterfall')
 const CID = require('cids')
 const log = require('debug')('ipfs:mfs:utils:load-node')
-const bs58 = require('bs58')
 
-const loadNode = (ipfs, cid, callback) => {
-  const multihash = cid && (cid.multihash || cid.hash)
+const loadNode = (ipfs, object, callback) => {
+  const multihash = object && (object.multihash || object.hash)
 
   if (!multihash) {
     log(`No multihash passed so cannot load DAGNode`)
 
     return callback()
   }
 
-  log(`Loading DAGNode for child ${bs58.encode(multihash)}`)
+  const cid = new CID(multihash)
+
+  log(`Loading DAGNode for child ${cid.toBaseEncodedString()}`)
 
   waterfall([
-    (cb) => ipfs.dag.get(new CID(multihash), cb),
+    (cb) => ipfs.dag.get(cid, cb),
     (result, cb) => cb(null, result.value)
   ], callback)
 }

src/core/utils/traverse-to.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const bs58 = require('bs58')
 const CID = require('cids')
 const log = require('debug')('ipfs:mfs:utils:traverse-to')
 const UnixFS = require('ipfs-unixfs')
@@ -77,8 +76,6 @@ const traverseToMfsObject = (ipfs, path, options, callback) => {
             node: rootNode,
             parent: null
           }, (parent, {pathSegment, index}, done) => {
-            log(`Looking for ${pathSegment} in ${parent.name} ${bs58.encode(parent.node.multihash)}`)
-
             const existingLink = parent.node.links.find(link => link.name === pathSegment)
 
             if (!existingLink) {
@@ -114,6 +111,7 @@ const traverseToMfsObject = (ipfs, path, options, callback) => {
                     next(error, {
                       name: pathSegment,
                       node: emptyDirectory,
+                      cid: new CID(emptyDirectory.multihash),
                       parent: parent
                     })
                   })
@@ -126,43 +124,31 @@ const traverseToMfsObject = (ipfs, path, options, callback) => {
             }
 
             let hash = existingLink.hash || existingLink.multihash
+            const cid = new CID(hash)
 
             // child existed, fetch it
-            ipfs.dag.get(new CID(hash), (error, result) => {
-              log(`Loaded ${bs58.encode(result.value.multihash)} from ${bs58.encode(hash)}`)
+            ipfs.dag.get(cid, (error, result) => {
+              if (error) {
+                return done(error)
+              }
+
+              const node = result.value
+
               const child = {
                 name: pathSegment,
-                node: result && result.value,
+                node,
                 parent: parent
               }
 
               trail.push(child)
 
-              done(error, child)
+              done(null, child)
             })
           }, cb)
         }
       ], done)
     }
-  ], (error, trail) => {
-    if (!error) {
-      let node = trail
-      let path = []
-
-      while (node) {
-        path.push(`${node.name} ${bs58.encode(node.node.multihash)}`)
-        node = node.parent
-      }
-
-      log('Path:')
-
-      path
-        .reverse()
-        .forEach((segment) => log(segment))
-    }
-
-    callback(error, trail)
-  })
+  ], callback)
 }
 
 module.exports = traverseTo

src/core/utils/update-mfs-root.js

@@ -1,8 +1,8 @@
 'use strict'
 
-const bs58 = require('bs58')
 const log = require('debug')('ipfs:mfs:utils:update-mfs:root')
 const waterfall = require('async/waterfall')
+const CID = require('cids')
 const {
   MFS_ROOT_KEY
 } = require('./constants')
@@ -15,11 +15,9 @@ const updateMfsRoot = (ipfs, buffer, callback) => {
     return callback(new Error('Please run jsipfs init first'))
   }
 
-  if (typeof buffer === 'string' || buffer instanceof String) {
-    buffer = bs58.decode(buffer)
-  }
+  const cid = new CID(buffer)
 
-  log(`New MFS root will be ${bs58.encode(buffer)}`)
+  log(`New MFS root will be ${cid.toBaseEncodedString()}`)
 
   waterfall([
     (cb) => {
@@ -31,8 +29,8 @@ const updateMfsRoot = (ipfs, buffer, callback) => {
       log('Datastore was already open')
       cb()
     },
-    (cb) => datastore.put(MFS_ROOT_KEY, buffer, (error) => cb(error))
-  ], (error) => callback(error, buffer))
+    (cb) => datastore.put(MFS_ROOT_KEY, cid.buffer, (error) => cb(error))
+  ], (error) => callback(error, cid))
 }
 
 module.exports = updateMfsRoot

src/core/utils/with-mfs-root.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const bs58 = require('bs58')
 const CID = require('cids')
 const log = require('debug')('ipfs:mfs:utils:with-mfs-root')
 const waterfall = require('async/waterfall')
@@ -32,21 +31,21 @@ const withMfsRoot = (ipfs, callback) => {
               path: '/'
             }, next),
             // Turn the hash into a Buffer
-            ([{hash}], next) => next(null, bs58.decode(hash)),
-            (buffer, next) => repo.closed ? datastore.open((error) => next(error, buffer)) : next(null, buffer),
+            ([{hash}], next) => next(null, new CID(hash)),
+            (cid, next) => repo.closed ? datastore.open((error) => next(error, cid)) : next(null, cid),
             // Store the Buffer in the datastore
-            (buffer, next) => datastore.put(MFS_ROOT_KEY, buffer, (error) => next(error, buffer))
+            (cid, next) => datastore.put(MFS_ROOT_KEY, cid.buffer, (error) => next(error, cid))
           ], cb)
         }
 
-        cb(error, result)
+        cb(error, new CID(result))
       })
     },
     // Turn the Buffer into a CID
-    (hash, cb) => {
-      log(`Fetched MFS root ${bs58.encode(hash)}`)
+    (cid, cb) => {
+      log(`Fetched MFS root ${cid.toBaseEncodedString()}`)
 
-      cb(null, new CID(hash))
+      cb(null, cid)
     }
     // Invoke the API function with the root CID
   ], callback)

src/core/write/import-node.js

@@ -5,8 +5,6 @@ const pull = require('pull-stream/pull')
 const values = require('pull-stream/sources/values')
 const collect = require('pull-stream/sinks/collect')
 const importer = require('ipfs-unixfs-engine').importer
-const bs58 = require('bs58')
-const log = require('debug')('ipfs:mfs:import-node')
 const {
   loadNode
 } = require('../utils')
@@ -19,19 +17,16 @@ const importStream = (ipfs, source, options, callback) => {
       }]),
       importer(ipfs._ipld, {
         progress: options.progress,
-        hashAlg: options.hash,
+        hashAlg: options.hashAlg,
         cidVersion: options.cidVersion,
         strategy: options.strategy,
-        rawLeafNodes: options.rawLeafNodes,
-        reduceSingleLeafToSelf: options.reduceSingleLeafToSelf
+        rawLeaves: options.rawLeaves,
+        reduceSingleLeafToSelf: options.reduceSingleLeafToSelf,
+        leafType: options.leafType
       }),
       collect(cb)
     ),
-    (results, cb) => {
-      log(`Imported file ${bs58.encode(results[0].multihash)}`)
-
-      return loadNode(ipfs, results[0], cb)
-    }
+    (results, cb) => loadNode(ipfs, results[0], cb)
   ], callback)
 }
 

src/core/write/index.js

@@ -14,37 +14,31 @@ const {
 } = require('../utils')
 const values = require('pull-stream/sources/values')
 const log = require('debug')('ipfs:mfs:write')
-const bs58 = require('bs58')
 const importNode = require('./import-node')
 const updateNode = require('./update-node')
 const toPull = require('stream-to-pull-stream')
 const isStream = require('is-stream')
-const isNode = require('detect-node')
 const fileReaderStream = require('filereader-stream')
 const isPullStream = require('is-pull-stream')
 const cat = require('pull-cat')
 const pull = require('pull-stream/pull')
-
-let fs
-
-if (isNode) {
-  fs = require('fs')
-}
+const fs = require('fs')
 
 const defaultOptions = {
   offset: 0, // the offset in the file to begin writing
   length: undefined, // how many bytes from the incoming buffer to write
   create: false, // whether to create the file if it does not exist
   truncate: false, // whether to truncate the file first
-  rawLeafNodes: true,
+  rawLeaves: false,
   reduceSingleLeafToSelf: false,
-  cidVersion: undefined,
+  cidVersion: 0,
   hashAlg: 'sha2-256',
   format: 'dag-pb',
   parents: false, // whether to create intermediate directories if they do not exist
   progress: undefined,
   strategy: 'trickle',
-  flush: true
+  flush: true,
+  leafType: 'raw'
 }
 
 const toPullSource = (content, options, callback) => {
@@ -160,10 +154,11 @@ const updateOrImport = (ipfs, options, path, source, containingFolder, callback)
       }, null)
 
       if (existingChild) {
-        log('Updating linked DAGNode', bs58.encode(existingChild.multihash))
+        const cid = new CID(existingChild.multihash)
+        log(`Updating linked DAGNode ${cid.toBaseEncodedString()}`)
 
         // overwrite the existing file or part of it, possibly truncating what's left
-        updateNode(ipfs, new CID(existingChild.multihash), source, options, next)
+        updateNode(ipfs, cid, source, options, next)
       } else {
         if (!options.create) {
           return next(new Error('file does not exist'))
@@ -185,10 +180,7 @@ const updateOrImport = (ipfs, options, path, source, containingFolder, callback)
         )
 
         log('Importing file', path.name)
-        importNode(ipfs, source, options, (error, result) => {
-          log(`Imported file ${path.name} ${bs58.encode(result.multihash)}`)
-          next(error, result)
-        })
+        importNode(ipfs, source, options, next)
       }
     },
 
@@ -205,12 +197,6 @@ const updateOrImport = (ipfs, options, path, source, containingFolder, callback)
       // Store new containing folder CID
       containingFolder.node = newContaingFolder
 
-      log(`New CID for the containing folder is ${bs58.encode(newContaingFolder.multihash)}`)
-
-      newContaingFolder.links.forEach(link => {
-        log(`${link.name} ${bs58.encode(link.multihash)}`)
-      })
-
       next(error)
     }),
 

src/core/write/truncate-node.js

@@ -29,8 +29,9 @@ const truncateNode = (ipfs, dagNode, newLength, options, callback) => {
           hashAlg: options.hash,
           cidVersion: options.cidVersion,
           strategy: options.strategy,
-          rawLeafNodes: true,
-          reduceSingleLeafToSelf: false
+          rawLeaves: options.rawLeaves,
+          reduceSingleLeafToSelf: options.reduceSingleLeafToSelf,
+          leafType: options.leafType
         }),
         collect(cb)
       )