feat: Add --cid-version option to ipfs files add + decodeURIComponent for file and directory names

Author: daviddias

package.json

@@ -73,8 +73,8 @@
     "expose-loader": "^0.7.3",
     "form-data": "^2.3.1",
     "gulp": "^3.9.1",
-    "interface-ipfs-core": "~0.31.13",
-    "ipfsd-ctl": "~0.21.0",
+    "interface-ipfs-core": "~0.31.16",
+    "ipfsd-ctl": "~0.22.0",
     "left-pad": "^1.1.3",
     "lodash": "^4.17.4",
     "mocha": "^3.5.0",
@@ -99,15 +99,15 @@
     "hapi": "^16.5.2",
     "hapi-set-header": "^1.0.2",
     "hoek": "^4.2.0",
-    "ipfs-api": "^14.1.2",
+    "ipfs-api": "^14.2.1",
     "ipfs-bitswap": "~0.16.1",
     "ipfs-block": "~0.6.0",
     "ipfs-block-service": "~0.12.0",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.17.0",
     "ipfs-unixfs": "~0.1.12",
     "ipfs-unixfs-engine": "~0.22.0",
-    "ipld-resolver": "~0.13.0",
+    "ipld-resolver": "~0.13.1",
     "is-ipfs": "^0.3.0",
     "is-stream": "^1.1.0",
     "joi": "^10.6.0",
@@ -128,7 +128,7 @@
     "mafmt": "^2.1.8",
     "mkdirp": "~0.5.1",
     "multiaddr": "^2.3.0",
-    "multihashes": "~0.4.8",
+    "multihashes": "~0.4.9",
     "once": "^1.4.0",
     "path-exists": "^3.0.0",
     "peer-book": "~0.5.0",
@@ -208,4 +208,4 @@
     "tcme <hi@this-connect.me>",
     "ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ <victorbjelkholm@gmail.com>"
   ]
-}
+}

src/cli/commands/files/add.js

@@ -112,6 +112,15 @@ module.exports = {
     'shard-split-threshold': {
       type: 'integer',
       default: 1000
+    },
+    'raw-leaves': {
+      type: 'boolean',
+      default: undefined,
+      describe: 'Use raw blocks for leaf nodes. (experimental)'
+    },
+    'cid-version': {
+      type: 'integer',
+      describe: 'Cid version. Non-zero value will change default of \'raw-leaves\' to true. (experimental)'
     }
   },
 
@@ -120,7 +129,25 @@ module.exports = {
     const index = inPath.lastIndexOf('/') + 1
     const options = {
       strategy: argv.trickle ? 'trickle' : 'balanced',
-      shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity
+      shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity,
+      'cid-version': argv['cid-version'],
+      'raw-leaves': argv['raw-leaves']
+    }
+
+    // Temporary restriction on raw-leaves:
+    // When cid-version=1 then raw-leaves MUST be present and false.
+    //
+    // This is because raw-leaves is not yet implemented in js-ipfs,
+    // and go-ipfs changes the value of raw-leaves to true when
+    // cid-version > 0 unless explicitly set to false.
+    //
+    // This retains feature parity without having to implement raw-leaves.
+    if (argv['cid-version'] > 0 && argv['raw-leaves'] !== false) {
+      throw new Error('Implied argument raw-leaves must be passed and set to false when cid-version is > 0')
+    }
+
+    if (argv['raw-leaves']) {
+      throw new Error('Not implemented: raw-leaves')
     }
 
     if (argv.enableShardingExperiment && utils.isDaemonOn()) {

src/core/components/files.js

@@ -4,7 +4,6 @@ const unixfsEngine = require('ipfs-unixfs-engine')
 const importer = unixfsEngine.importer
 const exporter = unixfsEngine.exporter
 const promisify = require('promisify-es6')
-const multihashes = require('multihashes')
 const pull = require('pull-stream')
 const sort = require('pull-sort')
 const pushable = require('pull-pushable')
@@ -13,6 +12,7 @@ const toPull = require('stream-to-pull-stream')
 const waterfall = require('async/waterfall')
 const isStream = require('is-stream')
 const Duplex = require('stream').Duplex
+const CID = require('cids')
 
 module.exports = function files (self) {
   const createAddPullStream = (options) => {
@@ -24,7 +24,7 @@ module.exports = function files (self) {
       pull.map(normalizeContent),
       pull.flatten(),
       importer(self._ipldResolver, opts),
-      pull.asyncMap(prepareFile.bind(null, self))
+      pull.asyncMap(prepareFile.bind(null, self, opts))
     )
   }
 
@@ -68,7 +68,7 @@ module.exports = function files (self) {
       pull(
         pull.values(normalizeContent(data)),
         importer(self._ipldResolver, options),
-        pull.asyncMap(prepareFile.bind(null, self)),
+        pull.asyncMap(prepareFile.bind(null, self, options)),
         sort((a, b) => {
           if (a.path < b.path) return 1
           if (a.path > b.path) return -1
@@ -114,15 +114,23 @@ module.exports = function files (self) {
   }
 }
 
-function prepareFile (self, file, callback) {
-  const bs58mh = multihashes.toB58String(file.multihash)
+function prepareFile (self, opts, file, callback) {
+  opts = opts || {}
 
   waterfall([
     (cb) => self.object.get(file.multihash, cb),
     (node, cb) => {
+      let cid = new CID(node.multihash)
+
+      if (opts['cid-version'] === 1) {
+        cid = cid.toV1()
+      }
+
+      const b58Hash = cid.toBaseEncodedString()
+
       cb(null, {
-        path: file.path || bs58mh,
-        hash: bs58mh,
+        path: file.path || b58Hash,
+        hash: b58Hash,
         size: node.size
       })
     }

src/http-api/resources/files.js

@@ -11,6 +11,7 @@ const toPull = require('stream-to-pull-stream')
 const pushable = require('pull-pushable')
 const EOL = require('os').EOL
 const toStream = require('pull-stream-to-stream')
+const Joi = require('joi')
 
 exports = module.exports
 
@@ -140,6 +141,28 @@ exports.get = {
 }
 
 exports.add = {
+  validate: {
+    query: Joi.object()
+      .keys({
+        'cid-version': Joi.number().integer().min(0).max(1),
+        // Temporary restriction on raw-leaves:
+        // When cid-version=1 then raw-leaves MUST be present and false.
+        //
+        // This is because raw-leaves is not yet implemented in js-ipfs,
+        // and go-ipfs changes the value of raw-leaves to true when
+        // cid-version > 0 unless explicitly set to false.
+        //
+        // This retains feature parity without having to implement raw-leaves.
+        'raw-leaves': Joi.any().when('cid-version', {
+          is: 1,
+          then: Joi.boolean().valid(false).required(),
+          otherwise: Joi.boolean().valid(false)
+        })
+      })
+      // TODO: Necessary until validate "recursive", "stream-channels" etc.
+      .options({ allowUnknown: true })
+  },
+
   handler: (request, reply) => {
     if (!request.payload) {
       return reply({
@@ -156,6 +179,7 @@ exports.add = {
     const fileAdder = pushable()
 
     parser.on('file', (fileName, fileStream) => {
+      fileName = decodeURIComponent(fileName)
       const filePair = {
         path: fileName,
         content: toPull(fileStream)
@@ -165,6 +189,8 @@ exports.add = {
     })
 
     parser.on('directory', (directory) => {
+      directory = decodeURIComponent(directory)
+
       fileAdder.push({
         path: directory,
         content: ''
@@ -181,9 +207,14 @@ exports.add = {
       fileAdder.end()
     })
 
+    const options = {
+      'cid-version': request.query['cid-version'],
+      'raw-leaves': request.query['raw-leaves']
+    }
+
     pull(
       fileAdder,
-      ipfs.files.createAddPullStream(),
+      ipfs.files.createAddPullStream(options),
       pull.map((file) => {
         return {
           Name: file.path ? file.path : file.hash,

src/http-api/routes/files.js

@@ -38,7 +38,8 @@ module.exports = (server) => {
         parse: false,
         output: 'stream'
       },
-      handler: resources.files.add.handler
+      handler: resources.files.add.handler,
+      validate: resources.files.add.validate
     }
   })
 }

test/cli/files.js

@@ -136,6 +136,60 @@ describe('files', () => runOnAndOff((thing) => {
     })
   })
 
+  it('add with cid-version=0', () => {
+    return ipfs('add src/init-files/init-docs/readme --cid-version=0').then((out) => {
+      expect(out)
+        .to.eql('added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme\n')
+    })
+  })
+
+  // Temporarily expect to fail as raw-leaves not yet implemented.
+  //
+  // When cid-version=1 then raw-leaves MUST be present and false.
+  //
+  // This is because raw-leaves is not yet implemented in js-ipfs,
+  // and go-ipfs changes the value of raw-leaves to true when
+  // cid-version > 0 unless explicitly set to false.
+  //
+  // This retains feature parity without having to implement raw-leaves.
+  it('add with cid-version=1', () => {
+    return new Promise((resolve, reject) => {
+      ipfs('add src/init-files/init-docs/readme --cid-version=1')
+        .then(() => reject(new Error('Raw leaves not expected to be implemented')))
+        .catch((err) => {
+          expect(err).to.exist()
+          resolve()
+        })
+    })
+  })
+
+  it('add with cid-version=1 and raw-leaves=false', () => {
+    return ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=false').then((out) => {
+      expect(out)
+        .to.eql('added zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 readme\n')
+    })
+  })
+
+  // Temporarily expect to fail as raw-leaves not yet implemented
+  //
+  // When cid-version=1 then raw-leaves MUST be present and false.
+  //
+  // This is because raw-leaves is not yet implemented in js-ipfs,
+  // and go-ipfs changes the value of raw-leaves to true when
+  // cid-version > 0 unless explicitly set to false.
+  //
+  // This retains feature parity without having to implement raw-leaves.
+  it('add with cid-version=1 and raw-leaves=true', () => {
+    return new Promise((resolve, reject) => {
+      ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=true')
+        .then(() => reject(new Error('Raw leaves not expected to be implemented')))
+        .catch((err) => {
+          expect(err).to.exist()
+          resolve()
+        })
+    })
+  })
+
   it('cat', () => {
     return ipfs('files cat QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB')
       .then((out) => {

test/http-api/index.js

@@ -41,7 +41,7 @@ describe('HTTP API', () => {
       .forEach((file) => require('./spec/' + file)(http))
   })
 
-  describe('## interface tests', () => {
+  describe.only('## interface tests', () => {
     fs.readdirSync(path.join(__dirname, '/interface'))
       .forEach((file) => require('./interface/' + file))
   })