Merge branch 'master' into automatic-ci-script-update

ipfs · Feb 22, 2018 · 5ebea99 · 5ebea99
2 parents 51ef64b + 7f69628
commit 5ebea99
Show file tree

Hide file tree

Showing 39 changed files with 354 additions and 146 deletions.
diff --git a/examples/README.md b/examples/README.md
@@ -20,6 +20,7 @@ Let us know if you find any issue or if you want to contribute and add a new tut
 - [js-ipfs in the browser with WebPack](./browser-webpack)
 - [js-ipfs in the browser with a `<script>` tag](./browser-script-tag)
 - [js-ipfs in electron](./run-in-electron)
+- [Using streams to add a directory of files to ipfs](./browser-add-readable-stream)
 
 ## Understanding the IPFS Stack
 

diff --git a/examples/browser-add-readable-stream/README.md b/examples/browser-add-readable-stream/README.md
@@ -0,0 +1,7 @@
+# Using duplex streams to add files to IPFS in the browser
+
+If you have a number of files that you'd like to add to IPFS and end up with a hash representing the directory containing your files, you can invoke [`ipfs.files.add`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#add) with an array of objects.
+
+But what if you don't know how many there will be in advance?  You can add multiple files to a directory in IPFS over time by using [`ipfs.files.addReadableStream`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#addreadablestream).
+
+See `index.js` for a working example and open `index.html` in your browser to see it run.
diff --git a/examples/browser-add-readable-stream/index.html b/examples/browser-add-readable-stream/index.html
@@ -0,0 +1,7 @@
+<html>
+  <body>
+    <pre id="output"></pre>
+    <script src="https://unpkg.com/ipfs/dist/index.js"></script>
+    <script src="index.js"></script>
+  </body>
+</html>
diff --git a/examples/browser-add-readable-stream/index.js b/examples/browser-add-readable-stream/index.js
@@ -0,0 +1,75 @@
+'use strict'
+
+/* global Ipfs */
+/* eslint-env browser */
+
+const repoPath = 'ipfs-' + Math.random()
+const ipfs = new Ipfs({ repo: repoPath })
+
+ipfs.on('ready', () => {
+  const directory = 'directory'
+
+  // Our list of files
+  const files = createFiles(directory)
+
+  streamFiles(directory, files, (err, directoryHash) => {
+    if (err) {
+      return log(`There was an error adding the files ${err}`)
+    }
+
+    ipfs.ls(directoryHash, (err, files) => {
+      if (err) {
+        return log(`There was an error listing the files ${err}`)
+      }
+
+      log(`
+--
+
+Directory contents:
+
+${directory}/ ${directoryHash}`)
+
+      files.forEach((file, index) => {
+        log(` ${index < files.length - 1 ? '\u251C' : '\u2514'}\u2500 ${file.name} ${file.path} ${file.hash}`)
+      })
+    })
+  })
+})
+
+const createFiles = (directory) => {
+  return [{
+    path: `${directory}/file1.txt`,
+
+    // content could be a stream, a url etc
+    content: ipfs.types.Buffer.from('one', 'utf8')
+  }, {
+    path: `${directory}/file2.txt`,
+    content: ipfs.types.Buffer.from('two', 'utf8')
+  }, {
+    path: `${directory}/file3.txt`,
+    content: ipfs.types.Buffer.from('three', 'utf8')
+  }]
+}
+
+const streamFiles = (directory, files, cb) => {
+  // Create a stream to write files to
+  const stream = ipfs.files.addReadableStream()
+  stream.on('data', function (data) {
+    log(`Added ${data.path} hash: ${data.hash}`)
+
+    // The last data event will contain the directory hash
+    if (data.path === directory) {
+      cb(null, data.hash)
+    }
+  })
+
+  // Add the files one by one
+  files.forEach(file => stream.write(file))
+
+  // When we have no more files to add, close the stream
+  stream.end()
+}
+
+const log = (line) => {
+  document.getElementById('output').appendChild(document.createTextNode(`${line}\r\n`))
+}
diff --git a/package.json b/package.json
@@ -67,14 +67,14 @@
     "detect-node": "^2.0.3",
     "dir-compare": "^1.4.0",
     "dirty-chai": "^2.0.1",
-    "eslint-plugin-react": "^7.6.1",
+    "eslint-plugin-react": "^7.7.0",
     "execa": "^0.9.0",
     "expose-loader": "^0.7.4",
     "form-data": "^2.3.2",
     "go-ipfs-dep": "^0.4.13",
     "hat": "0.0.3",
     "interface-ipfs-core": "~0.52.0",
-    "ipfsd-ctl": "~0.28.0",
+    "ipfsd-ctl": "~0.29.0",
     "left-pad": "^1.2.0",
     "lodash": "^4.17.5",
     "mocha": "^5.0.1",
@@ -98,7 +98,7 @@
     "byteman": "^1.3.5",
     "cids": "~0.5.2",
     "debug": "^3.1.0",
-    "file-type": "^7.5.0",
+    "file-type": "^7.6.0",
     "filesize": "^3.6.0",
     "fsm-event": "^2.1.0",
     "get-folder-size": "^1.0.1",
@@ -107,7 +107,7 @@
     "hapi-set-header": "^1.0.2",
     "hoek": "^5.0.3",
     "human-to-milliseconds": "^1.0.0",
-    "ipfs-api": "^18.0.0",
+    "ipfs-api": "^18.1.1",
     "ipfs-bitswap": "~0.19.0",
     "ipfs-block": "~0.6.1",
     "ipfs-block-service": "~0.13.0",
@@ -119,7 +119,7 @@
     "is-ipfs": "^0.3.2",
     "is-stream": "^1.1.0",
     "joi": "^13.1.2",
-    "libp2p": "~0.17.0",
+    "libp2p": "~0.18.0",
     "libp2p-circuit": "~0.1.4",
     "libp2p-floodsub": "~0.14.1",
     "libp2p-kad-dht": "~0.8.0",
@@ -128,16 +128,16 @@
     "libp2p-multiplex": "~0.5.1",
     "libp2p-railing": "~0.7.1",
     "libp2p-secio": "~0.9.2",
-    "libp2p-tcp": "~0.11.5",
-    "libp2p-webrtc-star": "~0.13.3",
-    "libp2p-websocket-star": "~0.7.6",
-    "libp2p-websockets": "~0.10.4",
+    "libp2p-tcp": "~0.11.6",
+    "libp2p-webrtc-star": "~0.13.4",
+    "libp2p-websocket-star": "~0.7.7",
+    "libp2p-websockets": "~0.10.5",
     "lodash.flatmap": "^4.5.0",
     "lodash.get": "^4.4.2",
     "lodash.sortby": "^4.7.0",
     "lodash.values": "^4.3.0",
     "mafmt": "^4.0.0",
-    "mime-types": "^2.1.17",
+    "mime-types": "^2.1.18",
     "mkdirp": "~0.5.1",
     "multiaddr": "^3.0.2",
     "multihashes": "~0.4.13",
@@ -153,9 +153,9 @@
     "pull-file": "^1.1.0",
     "pull-ndjson": "^0.1.1",
     "pull-paramap": "^1.2.2",
-    "pull-pushable": "^2.1.2",
+    "pull-pushable": "^2.2.0",
     "pull-sort": "^1.0.1",
-    "pull-stream": "^3.6.1",
+    "pull-stream": "^3.6.2",
     "pull-stream-to-stream": "^1.3.4",
     "pull-zip": "^2.0.1",
     "read-pkg-up": "^3.0.0",

diff --git a/src/cli/commands/file/ls.js b/src/cli/commands/file/ls.js
@@ -11,6 +11,8 @@ module.exports = {
 
   handler (argv) {
     let path = argv.key
+    // `ipfs file ls` is deprecated. See https://ipfs.io/docs/commands/#ipfs-file-ls
+    print(`This functionality is deprecated, and will be removed in future versions. If possible, please use 'ipfs ls' instead.`)
     argv.ipfs.ls(path, (err, links) => {
       if (err) {
         throw err

diff --git a/src/cli/commands/files.js b/src/cli/commands/files.js
@@ -1,5 +1,8 @@
 'use strict'
 
+const print = require('../utils').print
+const lsCmd = require('./ls')
+
 module.exports = {
   command: 'files <command>',
 
@@ -8,9 +11,10 @@ module.exports = {
   builder (yargs) {
     return yargs
       .commandDir('files')
+      .command(lsCmd)
   },
 
   handler (argv) {
-    console.log('Type `jsipfs bitswap --help` for more instructions')
+    print('Type `jsipfs files --help` for more instructions')
   }
 }
diff --git a/src/cli/commands/ls.js b/src/cli/commands/ls.js
@@ -14,6 +14,12 @@ module.exports = {
       type: 'boolean',
       default: false
     },
+    r: {
+      alias: 'recursive',
+      desc: 'List subdirectories recursively',
+      type: 'boolean',
+      default: false
+    },
     'resolve-type': {
       desc: 'Resolve linked objects to find out their types. (not implemented yet)',
       type: 'boolean',
@@ -27,7 +33,7 @@ module.exports = {
       path = path.replace('/ipfs/', '')
     }
 
-    argv.ipfs.ls(path, (err, links) => {
+    argv.ipfs.ls(path, { recursive: argv.recursive }, (err, links) => {
       if (err) {
         throw err
       }
@@ -36,20 +42,17 @@ module.exports = {
         links = [{hash: 'Hash', size: 'Size', name: 'Name'}].concat(links)
       }
 
-      links = links.filter((link) => link.path !== path)
-      links.forEach((link) => {
-        if (link.type === 'dir') {
-          // directory: add trailing "/"
-          link.name = (link.name || '') + '/'
-        }
-      })
       const multihashWidth = Math.max.apply(null, links.map((file) => file.hash.length))
       const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length))
 
-      links.forEach((file) => {
-        utils.print(utils.rightpad(file.hash, multihashWidth + 1) +
-          utils.rightpad(file.size || '', sizeWidth + 1) +
-            file.name)
+      links.forEach(link => {
+        const fileName = link.type === 'dir' ? `${link.name || ''}/` : link.name
+        const padding = link.depth - path.split('/').length
+        utils.print(
+          utils.rightpad(link.hash, multihashWidth + 1) +
+          utils.rightpad(link.size || '', sizeWidth + 1) +
+          '  '.repeat(padding) + fileName
+        )
       })
     })
   }

diff --git a/src/core/components/files.js b/src/core/components/files.js
@@ -161,14 +161,20 @@ module.exports = function files (self) {
     return d
   }
 
-  function _lsPullStreamImmutable (ipfsPath) {
+  function _lsPullStreamImmutable (ipfsPath, options) {
     const path = normalizePath(ipfsPath)
-    const depth = path.split('/').length
+    const recursive = options && options.recursive
+    const pathDepth = path.split('/').length
+    const maxDepth = recursive ? global.Infinity : pathDepth
+
     return pull(
-      exporter(ipfsPath, self._ipldResolver, { maxDepth: depth }),
-      pull.filter((node) => node.depth === depth),
-      pull.map((node) => {
-        node = Object.assign({}, node, { hash: toB58String(node.hash) })
+      exporter(ipfsPath, self._ipldResolver, { maxDepth: maxDepth }),
+      pull.filter(node =>
+        recursive ? node.depth >= pathDepth : node.depth === pathDepth
+      ),
+      pull.map(node => {
+        const cid = new CID(node.hash)
+        node = Object.assign({}, node, { hash: cid.toBaseEncodedString() })
         delete node.content
         return node
       })
@@ -278,20 +284,26 @@ module.exports = function files (self) {
       return exporter(ipfsPath, self._ipldResolver)
     },
 
-    lsImmutable: promisify((ipfsPath, callback) => {
+    lsImmutable: promisify((ipfsPath, options, callback) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
       pull(
-        _lsPullStreamImmutable(ipfsPath),
+        _lsPullStreamImmutable(ipfsPath, options),
         pull.collect((err, values) => {
           if (err) {
-            return callback(err)
+            callback(err)
+            return
           }
           callback(null, values)
         })
       )
     }),
 
-    lsReadableStreamImmutable: (ipfsPath) => {
-      return toStream.source(_lsPullStreamImmutable(ipfsPath))
+    lsReadableStreamImmutable: (ipfsPath, options) => {
+      return toStream.source(_lsPullStreamImmutable(ipfsPath, options))
     },
 
     lsPullStreamImmutable: _lsPullStreamImmutable

diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js
@@ -272,13 +272,14 @@ exports.immutableLs = {
   handler: (request, reply) => {
     const key = request.pre.args.key
     const ipfs = request.server.app.ipfs
+    const recursive = request.query && request.query.recursive === 'true'
 
-    ipfs.ls(key, (err, files) => {
+    ipfs.ls(key, { recursive: recursive }, (err, files) => {
       if (err) {
-        reply({
+        return reply({
           Message: 'Failed to list dir: ' + err.message,
           Code: 0
-        }).code(500)
+        }).code(500).takeover()
       }
 
       reply({
@@ -288,7 +289,8 @@ exports.immutableLs = {
             Name: file.name,
             Hash: file.hash,
             Size: file.size,
-            Type: toTypeCode(file.type)
+            Type: toTypeCode(file.type),
+            Depth: file.depth
           }))
         }]
       })

diff --git a/test/cli/file.js b/test/cli/file.js
@@ -17,12 +17,16 @@ describe('file ls', () => runOnAndOff((thing) => {
 
   it('prints a filename', () => {
     return ipfs(`file ls ${file}`)
-      .then((out) => expect(out).to.eql(`${file}\n`))
+      .then((out) => expect(out).to.eql(
+        `This functionality is deprecated, and will be removed in future versions. If possible, please use 'ipfs ls' instead.\n` +
+        `${file}\n`
+      ))
   })
 
   it('prints the filenames in a directory', () => {
     return ipfs(`file ls ${dir}`)
       .then((out) => expect(out).to.eql(
+        `This functionality is deprecated, and will be removed in future versions. If possible, please use 'ipfs ls' instead.\n` +
         'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9\n' +
         'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN\n' +
         'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz\n' +