feat: support UnixFSv1.5 metadata (#2621)

* feat: support UnixFSv1.5 metadata

Adds metadata support.

Specifying metadata:

```console
$ jsipfs add --mode=0777 --mtime=100000 ./foo.txt
```

Respecting exsting metadata:

```console
$ jsipfs add --preserve-mode --preserve-mtime ./foo.txt
```

Displaying metadata:

```console
$ jsipfs ls Qmfoo
-rw-r--r-- Nov 21, 2019, 09:46:23 AM CST bar.txt Qbar 0
```

* chore: update dep paths and jsdocs

* chore: fix linting

* fix: expose metadata in http api

* test: interface tests passing

* chore: update package.json with branches

* fix: support mtime as timespec

* fix: check mtime formatting in timezone agnostic way

* fix: do not run hrtime tests in the browser

* chore: upgrade interop tests

* chore: symlink ipfs dir for interop tests

* fix: build ipfs before running examples

* fix: expose mtime-nsecs to cli and add tests

* fix: export path to js-ipfs binary

* fix: support optional mtime

* fix: linting

* fix: pin datastore-pubsub version due to ipfs/js-datastore-pubsub#20

* chore: restore datastore-pubsub semver

Author: achingbrain

.travis.yml

@@ -76,28 +76,44 @@ jobs:
     - stage: test
       name: interop - node
       script:
+        - mkdir -p node_modules/ipfs-interop/node_modules
+        - ln -s `pwd` node_modules/ipfs-interop/node_modules/ipfs
+        - export IPFS_JS_EXEC=`pwd`/src/cli/bin.js
+        - export IPFS_REUSEPORT=false
         - cd node_modules/ipfs-interop
-        - IPFS_JS_EXEC=./../../src/cli/bin.js IPFS_REUSEPORT=false npx aegir test -t node --bail
+        - npx aegir test -t node --bail
 
     - stage: test
       name: interop - browser
       script:
+        - mkdir -p node_modules/ipfs-interop/node_modules
+        - ln -s `pwd` node_modules/ipfs-interop/node_modules/ipfs
+        - export IPFS_JS_EXEC=`pwd`/src/cli/bin.js
+        - export IPFS_REUSEPORT=false
         - cd node_modules/ipfs-interop
-        - IPFS_JS_EXEC=./../../src/cli/bin.js IPFS_REUSEPORT=false npx aegir test -t browser --bail
+        - npx aegir test -t browser --bail
 
     - stage: test
       name: interop - electron-main
       os: osx
       script:
+        - mkdir -p node_modules/ipfs-interop/node_modules
+        - ln -s `pwd` node_modules/ipfs-interop/node_modules/ipfs
+        - export IPFS_JS_EXEC=`pwd`/src/cli/bin.js
+        - export IPFS_REUSEPORT=false
         - cd node_modules/ipfs-interop
-        - IPFS_JS_EXEC=./../../src/cli/bin.js IPFS_REUSEPORT=false npx aegir test -t electron-main -f ./test/node.js --bail --timeout 10000
+        - npx aegir test -t electron-main -f ./test/node.js --bail --timeout 10000
 
     - stage: test
       name: interop - electron-renderer
       os: osx
       script:
+        - mkdir -p node_modules/ipfs-interop/node_modules
+        - ln -s `pwd` node_modules/ipfs-interop/node_modules/ipfs
+        - export IPFS_JS_EXEC=`pwd`/src/cli/bin.js
+        - export IPFS_REUSEPORT=false
         - cd node_modules/ipfs-interop
-        - IPFS_JS_EXEC=./../../src/cli/bin.js IPFS_REUSEPORT=false npx aegir test -t electron-renderer -f ./test/browser.js --bail --timeout 10000
+        - npx aegir test -t electron-renderer -f ./test/browser.js --bail --timeout 10000
 
     - stage: test
       name: external - ipfs-companion
@@ -142,125 +158,143 @@ jobs:
     - stage: test
       name: example - browser-add-readable-stream
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- browser-add-readable-stream
 
     - stage: test
       name: example - browser-browserify
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- browser-browserify
 
     - stage: test
       name: example - browser-create-react-app
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- browser-create-react-app
 
     - stage: test
       name: example - browser-mfs
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- browser-mfs
 
     - stage: test
       name: example - browser-parceljs
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- browser-parceljs
 
     - stage: test
       name: example - browser-readablestream
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- browser-readablestream
 
     - stage: test
       name: example - browser-script-tag
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- browser-script-tag
 
     - stage: test
       name: example - browser-video-streaming
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- browser-video-streaming
 
     - stage: test
       name: example - browser-vue
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- browser-vue
 
     - stage: test
       name: example - browser-webpack
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- browser-webpack
 
     - stage: test
       name: example - circuit-relaying
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- circuit-relaying
 
     - stage: test
       name: example - custom-ipfs-repo
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- custom-ipfs-repo
 
     - stage: test
       name: example - custom-libp2p
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- custom-libp2p
 
     - stage: test
       name: example - exchange-files-in-browser
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- exchange-files-in-browser
 
     - stage: test
       name: example - explore-ethereum-blockchain
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- explore-ethereum-blockchain
 
     - stage: test
       name: example - ipfs-101
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- ipfs-101
 
     - stage: test
       name: example - running-multiple-nodes
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- running-multiple-nodes
 
     - stage: test
       name: example - traverse-ipld-graphs
       script:
+        - npm run build
         - cd examples
         - npm install
         - npm run test -- traverse-ipld-graphs

examples/running-multiple-nodes/test.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const IPFS = require('ipfs')
+const IPFS = require('../../')
 
 const execa = require('execa')
 const os = require('os')

package.json

@@ -81,7 +81,7 @@
     "class-is": "^1.1.0",
     "dag-cbor-links": "^1.3.2",
     "datastore-core": "~0.7.0",
-    "datastore-pubsub": "^0.2.1",
+    "datastore-pubsub": "^0.2.3",
     "debug": "^4.1.0",
     "dlv": "^1.1.3",
     "err-code": "^2.0.0",
@@ -95,18 +95,18 @@
     "hashlru": "^2.3.0",
     "human-to-milliseconds": "^2.0.0",
     "interface-datastore": "~0.8.0",
-    "ipfs-bitswap": "^0.26.0",
+    "ipfs-bitswap": "^0.26.2",
     "ipfs-block": "~0.8.1",
     "ipfs-block-service": "~0.16.0",
-    "ipfs-http-client": "^40.1.0",
+    "ipfs-http-client": "^40.2.0",
     "ipfs-http-response": "~0.4.0",
-    "ipfs-mfs": "^0.13.2",
-    "ipfs-multipart": "^0.2.0",
+    "ipfs-mfs": "^0.14.0",
+    "ipfs-multipart": "^0.3.0",
     "ipfs-repo": "^0.30.0",
-    "ipfs-unixfs": "~0.1.16",
-    "ipfs-unixfs-exporter": "^0.38.0",
-    "ipfs-unixfs-importer": "^0.42.0",
-    "ipfs-utils": "~0.4.0",
+    "ipfs-unixfs": "^0.3.0",
+    "ipfs-unixfs-exporter": "^0.40.0",
+    "ipfs-unixfs-importer": "^0.43.0",
+    "ipfs-utils": "^0.4.2",
     "ipld": "~0.25.0",
     "ipld-bitcoin": "~0.3.0",
     "ipld-dag-cbor": "~0.15.0",
@@ -137,7 +137,7 @@
     "libp2p-floodsub": "^0.18.0",
     "libp2p-gossipsub": "~0.0.5",
     "libp2p-kad-dht": "~0.16.0",
-    "libp2p-keychain": "^0.5.2",
+    "libp2p-keychain": "^0.5.4",
     "libp2p-mdns": "~0.12.0",
     "libp2p-record": "~0.7.0",
     "libp2p-secio": "~0.11.0",
@@ -154,7 +154,7 @@
     "multiaddr": "^6.1.1",
     "multiaddr-to-uri": "^5.0.0",
     "multibase": "~0.6.0",
-    "multicodec": "~0.5.5",
+    "multicodec": "^1.0.0",
     "multihashes": "~0.4.14",
     "multihashing-async": "^0.8.0",
     "node-fetch": "^2.3.0",
@@ -204,8 +204,8 @@
     "execa": "^3.0.0",
     "form-data": "^3.0.0",
     "hat": "0.0.3",
-    "interface-ipfs-core": "^0.125.0",
-    "ipfs-interop": "^0.1.1",
+    "interface-ipfs-core": "^0.126.0",
+    "ipfs-interop": "^0.2.0",
     "ipfsd-ctl": "^1.0.2",
     "libp2p-websocket-star": "~0.10.2",
     "lodash": "^4.17.15",

src/cli/commands/add.js

@@ -122,6 +122,50 @@ module.exports = {
       type: 'boolean',
       default: false,
       describe: 'Include files that are hidden. Only takes effect on recursive add.'
+    },
+    'preserve-mode': {
+      type: 'boolean',
+      default: false,
+      describe: 'Apply permissions to created UnixFS entries'
+    },
+    'preserve-mtime': {
+      type: 'boolean',
+      default: false,
+      describe: 'Apply modification time to created UnixFS entries'
+    },
+    mode: {
+      type: 'string',
+      describe: 'File mode to apply to created UnixFS entries'
+    },
+    mtime: {
+      type: 'number',
+      coerce: (value) => {
+        value = parseInt(value)
+
+        if (isNaN(value)) {
+          throw new Error('mtime must be a number')
+        }
+
+        return value
+      },
+      describe: 'Modification time in seconds before or since the Unix Epoch to apply to created UnixFS entries'
+    },
+    'mtime-nsecs': {
+      type: 'number',
+      coerce: (value) => {
+        value = parseInt(value)
+
+        if (isNaN(value)) {
+          throw new Error('mtime-nsecs must be a number')
+        }
+
+        if (value < 0 || value > 999999999) {
+          throw new Error('mtime-nsecs must be in the range [0,999999999]')
+        }
+
+        return value
+      },
+      describe: 'Modification time fraction in nanoseconds'
     }
   },
 
@@ -171,9 +215,28 @@ module.exports = {
         }
       }
 
+      let mtime
+
+      if (argv.mtime != null) {
+        mtime = {
+          secs: argv.mtime
+        }
+
+        if (argv.mtimeNsecs != null) {
+          mtime.nsecs = argv.mtimeNsecs
+        }
+      }
+
       const source = argv.file
-        ? globSource(argv.file, { recursive: argv.recursive, hidden: argv.hidden })
-        : process.stdin // Pipe directly to ipfs.add
+        ? globSource(argv.file, {
+          recursive: argv.recursive,
+          hidden: argv.hidden,
+          preserveMode: argv.preserveMode,
+          preserveMtime: argv.preserveMtime,
+          mode: argv.mode,
+          mtime
+        })
+        : argv.getStdin() // Pipe directly to ipfs.add
 
       let finalHash
 

src/cli/commands/block/put.js

@@ -44,7 +44,7 @@ module.exports = {
         data = await promisify(fs.readFile)(argv.block)
       } else {
         data = await new Promise((resolve, reject) => {
-          process.stdin.pipe(bl((err, input) => {
+          argv.getStdin().pipe(bl((err, input) => {
             if (err) return reject(err)
             resolve(input)
           }))

src/cli/commands/daemon.js

@@ -48,6 +48,7 @@ module.exports = {
   handler (argv) {
     argv.resolve((async () => {
       const { print } = argv
+
       print('Initializing IPFS daemon...')
       print(`js-ipfs version: ${require('../../../package.json').version}`)
       print(`System version: ${os.arch()}/${os.platform()}`)

src/cli/commands/dag/put.js

@@ -79,7 +79,7 @@ module.exports = {
     }
   },
 
-  handler ({ data, format, inputEncoding, pin, hashAlg, cidVersion, cidBase, preload, onlyHash, getIpfs, print, resolve }) {
+  handler ({ data, format, inputEncoding, pin, hashAlg, cidVersion, cidBase, preload, onlyHash, getIpfs, print, resolve, getStdin }) {
     resolve((async () => {
       const ipfs = await getIpfs()
 
@@ -101,7 +101,7 @@ module.exports = {
         // pipe from stdin
         source = Buffer.alloc(0)
 
-        for await (const buf of process.stdin) {
+        for await (const buf of getStdin()) {
           source = Buffer.concat([source, buf])
         }
       } else {