feat: pass file name to add/addAll progress handler (#3372)

Since ipfs/js-ipfs-unixfs#87 landed we can now pass the file name to
the progress handler for adding files:

```js
await ipfs.addAll(..., {
  progress: (bytes, fileName) => {
    //...
  }
})
```

This should make showing progress a bit more usable.

Co-authored-by: Hugo Dias <hugomrdias@gmail.com>

docs/core-api/FILES.md

@@ -252,7 +252,7 @@ An optional object which may have the following keys:
 | hashAlg | `String` | `'sha2-256'` | multihash hashing algorithm to use |
 | onlyHash | `boolean` | `false` | If true, will not add blocks to the blockstore |
 | pin | `boolean` | `true` | pin this object when adding |
-| progress | function | `undefined` | a function that will be called with the byte length of chunks as a file is added to ipfs |
+| progress | function | `undefined` | a function that will be called with the number of bytes added as a file is added to ipfs and the name of the file being added |
 | rawLeaves | `boolean` | `false` | if true, DAG leaves will contain raw file data and not be wrapped in a protobuf |
 | shardSplitThreshold | `Number` | `1000` | Directories with more than this number of files will be created as HAMT-sharded directories |
 | trickle | `boolean` | `false` | if true will use the [trickle DAG](https://godoc.org/github.com/ipsn/go-ipfs/gxlibs/github.com/ipfs/go-unixfs/importer/trickle) format for DAG generation |

packages/interface-ipfs-core/src/add-all.js

@@ -170,6 +170,26 @@ module.exports = (common, options) => {
       expect(root.cid.toString()).to.equal(fixtures.directory.cid)
     })
 
+    it('should receive file name from progress event', async () => {
+      const receivedNames = []
+      function handler (p, name) {
+        receivedNames.push(name)
+      }
+
+      await drain(ipfs.addAll([{
+        content: 'hello',
+        path: 'foo.txt'
+      }, {
+        content: 'world',
+        path: 'bar.txt'
+      }], {
+        progress: handler,
+        wrapWithDirectory: true
+      }))
+
+      expect(receivedNames).to.deep.equal(['foo.txt', 'bar.txt'])
+    })
+
     it('should add files to a directory non sequentially', async function () {
       const content = path => ({
         path: `test-dir/${path}`,

packages/interface-ipfs-core/src/add.js

@@ -130,6 +130,20 @@ module.exports = (common, options) => {
       expect(accumProgress).to.equal(fixtures.emptyFile.data.length)
     })
 
+    it('should receive file name from progress event', async () => {
+      let receivedName
+      function handler (p, name) {
+        receivedName = name
+      }
+
+      await ipfs.add({
+        content: 'hello',
+        path: 'foo.txt'
+      }, { progress: handler })
+
+      expect(receivedName).to.equal('foo.txt')
+    })
+
     it('should add an empty file without progress enabled', async () => {
       const file = await ipfs.add(fixtures.emptyFile.data)
 

packages/ipfs-core/src/components/add-all/index.js

@@ -47,9 +47,9 @@ module.exports = ({ block, gcLock, preload, pin, options: constructorOptions })
       let total = 0
       const prog = opts.progress
 
-      opts.progress = (bytes) => {
+      opts.progress = (bytes, fileName) => {
         total += bytes
-        prog(total)
+        prog(total, fileName)
       }
     }
 
@@ -162,8 +162,8 @@ function pinFile (pin, opts) {
  * @property {boolean} [onlyHash=false] - If true, will not add blocks to the
  * blockstore.
  * @property {boolean} [pin=true] - Pin this object when adding.
- * @property {(bytes:number) => void} [progress] - A function that will be
- * called with the byte length of chunks as a file is added to ipfs.
+ * @property {(bytes:number, fileName:string) => void} [progress] - A function that will be
+ * called with the number of bytes added as a file is added to ipfs and the name of the file being added.
  * @property {boolean} [rawLeaves=false] - If true, DAG leaves will contain raw
  * file data and not be wrapped in a protobuf.
  * @property {number} [shardSplitThreshold=1000] - Directories with more than this

packages/ipfs-http-client/src/add-all.js

@@ -38,7 +38,7 @@ module.exports = configure((api) => {
       if (file.hash !== undefined) {
         yield toCoreInterface(file)
       } else if (progressFn) {
-        progressFn(file.bytes || 0)
+        progressFn(file.bytes || 0, file.name)
       }
     }
   }

packages/ipfs-http-client/src/index.js

@@ -68,7 +68,7 @@ module.exports = ipfsClient
  * derives API from it's return type and extends it last `options` parameter
  * with `HttpOptions`.
  *
- * This can be used to avoid (re)typing API interface when implemeting it in
+ * This can be used to avoid (re)typing API interface when implementing it in
  * http client e.g you can annotate `ipfs.addAll` implementation with
  *
  * `@type {Implements<typeof import('ipfs-core/src/components/add-all')>}`
@@ -83,7 +83,7 @@ module.exports = ipfsClient
 /**
  * @template Key
  * @template {(config:any) => any} APIFactory
- * @typedef {import('./interface').APIMethadWithExtraOptions<ReturnType<APIFactory>, Key, HttpOptions>} ImplementsMethod
+ * @typedef {import('./interface').APIMethodWithExtraOptions<ReturnType<APIFactory>, Key, HttpOptions>} ImplementsMethod
  */
 
 /**

packages/ipfs-http-client/src/interface.ts

@@ -1,5 +1,5 @@
 // This file contains some utility types that either can't be expressed in
-// JSDoc syntax or that result in a different behavior when typed in JSDoc.
+// JSDoc syntax or that result in a different behaviour when typed in JSDoc.
 
 /**
  * Utility type that takes IPFS Core API function type (with 0 to 4 arguments
@@ -51,7 +51,7 @@ type WithExtendedOptions<Params, Ext> = Params extends [...End]
   ? [a1?: A1, a2?: A2, a3?: A3, options?: Options & Ext]
   : never
 
-export type APIMethadWithExtraOptions <
+export type APIMethodWithExtraOptions <
   API,
   Key extends keyof API,
   Extra

packages/ipfs-message-port-client/package.json

@@ -50,6 +50,7 @@
     "cross-env": "^7.0.0",
     "interface-ipfs-core": "^0.141.0",
     "ipfs": "^0.51.0",
+    "ipfs-core": "^0.1.0",
     "ipfs-message-port-protocol": "^0.3.0",
     "ipfs-message-port-server": "^0.3.0",
     "ipld-dag-pb": "^0.20.0",

packages/ipfs-message-port-client/src/core.js

@@ -73,30 +73,7 @@ class CoreClient extends Client {
    * `transfer: [input.buffer]` which would allow transferring it instead of
    * copying.
    *
-   * @param {AddAllInput} input
-   * @param {Object} [options]
-   * @param {string} [options.chunker="size-262144"]
-   * @param {number} [options.cidVersion=0]
-   * @param {boolean} [options.enableShardingExperiment]
-   * @param {string} [options.hashAlg="sha2-256"]
-   * @param {boolean} [options.onlyHash=false]
-   * @param {boolean} [options.pin=true]
-   * @param {function(number):void} [options.progress]
-   * @param {boolean} [options.rawLeaves=false]
-   * @param {number} [options.shardSplitThreshold=1000]
-   * @param {boolean} [options.trickle=false]
-   * @param {boolean} [options.wrapWithDirectory=false]
-   * @param {number} [options.timeout]
-   * @param {Transferable[]} [options.transfer]
-   * @param {AbortSignal} [options.signal]
-   * @returns {AsyncIterable<AddedData>}
-   *
-   * @typedef {Object} AddedData
-   * @property {string} path
-   * @property {CID} cid
-   * @property {number} mode
-   * @property {number} size
-   * @property {Time} mtime
+   * @type {import('.').Implements<typeof import('ipfs-core/src/components/add-all')>}
    */
   async * addAll (input, options = {}) {
     const { timeout, signal } = options
@@ -123,23 +100,7 @@ class CoreClient extends Client {
    * `transfer: [input.buffer]` which would allow transferring it instead of
    * copying.
    *
-   * @param {AddInput} input
-   * @param {Object} [options]
-   * @param {string} [options.chunker="size-262144"]
-   * @param {number} [options.cidVersion=0]
-   * @param {boolean} [options.enableShardingExperiment]
-   * @param {string} [options.hashAlg="sha2-256"]
-   * @param {boolean} [options.onlyHash=false]
-   * @param {boolean} [options.pin=true]
-   * @param {function(number):void} [options.progress]
-   * @param {boolean} [options.rawLeaves=false]
-   * @param {number} [options.shardSplitThreshold=1000]
-   * @param {boolean} [options.trickle=false]
-   * @param {boolean} [options.wrapWithDirectory=false]
-   * @param {number} [options.timeout]
-   * @param {Transferable[]} [options.transfer]
-   * @param {AbortSignal} [options.signal]
-   * @returns {Promise<AddedData>}
+   * @type {import('.').Implements<typeof import('ipfs-core/src/components/add')>}
    */
   async add (input, options = {}) {
     const { timeout, signal } = options
@@ -200,7 +161,7 @@ class CoreClient extends Client {
  * Decodes values yield by `ipfs.add`.
  *
  * @param {AddedEntry} data
- * @returns {AddedData}
+ * @returns {import('ipfs-core/src/components/add-all').UnixFSEntry}
  */
 const decodeAddedData = ({ path, cid, mode, mtime, size }) => {
   return {

packages/ipfs-message-port-client/src/index.js

@@ -62,3 +62,39 @@ class IPFSClient extends CoreClient {
 }
 
 module.exports = IPFSClient
+
+/**
+ * @typedef {Object} MessagePortOptions
+ * @property {Array} [transfer] - A list of ArrayBuffers whose ownership will be transferred to the shared worker
+ *
+ * @typedef {import('ipfs-core/src/utils').AbortOptions} AbortOptions}
+ */
+
+/**
+ * This is an utility type that can be used to derive type of the HTTP Client
+ * API from the Core API. It takes type of the API factory (from ipfs-core),
+ * derives API from it's return type and extends it last `options` parameter
+ * with `HttpOptions`.
+ *
+ * This can be used to avoid (re)typing API interface when implementing it in
+ * http client e.g you can annotate `ipfs.addAll` implementation with
+ *
+ * `@type {Implements<typeof import('ipfs-core/src/components/add-all')>}`
+ *
+ * **Caution**: This supports APIs with up to four parameters and last optional
+ * `options` parameter, anything else will result to `never` type.
+ *
+ * @template {(config:any) => any} APIFactory
+ * @typedef {APIWithExtraOptions<ReturnType<APIFactory>, MessagePortOptions>} Implements
+ */
+
+/**
+ * @template Key
+ * @template {(config:any) => any} APIFactory
+ * @typedef {import('./interface').APIMethodWithExtraOptions<ReturnType<APIFactory>, Key, MessagePortOptions>} ImplementsMethod
+ */
+
+/**
+ * @template API, Extra
+ * @typedef {import('./interface').APIWithExtraOptions<API, Extra>} APIWithExtraOptions
+ */

packages/ipfs-message-port-client/src/interface.ts

@@ -0,0 +1,58 @@
+// This file contains some utility types that either can't be expressed in
+// JSDoc syntax or that result in a different behaviour when typed in JSDoc.
+
+/**
+ * Utility type that takes IPFS Core API function type (with 0 to 4 arguments
+ * & last **optional** `options` parameter) and derives a function type with
+ * `options` parameter extended with given `Extra` options.
+ *
+ * **Caution**: API Functions with more than for arguments ahead of `options`
+ * will result to `never` type. API function that does not take `options` will
+ * result in function whose last argument is extended with `Extra` which would
+ * be an error.
+ */
+// This is typed in TS file because otherwise TS unifies on the first parameter
+// regardless of number of parameters function has.
+export type APIWithExtraOptions<API extends (...args: any[]) => any, Extra> =
+  (...args: WithExtendedOptions<Parameters<API>, Extra>) => ReturnType<API>
+
+type End = never[]
+type WithExtendedOptions<Params, Ext> = Params extends [...End]
+  ? []
+  // (options?: Options) -> (options?: Options & Ext)
+  : Params extends [options?: infer Options, ...end: End]
+  ? [options?: Options & Ext]
+  // (a: A1, options?: Options) -> (a1: A1, options?: Options & Ext)
+  : Params extends [a1: infer A1, options?: infer Options, ...end: End]
+  ? [a1: A1, options?: Options & Ext]
+  // (a1?: A1, options?: Options) -> (a1?: A1, options?: Options & Ext)
+  : Params extends [a1?: infer A1, options?: infer Options, ...end: End]
+  ? [a1?: A1, options?: Options & Ext]
+  // (a1: A1, a2: A2, options?: Options) -> (a1: A1, a2: A2 options?: Options & Ext)
+  : Params extends [a1: infer A1, a2: infer A2, options?: infer Options, ...end: End]
+  ? [a1: A1, a2: A2, options?: Options & Ext]
+  // (a1: A1, a2?: A2, options?: Options) -> (a1: A1, a2?: A2 options?: Options & Ext)
+  : Params extends [a1: infer A1, a2?: infer A2, options?: infer Options, ...end: End]
+  ? [a1: A1, a2?: A2, options?: Options & Ext]
+  // (a1: A1, a2?: A2, options?: Options) -> (a1: A1, a2?: A2 options?: Options & Ext)
+  : Params extends [a1?: infer A1, a2?: infer A2, options?: infer Options, ...end: End]
+  ? [a1?: A1, a2?: A2, options?: Options & Ext]
+  // (a1: A1, a2: A2, a3:A3 options?: Options) -> (a1: A1, a2: A2, a3:A3, options?: Options & Ext)
+  : Params extends [a1: infer A1, a2: infer A2, a3:infer A3, options?: infer Options, ...end: End]
+  ? [a1: A1, a2: A2, a3: A3, options?: Options & Ext]
+  // (a1: A1, a2: A2, a3?:A3 options?: Options) -> (a1: A1, a2: A2, a3?:A3, options?: Options & Ext)
+  : Params extends [a1: infer A1, a2:infer A2, a3?: infer A3, options?: infer Options, ...end: End]
+  ? [a1: A1, a2: A2, a3?: A3, options?: Options & Ext]
+  // (a1: A1, a2?: A2, a3?:A3 options?: Options) -> (a1: A1, a2?: A2, a3?:A3, options?: Options & Ext)
+  : Params extends [a1: infer A1, a2?: infer A2, a3?: infer A3, options?: infer Options, ...end: End]
+  ? [a1: A1, a2?: A2, a3?: A3, options?: Options & Ext]
+  // (a1?: A1, a2?: A2, a3?:A3 options?: Options) -> (a1?: A1, a2?: A2, a3?:A3, options?: Options & Ext)
+  : Params extends [a1?: infer A1, a2?: infer A2, a3?: infer A3, options?: infer Options, ...end: End]
+  ? [a1?: A1, a2?: A2, a3?: A3, options?: Options & Ext]
+  : never
+
+export type APIMethodWithExtraOptions <
+  API,
+  Key extends keyof API,
+  Extra
+  > = API[Key] extends (...args: any[]) => any ? APIWithExtraOptions<API[Key], Extra> : never

packages/ipfs-message-port-client/tsconfig.json

@@ -13,6 +13,12 @@
     },
     {
       "path": "../ipfs-message-port-server"
+    },
+    {
+      "path": "../ipfs-core"
+    },
+    {
+      "path": "../ipfs-core-utils"
     }
   ]
 }

packages/ipfs-message-port-protocol/src/core.js

@@ -11,7 +11,6 @@ const { encodeError, decodeError } = require('./error')
  */
 
 /**
- * @template T
  * @typedef {Object} RemoteCallback
  * @property {'RemoteCallback'} type
  * @property {MessagePort} port
@@ -173,33 +172,32 @@ const toIterator = iterable => {
 }
 
 /**
- * @template T
- * @param {function(T):void} callback
+ * @param {Function} callback
  * @param {Transferable[]} transfer
- * @returns {RemoteCallback<T>}
+ * @returns {RemoteCallback}
  */
 const encodeCallback = (callback, transfer) => {
   // eslint-disable-next-line no-undef
   const { port1: port, port2: remote } = new MessageChannel()
-  port.onmessage = ({ data }) => callback(data)
+  port.onmessage = ({ data }) => callback.apply(null, data)
   transfer.push(remote)
   return { type: 'RemoteCallback', port: remote }
 }
 exports.encodeCallback = encodeCallback
 
 /**
  * @template T
- * @param {RemoteCallback<T>} remote
- * @returns {function(T):void | function(T, Transferable[]):void}
+ * @param {RemoteCallback} remote
+ * @returns {function(T[]):void | function(T[], Transferable[]):void}
  */
 const decodeCallback = ({ port }) => {
   /**
-   * @param {T} value
+   * @param {T[]} args
    * @param {Transferable[]} [transfer]
    * @returns {void}
    */
-  const callback = (value, transfer = []) => {
-    port.postMessage(value, transfer)
+  const callback = (args, transfer = []) => {
+    port.postMessage(args, transfer)
   }
 
   return callback

packages/ipfs-message-port-protocol/test/core.browser.js

@@ -32,10 +32,10 @@ describe('core', function () {
         await move(encodeCallback(callback, transfer), transfer)
       )
 
-      remote(54)
+      remote([54])
       expect(await receive()).to.be.equal(54)
 
-      remote({ hello: 'world' })
+      remote([{ hello: 'world' }])
 
       expect(await receive()).to.be.deep.equal({ hello: 'world' })
     })
@@ -55,11 +55,11 @@ describe('core', function () {
         await move(encodeCallback(callback, transfer), transfer)
       )
 
-      remote({ hello: uint8ArrayFromString('world') })
+      remote([{ hello: uint8ArrayFromString('world') }])
       expect(await receive()).to.be.deep.equal({ hello: uint8ArrayFromString('world') })
 
       const world = uint8ArrayFromString('world')
-      remote({ hello: world }, [world.buffer])
+      remote([{ hello: world }], [world.buffer])
 
       expect(await receive()).to.be.deep.equal({ hello: uint8ArrayFromString('world') })
       expect(world.buffer).property('byteLength', 0, 'buffer was cleared')