This repository has been archived by the owner on Feb 12, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: report ipfs.add progress over http (#3310)
The browser fetch api doesn't allow reading of any data until the whole request has been sent which means progress events only fire after the upload is complete which rather defeats the purpose of reporting upload progress. Here we switch to XHR for uploads with progress that does allow reading response data before the request is complete. Co-authored-by: achingbrain <alex@achingbrain.net>
- Loading branch information
1 parent
bba650d
commit 39cad4b
Showing
13 changed files
with
163 additions
and
95 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,87 +1,10 @@ | ||
| 'use strict' | ||
|
|
||
| const normaliseInput = require('ipfs-core-utils/src/files/normalise-input/index') | ||
| const { nanoid } = require('nanoid') | ||
| const modeToString = require('../lib/mode-to-string') | ||
| const mtimeToObject = require('../lib/mtime-to-object') | ||
| const merge = require('merge-options').bind({ ignoreUndefined: true }) | ||
| const toStream = require('it-to-stream') | ||
| const { isElectronRenderer } = require('ipfs-utils/src/env') | ||
|
|
||
| /** | ||
| * | ||
| * @param {Object} source | ||
| * @param {AbortController} abortController | ||
| * @param {Headers|Record<string, string>} [headers] | ||
| * @param {string} [boundary] | ||
| */ | ||
| async function multipartRequest (source = '', abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { | ||
| async function * streamFiles (source) { | ||
| try { | ||
| let index = 0 | ||
|
|
||
| for await (const { content, path, mode, mtime } of normaliseInput(source)) { | ||
| let fileSuffix = '' | ||
| const type = content ? 'file' : 'dir' | ||
|
|
||
| if (index > 0) { | ||
| yield '\r\n' | ||
|
|
||
| fileSuffix = `-${index}` | ||
| } | ||
|
|
||
| let fieldName = type + fileSuffix | ||
| const qs = [] | ||
|
|
||
| if (mode !== null && mode !== undefined) { | ||
| qs.push(`mode=${modeToString(mode)}`) | ||
| } | ||
|
|
||
| const time = mtimeToObject(mtime) | ||
| if (time != null) { | ||
| const { secs, nsecs } = time | ||
|
|
||
| qs.push(`mtime=${secs}`) | ||
|
|
||
| if (nsecs != null) { | ||
| qs.push(`mtime-nsecs=${nsecs}`) | ||
| } | ||
| } | ||
|
|
||
| if (qs.length) { | ||
| fieldName = `${fieldName}?${qs.join('&')}` | ||
| } | ||
|
|
||
| yield `--${boundary}\r\n` | ||
| yield `Content-Disposition: form-data; name="${fieldName}"; filename="${encodeURIComponent(path)}"\r\n` | ||
| yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n` | ||
| yield '\r\n' | ||
|
|
||
| if (content) { | ||
| yield * content | ||
| } | ||
|
|
||
| index++ | ||
| } | ||
| } catch (err) { | ||
| // workaround for https://github.com/node-fetch/node-fetch/issues/753 | ||
| // @ts-ignore - abort does not expect an arguments | ||
| abortController.abort(err) | ||
| } finally { | ||
| yield `\r\n--${boundary}--\r\n` | ||
| } | ||
| } | ||
|
|
||
| return { | ||
| headers: merge(headers, { | ||
| 'Content-Type': `multipart/form-data; boundary=${boundary}` | ||
| }), | ||
| body: await toStream(streamFiles(source)) | ||
| } | ||
| } | ||
|
|
||
| module.exports = multipartRequest | ||
|
|
||
| // In electron-renderer we use native fetch and should encode body using native | ||
| // form data. | ||
| if (isElectronRenderer) { | ||
| module.exports = require('./multipart-request.browser') | ||
| } else { | ||
| module.exports = require('./multipart-request.node') | ||
| } |
84 changes: 84 additions & 0 deletions
84
packages/ipfs-http-client/src/lib/multipart-request.node.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| 'use strict' | ||
|
|
||
| const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') | ||
| const { nanoid } = require('nanoid') | ||
| const modeToString = require('./mode-to-string') | ||
| const mtimeToObject = require('./mtime-to-object') | ||
| const merge = require('merge-options').bind({ ignoreUndefined: true }) | ||
| const toStream = require('it-to-stream') | ||
|
|
||
| /** | ||
| * | ||
| * @param {Object} source | ||
| * @param {AbortController} abortController | ||
| * @param {Headers|Record<string, string>} [headers] | ||
| * @param {string} [boundary] | ||
| */ | ||
| async function multipartRequest (source = '', abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { | ||
| async function * streamFiles (source) { | ||
| try { | ||
| let index = 0 | ||
|
|
||
| for await (const { content, path, mode, mtime } of normaliseInput(source)) { | ||
| let fileSuffix = '' | ||
| const type = content ? 'file' : 'dir' | ||
|
|
||
| if (index > 0) { | ||
| yield '\r\n' | ||
|
|
||
| fileSuffix = `-${index}` | ||
| } | ||
|
|
||
| let fieldName = type + fileSuffix | ||
| const qs = [] | ||
|
|
||
| if (mode !== null && mode !== undefined) { | ||
| qs.push(`mode=${modeToString(mode)}`) | ||
| } | ||
|
|
||
| const time = mtimeToObject(mtime) | ||
| if (time != null) { | ||
| const { secs, nsecs } = time | ||
|
|
||
| qs.push(`mtime=${secs}`) | ||
|
|
||
| if (nsecs != null) { | ||
| qs.push(`mtime-nsecs=${nsecs}`) | ||
| } | ||
| } | ||
|
|
||
| if (qs.length) { | ||
| fieldName = `${fieldName}?${qs.join('&')}` | ||
| } | ||
|
|
||
| yield `--${boundary}\r\n` | ||
| yield `Content-Disposition: form-data; name="${fieldName}"; filename="${encodeURIComponent(path)}"\r\n` | ||
| yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n` | ||
| yield '\r\n' | ||
|
|
||
| if (content) { | ||
| yield * content | ||
| } | ||
|
|
||
| index++ | ||
| } | ||
| } catch (err) { | ||
| // workaround for https://github.com/node-fetch/node-fetch/issues/753 | ||
| // @ts-ignore - abort does not expect an arguments | ||
| abortController.abort(err) | ||
| } finally { | ||
| yield `\r\n--${boundary}--\r\n` | ||
| } | ||
| } | ||
|
|
||
| return { | ||
| parts: null, | ||
| total: -1, | ||
| headers: merge(headers, { | ||
| 'Content-Type': `multipart/form-data; boundary=${boundary}` | ||
| }), | ||
| body: await toStream(streamFiles(source)) | ||
| } | ||
| } | ||
|
|
||
| module.exports = multipartRequest |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters