From 3e7baf77995cebee59e19347519ef3482f54484e Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 7 Jul 2020 16:28:42 -0700 Subject: [PATCH] feat: bufferring free multipart body encoder --- packages/ipfs-core-utils/package.json | 6 +- .../ipfs-core-utils/src/files/blob.browser.js | 24 + packages/ipfs-core-utils/src/files/blob.js | 186 +++++ .../ipfs-core-utils/src/files/file.browser.js | 4 + packages/ipfs-core-utils/src/files/file.js | 55 ++ .../src/files/normalise-input.js | 707 ++++++++++++------ packages/ipfs-http-client/package.json | 6 +- .../src/lib/async-iterable.js | 24 + .../src/lib/form-data-encoder.js | 116 +++ .../src/lib/mode-to-headers.js | 14 + .../src/lib/mtime-to-headers.js | 20 + .../src/lib/multipart-request.js | 74 +- .../src/lib/to-body.browser.js | 22 + packages/ipfs-http-client/src/lib/to-body.js | 15 + .../src/lib/to-stream.browser.js | 22 - .../ipfs-http-client/src/lib/to-stream.js | 7 - 16 files changed, 1000 insertions(+), 302 deletions(-) create mode 100644 packages/ipfs-core-utils/src/files/blob.browser.js create mode 100644 packages/ipfs-core-utils/src/files/blob.js create mode 100644 packages/ipfs-core-utils/src/files/file.browser.js create mode 100644 packages/ipfs-core-utils/src/files/file.js create mode 100644 packages/ipfs-http-client/src/lib/async-iterable.js create mode 100644 packages/ipfs-http-client/src/lib/form-data-encoder.js create mode 100644 packages/ipfs-http-client/src/lib/mode-to-headers.js create mode 100644 packages/ipfs-http-client/src/lib/mtime-to-headers.js create mode 100644 packages/ipfs-http-client/src/lib/to-body.browser.js create mode 100644 packages/ipfs-http-client/src/lib/to-body.js delete mode 100644 packages/ipfs-http-client/src/lib/to-stream.browser.js delete mode 100644 packages/ipfs-http-client/src/lib/to-stream.js diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index cca9645f9f..4d26f4a260 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -11,6 +11,10 @@ "src", "dist" ], + "browser": { + "./src/lib/blob.js": "./src/lib/blob.browser.js", + "./src/lib/file.js": "./src/lib/file.browser.js" + }, "repository": { "type": "git", "url": "git+https://github.com/ipfs/js-ipfs.git" @@ -40,4 +44,4 @@ "dirty-chai": "^2.0.1", "it-all": "^1.0.1" } -} +} \ No newline at end of file diff --git a/packages/ipfs-core-utils/src/files/blob.browser.js b/packages/ipfs-core-utils/src/files/blob.browser.js new file mode 100644 index 0000000000..3d78845033 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/blob.browser.js @@ -0,0 +1,24 @@ +// @ts-check +'use strict' +/* eslint-env browser */ + +exports.Blob = Blob + +/** + * Universal blob reading function + * @param {Blob} blob + * @returns {AsyncIterable} + */ +const readBlob = async function * (blob) { + const { body } = new Response(blob) + const reader = body.getReader() + while (true) { + const next = await reader.read() + if (next.done) { + return + } else { + yield next.value + } + } +} +exports.readBlob = readBlob diff --git a/packages/ipfs-core-utils/src/files/blob.js b/packages/ipfs-core-utils/src/files/blob.js new file mode 100644 index 0000000000..54f9dd9813 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/blob.js @@ -0,0 +1,186 @@ +// @ts-check +'use strict' + +const { TextEncoder, TextDecoder } = require('util') + +class Blob { + /** + * + * @param {BlobPart[]} init + * @param {Object} [options] + * @param {string} [options.type] + * + */ + constructor (init, options = {}) { + /** @type {Uint8Array[]} */ + const parts = [] + + let size = 0 + for (const part of init) { + if (typeof part === 'string') { + const bytes = new TextEncoder().encode(part) + parts.push(bytes) + size += bytes.byteLength + } else if (part instanceof Blob) { + size += part.size + // @ts-ignore - `_parts` is marked private so TS will complain about + // accessing it. + parts.push(...part._parts) + } else if (part instanceof ArrayBuffer) { + parts.push(new Uint8Array(part)) + size += part.byteLength + } else if (part instanceof Uint8Array) { + parts.push(part) + size += part.byteLength + } else if (ArrayBuffer.isView(part)) { + const { buffer, byteOffset, byteLength } = part + parts.push(new Uint8Array(buffer, byteOffset, byteLength)) + size += byteLength + } else { + throw new TypeError(`Can not convert ${part} value to a BlobPart`) + } + } + + /** @private */ + this._size = size + /** @private */ + this._type = options.type || '' + /** @private */ + this._parts = parts + } + + /** + * A string indicating the MIME type of the data contained in the Blob. + * If the type is unknown, this string is empty. + * @type {string} + */ + get type () { + return this._type + } + + /** + * The size, in bytes, of the data contained in the Blob object. + * @type {number} + */ + get size () { + return this._size + } + + /** + * Returns a new Blob object containing the data in the specified range of + * bytes of the blob on which it's called. + * @param {number} [start=0] - An index into the Blob indicating the first + * byte to include in the new Blob. If you specify a negative value, it's + * treated as an offset from the end of the Blob toward the beginning. For + * example, `-10` would be the 10th from last byte in the Blob. The default + * value is `0`. If you specify a value for start that is larger than the + * size of the source Blob, the returned Blob has size 0 and contains no + * data. + * @param {number} [end] - An index into the `Blob` indicating the first byte + * that will *not* be included in the new `Blob` (i.e. the byte exactly at + * this index is not included). If you specify a negative value, it's treated + * as an offset from the end of the Blob toward the beginning. For example, + * `-10` would be the 10th from last byte in the `Blob`. The default value is + * size. + * @param {string} [type] - The content type to assign to the new Blob; + * this will be the value of its type property. The default value is an empty + * string. + * @returns {Blob} + */ + slice (start = 0, end = this.size, type = '') { + const { size, _parts } = this + let offset = start < 0 + ? Math.max(size + start, 0) + : Math.min(start, size) + + let limit = (end < 0 ? Math.max(size + end, 0) : Math.min(end, size)) + const span = Math.max(limit - offset, 0) + + let blobSize = 0 + const blobParts = [] + for (const part of _parts) { + const { byteLength } = part + if (offset > 0 && byteLength <= offset) { + offset -= byteLength + limit -= byteLength + } else { + const chunk = part.subarray(offset, Math.min(byteLength, limit)) + blobParts.push(chunk) + blobSize += chunk.byteLength + // no longer need to take that into account + offset = 0 + + // don't add the overflow to new blobParts + if (blobSize >= span) { + break + } + } + } + + const blob = new Blob([], { type }) + blob._parts = blobParts + blob._size = blobSize + + return blob + } + + /** + * Returns a promise that resolves with an ArrayBuffer containing the entire + * contents of the Blob as binary data. + * @returns {Promise} + */ + // eslint-disable-next-line require-await + async arrayBuffer () { + const buffer = new ArrayBuffer(this.size) + const bytes = new Uint8Array(buffer) + let offset = 0 + for (const part of this._parts) { + bytes.set(part, offset) + offset += part.byteLength + } + return buffer + } + + /** + * Returns a promise that resolves with a USVString containing the entire + * contents of the Blob interpreted as UTF-8 text. + * @returns {Promise} + */ + // eslint-disable-next-line require-await + async text () { + const decoder = new TextDecoder() + let text = '' + for (const part of this._parts) { + text += decoder.decode(part) + } + return text + } + + /** + * @returns {never} + */ + // eslint-disable-next-line valid-jsdoc + stream () { + throw Error('Not implemented') + } + + /** + * Non standard, but if `ReadableStream`s are extended to be + * made async iterable why not blobs. + * @returns {AsyncIterator} + */ + // eslint-disable-next-line require-await + async * [Symbol.asyncIterator] () { + yield * this._parts + } +} + +exports.Blob = Blob + +/** + * Universal blob reading function + * @param {Blob} blob + * @returns {AsyncIterable} + */ +const readBlob = (blob) => blob +exports.readBlob = readBlob diff --git a/packages/ipfs-core-utils/src/files/file.browser.js b/packages/ipfs-core-utils/src/files/file.browser.js new file mode 100644 index 0000000000..3e8fa08f23 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/file.browser.js @@ -0,0 +1,4 @@ +'use strict' +/* eslint-env browser */ + +exports.File = File diff --git a/packages/ipfs-core-utils/src/files/file.js b/packages/ipfs-core-utils/src/files/file.js new file mode 100644 index 0000000000..222214d31f --- /dev/null +++ b/packages/ipfs-core-utils/src/files/file.js @@ -0,0 +1,55 @@ +// @ts-check +'use strict' + +const { Blob } = require('./blob') + +class File extends Blob { + /** + * + * @param {BlobPart[]} init + * @param {string} name - A USVString representing the file name or the path + * to the file. + * @param {Object} [options] + * @param {string} [options.type] - A DOMString representing the MIME type + * of the content that will be put into the file. Defaults to a value of "". + * @param {number} [options.lastModified] - A number representing the number + * of milliseconds between the Unix time epoch and when the file was last + * modified. Defaults to a value of Date.now(). + */ + constructor (init, name, options = {}) { + super(init, options) + /** @private */ + this._name = name.replace(/\//g, ':') + this._lastModified = options.lastModified || Date.now() + } + + /** + * The name of the file referenced by the File object. + * @type {string} + */ + get name () { + return this._name + } + + /** + * The path the URL of the File is relative to. + * @type {string} + */ + get webkitRelativePath () { + return '' + } + + /** + * Returns the last modified time of the file, in millisecond since the UNIX + * epoch (January 1st, 1970 at Midnight). + * @returns {number} + */ + get lastModified () { + return this._lastModified + } + + get [Symbol.toStringTag] () { + return 'File' + } +} +exports.File = File diff --git a/packages/ipfs-core-utils/src/files/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input.js index 63daa833f3..5d954f2c91 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input.js +++ b/packages/ipfs-core-utils/src/files/normalise-input.js @@ -1,258 +1,360 @@ +// @ts-check 'use strict' const errCode = require('err-code') -const { Buffer } = require('buffer') -const globalThis = require('ipfs-utils/src/globalthis') +const { File } = require('./file') +const { Blob, readBlob } = require('./blob') -/* - * Transform one of: +/** + * @template T + * @typedef {Iterable|AsyncIterable|ReadableStream} Multiple + */ + +/** + * + * @typedef {ExtendedFile | FileStream | Directory} FSEntry + */ + +/** + * Normalizes input into async iterable of extended File or custom FileStream + * objects. + * + * @param {Input} input + * @return {AsyncIterable} * - * ``` - * Bytes (Buffer|ArrayBuffer|TypedArray) [single file] - * Bloby (Blob|File) [single file] - * String [single file] - * { path, content: Bytes } [single file] - * { path, content: Bloby } [single file] - * { path, content: String } [single file] - * { path, content: Iterable } [single file] - * { path, content: Iterable } [single file] - * { path, content: AsyncIterable } [single file] - * Iterable [single file] - * Iterable [single file] - * Iterable [multiple files] - * Iterable [multiple files] - * Iterable<{ path, content: Bytes }> [multiple files] - * Iterable<{ path, content: Bloby }> [multiple files] - * Iterable<{ path, content: String }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: AsyncIterable }> [multiple files] - * AsyncIterable [single file] - * AsyncIterable [multiple files] - * AsyncIterable [multiple files] - * AsyncIterable<{ path, content: Bytes }> [multiple files] - * AsyncIterable<{ path, content: Bloby }> [multiple files] - * AsyncIterable<{ path, content: String }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] - * ``` - * Into: + * @typedef {SingleFileInput | MultiFileInput} Input + * @typedef {Blob|Bytes|string|FileObject|Iterable|Multiple} SingleFileInput + * @typedef {Multiple|Multiple|Multiple} MultiFileInput * - * ``` - * AsyncIterable<{ path, content: AsyncIterable }> - * ``` + * @typedef {Object} FileObject + * @property {string} [path] + * @property {FileContent} [content] + * @property {string|number} [mode] + * @property {UnixFSTime} [mtime] + * @typedef {Blob|Bytes|string|Iterable|Multiple} FileContent * - * @param input Object - * @return AsyncInterable<{ path, content: AsyncIterable }> + * @typedef {ArrayBuffer|ArrayBufferView} Bytes + * + * @typedef {Object} UnixFSTime + * @property {number} secs + * @property {number} [nsecs] */ -module.exports = function normaliseInput (input) { +// eslint-disable-next-line complexity +module.exports = async function * normaliseInput (input) { // must give us something - if (input === null || input === undefined) { - throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT')) + if (input == null) { + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') } - // String - if (typeof input === 'string' || input instanceof String) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() + // If input is a one of the following types + // - string + // - ArrayBuffer + // - ArrayBufferView + // - Blob + // - FileObject + // It is turned into collection of one file (with that content) + const file = asFile(input) + if (file != null) { + yield file + return } - // Buffer|ArrayBuffer|TypedArray - // Blob|File - if (isBytes(input) || isBloby(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() + // If input is sync iterable we expect it to be a homogenous collection & + // need to probe it's first item to tell if input to be interpreted as single + // file with multiple chunks or multiple files. + // NOTE: We had to ensure that input was not string or arraybuffer view + // because those are also iterables. + /** @type {null|Iterable<*>} */ + const iterable = asIterable(input) + if (iterable != null) { + yield * normilizeIterableInput(iterable) + + // Return here since we have have exhasted an input iterator. + return } - // Iterable - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - // Iterable - if (Number.isInteger(first.value) || isBytes(first.value)) { - yield toFileObject((function * () { - yield first.value - yield * iterator - })()) - return - } - - // Iterable - // Iterable - // Iterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // window.ReadableStream - if (typeof input.getReader === 'function') { - return (async function * () { - for await (const obj of browserStreamToIt(input)) { - yield toFileObject(obj) - } - })() - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - const iterator = input[Symbol.asyncIterator]() - const first = await iterator.next() - if (first.done) return iterator - - // AsyncIterable - if (isBytes(first.value)) { - yield toFileObject((async function * () { // eslint-disable-line require-await - yield first.value - yield * iterator - })()) - return - } + // If we got here than we are dealing with async input, which can be either + // readable stream or an async iterable (casting former to later) + const stream = asReadableStream(input) + const asyncIterable = stream + ? iterateReadableStream(stream) + : asAsyncIterable(input) + + // Async iterable (whech we assume to be homogenous) may represent single file + // with multilpe chunks or multiple files, to decide we probe it's first item. + if (asyncIterable != null) { + // Create peekable to be able to probe head without consuming it. + const peekable = AsyncPeekable.from(asyncIterable) + const { done, value } = await peekable.peek() + // If done input was empty so we return early. + if (done) { + return + } - // AsyncIterable - // AsyncIterable - // AsyncIterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for await (const obj of iterator) { - yield toFileObject(obj) + // If first item is array buffer or one of it's views input represents a + // single file with multiple chunks. + if (ArrayBuffer.isView(value) || value instanceof ArrayBuffer) { + yield new FileStream(peekable, '') + // Otherwise we interpret input as async collection of multiple files. + // In that case itemss of input can be either `string`, `Blob` or + // `FileObject`, so we normalize each to a file. If item is anything else + // we throw an exception. + } else { + for await (const content of peekable) { + // Note: If content here is `ArrayBuffer` or a view this will turn it + // into a file, but that can only occur if async iterable contained + // variadic chunks which is not supported. + const file = asFile(content) + if (file) { + yield file + } else { + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') } - return } + } - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // { path, content: ? } - // Note: Detected _after_ AsyncIterable because Node.js streams have a - // `path` property that passes this check. - if (isFileObject(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() + return } throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') } -function toFileObject (input) { - const obj = { - path: input.path || '', - mode: input.mode, - mtime: input.mtime - } - - if (input.content) { - obj.content = toAsyncIterable(input.content) - } else if (!input.path) { // Not already a file object with path or content prop - obj.content = toAsyncIterable(input) +/** + * + * @param {Iterable|Iterable} iterable + * @returns {Iterable} + * @typedef {Iterable|Iterable|Iterable} IterableFileContent + * @typedef {Iterable|Iterable|Iterable} IterableFiles + */ +const normilizeIterableInput = function * (iterable) { + // In order to peek at first without loosing capablitiy to iterate, we + // create peekable which allows us to do that. + const peekable = Peekable.from(iterable) + // First try to interpret it a single file content chunks. + const bytes = asIterableBytes(peekable) + if (bytes != null) { + yield new ExtendedFile(bytes, '') + // If first item is a `Blob`, `string`, or a `FileObject` we treat this + // input as collection of files. We iterate and normalize each each value + // into a file. + } else { + for (const content of peekable) { + const file = asFile(content) + if (file) { + yield file + } else { + throw errCode(new Error('Unexpected input: ' + typeof content), 'ERR_UNEXPECTED_INPUT') + } + } } - return obj + // Otherwise eslint complains about lack of return + return undefined } -function toAsyncIterable (input) { - // Bytes | String - if (isBytes(input) || typeof input === 'string') { - return (async function * () { // eslint-disable-line require-await - yield toBuffer(input) - })() +/** + * Utility function takes any input and returns a `File|FileStream|Directoriy` + * (containing that input) if input was one of the following types (or `null` + * otherwise): + * - `ArrayBuffer` + * - `ArrayBufferView` + * - `string` + * - `Blob` + * - `FileObject` + * It will return `File` instance when content is of known size (not a stream) + * other it returns a `FileStream`. If input is `FileObject` with no `content` + * returns `Directory`. + * @param {any} input + * @param {string} [name] - optional name for the file + * @returns {null|ExtendedFile|FileStream|Directory} + */ +const asFile = (input, name) => { + const file = asFileFromBlobPart(input, name) + if (file) { + return file + } else { + // If input is a `FileObject` + const fileObject = asFileObject(input) + if (fileObject) { + return fileFromFileObject(fileObject) + } else { + return null + } } +} - // Bloby - if (isBloby(input)) { - return blobToAsyncGenerator(input) +/** + * Utility function takes any input and returns a `File` (containing it) + * if `input` is of `BlobPart` type, otherwise returns `null`. If optional + * `name` is passed it will be used as a file name. + * @param {any} content + * @param {string} [name] + * @returns {ExtendedFile|null} + */ +const asFileFromBlobPart = (content, name) => { + if ( + typeof content === 'string' || + ArrayBuffer.isView(content) || + content instanceof ArrayBuffer + ) { + return new ExtendedFile([content], name || '') + } else if (content instanceof Blob) { + // Third argument is passed to preserve a mime type. + return new ExtendedFile([content], name || '', content) + } else { + return null } +} - // Browser stream - if (typeof input.getReader === 'function') { - return browserStreamToIt(input) - } +/** + * Utility function takes a `FileObject` and returns a web `File` (with extended) + * attributes if content is of known size or a `FileStream` if content is an + * async stream or `Directory` if it has no content. + * @param {FileObject} fileObject + * @returns {null|ExtendedFile|FileStream|Directory} + */ +const fileFromFileObject = (fileObject) => { + const { path, mtime, mode, content } = fileObject + const ext = { mtime, mode, path } + const name = path == null ? undefined : basename(path) + const file = asFileFromBlobPart(content, name) + if (file) { + return Object.assign(file, ext) + } else { + // If content is empty it is a diretory + if (content == null) { + return new Directory(name, ext) + } - // Iterator - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - if (Number.isInteger(first.value)) { - yield toBuffer(Array.from((function * () { - yield first.value - yield * iterator - })())) - return + // First try to interpret it a single file content chunks. + const iterable = asIterable(content) + if (iterable != null) { + const peekable = Peekable.from(iterable) + // File object content can only contain iterable of numbers or array + // buffers (or it's views). If so we create an object otherwise + // throw an exception. + const bytes = asIterableBytes(peekable) + if (bytes != null) { + return new ExtendedFile(bytes, name, ext) + } else { + throw errCode(new Error('Unexpected input: ' + typeof content), 'ERR_UNEXPECTED_INPUT') } + } - // Iterable - if (isBytes(first.value)) { - yield toBuffer(first.value) - for (const chunk of iterator) { - yield toBuffer(chunk) - } - return - } + // If we got here than we are dealing with async input, which can be either + // readable stream or an async iterable (casting former to later) + const stream = asReadableStream(content) + const asyncIterable = stream + ? iterateReadableStream(stream) + : asAsyncIterable(content) + if (asyncIterable != null) { + return new FileStream(asyncIterable, name, ext) + } - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() + throw errCode(new Error(`Unexpected FileObject content: ${content}`), 'ERR_UNEXPECTED_INPUT') } +} - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - for await (const chunk of input) { - yield toBuffer(chunk) - } - })() +/** + * @param {Peekable} content + * @returns {ArrayBufferView[]|ArrayBuffer[]|null} + */ +const asIterableBytes = (content) => { + const { done, value } = content.peek() + // If it is done input was empty collection so we return early. + if (done) { + return [] } - throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') -} - -function toBuffer (chunk) { - return isBytes(chunk) ? chunk : Buffer.from(chunk) + // If first item is an integer we treat input as a byte array and result + // will be collection of one file contaning those bytes. + if (Number.isInteger(value)) { + const bytes = new Uint8Array(content) + return [bytes] + + // If first item is array buffer or it's view, it is interpreted as chunks + // of one file. In that case we collect all chunks and normalize input into + // collection with a single file containing those chunks. + // Note: Since this is a synchronous iterator all chunks are already in + // memory so by by collecting them into a single file we are not allocate + // new memory (unless iterator is generating content, but that is exotic + // enough use case that we prefer to go with File over FileStream). + } else if (ArrayBuffer.isView(value) || value instanceof ArrayBuffer) { + return [...content] + } else { + return null + } } -function isBytes (obj) { - return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer +/** + * Pattern matches given `input` as `ReadableStream` and return back either + * matched input or `null`. + * + * @param {any} input + * @returns {ReadableStream|null} + */ +const asReadableStream = input => { + if (input && typeof input.getReader === 'function') { + return input + } else { + return null + } } -function isBloby (obj) { - return typeof globalThis.Blob !== 'undefined' && obj instanceof globalThis.Blob +/** + * Pattern matches given `input` as `AsyncIterable` and returns back either + * matched `AsyncIterable` or `null`. + * @template I + * @param {AsyncIterable|Input} input + * @returns {AsyncIterable|null} + */ +const asAsyncIterable = input => { + /** @type {*} */ + const object = input + if (object && typeof object[Symbol.asyncIterator] === 'function') { + return object + } else { + return null + } } -// An object with a path or content property -function isFileObject (obj) { - return typeof obj === 'object' && (obj.path || obj.content) +/** + * Pattern matches given input as `Iterable` and returns back either matched + * iterable or `null`. + * @template I + * @param {Iterable|Input} input + * @returns {Iterable|null} + */ +const asIterable = input => { + /** @type {*} */ + const object = input + if (object && typeof object[Symbol.iterator] === 'function') { + return object + } else { + return null + } } -function blobToAsyncGenerator (blob) { - if (typeof blob.stream === 'function') { - // firefox < 69 does not support blob.stream() - return browserStreamToIt(blob.stream()) +/** + * Pattern matches given input as "FileObject" and returns back eithr matched + * input or `null`. + * @param {*} input + * @returns {FileObject|null} + */ +const asFileObject = input => { + if (typeof input === 'object' && (input.path || input.content)) { + return input + } else { + return null } - - return readBlob(blob) } +/** + * @template T + * @param {ReadableStream} stream + * @returns {AsyncIterable} + */ -async function * browserStreamToIt (stream) { +const iterateReadableStream = async function * (stream) { const reader = stream.getReader() while (true) { @@ -266,33 +368,186 @@ async function * browserStreamToIt (stream) { } } -async function * readBlob (blob, options) { - options = options || {} +/** + * @template T + */ +class Peekable { + /** + * @template T + * @template {Iterable} I + * @param {I} iterable + * @returns {Peekable} + */ + static from (iterable) { + return new Peekable(iterable) + } - const reader = new globalThis.FileReader() - const chunkSize = options.chunkSize || 1024 * 1024 - let offset = options.offset || 0 + /** + * @private + * @param {Iterable} iterable + */ + constructor (iterable) { + const iterator = iterable[Symbol.iterator]() + /** @private */ + this.first = iterator.next() + /** @private */ + this.rest = iterator + } - const getNextChunk = () => new Promise((resolve, reject) => { - reader.onloadend = e => { - const data = e.target.result - resolve(data.byteLength === 0 ? null : data) - } - reader.onerror = reject + peek () { + return this.first + } - const end = offset + chunkSize - const slice = blob.slice(offset, end) - reader.readAsArrayBuffer(slice) - offset = end - }) + next () { + const { first, rest } = this + this.first = rest.next() + return first + } - while (true) { - const data = await getNextChunk() + [Symbol.iterator] () { + return this + } - if (data == null) { - return - } + [Symbol.asyncIterator] () { + return this + } +} + +/** + * @template T + */ +class AsyncPeekable { + /** + * @template T + * @template {AsyncIterable} I + * @param {I} iterable + * @returns {AsyncPeekable} + */ + static from (iterable) { + return new AsyncPeekable(iterable) + } + + /** + * @private + * @param {AsyncIterable} iterable + */ + constructor (iterable) { + const iterator = iterable[Symbol.asyncIterator]() + /** @private */ + this.first = iterator.next() + /** @private */ + this.rest = iterator + } + + peek () { + return this.first + } + + next () { + const { first, rest } = this + this.first = rest.next() + return first + } + + [Symbol.asyncIterator] () { + return this + } +} + +/** + * @param {string} path + * @returns {string} + */ +const basename = (path) => + path.split(/\\|\//).pop() + +class ExtendedFile extends File { + /** + * @param {BlobPart[]} init + * @param {string} name - A USVString representing the file name or the path + * to the file. + * @param {Object} [options] + * @param {string} [options.type] - A DOMString representing the MIME type + * of the content that will be put into the file. Defaults to a value of "". + * @param {number} [options.lastModified] - A number representing the number + * of milliseconds between the Unix time epoch and when the file was last + * modified. Defaults to a value of Date.now(). + * @param {string} [options.path] + * @param {string|number} [options.mode] + * @param {UnixFSTime} [options.mtime] + */ + constructor (init, name, options = {}) { + super(init, name, options) + const { path, mode, mtime } = options + this.path = path + this.mode = mode + this.mtime = mtime + + /** @type {'file'} */ + this.kind = 'file' + } - yield Buffer.from(data) + /** + * @returns {AsyncIterable} + */ + get content () { + return readBlob(this) + } +} +module.exports.ExtendedFile = ExtendedFile + +class FileStream { + /** + * @param {AsyncIterable} content + * @param {string} name + * @param {Object} [options] + * @param {string} [options.type] + * @param {number} [options.lastModified] + * @param {string} [options.path] + * @param {UnixFSTime} [options.mtime] + * @param {string|number} [options.mode] + */ + constructor (content, name, options = {}) { + this.content = content + this.name = name + this.type = options.type || '' + this.lastModified = options.lastModified || Date.now() + this.path = options.path || '' + this.mtime = options.mtime + this.mode = options.mode + + /** @type {'file-stream'} */ + this.kind = 'file-stream' + } + + get size () { + throw Error('File size is unknown') + } +} +module.exports.FileStream = FileStream + +class Directory { + /** + * @param {string} name + * @param {Object} [options] + * @param {string} [options.type] + * @param {number} [options.lastModified] + * @param {string} [options.path] + * @param {UnixFSTime} [options.mtime] + * @param {string|number} [options.mode] + */ + constructor (name, options = {}) { + this.name = name + this.type = options.type || '' + this.lastModified = options.lastModified || Date.now() + this.path = options.path || '' + this.mtime = options.mtime + this.mode = options.mode + + /** @type {'directory'} */ + this.kind = 'directory' + /** @type {void} */ + this.content = undefined } } +module.exports.Directory = Directory diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 067f9a3156..00bcd8c5dd 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -15,7 +15,8 @@ ], "main": "src/index.js", "browser": { - "./src/lib/to-stream.js": "./src/lib/to-stream.browser.js", + "./src/lib/to-body.js": "./src/lib/to-body.browser.js", + "ipfs-core-utils/src/files/blob.js": "ipfs-core-utils/src/files/blob.browser.js", "ipfs-utils/src/files/glob-source": false, "go-ipfs-dep": false }, @@ -54,7 +55,6 @@ "ipld-raw": "^5.0.0", "iso-url": "^0.4.7", "it-tar": "^1.2.2", - "it-to-buffer": "^1.0.0", "it-to-stream": "^0.1.1", "merge-options": "^2.0.0", "multiaddr": "^7.4.3", @@ -184,4 +184,4 @@ "Łukasz Magiera ", "Łukasz Magiera " ] -} +} \ No newline at end of file diff --git a/packages/ipfs-http-client/src/lib/async-iterable.js b/packages/ipfs-http-client/src/lib/async-iterable.js new file mode 100644 index 0000000000..8867b80ed6 --- /dev/null +++ b/packages/ipfs-http-client/src/lib/async-iterable.js @@ -0,0 +1,24 @@ +// @ts-check +'use strict' + +/** + * @template T + * @param {Iterable|AsyncIterable} iterable + * @returns {AsyncIterable} + */ +// eslint-disable-next-line require-await +const from = async function * AsyncIterableFrom (iterable) { + yield * iterable +} +exports.from = from + +/** + * @template T + * @param {...T} items + * @returns {AsyncIterable} + */ +// eslint-disable-next-line require-await +const of = async function * AsyncIterableOf (...items) { + yield * items +} +exports.of = of diff --git a/packages/ipfs-http-client/src/lib/form-data-encoder.js b/packages/ipfs-http-client/src/lib/form-data-encoder.js new file mode 100644 index 0000000000..a210bcebe0 --- /dev/null +++ b/packages/ipfs-http-client/src/lib/form-data-encoder.js @@ -0,0 +1,116 @@ +// @ts-check +'use strict' + +const { nanoid } = require('nanoid') +const { Blob } = require('ipfs-core-utils/src/files/blob') +const { from } = require('./async-iterable') + +class FormDataEncoder { +/** + * @param {Object} [options] + * @param {string} [options.boundary] + */ + constructor (options = {}) { + this.boundary = getBoundary(options) + this.type = `multipart/form-data; boundary=${this.boundary}` + } + + /** + * @param {AsyncIterable|Iterable} source + * @returns {AsyncIterable} + */ + async * encode (source) { + const { boundary } = this + let first = true + for await (const part of from(source)) { + if (!first) { + yield '\r\n' + first = false + } + + yield `--${boundary}\r\n` + yield * encodeHead(part) + yield '\r\n' + yield * encodeBody(part.content) + } + + yield `\r\n--${boundary}--\r\n` + } +} +exports.FormDataEncoder = FormDataEncoder + +/** + * @param {void|Blob|AsyncIterable} content + * @returns {Iterable|AsyncIterable} + */ +function encodeBody (content) { + if (content == null) { + return [] + } else if (content instanceof Blob) { + return [content] + } else { + /** @type {AsyncIterable|AsyncIterable} */ + const chunks = (content) + return chunks + } +} + +/** + * @typedef {Object} Part + * @property {string} name + * @property {void|Blob|AsyncIterable} content + * @property {string} [filename] + * @property {Record} [headers] + */ + +/** + * @param {Part} part + * @returns {Iterable} + */ +function * encodeHead ({ name, content, filename, headers }) { + const file = filename || getFileName(content) + const contentDisposition = + file == null + ? `form-data; name="${name}"` + : `form-data; name="${name}"; filename="${encodeURIComponent(file)}"` + + yield `Content-Disposition: ${contentDisposition}\r\n` + + let hasContentType = false + if (headers) { + for (const [name, value] of Object.entries(headers)) { + // if content type is provided we do no want to derive + if (name === 'Content-Type' || name === 'content-type') { + hasContentType = true + } + + yield `${name}: ${value}\r\n` + } + } + + const contentType = !hasContentType ? getContentType(content) : null + if (contentType != null) { + yield `Content-Type: ${contentType}\r\n` + } + + // Otherwise jslint is unhappy. + return undefined +} + +/** + * @param {any} content + * @returns {string|null} + */ +const getFileName = (content) => + content.filepath || content.webkitRelativePath || content.name || null + +const getContentType = (content) => + content.type || null + +/** + * @param {Object} options + * @param {string} [options.boundary] + * @returns {string} + */ +const getBoundary = ({ boundary }) => + (boundary || `-----------------------------${nanoid()}`).toLowerCase() diff --git a/packages/ipfs-http-client/src/lib/mode-to-headers.js b/packages/ipfs-http-client/src/lib/mode-to-headers.js new file mode 100644 index 0000000000..92cff1c321 --- /dev/null +++ b/packages/ipfs-http-client/src/lib/mode-to-headers.js @@ -0,0 +1,14 @@ +'use strict' + +const modeToString = require('./mode-to-string') + +const modeToHeaders = (mode) => { + const value = modeToString(mode) + if (value != null) { + return { mode: value } + } else { + return undefined + } +} + +module.exports = modeToHeaders diff --git a/packages/ipfs-http-client/src/lib/mtime-to-headers.js b/packages/ipfs-http-client/src/lib/mtime-to-headers.js new file mode 100644 index 0000000000..8b0a501c05 --- /dev/null +++ b/packages/ipfs-http-client/src/lib/mtime-to-headers.js @@ -0,0 +1,20 @@ +'use strict' + +const mtimeToObject = require('./mtime-to-object') + +const mtimeToHeaders = (mtime) => { + const data = mtimeToObject(mtime) + if (data) { + const headers = {} + const { secs, nsecs } = data + headers.mtime = secs + if (nsecs != null) { + headers[mtime - nsecs] = nsecs + } + return headers + } else { + return undefined + } +} + +module.exports = mtimeToHeaders diff --git a/packages/ipfs-http-client/src/lib/multipart-request.js b/packages/ipfs-http-client/src/lib/multipart-request.js index eee4e26b1b..cb9d2a09f1 100644 --- a/packages/ipfs-http-client/src/lib/multipart-request.js +++ b/packages/ipfs-http-client/src/lib/multipart-request.js @@ -1,51 +1,35 @@ +// @ts-check 'use strict' const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') -const toStream = require('./to-stream') -const { nanoid } = require('nanoid') -const modeToString = require('../lib/mode-to-string') -const mtimeToObject = require('../lib/mtime-to-object') +const toBody = require('./to-body') +const modeToHeaders = require('../lib/mode-to-headers') +const mtimeToHeaders = require('../lib/mtime-to-headers') const merge = require('merge-options').bind({ ignoreUndefined: true }) +const { FormDataEncoder } = require('./form-data-encoder') -async function multipartRequest (source = '', abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { - async function * streamFiles (source) { +async function multipartRequest (source = '', abortController, headers = {}, boundary) { + async function * parts (source, abortController) { try { let index = 0 - - for await (const { content, path, mode, mtime } of normaliseInput(source)) { - let fileSuffix = '' - const type = content ? 'file' : 'dir' - - if (index > 0) { - yield '\r\n' - - fileSuffix = `-${index}` - } - - yield `--${boundary}\r\n` - yield `Content-Disposition: form-data; name="${type}${fileSuffix}"; filename="${encodeURIComponent(path)}"\r\n` - yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n` - - if (mode !== null && mode !== undefined) { - yield `mode: ${modeToString(mode)}\r\n` + for await (const input of normaliseInput(source)) { + const { kind, path, mode, mtime } = input + const type = kind === 'directory' ? 'dir' : 'file' + const suffix = index > 0 ? `-${index}` : '' + const name = `${type}${suffix}` + const filename = path !== '' ? encodeURIComponent(path) : '' + const headers = { + 'Content-Type': type === 'file' ? 'application/octet-stream' : 'application/x-directory', + ...(mtime && mtimeToHeaders(mtime)), + ...(mode && modeToHeaders(mode)) } + const content = input.kind === 'file' ? input : input.content - if (mtime != null) { - const { - secs, nsecs - } = mtimeToObject(mtime) - - yield `mtime: ${secs}\r\n` - - if (nsecs != null) { - yield `mtime-nsecs: ${nsecs}\r\n` - } - } - - yield '\r\n' - - if (content) { - yield * content + yield { + name, + content, + filename, + headers } index++ @@ -53,16 +37,20 @@ async function multipartRequest (source = '', abortController, headers = {}, bou } catch (err) { // workaround for https://github.com/node-fetch/node-fetch/issues/753 abortController.abort(err) - } finally { - yield `\r\n--${boundary}--\r\n` } } + const encoder = new FormDataEncoder({ boundary }) + const data = encoder.encode(parts(source, abortController)) + // In node this will produce readable stream, in browser it will + // produce a blob instance. + const body = await toBody(data) + return { headers: merge(headers, { - 'Content-Type': `multipart/form-data; boundary=${boundary}` + 'Content-Type': encoder.type }), - body: await toStream(streamFiles(source)) + body } } diff --git a/packages/ipfs-http-client/src/lib/to-body.browser.js b/packages/ipfs-http-client/src/lib/to-body.browser.js new file mode 100644 index 0000000000..7170677cec --- /dev/null +++ b/packages/ipfs-http-client/src/lib/to-body.browser.js @@ -0,0 +1,22 @@ +// @ts-check +'use strict' +/* eslint-env browser */ + +// browsers can't stream. When the 'Send ReadableStream in request body' row +// is green here: https://developer.mozilla.org/en-US/docs/Web/API/Request/Request#Browser_compatibility +// we'll be able to wrap the passed iterator in the it-to-browser-readablestream module +// in the meantime we create Blob out of all parts. + +/** + * Turns async iterable of the `BlobPart`s into an aggregate `Blob`. + * @param {AsyncIterable} source + * @returns {Promise} + */ +module.exports = async (source) => { + const parts = [] + for await (const chunk of source) { + parts.push(chunk) + } + + return new Blob(parts) +} diff --git a/packages/ipfs-http-client/src/lib/to-body.js b/packages/ipfs-http-client/src/lib/to-body.js new file mode 100644 index 0000000000..843def0a8d --- /dev/null +++ b/packages/ipfs-http-client/src/lib/to-body.js @@ -0,0 +1,15 @@ +// @ts-check +'use strict' + +const toStream = require('it-to-stream') + +/** + * @typedef {import('stream').Readable} Readable + */ + +/** + * @param {AsyncIterable} it + * @returns {Readable} + */ +module.exports = (it) => + toStream.readable(it) diff --git a/packages/ipfs-http-client/src/lib/to-stream.browser.js b/packages/ipfs-http-client/src/lib/to-stream.browser.js deleted file mode 100644 index 9f5784fedb..0000000000 --- a/packages/ipfs-http-client/src/lib/to-stream.browser.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict' - -// browsers can't stream. When the 'Send ReadableStream in request body' row -// is green here: https://developer.mozilla.org/en-US/docs/Web/API/Request/Request#Browser_compatibility -// we'll be able to wrap the passed iterator in the it-to-browser-readablestream module -// in the meantime we have to convert the whole thing to a BufferSource of some sort -const toBuffer = require('it-to-buffer') -const { Buffer } = require('buffer') - -module.exports = (it) => { - async function * bufferise (source) { - for await (const chunk of source) { - if (Buffer.isBuffer(chunk)) { - yield chunk - } else { - yield Buffer.from(chunk) - } - } - } - - return toBuffer(bufferise(it)) -} diff --git a/packages/ipfs-http-client/src/lib/to-stream.js b/packages/ipfs-http-client/src/lib/to-stream.js deleted file mode 100644 index f0f59ffc50..0000000000 --- a/packages/ipfs-http-client/src/lib/to-stream.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict' - -const toStream = require('it-to-stream') - -module.exports = (it) => { - return toStream.readable(it) -}