Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

@uppy/aws-s3-multipart: refactor to TS #4902

Merged
merged 22 commits into from
Mar 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
427 changes: 427 additions & 0 deletions packages/@uppy/aws-s3-multipart/src/HTTPCommunicationQueue.ts

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -1,24 +1,53 @@
import type { Uppy } from '@uppy/core'
import { AbortController } from '@uppy/utils/lib/AbortController'
import type { Meta, UppyFile } from '@uppy/utils/lib/UppyFile'
import type { HTTPCommunicationQueue } from './HTTPCommunicationQueue'
import type { Body } from './utils'

const MB = 1024 * 1024

interface MultipartUploaderOptions<M extends Meta, B extends Body> {
getChunkSize?: (file: { size: number }) => number
onProgress?: (bytesUploaded: number, bytesTotal: number) => void
onPartComplete?: (part: { PartNumber: number; ETag: string }) => void
shouldUseMultipart?: boolean | ((file: UppyFile<M, B>) => boolean)
onSuccess?: (result: B) => void
onError?: (err: unknown) => void
companionComm: HTTPCommunicationQueue<M, B>
file: UppyFile<M, B>
log: Uppy<M, B>['log']

uploadId: string
key: string
}

const defaultOptions = {
getChunkSize (file) {
getChunkSize(file: { size: number }) {
return Math.ceil(file.size / 10000)
},
onProgress () {},
onPartComplete () {},
onSuccess () {},
onError (err) {
onProgress() {},
onPartComplete() {},
onSuccess() {},
onError(err: unknown) {
throw err
},
} satisfies Partial<MultipartUploaderOptions<any, any>>

export interface Chunk {
getData: () => Blob
onProgress: (ev: ProgressEvent) => void
onComplete: (etag: string) => void
shouldUseMultipart: boolean
setAsUploaded?: () => void
}

function ensureInt (value) {
function ensureInt<T>(value: T): T extends number | string ? number : never {
if (typeof value === 'string') {
// @ts-expect-error TS is not able to recognize it's fine.
return parseInt(value, 10)
}
if (typeof value === 'number') {
// @ts-expect-error TS is not able to recognize it's fine.
return value
}
throw new TypeError('Expected a number')
Expand All @@ -32,47 +61,41 @@ export const pausingUploadReason = Symbol('pausing upload, not an actual error')
* (based on the user-provided `shouldUseMultipart` option value) and to manage
* the chunk splitting.
*/
class MultipartUploader {
class MultipartUploader<M extends Meta, B extends Body> {
options: MultipartUploaderOptions<M, B> &
Required<Pick<MultipartUploaderOptions<M, B>, keyof typeof defaultOptions>>

#abortController = new AbortController()

/** @type {import("../types/chunk").Chunk[]} */
#chunks
#chunks: Array<Chunk | null>

/** @type {{ uploaded: number, etag?: string, done?: boolean }[]} */
#chunkState
#chunkState: { uploaded: number; etag?: string; done?: boolean }[]

/**
* The (un-chunked) data to upload.
*
* @type {Blob}
*/
#data
#data: Blob

/** @type {import("@uppy/core").UppyFile} */
#file
#file: UppyFile<M, B>

/** @type {boolean} */
#uploadHasStarted = false

/** @type {(err?: Error | any) => void} */
#onError
#onError: (err: unknown) => void

/** @type {() => void} */
#onSuccess
#onSuccess: (result: B) => void

/** @type {import('../types/index').AwsS3MultipartOptions["shouldUseMultipart"]} */
#shouldUseMultipart
#shouldUseMultipart: MultipartUploaderOptions<M, B>['shouldUseMultipart']

/** @type {boolean} */
#isRestoring
#isRestoring: boolean

#onReject = (err) => (err?.cause === pausingUploadReason ? null : this.#onError(err))
#onReject = (err: unknown) =>
(err as any)?.cause === pausingUploadReason ? null : this.#onError(err)

#maxMultipartParts = 10_000

#minPartSize = 5 * MB

constructor (data, options) {
constructor(data: Blob, options: MultipartUploaderOptions<M, B>) {
this.options = {
...defaultOptions,
...options,
Expand All @@ -89,7 +112,7 @@ class MultipartUploader {
// When we are restoring an upload, we already have an UploadId and a Key. Otherwise
// we need to call `createMultipartUpload` to get an `uploadId` and a `key`.
// Non-multipart uploads are not restorable.
this.#isRestoring = options.uploadId && options.key
this.#isRestoring = (options.uploadId && options.key) as any as boolean

this.#initChunks()
}
Expand All @@ -98,15 +121,19 @@ class MultipartUploader {
// and calculates the optimal part size. When using multipart part uploads every part except for the last has
// to be at least 5 MB and there can be no more than 10K parts.
// This means we sometimes need to change the preferred part size from the user in order to meet these requirements.
#initChunks () {
#initChunks() {
const fileSize = this.#data.size
const shouldUseMultipart = typeof this.#shouldUseMultipart === 'function'
? this.#shouldUseMultipart(this.#file)
const shouldUseMultipart =
typeof this.#shouldUseMultipart === 'function' ?
this.#shouldUseMultipart(this.#file)
: Boolean(this.#shouldUseMultipart)

if (shouldUseMultipart && fileSize > this.#minPartSize) {
// At least 5MB per request:
let chunkSize = Math.max(this.options.getChunkSize(this.#data), this.#minPartSize)
let chunkSize = Math.max(
this.options.getChunkSize(this.#data),
this.#minPartSize,
)
let arraySize = Math.floor(fileSize / chunkSize)

// At most 10k requests per file:
Expand All @@ -132,41 +159,48 @@ class MultipartUploader {
shouldUseMultipart,
}
if (this.#isRestoring) {
const size = offset + chunkSize > fileSize ? fileSize - offset : chunkSize
const size =
offset + chunkSize > fileSize ? fileSize - offset : chunkSize
// setAsUploaded is called by listPart, to keep up-to-date the
// quantity of data that is left to actually upload.
this.#chunks[j].setAsUploaded = () => {
this.#chunks[j]!.setAsUploaded = () => {
this.#chunks[j] = null
this.#chunkState[j].uploaded = size
}
}
}
} else {
this.#chunks = [{
getData: () => this.#data,
onProgress: this.#onPartProgress(0),
onComplete: this.#onPartComplete(0),
shouldUseMultipart,
}]
this.#chunks = [
{
getData: () => this.#data,
onProgress: this.#onPartProgress(0),
onComplete: this.#onPartComplete(0),
shouldUseMultipart,
},
]
}

this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 }))
}

#createUpload () {
this
.options.companionComm.uploadFile(this.#file, this.#chunks, this.#abortController.signal)
#createUpload() {
this.options.companionComm
.uploadFile(
this.#file,
this.#chunks as Chunk[],
this.#abortController.signal,
)
.then(this.#onSuccess, this.#onReject)
this.#uploadHasStarted = true
}

#resumeUpload () {
this
.options.companionComm.resumeUploadFile(this.#file, this.#chunks, this.#abortController.signal)
#resumeUpload() {
this.options.companionComm
.resumeUploadFile(this.#file, this.#chunks, this.#abortController.signal)
.then(this.#onSuccess, this.#onReject)
}

#onPartProgress = (index) => (ev) => {
#onPartProgress = (index: number) => (ev: ProgressEvent) => {
if (!ev.lengthComputable) return

this.#chunkState[index].uploaded = ensureInt(ev.loaded)
Expand All @@ -175,7 +209,7 @@ class MultipartUploader {
this.options.onProgress(totalUploaded, this.#data.size)
}

#onPartComplete = (index) => (etag) => {
#onPartComplete = (index: number) => (etag: string) => {
// This avoids the net::ERR_OUT_OF_MEMORY in Chromium Browsers.
this.#chunks[index] = null
this.#chunkState[index].etag = etag
Expand All @@ -188,37 +222,44 @@ class MultipartUploader {
this.options.onPartComplete(part)
}

#abortUpload () {
#abortUpload() {
this.#abortController.abort()
this.options.companionComm.abortFileUpload(this.#file).catch((err) => this.options.log(err))
this.options.companionComm
.abortFileUpload(this.#file)
.catch((err: unknown) => this.options.log(err as Error))
}

start () {
start(): void {
if (this.#uploadHasStarted) {
if (!this.#abortController.signal.aborted) this.#abortController.abort(pausingUploadReason)
if (!this.#abortController.signal.aborted)
this.#abortController.abort(pausingUploadReason)
this.#abortController = new AbortController()
this.#resumeUpload()
} else if (this.#isRestoring) {
this.options.companionComm.restoreUploadFile(this.#file, { uploadId: this.options.uploadId, key: this.options.key })
this.options.companionComm.restoreUploadFile(this.#file, {
uploadId: this.options.uploadId,
key: this.options.key,
})
this.#resumeUpload()
} else {
this.#createUpload()
}
}

pause () {
pause(): void {
this.#abortController.abort(pausingUploadReason)
// Swap it out for a new controller, because this instance may be resumed later.
this.#abortController = new AbortController()
}

abort (opts = undefined) {
abort(opts?: { really?: boolean }): void {
if (opts?.really) this.#abortUpload()
else this.pause()
}

// TODO: remove this in the next major
get chunkState () {
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
get chunkState() {
return this.#chunkState
}
}
Expand Down
113 changes: 0 additions & 113 deletions packages/@uppy/aws-s3-multipart/src/createSignedURL.test.js

This file was deleted.