From ba1b1b21189ab3cd6ba72962bf5daac367a97ac4 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 21 May 2026 17:20:04 +0400 Subject: [PATCH 1/4] feat: support concurrent chunk uploads --- src/client.ts | 214 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 170 insertions(+), 44 deletions(-) diff --git a/src/client.ts b/src/client.ts index 37bb016..42e0598 100644 --- a/src/client.ts +++ b/src/client.ts @@ -415,82 +415,208 @@ class Client { return await this.call(method, url, headers, payload); } - let start = 0; - let response = null; + const totalChunks = Math.ceil(size / Client.CHUNK_SIZE); - while (start < size) { - let end = start + Client.CHUNK_SIZE; - if (end >= size) { - end = size; - } + // Upload first chunk alone to get the upload ID + const firstChunkEnd = Math.min(Client.CHUNK_SIZE, size); + const firstChunkHeaders = { ...headers, 'content-range': `bytes 0-${firstChunkEnd - 1}/${size}` }; + const firstChunk = await file.slice(0, firstChunkEnd); + const firstPayload = { ...originalPayload }; + firstPayload[fileParam] = new File([firstChunk], file.filename); - headers['content-range'] = `bytes ${start}-${end - 1}/${size}`; - const chunk = await file.slice(start, end); + let response = await this.call(method, url, firstChunkHeaders, firstPayload); + const uploadId = response?.$id; - const payload = { ...originalPayload }; - payload[fileParam] = new File([chunk], file.filename); + if (onProgress && typeof onProgress === 'function') { + onProgress({ + $id: uploadId, + progress: Math.round((firstChunkEnd / size) * 100), + sizeUploaded: firstChunkEnd, + chunksTotal: totalChunks, + chunksUploaded: 1 + }); + } + + if (totalChunks === 1) { + return response; + } - response = await this.call(method, url, headers, payload); + // Prepare remaining chunks + const chunks: { index: number; start: number; end: number }[] = []; + for (let i = 1; i < totalChunks; i++) { + const start = i * Client.CHUNK_SIZE; + const end = Math.min(start + Client.CHUNK_SIZE, size); + chunks.push({ index: i, start, end }); + } + + // Upload remaining chunks with max concurrency of 8 + const CONCURRENCY = 8; + let completedCount = 1; + let uploadedBytes = firstChunkEnd; + let lastResponse = response; + + const isUploadComplete = (chunkResponse: any) => { + const chunksUploaded = chunkResponse?.chunksUploaded; + const chunksTotal = chunkResponse?.chunksTotal ?? totalChunks; + return typeof chunksUploaded === 'number' && typeof chunksTotal === 'number' && chunksUploaded >= chunksTotal; + }; + + const uploadChunk = async (chunk: typeof chunks[0]) => { + const chunkHeaders = { ...headers }; + if (uploadId) { + chunkHeaders['x-appwrite-id'] = uploadId; + } + chunkHeaders['content-range'] = `bytes ${chunk.start}-${chunk.end - 1}/${size}`; + + const chunkBlob = await file.slice(chunk.start, chunk.end); + const chunkPayload = { ...originalPayload }; + chunkPayload[fileParam] = new File([chunkBlob], file.filename); + + const chunkResponse = await this.call(method, url, chunkHeaders, chunkPayload); + + completedCount++; + uploadedBytes += (chunk.end - chunk.start); + + if (isUploadComplete(chunkResponse)) { + lastResponse = chunkResponse; + } if (onProgress && typeof onProgress === 'function') { onProgress({ - $id: response.$id, - progress: Math.round((end / size) * 100), - sizeUploaded: end, - chunksTotal: Math.ceil(size / Client.CHUNK_SIZE), - chunksUploaded: Math.ceil(end / Client.CHUNK_SIZE) + $id: uploadId, + progress: Math.round((uploadedBytes / size) * 100), + sizeUploaded: uploadedBytes, + chunksTotal: totalChunks, + chunksUploaded: completedCount }); } - if (response && response.$id) { - headers['x-appwrite-id'] = response.$id; - } + return chunkResponse; + }; - start = end; + // Process with limited concurrency using a worker pool + const queue = [...chunks]; + const workers: Promise[] = []; + const workerCount = Math.min(CONCURRENCY, queue.length); + + for (let i = 0; i < workerCount; i++) { + workers.push( + (async () => { + while (queue.length > 0) { + const chunk = queue.shift()!; + await uploadChunk(chunk); + } + })() + ); } - return response; + await Promise.all(workers); + + return lastResponse; } if (file.size <= Client.CHUNK_SIZE) { return await this.call(method, url, headers, originalPayload); } - let start = 0; - let response = null; + const totalChunks = Math.ceil(file.size / Client.CHUNK_SIZE); + + // Upload first chunk alone to get the upload ID + const firstChunkEnd = Math.min(Client.CHUNK_SIZE, file.size); + const firstChunkHeaders = { ...headers, 'content-range': `bytes 0-${firstChunkEnd - 1}/${file.size}` }; + const firstChunk = file.slice(0, firstChunkEnd); + const firstPayload = { ...originalPayload }; + firstPayload[fileParam] = new File([firstChunk], file.name); + + let response = await this.call(method, url, firstChunkHeaders, firstPayload); + const uploadId = response?.$id; + + if (onProgress && typeof onProgress === 'function') { + onProgress({ + $id: uploadId, + progress: Math.round((firstChunkEnd / file.size) * 100), + sizeUploaded: firstChunkEnd, + chunksTotal: totalChunks, + chunksUploaded: 1 + }); + } + + if (totalChunks === 1) { + return response; + } - while (start < file.size) { - let end = start + Client.CHUNK_SIZE; // Prepare end for the next chunk - if (end >= file.size) { - end = file.size; // Adjust for the last chunk to include the last byte - } + // Prepare remaining chunks + const chunks: { index: number; start: number; end: number }[] = []; + for (let i = 1; i < totalChunks; i++) { + const start = i * Client.CHUNK_SIZE; + const end = Math.min(start + Client.CHUNK_SIZE, file.size); + chunks.push({ index: i, start, end }); + } - headers['content-range'] = `bytes ${start}-${end-1}/${file.size}`; - const chunk = file.slice(start, end); + // Upload remaining chunks with max concurrency of 8 + const CONCURRENCY = 8; + let completedCount = 1; + let uploadedBytes = firstChunkEnd; + let lastResponse = response; - let payload = { ...originalPayload }; - payload[fileParam] = new File([chunk], file.name); + const isUploadComplete = (chunkResponse: any) => { + const chunksUploaded = chunkResponse?.chunksUploaded; + const chunksTotal = chunkResponse?.chunksTotal ?? totalChunks; + return typeof chunksUploaded === 'number' && typeof chunksTotal === 'number' && chunksUploaded >= chunksTotal; + }; - response = await this.call(method, url, headers, payload); + const uploadChunk = async (chunk: typeof chunks[0]) => { + const chunkHeaders = { ...headers }; + if (uploadId) { + chunkHeaders['x-appwrite-id'] = uploadId; + } + chunkHeaders['content-range'] = `bytes ${chunk.start}-${chunk.end - 1}/${file.size}`; + + const chunkBlob = file.slice(chunk.start, chunk.end); + const chunkPayload = { ...originalPayload }; + chunkPayload[fileParam] = new File([chunkBlob], file.name); + + const chunkResponse = await this.call(method, url, chunkHeaders, chunkPayload); + + completedCount++; + uploadedBytes += (chunk.end - chunk.start); + + if (isUploadComplete(chunkResponse)) { + lastResponse = chunkResponse; + } if (onProgress && typeof onProgress === 'function') { onProgress({ - $id: response.$id, - progress: Math.round((end / file.size) * 100), - sizeUploaded: end, - chunksTotal: Math.ceil(file.size / Client.CHUNK_SIZE), - chunksUploaded: Math.ceil(end / Client.CHUNK_SIZE) + $id: uploadId, + progress: Math.round((uploadedBytes / file.size) * 100), + sizeUploaded: uploadedBytes, + chunksTotal: totalChunks, + chunksUploaded: completedCount }); } - if (response && response.$id) { - headers['x-appwrite-id'] = response.$id; - } + return chunkResponse; + }; - start = end; + // Process with limited concurrency using a worker pool + const queue = [...chunks]; + const workers: Promise[] = []; + const workerCount = Math.min(CONCURRENCY, queue.length); + + for (let i = 0; i < workerCount; i++) { + workers.push( + (async () => { + while (queue.length > 0) { + const chunk = queue.shift()!; + await uploadChunk(chunk); + } + })() + ); } - return response; + await Promise.all(workers); + + return lastResponse; } async ping(): Promise { From 5c0095babde8370d460469b9d1ea67b8a35c6be8 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 21 May 2026 20:46:02 +0400 Subject: [PATCH 2/4] feat: support concurrent chunk uploads --- src/client.ts | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/client.ts b/src/client.ts index 42e0598..766776c 100644 --- a/src/client.ts +++ b/src/client.ts @@ -442,11 +442,11 @@ class Client { } // Prepare remaining chunks - const chunks: { index: number; start: number; end: number }[] = []; + const chunks: { start: number; end: number }[] = []; for (let i = 1; i < totalChunks; i++) { const start = i * Client.CHUNK_SIZE; const end = Math.min(start + Client.CHUNK_SIZE, size); - chunks.push({ index: i, start, end }); + chunks.push({ start, end }); } // Upload remaining chunks with max concurrency of 8 @@ -454,6 +454,7 @@ class Client { let completedCount = 1; let uploadedBytes = firstChunkEnd; let lastResponse = response; + let finalResponse = null; const isUploadComplete = (chunkResponse: any) => { const chunksUploaded = chunkResponse?.chunksUploaded; @@ -477,8 +478,9 @@ class Client { completedCount++; uploadedBytes += (chunk.end - chunk.start); + lastResponse = chunkResponse; if (isUploadComplete(chunkResponse)) { - lastResponse = chunkResponse; + finalResponse = chunkResponse; } if (onProgress && typeof onProgress === 'function') { @@ -512,7 +514,7 @@ class Client { await Promise.all(workers); - return lastResponse; + return finalResponse ?? lastResponse; } if (file.size <= Client.CHUNK_SIZE) { @@ -546,11 +548,11 @@ class Client { } // Prepare remaining chunks - const chunks: { index: number; start: number; end: number }[] = []; + const chunks: { start: number; end: number }[] = []; for (let i = 1; i < totalChunks; i++) { const start = i * Client.CHUNK_SIZE; const end = Math.min(start + Client.CHUNK_SIZE, file.size); - chunks.push({ index: i, start, end }); + chunks.push({ start, end }); } // Upload remaining chunks with max concurrency of 8 @@ -558,6 +560,7 @@ class Client { let completedCount = 1; let uploadedBytes = firstChunkEnd; let lastResponse = response; + let finalResponse = null; const isUploadComplete = (chunkResponse: any) => { const chunksUploaded = chunkResponse?.chunksUploaded; @@ -581,8 +584,9 @@ class Client { completedCount++; uploadedBytes += (chunk.end - chunk.start); + lastResponse = chunkResponse; if (isUploadComplete(chunkResponse)) { - lastResponse = chunkResponse; + finalResponse = chunkResponse; } if (onProgress && typeof onProgress === 'function') { @@ -616,10 +620,10 @@ class Client { await Promise.all(workers); - return lastResponse; + return finalResponse ?? lastResponse; } - async ping(): Promise { + async ping(): Promise { return this.call('GET', new URL(this.config.endpoint + '/ping')); } From 46c3e211b1723132c90ef35a77ede006d14b4999 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 21 May 2026 21:10:31 +0400 Subject: [PATCH 3/4] feat: support concurrent chunk uploads --- src/client.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/client.ts b/src/client.ts index 766776c..585cc3e 100644 --- a/src/client.ts +++ b/src/client.ts @@ -500,13 +500,19 @@ class Client { const queue = [...chunks]; const workers: Promise[] = []; const workerCount = Math.min(CONCURRENCY, queue.length); + let failed = false; for (let i = 0; i < workerCount; i++) { workers.push( (async () => { - while (queue.length > 0) { + while (!failed && queue.length > 0) { const chunk = queue.shift()!; - await uploadChunk(chunk); + try { + await uploadChunk(chunk); + } catch (error) { + failed = true; + throw error; + } } })() ); @@ -606,13 +612,19 @@ class Client { const queue = [...chunks]; const workers: Promise[] = []; const workerCount = Math.min(CONCURRENCY, queue.length); + let failed = false; for (let i = 0; i < workerCount; i++) { workers.push( (async () => { - while (queue.length > 0) { + while (!failed && queue.length > 0) { const chunk = queue.shift()!; - await uploadChunk(chunk); + try { + await uploadChunk(chunk); + } catch (error) { + failed = true; + throw error; + } } })() ); From 294b096250abd5ce8186931ccda26464bb160a82 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 21 May 2026 21:33:01 +0400 Subject: [PATCH 4/4] feat: support concurrent chunk uploads --- src/client.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/client.ts b/src/client.ts index 585cc3e..50724f6 100644 --- a/src/client.ts +++ b/src/client.ts @@ -455,6 +455,7 @@ class Client { let uploadedBytes = firstChunkEnd; let lastResponse = response; let finalResponse = null; + let failed = false; const isUploadComplete = (chunkResponse: any) => { const chunksUploaded = chunkResponse?.chunksUploaded; @@ -474,6 +475,10 @@ class Client { chunkPayload[fileParam] = new File([chunkBlob], file.filename); const chunkResponse = await this.call(method, url, chunkHeaders, chunkPayload); + + if (failed) { + return chunkResponse; + } completedCount++; uploadedBytes += (chunk.end - chunk.start); @@ -500,7 +505,6 @@ class Client { const queue = [...chunks]; const workers: Promise[] = []; const workerCount = Math.min(CONCURRENCY, queue.length); - let failed = false; for (let i = 0; i < workerCount; i++) { workers.push( @@ -567,6 +571,7 @@ class Client { let uploadedBytes = firstChunkEnd; let lastResponse = response; let finalResponse = null; + let failed = false; const isUploadComplete = (chunkResponse: any) => { const chunksUploaded = chunkResponse?.chunksUploaded; @@ -586,6 +591,10 @@ class Client { chunkPayload[fileParam] = new File([chunkBlob], file.name); const chunkResponse = await this.call(method, url, chunkHeaders, chunkPayload); + + if (failed) { + return chunkResponse; + } completedCount++; uploadedBytes += (chunk.end - chunk.start); @@ -612,7 +621,6 @@ class Client { const queue = [...chunks]; const workers: Promise[] = []; const workerCount = Math.min(CONCURRENCY, queue.length); - let failed = false; for (let i = 0; i < workerCount; i++) { workers.push( @@ -635,7 +643,7 @@ class Client { return finalResponse ?? lastResponse; } - async ping(): Promise { + async ping(): Promise { return this.call('GET', new URL(this.config.endpoint + '/ping')); }