From 9e667533e972e1c2f5247f1350185627f11e495e Mon Sep 17 00:00:00 2001
From: Florent BEAUCHAMP <florent.beauchamp@vates.fr>
Date: Mon, 2 Oct 2023 14:31:01 +0200
Subject: [PATCH] fix(fs/s3): throw error if upload >50GB

---
 @xen-orchestra/fs/src/s3.js | 25 +++++++++++++++++++++++--
 CHANGELOG.unreleased.md     |  2 ++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/@xen-orchestra/fs/src/s3.js b/@xen-orchestra/fs/src/s3.js
index ab5a4edd353..adce0bb97ab 100644
--- a/@xen-orchestra/fs/src/s3.js
+++ b/@xen-orchestra/fs/src/s3.js
@@ -17,7 +17,7 @@ import { getApplyMd5BodyChecksumPlugin } from '@aws-sdk/middleware-apply-body-ch
 import { Agent as HttpAgent } from 'http'
 import { Agent as HttpsAgent } from 'https'
 import { createLogger } from '@xen-orchestra/log'
-import { PassThrough, pipeline } from 'stream'
+import { PassThrough, Transform, pipeline } from 'stream'
 import { parse } from 'xo-remote-parser'
 import copyStreamToBuffer from './_copyStreamToBuffer.js'
 import guessAwsRegion from './_guessAwsRegion.js'
@@ -224,10 +224,29 @@ export default class S3Handler extends RemoteHandlerAbstract {
   }
 
   async _outputStream(path, input, { validator }) {
+    // S3 storage is limited to 10K part, each part is limited to 5GB. And the total upload must be smaller than 5TB
+    // a bigger partSize increase the memory consumption of aws/lib-storage exponentially
+    const MAX_PART = 10000
+    const PART_SIZE = 5 * 1024 * 1024
+    const MAX_SIZE = MAX_PART * PART_SIZE
+
+    // ensure we don't try to upload a stream to big for this partSize
+    let readCounter = 0
+    const streamCutter = new Transform({
+      transform(chunk, encoding, callback) {
+        readCounter += chunk.length
+        if (readCounter > MAX_SIZE) {
+          callback(new Error(`read ${readCounter} bytes, maximum size allowed  is ${MAX_SIZE} `))
+        } else {
+          callback(null, chunk)
+        }
+      },
+    })
+
     // Workaround for "ReferenceError: ReadableStream is not defined"
     // https://github.com/aws/aws-sdk-js-v3/issues/2522
     const Body = new PassThrough()
-    pipeline(input, Body, () => {})
+    pipeline(input, streamCutter, Body, () => {})
 
     const upload = new Upload({
       client: this.#s3,
@@ -235,6 +254,8 @@ export default class S3Handler extends RemoteHandlerAbstract {
         ...this.#createParams(path),
         Body,
       },
+      partSize: PART_SIZE,
+      leavePartsOnError: false,
     })
 
     await upload.done()
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
index 261055fd818..4e2f507affc 100644
--- a/CHANGELOG.unreleased.md
+++ b/CHANGELOG.unreleased.md
@@ -43,6 +43,8 @@
 
 <!--packages-start-->
 
+- @xen-orchestra/backups patch
+- @xen-orchestra/fs patch
 - @xen-orchestra/mixins minor
 - @xen-orchestra/xapi minor
 - xo-server minor