feat(fs/S3#_writeFile): retry on remote internal error

@xen-orchestra/fs/package.json

@@ -21,7 +21,9 @@
     "@sindresorhus/df": "^3.1.1",
     "@sullux/aws-sdk": "^1.0.5",
     "@vates/coalesce-calls": "^0.1.0",
+    "@vates/decorate-with": "^1.0.0",
     "@xen-orchestra/async-map": "^0.1.2",
+    "@xen-orchestra/log": "^0.3.0",
     "aws-sdk": "^2.686.0",
     "decorator-synchronized": "^0.6.0",
     "execa": "^5.0.0",

@xen-orchestra/fs/src/s3.js

@@ -2,6 +2,9 @@ import aws from '@sullux/aws-sdk'
 import assert from 'assert'
 import http from 'http'
 import https from 'https'
+import pRetry from 'promise-toolbox/retry'
+import { createLogger } from '@xen-orchestra/log'
+import { decorateWith } from '@vates/decorate-with'
 import { parse } from 'xo-remote-parser'
 
 import RemoteHandlerAbstract from './abstract'
@@ -14,6 +17,9 @@ const MAX_PART_SIZE = 1024 * 1024 * 1024 * 5 // 5GB
 const MAX_PARTS_COUNT = 10000
 const MAX_OBJECT_SIZE = 1024 * 1024 * 1024 * 1024 * 5 // 5TB
 const IDEAL_FRAGMENT_SIZE = Math.ceil(MAX_OBJECT_SIZE / MAX_PARTS_COUNT) // the smallest fragment size that still allows a 5TB upload in 10000 fragments, about 524MB
+
+const { warn } = createLogger('xo:fs:s3')
+
 export default class S3Handler extends RemoteHandlerAbstract {
   constructor(remote, _opts) {
     super(remote)
@@ -117,6 +123,21 @@ export default class S3Handler extends RemoteHandlerAbstract {
     }
   }
 
+  // some objectstorage provider like backblaze, can answer a 500/503 routinely
+  // in this case we should retry,  and let their load balancing do its magic
+  // https://www.backblaze.com/b2/docs/calling.html#error_handling
+  @decorateWith(pRetry.wrap, {
+    delays: [100, 200, 500, 1000, 2000],
+    when: e => e.code === 'InternalError',
+    onRetry(error) {
+      warn('retrying writing file', {
+        attemptNumber: this.attemptNumber,
+        delay: this.delay,
+        error,
+        file: this.arguments[0],
+      })
+    },
+  })
   async _writeFile(file, data, options) {
     return this._s3.putObject({ ...this._createParams(file), Body: data })
   }

CHANGELOG.unreleased.md

@@ -10,6 +10,7 @@
 - Limit number of concurrent VM migrations per pool to `3` [#6065](https://github.com/vatesfr/xen-orchestra/issues/6065) (PR [#6076](https://github.com/vatesfr/xen-orchestra/pull/6076))
   Can be changed in `xo-server`'s configuration file: `xapiOptions.vmMigrationConcurrency`
 - [Proxy] Now ships a reverse proxy [PR#6072](https://github.com/vatesfr/xen-orchestra/pull/6072)
+- [Delta Backup] When using S3 remote, retry uploading VHD parts on Internal Error to support [Blackblaze](https://www.backblaze.com/b2/docs/calling.html#error_handling) (PR [#6086](https://github.com/vatesfr/xen-orchestra/issues/6086)) (Forum [5397](https://xcp-ng.org/forum/topic/5397/delta-backups-failing-aws-s3-uploadpartcopy-cpu-too-busy/5))
 
 ### Bug fixes
 
@@ -34,6 +35,8 @@
 >
 > In case of conflict, the highest (lowest in previous list) `$version` wins.
 
+- @xen-orchestra/fs minor
+- vhd-lib minor
 - @xen-orchestra/backups minor
 - @xen-orchestra/backups-cli minor
 - @xen-orchestra/proxy minor