feat(backups): can limit parallel VDI transfers per VM per job

@xen-orchestra/backups/_runners/VmsXapi.js

@@ -19,6 +19,7 @@ const DEFAULT_XAPI_VM_SETTINGS = {
   concurrency: 2,
   copyRetention: 0,
   deleteFirst: false,
+  diskPerVmConcurrency: 0, // not limited by default
   exportRetention: 0,
   fullInterval: 0,
   healthCheckSr: undefined,

@xen-orchestra/backups/_runners/_writers/IncrementalRemoteWriter.js

@@ -1,9 +1,9 @@
 'use strict'
 
 const assert = require('assert')
-const map = require('lodash/map.js')
 const mapValues = require('lodash/mapValues.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
+const { asyncEach } = require('@vates/async-each')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')
@@ -138,7 +138,7 @@ class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWrite
     const adapter = this._adapter
     const job = this._job
     const scheduleId = this._scheduleId
-
+    const settings = this._settings
     const jobId = job.id
     const handler = adapter.handler
 
@@ -176,8 +176,9 @@ class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWrite
     }
     const { size } = await Task.run({ name: 'transfer' }, async () => {
       let transferSize = 0
-      await Promise.all(
-        map(deltaExport.vdis, async (vdi, id) => {
+      await asyncEach(
+        Object.entries(deltaExport.vdis),
+        async ([id, vdi]) => {
           const path = `${this._vmBackupDir}/${vhds[id]}`
 
           const isDelta = differentialVhds[`${id}.vhd`]
@@ -223,8 +224,12 @@ class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWrite
             await vhd.readBlockAllocationTable() // required by writeFooter()
             await vhd.writeFooter()
           })
-        })
+        },
+        {
+          concurrency: settings.diskPerVmConcurrency,
+        }
       )
+
       return { size: transferSize }
     })
     metadataContent.size = size

CHANGELOG.unreleased.md

@@ -13,6 +13,7 @@
 - [Dashboard/Health] Button to copy UUID of an orphan VDI to the clipboard (PR [#6893](https://github.com/vatesfr/xen-orchestra/pull/6893))
 - [Kubernetes recipe] Add the possibility to choose the version for the cluster [#6842](https://github.com/vatesfr/xen-orchestra/issues/6842) (PR [#6880](https://github.com/vatesfr/xen-orchestra/pull/6880))
 - [New VM] cloud-init drives are now bootable in a Windows VM (PR [#6889](https://github.com/vatesfr/xen-orchestra/pull/6889))
+- [Backups] Add setting `backups.metadata.defaultSettings.diskPerVmConcurrency` in xo-server's configuration file to limit the number of disks transferred in parallel per VM, this is useful to avoid transfer overloading remote and Sr (PR [#6787](https://github.com/vatesfr/xen-orchestra/pull/6787))
 
 ### Bug fixes