feat(backup): make merge block concurrency and write block concurrenc…

…y configurable
vatesfr · Sep 12, 2022 · bc9bfe4 · bc9bfe4
1 parent 9d09a3a
commit bc9bfe4
Show file tree

Hide file tree

Showing 6 changed files with 37 additions and 7 deletions.
diff --git a/@xen-orchestra/backups/RemoteAdapter.js b/@xen-orchestra/backups/RemoteAdapter.js
@@ -634,13 +634,13 @@ class RemoteAdapter {
     return path
   }
 
-  async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
+  async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency } = {}) {
     const handler = this._handler
 
     if (this.#useVhdDirectory()) {
       const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
       await createVhdDirectoryFromStream(handler, dataPath, input, {
-        concurrency: 16,
+        concurrency: writeBlockConcurrency,
         compression: this.#getCompressionType(),
         async validator() {
           await input.task

diff --git a/@xen-orchestra/backups/_cleanVm.js b/@xen-orchestra/backups/_cleanVm.js
@@ -37,7 +37,7 @@ const computeVhdsSize = (handler, vhdPaths) =>
   )
 
 // chain is [ ancestor, child_1, ..., child_n ]
-async function _mergeVhdChain(handler, chain, { logInfo, remove, merge }) {
+async function _mergeVhdChain(handler, chain, { logInfo, remove, merge, mergeBlockConcurrency }) {
   if (merge) {
     logInfo(`merging VHD chain`, { chain })
 
@@ -55,6 +55,7 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, merge }) {
     try {
       return await mergeVhdChain(handler, chain, {
         logInfo,
+        mergeBlockConcurrency,
         onProgress({ done: d, total: t }) {
           done = d
           total = t
@@ -181,7 +182,15 @@ const defaultMergeLimiter = limitConcurrency(1)
 
 exports.cleanVm = async function cleanVm(
   vmDir,
-  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, logInfo = noop, logWarn = console.warn }
+  {
+    fixMetadata,
+    remove,
+    merge,
+    mergeBlockConcurrency,
+    mergeLimiter = defaultMergeLimiter,
+    logInfo = noop,
+    logWarn = console.warn,
+  }
 ) {
   const limitedMergeVhdChain = mergeLimiter(_mergeVhdChain)
 
@@ -447,7 +456,13 @@ exports.cleanVm = async function cleanVm(
   const metadataWithMergedVhd = {}
   const doMerge = async () => {
     await asyncMap(toMerge, async chain => {
-      const merged = await limitedMergeVhdChain(handler, chain, { logInfo, logWarn, remove, merge })
+      const merged = await limitedMergeVhdChain(handler, chain, {
+        logInfo,
+        logWarn,
+        remove,
+        merge,
+        mergeBlockConcurrency,
+      })
       if (merged !== undefined) {
         const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
         metadataWithMergedVhd[metadataPath] = true

diff --git a/@xen-orchestra/backups/writers/DeltaBackupWriter.js b/@xen-orchestra/backups/writers/DeltaBackupWriter.js
@@ -204,6 +204,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
             // merges and chainings
             checksum: false,
             validator: tmpPath => checkVhd(handler, tmpPath),
+            writeBlockConcurrency: this._backup.config.writeBlockConcurrency,
           })
 
           if (isDelta) {

diff --git a/@xen-orchestra/backups/writers/_MixinBackupWriter.js b/@xen-orchestra/backups/writers/_MixinBackupWriter.js
@@ -39,6 +39,7 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
               Task.warning(message, data)
             },
             lock: false,
+            mergeBlockConcurrency: this._backup.config.mergeBlockConcurrency,
           })
         })
       } catch (error) {

diff --git a/packages/vhd-lib/merge.js b/packages/vhd-lib/merge.js
@@ -78,7 +78,7 @@ module.exports._cleanupVhds = cleanupVhds
 module.exports.mergeVhdChain = limitConcurrency(2)(async function mergeVhdChain(
   handler,
   chain,
-  { onProgress = noop, logInfo = noop, removeUnused = false } = {}
+  { onProgress = noop, logInfo = noop, removeUnused = false, mergeBlockConcurrency = 2 } = {}
 ) {
   assert(chain.length >= 2)
 
@@ -123,7 +123,8 @@ module.exports.mergeVhdChain = limitConcurrency(2)(async function mergeVhdChain(
       childIsVhdDirectory = childVhd instanceof VhdDirectory
     }
 
-    const concurrency = parentIsVhdDirectory && childIsVhdDirectory ? 2 : 1
+    // merging vhdFile must not be concurrently with the potential block reordering after a change
+    const concurrency = parentIsVhdDirectory && childIsVhdDirectory ? mergeBlockConcurrency : 1
     if (mergeState === undefined) {
       // merge should be along a vhd chain
       assert.strictEqual(UUID.stringify(childVhd.header.parentUuid), UUID.stringify(parentVhd.footer.uuid))

diff --git a/packages/xo-server/config.toml b/packages/xo-server/config.toml
@@ -78,6 +78,7 @@ throttlingDelay = '2 seconds'
 [backups]
 disableMergeWorker = false
 
+
 # Mode to use for newly created backup directories
 #
 # https://en.wikipedia.org/wiki/File-system_permissions#Numeric_notation
@@ -87,8 +88,19 @@ snapshotNameLabelTpl = '[XO Backup {job.name}] {vm.name_label}'
 # Delay for which backups listing on a remote is cached
 listingDebounce = '1 min'
 
+# settings when using Vhd directories ( s3 , encryption )
+# you should use 'none' if your fs is already compressed
+# changing this setting will generate new full backups
 vhdDirectoryCompression = 'brotli'
 
+# how many block can be merged in parallel per backup running
+# increase to increase performance, reduce if you have timeout during merge
+mergeBlockConcurrency = 2
+
+# how many block can be uploaded in parallel
+# increase to in rease performance, reduce if you have timeout or memory error during transfer
+writeBlockConcurrency = 16
+
 # This is a work-around.
 #
 # See https://github.com/vatesfr/xen-orchestra/pull/4674