feat(backups): add cache for backup metadata (#6220)

Fixes zammad#5747

Listing all the backup can be slow. To speed it up, the metadata of all the backups of each VM is cached in a single gzipped JSON file.

The cache is invalidated when a backup is deleted or created.

@xen-orchestra/backups/RemoteAdapter.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const { synchronized } = require('decorator-synchronized')
 const Disposable = require('promise-toolbox/Disposable')
 const fromCallback = require('promise-toolbox/fromCallback')
 const fromEvent = require('promise-toolbox/fromEvent')
@@ -17,6 +18,7 @@ const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
 const { v4: uuidv4 } = require('uuid')
 const { ZipFile } = require('yazl')
+const zlib = require('zlib')
 
 const { BACKUP_DIR } = require('./_getVmBackupDir.js')
 const { cleanVm } = require('./_cleanVm.js')
@@ -78,6 +80,7 @@ class RemoteAdapter {
     this._dirMode = dirMode
     this._handler = handler
     this._vhdDirectoryCompression = vhdDirectoryCompression
+    this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups)
   }
 
   get handler() {
@@ -261,7 +264,8 @@ class RemoteAdapter {
   }
 
   async deleteVmBackups(files) {
-    const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
+    const metadatas = await asyncMap(files, file => this.readVmBackupMetadata(file))
+    const { delta, full, ...others } = groupBy(metadatas, 'mode')
 
     const unsupportedModes = Object.keys(others)
     if (unsupportedModes.length !== 0) {
@@ -278,6 +282,9 @@ class RemoteAdapter {
       // don't merge in main process, unused VHDs will be merged in the next backup run
       await this.cleanVm(dir, { remove: true, onLog: warn })
     }
+
+    const dedupedVmUuid = new Set(metadatas.map(_ => _.vm.uuid))
+    await asyncMap(dedupedVmUuid, vmUuid => this.invalidateVmBackupListCache(vmUuid))
   }
 
   #getCompressionType() {
@@ -448,34 +455,94 @@ class RemoteAdapter {
     return backupsByPool
   }
 
-  async listVmBackups(vmUuid, predicate) {
+  async invalidateVmBackupListCache(vmUuid) {
+    await this.handler.unlink(`${BACKUP_DIR}/${vmUuid}/cache.json.gz`)
+  }
+
+  async #getCachabledDataListVmBackups(dir) {
     const handler = this._handler
-    const backups = []
+    const backups = {}
 
     try {
-      const files = await handler.list(`${BACKUP_DIR}/${vmUuid}`, {
+      const files = await handler.list(dir, {
         filter: isMetadataFile,
         prependDir: true,
       })
       await asyncMap(files, async file => {
         try {
           const metadata = await this.readVmBackupMetadata(file)
-          if (predicate === undefined || predicate(metadata)) {
-            // inject an id usable by importVmBackupNg()
-            metadata.id = metadata._filename
-
-            backups.push(metadata)
-          }
+          // inject an id usable by importVmBackupNg()
+          metadata.id = metadata._filename
+          backups[file] = metadata
         } catch (error) {
-          warn(`listVmBackups ${file}`, { error })
+          warn(`can't read vm backup metadata`, { error, file, dir })
         }
       })
+      return backups
     } catch (error) {
       let code
       if (error == null || ((code = error.code) !== 'ENOENT' && code !== 'ENOTDIR')) {
         throw error
       }
     }
+  }
+
+  // use _ to mark this method as private by convention
+  // since we decorate it with synchronized.withKey in the constructor
+  // and # function are not writeable.
+  //
+  // read the list of backup of a Vm from cache
+  // if cache is missing  or broken  => regenerate it and return
+
+  async _readCacheListVmBackups(vmUuid) {
+    const dir = `${BACKUP_DIR}/${vmUuid}`
+    const path = `${dir}/cache.json.gz`
+
+    try {
+      const gzipped = await this.handler.readFile(path)
+      const text = await fromCallback(zlib.gunzip, gzipped)
+      return JSON.parse(text)
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        warn('Cache file was unreadable', { vmUuid, error })
+      }
+    }
+
+    // nothing cached, or cache unreadable => regenerate it
+    const backups = await this.#getCachabledDataListVmBackups(dir)
+    if (backups === undefined) {
+      return
+    }
+
+    // detached async action, will not reject
+    this.#writeVmBackupsCache(path, backups)
+
+    return backups
+  }
+
+  async #writeVmBackupsCache(cacheFile, backups) {
+    try {
+      const text = JSON.stringify(backups)
+      const zipped = await fromCallback(zlib.gzip, text)
+      await this.handler.writeFile(cacheFile, zipped, { flags: 'w' })
+    } catch (error) {
+      warn('writeVmBackupsCache', { cacheFile, error })
+    }
+  }
+
+  async listVmBackups(vmUuid, predicate) {
+    const backups = []
+    const cached = await this._readCacheListVmBackups(vmUuid)
+
+    if (cached === undefined) {
+      return []
+    }
+
+    Object.values(cached).forEach(metadata => {
+      if (predicate === undefined || predicate(metadata)) {
+        backups.push(metadata)
+      }
+    })
 
     return backups.sort(compareTimestamp)
   }
@@ -584,7 +651,10 @@ class RemoteAdapter {
   }
 
   async readVmBackupMetadata(path) {
-    return Object.defineProperty(JSON.parse(await this._handler.readFile(path)), '_filename', { value: path })
+    // _filename is a private field used to compute the backup id
+    //
+    // it's enumerable to make it cacheable
+    return { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
   }
 }
 

@xen-orchestra/backups/package.json

@@ -27,6 +27,7 @@
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^4.0.1",
     "d3-time-format": "^3.0.0",
+    "decorator-synchronized": "^0.6.0",
     "end-of-stream": "^1.4.4",
     "fs-extra": "^10.0.0",
     "golike-defer": "^0.5.1",

@xen-orchestra/backups/writers/_MixinBackupWriter.js

@@ -64,5 +64,6 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
         const remotePath = handler._getRealPath()
         await MergeWorker.run(remotePath)
       }
+      await this._adapter.invalidateVmBackupListCache(this._backup.vm.uuid)
     }
   }

CHANGELOG.unreleased.md

@@ -8,6 +8,7 @@
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
 - [Backup] Merge multiple VHDs at once which will speed up the merging ĥase after reducing the retention of a backup job(PR [#6184](https://github.com/vatesfr/xen-orchestra/pull/6184))
+- [Backup] Implement file cache for listing the backups of a VM (PR [#6220](https://github.com/vatesfr/xen-orchestra/pull/6220))
 
 ### Bug fixes
 
@@ -38,8 +39,6 @@
 - vhd-lib patch
 - @xen-orchestra/fs patch
 - vhd-cli patch
-- @xen-orchestra/backups patch
-- xo-server patch
 - xo-vmdk-to-vhd minor
 - @xen-orchestra/upload-ova patch
 - @xen-orchestra/backups minor