From 8bf3a747f0d5ae99a170b989eb44408bff99f1f6 Mon Sep 17 00:00:00 2001 From: Florent BEAUCHAMP Date: Tue, 17 May 2022 10:43:00 +0200 Subject: [PATCH] feat(backups): add cache for backup metadata (#6220) Fixes zammad#5747 Listing all the backup can be slow. To speed it up, the metadata of all the backups of each VM is cached in a single gzipped JSON file. The cache is invalidated when a backup is deleted or created. --- @xen-orchestra/backups/RemoteAdapter.js | 94 ++++++++++++++++--- @xen-orchestra/backups/package.json | 1 + .../backups/writers/_MixinBackupWriter.js | 1 + CHANGELOG.unreleased.md | 3 +- 4 files changed, 85 insertions(+), 14 deletions(-) diff --git a/@xen-orchestra/backups/RemoteAdapter.js b/@xen-orchestra/backups/RemoteAdapter.js index 0d947916e72..605e3335410 100644 --- a/@xen-orchestra/backups/RemoteAdapter.js +++ b/@xen-orchestra/backups/RemoteAdapter.js @@ -1,6 +1,7 @@ 'use strict' const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map') +const { synchronized } = require('decorator-synchronized') const Disposable = require('promise-toolbox/Disposable') const fromCallback = require('promise-toolbox/fromCallback') const fromEvent = require('promise-toolbox/fromEvent') @@ -17,6 +18,7 @@ const { execFile } = require('child_process') const { readdir, stat } = require('fs-extra') const { v4: uuidv4 } = require('uuid') const { ZipFile } = require('yazl') +const zlib = require('zlib') const { BACKUP_DIR } = require('./_getVmBackupDir.js') const { cleanVm } = require('./_cleanVm.js') @@ -78,6 +80,7 @@ class RemoteAdapter { this._dirMode = dirMode this._handler = handler this._vhdDirectoryCompression = vhdDirectoryCompression + this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups) } get handler() { @@ -261,7 +264,8 @@ class RemoteAdapter { } async deleteVmBackups(files) { - const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode') + const metadatas = await asyncMap(files, file => this.readVmBackupMetadata(file)) + const { delta, full, ...others } = groupBy(metadatas, 'mode') const unsupportedModes = Object.keys(others) if (unsupportedModes.length !== 0) { @@ -278,6 +282,9 @@ class RemoteAdapter { // don't merge in main process, unused VHDs will be merged in the next backup run await this.cleanVm(dir, { remove: true, onLog: warn }) } + + const dedupedVmUuid = new Set(metadatas.map(_ => _.vm.uuid)) + await asyncMap(dedupedVmUuid, vmUuid => this.invalidateVmBackupListCache(vmUuid)) } #getCompressionType() { @@ -448,34 +455,94 @@ class RemoteAdapter { return backupsByPool } - async listVmBackups(vmUuid, predicate) { + async invalidateVmBackupListCache(vmUuid) { + await this.handler.unlink(`${BACKUP_DIR}/${vmUuid}/cache.json.gz`) + } + + async #getCachabledDataListVmBackups(dir) { const handler = this._handler - const backups = [] + const backups = {} try { - const files = await handler.list(`${BACKUP_DIR}/${vmUuid}`, { + const files = await handler.list(dir, { filter: isMetadataFile, prependDir: true, }) await asyncMap(files, async file => { try { const metadata = await this.readVmBackupMetadata(file) - if (predicate === undefined || predicate(metadata)) { - // inject an id usable by importVmBackupNg() - metadata.id = metadata._filename - - backups.push(metadata) - } + // inject an id usable by importVmBackupNg() + metadata.id = metadata._filename + backups[file] = metadata } catch (error) { - warn(`listVmBackups ${file}`, { error }) + warn(`can't read vm backup metadata`, { error, file, dir }) } }) + return backups } catch (error) { let code if (error == null || ((code = error.code) !== 'ENOENT' && code !== 'ENOTDIR')) { throw error } } + } + + // use _ to mark this method as private by convention + // since we decorate it with synchronized.withKey in the constructor + // and # function are not writeable. + // + // read the list of backup of a Vm from cache + // if cache is missing or broken => regenerate it and return + + async _readCacheListVmBackups(vmUuid) { + const dir = `${BACKUP_DIR}/${vmUuid}` + const path = `${dir}/cache.json.gz` + + try { + const gzipped = await this.handler.readFile(path) + const text = await fromCallback(zlib.gunzip, gzipped) + return JSON.parse(text) + } catch (error) { + if (error.code !== 'ENOENT') { + warn('Cache file was unreadable', { vmUuid, error }) + } + } + + // nothing cached, or cache unreadable => regenerate it + const backups = await this.#getCachabledDataListVmBackups(dir) + if (backups === undefined) { + return + } + + // detached async action, will not reject + this.#writeVmBackupsCache(path, backups) + + return backups + } + + async #writeVmBackupsCache(cacheFile, backups) { + try { + const text = JSON.stringify(backups) + const zipped = await fromCallback(zlib.gzip, text) + await this.handler.writeFile(cacheFile, zipped, { flags: 'w' }) + } catch (error) { + warn('writeVmBackupsCache', { cacheFile, error }) + } + } + + async listVmBackups(vmUuid, predicate) { + const backups = [] + const cached = await this._readCacheListVmBackups(vmUuid) + + if (cached === undefined) { + return [] + } + + Object.values(cached).forEach(metadata => { + if (predicate === undefined || predicate(metadata)) { + backups.push(metadata) + } + }) return backups.sort(compareTimestamp) } @@ -584,7 +651,10 @@ class RemoteAdapter { } async readVmBackupMetadata(path) { - return Object.defineProperty(JSON.parse(await this._handler.readFile(path)), '_filename', { value: path }) + // _filename is a private field used to compute the backup id + // + // it's enumerable to make it cacheable + return { ...JSON.parse(await this._handler.readFile(path)), _filename: path } } } diff --git a/@xen-orchestra/backups/package.json b/@xen-orchestra/backups/package.json index 4a62d478575..9a17470d7aa 100644 --- a/@xen-orchestra/backups/package.json +++ b/@xen-orchestra/backups/package.json @@ -27,6 +27,7 @@ "@xen-orchestra/template": "^0.1.0", "compare-versions": "^4.0.1", "d3-time-format": "^3.0.0", + "decorator-synchronized": "^0.6.0", "end-of-stream": "^1.4.4", "fs-extra": "^10.0.0", "golike-defer": "^0.5.1", diff --git a/@xen-orchestra/backups/writers/_MixinBackupWriter.js b/@xen-orchestra/backups/writers/_MixinBackupWriter.js index 159f8400bc4..d5817a417e1 100644 --- a/@xen-orchestra/backups/writers/_MixinBackupWriter.js +++ b/@xen-orchestra/backups/writers/_MixinBackupWriter.js @@ -64,5 +64,6 @@ exports.MixinBackupWriter = (BaseClass = Object) => const remotePath = handler._getRealPath() await MergeWorker.run(remotePath) } + await this._adapter.invalidateVmBackupListCache(this._backup.vm.uuid) } } diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index f90448d1a6e..1e36721b6e6 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -8,6 +8,7 @@ > Users must be able to say: “Nice enhancement, I'm eager to test it” - [Backup] Merge multiple VHDs at once which will speed up the merging ĥase after reducing the retention of a backup job(PR [#6184](https://github.com/vatesfr/xen-orchestra/pull/6184)) +- [Backup] Implement file cache for listing the backups of a VM (PR [#6220](https://github.com/vatesfr/xen-orchestra/pull/6220)) ### Bug fixes @@ -38,8 +39,6 @@ - vhd-lib patch - @xen-orchestra/fs patch - vhd-cli patch -- @xen-orchestra/backups patch -- xo-server patch - xo-vmdk-to-vhd minor - @xen-orchestra/upload-ova patch - @xen-orchestra/backups minor