diff --git a/@xen-orchestra/backups/RemoteAdapter.js b/@xen-orchestra/backups/RemoteAdapter.js index 7039877b4ce..c8f16e9f08b 100644 --- a/@xen-orchestra/backups/RemoteAdapter.js +++ b/@xen-orchestra/backups/RemoteAdapter.js @@ -22,6 +22,7 @@ const zlib = require('zlib') const { BACKUP_DIR } = require('./_getVmBackupDir.js') const { cleanVm } = require('./_cleanVm.js') +const { formatFilenameDate } = require('./_filenameDate.js') const { getTmpDir } = require('./_getTmpDir.js') const { isMetadataFile } = require('./_backupType.js') const { isValidXva } = require('./_isValidXva.js') @@ -224,11 +225,31 @@ class RemoteAdapter { return promise } + #removeVmBackupsFromCache(backups) { + // will not throw + asyncMap( + Object.entries( + groupBy( + backups.map(_ => _._filename), + dirname + ) + ), + ([dir, filenames]) => + this.#updateCache(dir + '/cache.json.gz', backups => { + for (const filename of filenames) { + delete backups[filename] + } + }) + ) + } + async deleteDeltaVmBackups(backups) { const handler = this._handler // this will delete the json, unused VHDs will be detected by `cleanVm` await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename)) + + this.#removeVmBackupsFromCache(backups) } async deleteMetadataBackup(backupId) { @@ -256,6 +277,8 @@ class RemoteAdapter { await asyncMapSettled(backups, ({ _filename, xva }) => Promise.all([handler.unlink(_filename), handler.unlink(resolveRelativeFromFile(_filename, xva))]) ) + + this.#removeVmBackupsFromCache(backups) } deleteVmBackup(file) { @@ -281,9 +304,6 @@ class RemoteAdapter { // don't merge in main process, unused VHDs will be merged in the next backup run await this.cleanVm(dir, { remove: true, logWarn: warn }) } - - const dedupedVmUuid = new Set(metadatas.map(_ => _.vm.uuid)) - await asyncMap(dedupedVmUuid, vmUuid => this.invalidateVmBackupListCache(vmUuid)) } #getCompressionType() { @@ -458,8 +478,39 @@ class RemoteAdapter { return backupsByPool } + #getVmBackupsCache(vmUuid) { + return `${BACKUP_DIR}/${vmUuid}/cache.json.gz` + } + + async #readCache(path) { + try { + return JSON.parse(await fromCallback(zlib.gunzip, await this.handler.readFile(path))) + } catch (error) { + if (error.code !== 'ENOENT') { + warn('#readCache', { error, path }) + } + } + } + + async #updateCache(path, fn) { + const cache = await this.#readCache(path) + if (cache !== undefined) { + fn(cache) + + await this.#writeCache(path, cache) + } + } + + async #writeCache(path, data) { + try { + await this.handler.writeFile(path, await fromCallback(zlib.gzip, JSON.stringify(data)), { flags: 'w' }) + } catch (error) { + warn('#writeCache', { error, path }) + } + } + async invalidateVmBackupListCache(vmUuid) { - await this.handler.unlink(`${BACKUP_DIR}/${vmUuid}/cache.json.gz`) + await this.handler.unlink(this.#getVmBackupsCache(vmUuid)) } async #getCachabledDataListVmBackups(dir) { @@ -498,41 +549,25 @@ class RemoteAdapter { // if cache is missing or broken => regenerate it and return async _readCacheListVmBackups(vmUuid) { - const dir = `${BACKUP_DIR}/${vmUuid}` - const path = `${dir}/cache.json.gz` + const path = this.#getVmBackupsCache(vmUuid) - try { - const gzipped = await this.handler.readFile(path) - const text = await fromCallback(zlib.gunzip, gzipped) - return JSON.parse(text) - } catch (error) { - if (error.code !== 'ENOENT') { - warn('Cache file was unreadable', { vmUuid, error }) - } + const cache = await this.#readCache(path) + if (cache !== undefined) { + return cache } // nothing cached, or cache unreadable => regenerate it - const backups = await this.#getCachabledDataListVmBackups(dir) + const backups = await this.#getCachabledDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`) if (backups === undefined) { return } // detached async action, will not reject - this.#writeVmBackupsCache(path, backups) + this.#writeCache(path, backups) return backups } - async #writeVmBackupsCache(cacheFile, backups) { - try { - const text = JSON.stringify(backups) - const zipped = await fromCallback(zlib.gzip, text) - await this.handler.writeFile(cacheFile, zipped, { flags: 'w' }) - } catch (error) { - warn('writeVmBackupsCache', { cacheFile, error }) - } - } - async listVmBackups(vmUuid, predicate) { const backups = [] const cached = await this._readCacheListVmBackups(vmUuid) @@ -571,6 +606,21 @@ class RemoteAdapter { return backups.sort(compareTimestamp) } + async writeVmBackupMetadata(vmUuid, metadata) { + const path = `/${BACKUP_DIR}/${vmUuid}/${formatFilenameDate(metadata.timestamp)}.json` + + await this.handler.outputFile(path, JSON.stringify(metadata), { + dirMode: this._dirMode, + }) + + // will not throw + this.#updateCache(this.#getVmBackupsCache(vmUuid), backups => { + backups[path] = metadata + }) + + return path + } + async writeVhd(path, input, { checksum = true, validator = noop } = {}) { const handler = this._handler diff --git a/@xen-orchestra/backups/writers/DeltaBackupWriter.js b/@xen-orchestra/backups/writers/DeltaBackupWriter.js index 217bbbb4a5e..0b867997adb 100644 --- a/@xen-orchestra/backups/writers/DeltaBackupWriter.js +++ b/@xen-orchestra/backups/writers/DeltaBackupWriter.js @@ -189,7 +189,6 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab }/${adapter.getVhdFileName(basename)}` ) - const metadataFilename = (this._metadataFileName = `${backupDir}/${basename}.json`) const metadataContent = { jobId, mode: job.mode, @@ -254,9 +253,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab } }) metadataContent.size = size - await handler.outputFile(metadataFilename, JSON.stringify(metadataContent), { - dirMode: backup.config.dirMode, - }) + this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent) // TODO: run cleanup? } diff --git a/@xen-orchestra/backups/writers/FullBackupWriter.js b/@xen-orchestra/backups/writers/FullBackupWriter.js index e965aed04f1..079990928f7 100644 --- a/@xen-orchestra/backups/writers/FullBackupWriter.js +++ b/@xen-orchestra/backups/writers/FullBackupWriter.js @@ -34,7 +34,6 @@ exports.FullBackupWriter = class FullBackupWriter extends MixinBackupWriter(Abst const { job, scheduleId, vm } = backup const adapter = this._adapter - const handler = adapter.handler const backupDir = getVmBackupDir(vm.uuid) // TODO: clean VM backup directory @@ -50,7 +49,6 @@ exports.FullBackupWriter = class FullBackupWriter extends MixinBackupWriter(Abst const dataBasename = basename + '.xva' const dataFilename = backupDir + '/' + dataBasename - const metadataFilename = `${backupDir}/${basename}.json` const metadata = { jobId: job.id, mode: job.mode, @@ -74,9 +72,7 @@ exports.FullBackupWriter = class FullBackupWriter extends MixinBackupWriter(Abst return { size: sizeContainer.size } }) metadata.size = sizeContainer.size - await handler.outputFile(metadataFilename, JSON.stringify(metadata), { - dirMode: backup.config.dirMode, - }) + await adapter.writeVmBackupMetadata(vm.uuid, metadata) if (!deleteFirst) { await deleteOldBackups() diff --git a/@xen-orchestra/backups/writers/_MixinBackupWriter.js b/@xen-orchestra/backups/writers/_MixinBackupWriter.js index d5ddf6af5b9..baeb283bba2 100644 --- a/@xen-orchestra/backups/writers/_MixinBackupWriter.js +++ b/@xen-orchestra/backups/writers/_MixinBackupWriter.js @@ -71,6 +71,5 @@ exports.MixinBackupWriter = (BaseClass = Object) => const remotePath = handler._getRealPath() await MergeWorker.run(remotePath) } - await this._adapter.invalidateVmBackupListCache(this._backup.vm.uuid) } }