From fe0b42241ae84f41af5dc74378867f0b9e2076a8 Mon Sep 17 00:00:00 2001 From: Florent Beauchamp Date: Mon, 2 Jan 2023 16:25:37 +0100 Subject: [PATCH] fix(@xen-orchestra/backups): compute size of block based backups --- @xen-orchestra/backups/RemoteAdapter.js | 7 ++++--- .../backups/writers/DeltaBackupWriter.js | 2 +- @xen-orchestra/fs/src/abstract.js | 3 +++ packages/vhd-lib/Vhd/VhdAbstract.js | 20 +++++++++++++++++++ .../vhd-lib/createVhdDirectoryFromStream.js | 4 +++- 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/@xen-orchestra/backups/RemoteAdapter.js b/@xen-orchestra/backups/RemoteAdapter.js index 93fa26dd66d..0e411f35c08 100644 --- a/@xen-orchestra/backups/RemoteAdapter.js +++ b/@xen-orchestra/backups/RemoteAdapter.js @@ -661,7 +661,7 @@ class RemoteAdapter { const handler = this._handler if (this.#useVhdDirectory()) { const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd` - await createVhdDirectoryFromStream(handler, dataPath, input, { + const size = await createVhdDirectoryFromStream(handler, dataPath, input, { concurrency: writeBlockConcurrency, compression: this.#getCompressionType(), async validator() { @@ -671,13 +671,14 @@ class RemoteAdapter { nbdClient, }) await VhdAbstract.createAlias(handler, path, dataPath) + return size } else { - await this.outputStream(path, input, { checksum, validator }) + return this.outputStream(path, input, { checksum, validator }) } } async outputStream(path, input, { checksum = true, validator = noop } = {}) { - await this._handler.outputStream(path, input, { + return this._handler.outputStream(path, input, { checksum, dirMode: this._dirMode, async validator() { diff --git a/@xen-orchestra/backups/writers/DeltaBackupWriter.js b/@xen-orchestra/backups/writers/DeltaBackupWriter.js index c330132cbc6..5c36d826749 100644 --- a/@xen-orchestra/backups/writers/DeltaBackupWriter.js +++ b/@xen-orchestra/backups/writers/DeltaBackupWriter.js @@ -214,7 +214,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab } } - await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], { + sizeContainers[`${id}.vhd`].size = await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], { // no checksum for VHDs, because they will be invalidated by // merges and chainings checksum: false, diff --git a/@xen-orchestra/fs/src/abstract.js b/@xen-orchestra/fs/src/abstract.js index 88796943453..a984576188c 100644 --- a/@xen-orchestra/fs/src/abstract.js +++ b/@xen-orchestra/fs/src/abstract.js @@ -13,6 +13,7 @@ import { synchronized } from 'decorator-synchronized' import { basename, dirname, normalize as normalizePath } from './path' import { createChecksumStream, validChecksumOfReadStream } from './checksum' import { DEFAULT_ENCRYPTION_ALGORITHM, _getEncryptor } from './_encryptor' +import { watchStreamSize } from '@xen-orchestra/backups/_watchStreamSize' const { info, warn } = createLogger('xo:fs:abstract') @@ -185,6 +186,7 @@ export default class RemoteHandlerAbstract { async outputStream(path, input, { checksum = true, dirMode, validator } = {}) { path = normalizePath(path) let checksumStream + const container = watchStreamSize(input) input = this._encryptor.encryptStream(input) if (checksum) { @@ -201,6 +203,7 @@ export default class RemoteHandlerAbstract { // it is by design to allow checking of encrypted files without the key await this._outputFile(checksumFile(path), await checksumStream.checksum, { dirMode, flags: 'wx' }) } + return container.size } // Free the resources possibly dedicated to put the remote at work, when it diff --git a/packages/vhd-lib/Vhd/VhdAbstract.js b/packages/vhd-lib/Vhd/VhdAbstract.js index 6b76726784a..b832298c984 100644 --- a/packages/vhd-lib/Vhd/VhdAbstract.js +++ b/packages/vhd-lib/Vhd/VhdAbstract.js @@ -239,6 +239,26 @@ exports.VhdAbstract = class VhdAbstract { await handler.writeFile(aliasPath, relativePathToTarget) } + size() { + const { header, batSize } = this + let fileSize = FOOTER_SIZE + HEADER_SIZE + batSize + FOOTER_SIZE /* the footer at the end */ + + // add parentlocator size + for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) { + fileSize += header.parentLocatorEntry[i].platformDataSpace * SECTOR_SIZE + } + + // add block size + for (let i = 0; i < header.maxTableEntries; i++) { + if (this.containsBlock(i)) { + fileSize += this.fullBlockSize + } + } + + assert.strictEqual(fileSize % SECTOR_SIZE, 0) + return fileSize + } + stream() { const { footer, batSize } = this const { ...header } = this.header // copy since we don't ant to modifiy the current header diff --git a/packages/vhd-lib/createVhdDirectoryFromStream.js b/packages/vhd-lib/createVhdDirectoryFromStream.js index 1840cb4d9ae..dd61e0da3fc 100644 --- a/packages/vhd-lib/createVhdDirectoryFromStream.js +++ b/packages/vhd-lib/createVhdDirectoryFromStream.js @@ -38,6 +38,7 @@ const buildVhd = Disposable.wrap(async function* (handler, path, inputStream, { } ) await Promise.all([vhd.writeFooter(), vhd.writeHeader(), vhd.writeBlockAllocationTable()]) + return vhd.size() }) exports.createVhdDirectoryFromStream = async function createVhdDirectoryFromStream( @@ -47,10 +48,11 @@ exports.createVhdDirectoryFromStream = async function createVhdDirectoryFromStre { validator, concurrency = 16, compression, nbdClient } = {} ) { try { - await buildVhd(handler, path, inputStream, { concurrency, compression, nbdClient }) + const size = await buildVhd(handler, path, inputStream, { concurrency, compression, nbdClient }) if (validator !== undefined) { await validator.call(this, path) } + return size } catch (error) { // cleanup on error await handler.rmtree(path).catch(warn)