From d056dfb18020f0249a184ae421a68fe2c7482096 Mon Sep 17 00:00:00 2001 From: Florent Beauchamp Date: Wed, 11 Jan 2023 15:39:32 +0100 Subject: [PATCH] fix following review --- @xen-orchestra/backups/RemoteAdapter.js | 5 ++++- @xen-orchestra/backups/writers/DeltaBackupWriter.js | 11 +++++------ @xen-orchestra/fs/src/abstract.js | 3 --- CHANGELOG.unreleased.md | 2 ++ packages/vhd-lib/Vhd/VhdAbstract.integ.spec.js | 11 +++++++++++ packages/vhd-lib/Vhd/VhdAbstract.js | 2 +- packages/vhd-lib/createVhdDirectoryFromStream.js | 2 +- 7 files changed, 24 insertions(+), 12 deletions(-) diff --git a/@xen-orchestra/backups/RemoteAdapter.js b/@xen-orchestra/backups/RemoteAdapter.js index 0e411f35c08..e6be9b97466 100644 --- a/@xen-orchestra/backups/RemoteAdapter.js +++ b/@xen-orchestra/backups/RemoteAdapter.js @@ -28,6 +28,7 @@ const { isMetadataFile } = require('./_backupType.js') const { isValidXva } = require('./_isValidXva.js') const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js') const { lvs, pvs } = require('./_lvm.js') +const { watchStreamSize } = require('./_watchStreamSize') // @todo : this import is marked extraneous , sould be fixed when lib is published const { mount } = require('@vates/fuse-vhd') const { asyncEach } = require('@vates/async-each') @@ -678,7 +679,8 @@ class RemoteAdapter { } async outputStream(path, input, { checksum = true, validator = noop } = {}) { - return this._handler.outputStream(path, input, { + const container = watchStreamSize(input) + await this._handler.outputStream(path, input, { checksum, dirMode: this._dirMode, async validator() { @@ -686,6 +688,7 @@ class RemoteAdapter { return validator.apply(this, arguments) }, }) + return container.size } // open the hierarchy of ancestors until we find a full one diff --git a/@xen-orchestra/backups/writers/DeltaBackupWriter.js b/@xen-orchestra/backups/writers/DeltaBackupWriter.js index 5c36d826749..dcefc07564e 100644 --- a/@xen-orchestra/backups/writers/DeltaBackupWriter.js +++ b/@xen-orchestra/backups/writers/DeltaBackupWriter.js @@ -171,7 +171,8 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab vmSnapshot: this._backup.exportedVm, } - const { size } = await Task.run({ name: 'transfer' }, async () => { + const transferSize = await Task.run({ name: 'transfer' }, async () => { + let transferSize = 0 await Promise.all( map(deltaExport.vdis, async (vdi, id) => { const path = `${this._vmBackupDir}/${vhds[id]}` @@ -214,7 +215,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab } } - sizeContainers[`${id}.vhd`].size = await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], { + transferSize += await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], { // no checksum for VHDs, because they will be invalidated by // merges and chainings checksum: false, @@ -235,11 +236,9 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab }) }) ) - return { - size: Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0), - } + return transferSize }) - metadataContent.size = size + metadataContent.size = transferSize this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent) // TODO: run cleanup? diff --git a/@xen-orchestra/fs/src/abstract.js b/@xen-orchestra/fs/src/abstract.js index a984576188c..88796943453 100644 --- a/@xen-orchestra/fs/src/abstract.js +++ b/@xen-orchestra/fs/src/abstract.js @@ -13,7 +13,6 @@ import { synchronized } from 'decorator-synchronized' import { basename, dirname, normalize as normalizePath } from './path' import { createChecksumStream, validChecksumOfReadStream } from './checksum' import { DEFAULT_ENCRYPTION_ALGORITHM, _getEncryptor } from './_encryptor' -import { watchStreamSize } from '@xen-orchestra/backups/_watchStreamSize' const { info, warn } = createLogger('xo:fs:abstract') @@ -186,7 +185,6 @@ export default class RemoteHandlerAbstract { async outputStream(path, input, { checksum = true, dirMode, validator } = {}) { path = normalizePath(path) let checksumStream - const container = watchStreamSize(input) input = this._encryptor.encryptStream(input) if (checksum) { @@ -203,7 +201,6 @@ export default class RemoteHandlerAbstract { // it is by design to allow checking of encrypted files without the key await this._outputFile(checksumFile(path), await checksumStream.checksum, { dirMode, flags: 'wx' }) } - return container.size } // Free the resources possibly dedicated to put the remote at work, when it diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 3ea238e1719..6bef217c448 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -11,6 +11,8 @@ > Users must be able to say: “I had this issue, happy to know it's fixed” +- [NBD Backups] Fix transfer size [#6599](https://github.com/vatesfr/xen-orchestra/issues/6599) + ### Packages to release > When modifying a package, add it here with its release type. diff --git a/packages/vhd-lib/Vhd/VhdAbstract.integ.spec.js b/packages/vhd-lib/Vhd/VhdAbstract.integ.spec.js index b2e2e4dd593..14aef7996f2 100644 --- a/packages/vhd-lib/Vhd/VhdAbstract.integ.spec.js +++ b/packages/vhd-lib/Vhd/VhdAbstract.integ.spec.js @@ -166,6 +166,17 @@ test('it can create a vhd stream', async () => { await vhd.writeFooter() const stream = vhd.stream() + // size and stream must have the same result + expect(stream.length).toEqual(vhd.streamSize()) + + expect(stream.length).toEqual( + 512 /* footer */ + + 1024 /* header */ + + 512 /* BAT */ + + 512 /* parentlocator */ + + 3 * (2 * 1024 * 1024 + 512) /* blocs */ + + 512 /* end footer */ + ) // read all the stream into a buffer const buffer = await streamToBuffer(stream) diff --git a/packages/vhd-lib/Vhd/VhdAbstract.js b/packages/vhd-lib/Vhd/VhdAbstract.js index b832298c984..fc78bdf56a3 100644 --- a/packages/vhd-lib/Vhd/VhdAbstract.js +++ b/packages/vhd-lib/Vhd/VhdAbstract.js @@ -239,7 +239,7 @@ exports.VhdAbstract = class VhdAbstract { await handler.writeFile(aliasPath, relativePathToTarget) } - size() { + streamSize() { const { header, batSize } = this let fileSize = FOOTER_SIZE + HEADER_SIZE + batSize + FOOTER_SIZE /* the footer at the end */ diff --git a/packages/vhd-lib/createVhdDirectoryFromStream.js b/packages/vhd-lib/createVhdDirectoryFromStream.js index dd61e0da3fc..645c83d52aa 100644 --- a/packages/vhd-lib/createVhdDirectoryFromStream.js +++ b/packages/vhd-lib/createVhdDirectoryFromStream.js @@ -38,7 +38,7 @@ const buildVhd = Disposable.wrap(async function* (handler, path, inputStream, { } ) await Promise.all([vhd.writeFooter(), vhd.writeHeader(), vhd.writeBlockAllocationTable()]) - return vhd.size() + return vhd.streamSize() }) exports.createVhdDirectoryFromStream = async function createVhdDirectoryFromStream(