Feat(vhd): implement copyless merge

@xen-orchestra/backups/RemoteAdapter.js

@@ -299,6 +299,10 @@ class RemoteAdapter {
     return this.#useVhdDirectory()
   }
 
+  getMergeMode() {
+    return this._handler.useRenameMerge ? VhdAbstract.MERGE_MODE_RENAME : VhdAbstract.MERGE_MODE_COPY
+  }
+
   async *getDisk(diskId) {
     const handler = this._handler
 

@xen-orchestra/backups/_cleanVm.js

@@ -47,7 +47,7 @@ const computeVhdsSize = (handler, vhdPaths) =>
 //         |                              |
 //         \___________rename_____________/
 
-async function mergeVhdChain(chain, { handler, logInfo, remove, merge }) {
+async function mergeVhdChain(chain, { handler, logInfo, remove, merge, mergeMode }) {
   assert(chain.length >= 2)
   const chainCopy = [...chain]
   const parent = chainCopy.pop()
@@ -59,11 +59,12 @@ async function mergeVhdChain(chain, { handler, logInfo, remove, merge }) {
     let done, total
     const handle = setInterval(() => {
       if (done !== undefined) {
-        logInfo(`merging children in progress`, { children, parent, doneCount: done, totalCount: total})
+        logInfo(`merging children in progress`, { children, parent, doneCount: done, totalCount: total })
       }
     }, 10e3)
 
     const mergedSize = await mergeVhd(handler, parent, handler, children, {
+      mergeMode,
       onProgress({ done: d, total: t }) {
         done = d
         total = t
@@ -190,7 +191,15 @@ const defaultMergeLimiter = limitConcurrency(1)
 
 exports.cleanVm = async function cleanVm(
   vmDir,
-  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, logInfo = noop, logWarn = console.warn }
+  {
+    fixMetadata,
+    remove,
+    merge,
+    mergeLimiter = defaultMergeLimiter,
+    mergeMode = VhdAbstract.MERGE_MODE_COPY,
+    logInfo = noop,
+    logWarn = console.warn,
+  }
 ) {
   const limitedMergeVhdChain = mergeLimiter(mergeVhdChain)
 
@@ -427,7 +436,7 @@ exports.cleanVm = async function cleanVm(
   const metadataWithMergedVhd = {}
   const doMerge = async () => {
     await asyncMap(toMerge, async chain => {
-      const merged = await limitedMergeVhdChain(chain, { handler, logInfo, logWarn, remove, merge })
+      const merged = await limitedMergeVhdChain(chain, { handler, logInfo, logWarn, remove, merge, mergeMode })
       if (merged !== undefined) {
         const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
         metadataWithMergedVhd[metadataPath] = true

@xen-orchestra/backups/writers/_MixinBackupWriter.js

@@ -36,6 +36,7 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
               Task.warning(message, data)
             },
             lock: false,
+            mergeMode: this._adapter.getMergeMode(),
           })
         })
       } catch (error) {

@xen-orchestra/fs/src/abstract.js

@@ -102,6 +102,10 @@ export default class RemoteHandlerAbstract {
     throw new Error('Not implemented')
   }
 
+  get useRenameMerge() {
+    return this._remote.useRenameMerge ?? false
+  }
+
   addPrefix(prefix) {
     prefix = normalizePath(prefix)
     return prefix === '/' ? this : new PrefixWrapper(this, prefix)

CHANGELOG.unreleased.md

@@ -31,6 +31,8 @@
 <!--packages-start-->
 
 - xen-api patch
+- @xen-orchestra/backups minor
+- vhd-lib minor
 - xo-cli minor
 - @xen-orchestra/xapi minor
 - xo-server minor

packages/vhd-lib/Vhd/VhdAbstract.js

@@ -22,7 +22,7 @@ const asyncIteratorToStream = require('async-iterator-to-stream')
 const { checksumStruct, fuFooter, fuHeader } = require('../_structs')
 const { isVhdAlias, resolveVhdAlias } = require('../aliases')
 
-exports.VhdAbstract = class VhdAbstract {
+class VhdAbstract {
   get bitmapSize() {
     return sectorsToBytes(this.sectorsOfBitmap)
   }
@@ -335,3 +335,8 @@ exports.VhdAbstract = class VhdAbstract {
     return stream
   }
 }
+
+VhdAbstract.MERGE_MODE_COPY = 1
+VhdAbstract.MERGE_MODE_RENAME = 2
+
+exports.VhdAbstract = VhdAbstract

packages/vhd-lib/Vhd/VhdDirectory.integ.spec.js

@@ -10,6 +10,7 @@ const { Disposable, pFromCallback } = require('promise-toolbox')
 
 const { openVhd, VhdDirectory } = require('../')
 const { createRandomFile, convertFromRawToVhd, convertToVhdDirectory } = require('../tests/utils')
+const { VhdAbstract } = require('./VhdAbstract')
 
 let tempDir = null
 
@@ -23,24 +24,26 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
-test('Can coalesce block', async () => {
-  const initalSize = 4
+test('Can coalesce block from file and directory', async () => {
+  const parentNbBlocks = 4
+  const childFileNbBlocks = 4
+  const childDirectoryNbBlocks = 4
   const parentrawFileName = `${tempDir}/randomfile`
   const parentFileName = `${tempDir}/parent.vhd`
   const parentDirectoryName = `${tempDir}/parent.dir.vhd`
 
-  await createRandomFile(parentrawFileName, initalSize)
+  await createRandomFile(parentrawFileName, parentNbBlocks * 2)
   await convertFromRawToVhd(parentrawFileName, parentFileName)
   await convertToVhdDirectory(parentrawFileName, parentFileName, parentDirectoryName)
 
   const childrawFileName = `${tempDir}/randomfile`
   const childFileName = `${tempDir}/childFile.vhd`
-  await createRandomFile(childrawFileName, initalSize)
+  await createRandomFile(childrawFileName, childFileNbBlocks * 2)
   await convertFromRawToVhd(childrawFileName, childFileName)
   const childRawDirectoryName = `${tempDir}/randomFile2.vhd`
   const childDirectoryFileName = `${tempDir}/childDirFile.vhd`
   const childDirectoryName = `${tempDir}/childDir.vhd`
-  await createRandomFile(childRawDirectoryName, initalSize)
+  await createRandomFile(childRawDirectoryName, childDirectoryNbBlocks * 2)
   await convertFromRawToVhd(childRawDirectoryName, childDirectoryFileName)
   await convertToVhdDirectory(childRawDirectoryName, childDirectoryFileName, childDirectoryName)
 
@@ -53,19 +56,24 @@ test('Can coalesce block', async () => {
     const childDirectoryVhd = yield openVhd(handler, childDirectoryName)
     await childDirectoryVhd.readBlockAllocationTable()
 
+    let childBlockData = (await childFileVhd.readBlock(0)).data
     await parentVhd.coalesceBlock(childFileVhd, 0)
     await parentVhd.writeFooter()
     await parentVhd.writeBlockAllocationTable()
     let parentBlockData = (await parentVhd.readBlock(0)).data
-    let childBlockData = (await childFileVhd.readBlock(0)).data
+    expect(parentBlockData.equals(childBlockData)).toEqual(true)
+    // block should still be present in child
+    childBlockData = (await childFileVhd.readBlock(0)).data
     expect(parentBlockData.equals(childBlockData)).toEqual(true)
 
-    await parentVhd.coalesceBlock(childDirectoryVhd, 0)
+    childBlockData = (await childDirectoryVhd.readBlock(1)).data
+    await parentVhd.coalesceBlock(childDirectoryVhd, 1, VhdAbstract.MERGE_MODE_RENAME)
     await parentVhd.writeFooter()
     await parentVhd.writeBlockAllocationTable()
-    parentBlockData = (await parentVhd.readBlock(0)).data
-    childBlockData = (await childDirectoryVhd.readBlock(0)).data
-    expect(parentBlockData).toEqual(childBlockData)
+    parentBlockData = (await parentVhd.readBlock(1)).data
+    expect(parentBlockData.equals(childBlockData)).toEqual(true)
+    // block should not be in child
+    await expect(childDirectoryVhd.readBlock(1)).rejects.toThrowError()
   })
 })
 

packages/vhd-lib/Vhd/VhdDirectory.js

@@ -242,18 +242,25 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
   // only works if data are in the same handler
   // and if the full block is modified in child ( which is the case whit xcp)
   // and if the compression type is same on both sides
-  async coalesceBlock(child, blockId) {
+  async coalesceBlock(child, blockId, mergeMode = VhdAbstract.MERGE_MODE_COPY) {
     if (
       !(child instanceof VhdDirectory) ||
       this._handler !== child._handler ||
       child.compressionType !== this.compressionType
     ) {
       return super.coalesceBlock(child, blockId)
     }
-    await this._handler.copy(
-      child._getChunkPath(child._getBlockPath(blockId)),
-      this._getChunkPath(this._getBlockPath(blockId))
-    )
+    if (mergeMode === VhdAbstract.MERGE_MODE_RENAME) {
+      await this._handler.rename(
+        child._getChunkPath(child._getBlockPath(blockId)),
+        this._getChunkPath(this._getBlockPath(blockId))
+      )
+    } else {
+      await this._handler.copy(
+        child._getChunkPath(child._getBlockPath(blockId)),
+        this._getChunkPath(this._getBlockPath(blockId))
+      )
+    }
     return sectorsToBytes(this.sectorsPerBlock)
   }
 

packages/vhd-lib/merge.js

@@ -14,6 +14,7 @@ const { Disposable } = require('promise-toolbox')
 const { asyncEach } = require('@vates/async-each')
 const { VhdDirectory } = require('./Vhd/VhdDirectory')
 const { VhdSynthetic } = require('./Vhd/VhdSynthetic')
+const { VhdAbstract } = require('./Vhd/VhdAbstract')
 
 const { warn } = createLogger('vhd-lib:merge')
 
@@ -37,12 +38,13 @@ module.exports = limitConcurrency(2)(async function merge(
   parentPath,
   childHandler,
   childPath,
-  { onProgress = noop } = {}
+  { onProgress = noop, mergeMode = VhdAbstract.MERGE_MODE_COPY } = {}
 ) {
   const mergeStatePath = dirname(parentPath) + '/' + '.' + basename(parentPath) + '.merge.json'
 
   return await Disposable.use(async function* () {
     let mergeState
+    let isResuming = false
     try {
       const mergeStateContent = await parentHandler.readFile(mergeStatePath)
       mergeState = JSON.parse(mergeStateContent)
@@ -75,6 +77,7 @@ module.exports = limitConcurrency(2)(async function merge(
       assert.strictEqual(childVhd.footer.diskType, DISK_TYPES.DIFFERENCING)
       assert.strictEqual(childVhd.header.blockSize, parentVhd.header.blockSize)
     } else {
+      isResuming = true
       // vhd should not have changed to resume
       assert.strictEqual(parentVhd.header.checksum, mergeState.parent.header)
       assert.strictEqual(childVhd.header.checksum, mergeState.child.header)
@@ -120,7 +123,20 @@ module.exports = limitConcurrency(2)(async function merge(
       toMerge,
       async blockId => {
         merging.add(blockId)
-        mergeState.mergedDataSize += await parentVhd.coalesceBlock(childVhd, blockId)
+        try {
+          mergeState.mergedDataSize += await parentVhd.coalesceBlock(childVhd, blockId, mergeMode)
+        } catch (error) {
+          if (mergeMode === VhdAbstract.MERGE_MODE_RENAME && error.code === 'ENOENT' && isResuming === true) {
+            // when resuming, the blocks moved since the last merge state write are
+            // not in the child anymore but it's ok
+
+            // @todo , should I read the parent block to ensure it's really here and readable
+            mergeState.mergedDataSize += parentVhd.header.blockSize
+          } else {
+            throw error
+          }
+        }
+
         merging.delete(blockId)
 
         onProgress({

packages/vhd-lib/tests/utils.js

@@ -85,10 +85,9 @@ async function convertToVhdDirectory(rawFileName, vhdFileName, path) {
   await fs.mkdir(path + '/blocks/0/')
   const stats = await fs.stat(rawFileName)
 
-  const sizeMB = stats.size / 1024 / 1024
-  for (let i = 0, offset = 0; i < sizeMB; i++, offset += blockDataSize) {
+  for (let i = 0, offset = 0; offset < stats.size; i++, offset += blockDataSize) {
     const blockData = Buffer.alloc(blockDataSize)
-    await fs.read(srcRaw, blockData, offset)
+    await fs.read(srcRaw, blockData, 0, blockData.length, offset)
     await fs.writeFile(path + '/blocks/0/' + i, Buffer.concat([bitmap, blockData]))
   }
   await fs.close(srcRaw)