wip

@xen-orchestra/backups/RemoteAdapter.js

@@ -17,6 +17,7 @@ const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
 const { v4: uuidv4 } = require('uuid')
 const { ZipFile } = require('yazl')
+const zlib = require('zlib')
 
 const { BACKUP_DIR } = require('./_getVmBackupDir.js')
 const { cleanVm } = require('./_cleanVm.js')
@@ -261,7 +262,8 @@ class RemoteAdapter {
   }
 
   async deleteVmBackups(files) {
-    const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
+    const metadatas = await asyncMap(files, file => this.readVmBackupMetadata(file))
+    const { delta, full, ...others } = groupBy(metadatas, 'mode')
 
     const unsupportedModes = Object.keys(others)
     if (unsupportedModes.length !== 0) {
@@ -278,6 +280,7 @@ class RemoteAdapter {
       // don't merge in main process, unused VHDs will be merged in the next backup run
       await this.cleanVm(dir, { remove: true, onLog: warn })
     }
+    await asyncMap(metadatas, metadata => this.invalidateVmBackupListCache(metadata.vm.uuid))
   }
 
   #getCompressionType() {
@@ -448,9 +451,22 @@ class RemoteAdapter {
     return backupsByPool
   }
 
-  async listVmBackups(vmUuid, predicate) {
+  async invalidateVmBackupListCache(vmUuid) {
+    try {
+      await this.handler.unlink(`${BACKUP_DIR}/${vmUuid}/cache.json.gz`)
+
+      // remove any pending loc
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        return
+      }
+      throw error
+    }
+  }
+
+  async #getCachabledDataListVmBackups(vmUuid) {
     const handler = this._handler
-    const backups = []
+    const backups = {}
 
     try {
       const files = await handler.list(`${BACKUP_DIR}/${vmUuid}`, {
@@ -460,22 +476,110 @@ class RemoteAdapter {
       await asyncMap(files, async file => {
         try {
           const metadata = await this.readVmBackupMetadata(file)
-          if (predicate === undefined || predicate(metadata)) {
-            // inject an id usable by importVmBackupNg()
-            metadata.id = metadata._filename
-
-            backups.push(metadata)
-          }
+          // inject an id usable by importVmBackupNg()
+          metadata.id = metadata._filename
+          backups[file] = metadata
         } catch (error) {
-          warn(`listVmBackups ${file}`, { error })
+          warn(`createCacheListVmBackups ${file}`, { error })
         }
       })
+      return backups
     } catch (error) {
       let code
       if (error == null || ((code = error.code) !== 'ENOENT' && code !== 'ENOTDIR')) {
         throw error
       }
     }
+  }
+
+  async #createCacheListVmBackups(vmUuid) {
+    const path = `${BACKUP_DIR}/${vmUuid}/cache.json.gz`
+    let lock
+    // const handler = this._handler
+    try {
+      const interval = setTimeout(async () => {
+        console.log(`NO LOCK FOR ${BACKUP_DIR}/${vmUuid}/cache.json.gz`)
+      }, 10000)
+      console.log('will acquire lock ')
+      // lock  = await handler.lock(path)
+
+      console.log('got lock  ')
+      clearTimeout(interval)
+      const cached = await this.#readCacheListVmBackups(vmUuid)
+      if (cached !== undefined) {
+        console.log(' file got created ', cached)
+        await lock.dispose()
+        return cached
+      }
+      // file did not get created during lock acquisition
+
+      const backups = await this.#getCachabledDataListVmBackups(vmUuid)
+      if (backups === undefined) {
+        return
+      }
+      const text = JSON.stringify(backups)
+      const zipped = await new Promise((resolve, reject) => {
+        zlib.gzip(text, (err, buffer) => {
+          if (err !== null) {
+            reject(err)
+          } else {
+            resolve(buffer)
+          }
+        })
+      })
+      await this.handler.writeFile(path, zipped, { flags: 'w' })
+
+      return backups
+    } catch (error) {
+      let code
+      if (error == null || ((code = error.code) !== 'ENOENT' && code !== 'ENOTDIR')) {
+        throw error
+      }
+    } finally {
+      // await lock?.dispose()
+    }
+  }
+
+  async #readCacheListVmBackups(vmUuid) {
+    try {
+      const gzipped = await this.handler.readFile(`${BACKUP_DIR}/${vmUuid}/cache.json.gz`)
+      const text = await new Promise((resolve, reject) => {
+        zlib.gunzip(gzipped, (err, buffer) => {
+          if (err !== null) {
+            reject(err)
+          } else {
+            resolve(buffer)
+          }
+        })
+      })
+      return JSON.parse(text)
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        return
+      }
+      throw error
+    }
+  }
+
+  async listVmBackups(vmUuid, predicate) {
+    const backups = []
+    // await this.invalidateVmBackupListCache(vmUuid)
+    let cached = await this.#readCacheListVmBackups(vmUuid)
+
+    // nothing cached, update cache
+    if (cached === undefined) {
+      cached = await this.#createCacheListVmBackups(vmUuid)
+    }
+
+    if (cached === undefined) {
+      return []
+    }
+
+    Object.values(cached).forEach(metadata => {
+      if (predicate === undefined || predicate(metadata)) {
+        backups.push(metadata)
+      }
+    })
 
     return backups.sort(compareTimestamp)
   }
@@ -603,7 +707,10 @@ class RemoteAdapter {
   }
 
   async readVmBackupMetadata(path) {
-    return Object.defineProperty(JSON.parse(await this._handler.readFile(path)), '_filename', { value: path })
+    // @todo : I really want to be able to stringify _filename
+    return { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
+
+    // Object.defineProperty(JSON.parse(await this._handler.readFile(path)), '_filename', { value: path })
   }
 }
 

@xen-orchestra/backups/writers/_MixinBackupWriter.js

@@ -63,6 +63,8 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
         await handler.outputFile(taskFile, this._backup.vm.uuid)
         const remotePath = handler._getRealPath()
         await MergeWorker.run(remotePath)
+
+        await handler.invalidateVmBackupListCache(this._backup.vm.uuid)
       }
     }
   }