Send updates to client during compaction so client has a chance to ap…

…ply missing remote records

client/requestUtil.js

@@ -184,7 +184,8 @@ RequestUtil.prototype.parseAWSResponse = function (bytes) {
  * @param {number=} maxRecords Limit response to a given number of recods. By default the Sync lib will fetch all matching records, which might take a long time. If falsey, fetch all records.
  * @param {{
  *    compaction {boolean} // compact records while list object from S3
- *    compactionCb {function} // callback when compaction is done
+ *    compactionDoneCb {function} // callback when compaction is done
+ *    compactionUpdateCb {function} // callback for client to update missing local records
  *  }} opts
  * @returns {Promise(Array.<Object>)}
  */
@@ -207,16 +208,19 @@ RequestUtil.prototype.list = function (category, startAt, maxRecords, nextContin
       }
       return new Promise((resolve, reject) => {
         s3ObjectsPromise.then((s3Objects) => {
-          this.compactObjects(s3Objects.contents)
+          const compactedRecords = this.compactObjects(s3Objects.contents)
+          if (opts.compactionUpdateCb) {
+            opts.compactionUpdateCb(compactedRecords)
+          }
           // wait for 15 seconds between batches
           setTimeout(() => {
             if (s3Objects.isTruncated) {
               return this.list(category, startAt, maxRecords, s3Objects.nextContinuationToken, opts)
             }
             return new Promise((resolve, reject) => {
               // compaction is done
-              if (opts.compactionCb) {
-                opts.compactionCb()
+              if (opts.compactionDoneCb) {
+                opts.compactionDoneCb()
               }
               resolve()
             })
@@ -468,9 +472,11 @@ RequestUtil.prototype.put = function (category, record) {
 /**
  * Compact all records in a category
  * @param {string=} category - the category ID
+ * @returns {Array.<Object>} - records after compaction
  */
 RequestUtil.prototype.compactObjects = function (s3Objects) {
   let s3ObjectsToDelete = []
+  const compactedRecords = {}
   const recordObjects = this.s3ObjectsToRecords(s3Objects)
   recordObjects.forEach((recordObject) => {
     const record = recordObject.record
@@ -482,15 +488,18 @@ RequestUtil.prototype.compactObjects = function (s3Objects) {
         s3ObjectsToDelete = s3ObjectsToDelete.concat(cacheRecordObject.objects)
         console.log(cacheRecordObject.record)
         this.latestRecordsCache.set(id, recordObject)
+        compactedRecords[id] = record
       } else {
         s3ObjectsToDelete = s3ObjectsToDelete.concat(recordObject.objects)
         console.log(record)
       }
     } else {
       this.latestRecordsCache.set(id, recordObject)
+      compactedRecords[id] = record
     }
   })
   s3Helper.deleteObjects(this.s3, this.bucket, s3ObjectsToDelete)
+  return Object.values(compactedRecords)
 }
 
 RequestUtil.prototype.s3PostFormData = function (objectKey) {

client/sync.js

@@ -195,9 +195,19 @@ const startSync = (requester) => {
       ipc.send(messages.COMPACTED_SYNC_CATEGORY, category)
       isCompactionInProgress = false
     }
+    const compactionUpdate = (records) => {
+      let jsRecords = []
+      for (let record of records) {
+        const jsRecord = recordUtil.syncRecordAsJS(record)
+        jsRecord.syncTimestamp = record.syncTimestamp
+        jsRecords.push(jsRecord)
+      }
+      logSync(`Compaction records update category: ${category}`)
+      ipc.send(messages.GET_EXISTING_OBJECTS, category, jsRecords, 0, false)
+    }
     if (!isCompactionInProgress) {
       requester.list(proto.categories[category], 0, 1000, '',
-        {compaction: true, compactionCb: compactionDone}).then(() => {
+        {compaction: true, compactionDoneCb: compactionDone, compactionUpdateCb: compactionUpdate}).then(() => {
         logSync(`Compacting category: ${category}`)
         isCompactionInProgress = true
       })

test/client/requestUtil.js

@@ -455,7 +455,7 @@ test('client RequestUtil', (t) => {
         // limit batch size to 10 to test cross batch compaction for around 40
         // objects
         requestUtil.list(proto.categories.BOOKMARKS, 0, 10, '', {compaction: true,
-          compactionCb: () => {
+          compactionDoneCb: () => {
             console.log = consoleLogBak
             console.log('compaction is done')
             //  we already have 15 second timeout for each batch so no need to