Provide compaction complete callback

client/constants/messages.js

@@ -104,6 +104,11 @@ const messages = {
    * browser sends this to compact records in a category
    */
   COMPACT_SYNC_CATEGORY: _, /* @param {string} categoryName */
+  /**
+   * webview -> browser
+   * sent after compaction is competed.
+   */
+  COMPACTED_SYNC_CATEGORY: _, /* @param {string} categoryName */
   /**
    * webview -> browser
    * webview sends this to delete all site settings.

client/requestUtil.js

@@ -184,6 +184,7 @@ RequestUtil.prototype.parseAWSResponse = function (bytes) {
  * @param {number=} maxRecords Limit response to a given number of recods. By default the Sync lib will fetch all matching records, which might take a long time. If falsey, fetch all records.
  * @param {{
  *    compaction {boolean} // compact records while list object from S3
+ *    compactionCb {function} // callback when compaction is done
  *  }} opts
  * @returns {Promise(Array.<Object>)}
  */
@@ -206,13 +207,17 @@ RequestUtil.prototype.list = function (category, startAt, maxRecords, nextContin
       }
       return new Promise((resolve, reject) => {
         s3ObjectsPromise.then((s3Objects) => {
+          this.compactObjects(s3Objects.contents)
+          // wait for 15 seconds between batches
           setTimeout(() => {
-            this.compactObjects(s3Objects.contents)
             if (s3Objects.isTruncated) {
               return this.list(category, startAt, maxRecords, s3Objects.nextContinuationToken, opts)
             }
             return new Promise((resolve, reject) => {
               // compaction is done
+              if (opts.compactionCb) {
+                opts.compactionCb()
+              }
               resolve()
             })
           }, 15000)

client/sync.js

@@ -23,6 +23,7 @@ var clientKeys = {}
 var config = {}
 var seed
 var nextContinuationTokens = {}
+var isCompactionInProgress = false
 
 /**
  * Logs stuff on the visible HTML page.
@@ -190,9 +191,17 @@ const startSync = (requester) => {
     if (!proto.categories[category]) {
       throw new Error(`Unsupported sync category: ${category}`)
     }
-    requester.list(proto.categories[category], 0, 1000, '', {compaction: true}).then(() => {
-      logSync(`Compacting category: ${category}`)
-    })
+    const compactionDone = () => {
+      ipc.send(messages.COMPACTED_SYNC_CATEGORY, category)
+      isCompactionInProgress = false
+    }
+    if (!isCompactionInProgress) {
+      requester.list(proto.categories[category], 0, 1000, '',
+        {compaction: true, compactionCb: compactionDone}).then(() => {
+        logSync(`Compacting category: ${category}`)
+        isCompactionInProgress = true
+      })
+    }
   })
   ipc.on(messages.DELETE_SYNC_SITE_SETTINGS, (e) => {
     logSync(`Deleting siteSettings`)

test/client/requestUtil.js

@@ -454,29 +454,32 @@ test('client RequestUtil', (t) => {
         const consoleLogBak = console.log
         // limit batch size to 10 to test cross batch compaction for around 40
         // objects
-        requestUtil.list(proto.categories.BOOKMARKS, 0, 10, '', {compaction: true}).then(() => {
+        requestUtil.list(proto.categories.BOOKMARKS, 0, 10, '', {compaction: true,
+          compactionCb: () => {
+            console.log = consoleLogBak
+            console.log('compaction is done')
+            //  we already have 15 second timeout for each batch so no need to
+            //  do another wait after compaction is done
+            requestUtil.list(proto.categories.BOOKMARKS, 0, 0)
+              .then(s3Objects => requestUtil.s3ObjectsToRecords(s3Objects.contents))
+              .then((response) => {
+                t.equals(response.length, 2, `${t.name} check records number`)
+                const s3Record = response[0].record
+                const s3Record2 = response[1].record
+                t.deepEquals(s3Record.objectId, record.objectId, `${t.name}: objectId`)
+                t.deepEquals(s3Record.bookmark.site.title, record_update.bookmark.site.title, `${t.name}: bookmark.title`)
+                requestUtil.deleteCategory(proto.categories.BOOKMARKS)
+                  .then((_response) => {
+                    testCanLimitResponse(t)
+                  })
+                  .catch((error) => { t.fail(error) })
+              })
+              .catch((error) => { t.fail(error) })
+          }
+        }).then(() => {
+          // compaction is still in progress
           console.log = function() {}
         })
-        // takes about 1.5 minute for delete to be completely done and we also
-        // have 15 second timeout for each batch
-        setTimeout(() => {
-          console.log = consoleLogBak
-          requestUtil.list(proto.categories.BOOKMARKS, 0, 0)
-            .then(s3Objects => requestUtil.s3ObjectsToRecords(s3Objects.contents))
-            .then((response) => {
-              t.equals(response.length, 2, `${t.name} check records number`)
-              const s3Record = response[0].record
-              const s3Record2 = response[1].record
-              t.deepEquals(s3Record.objectId, record.objectId, `${t.name}: objectId`)
-              t.deepEquals(s3Record.bookmark.site.title, record_update.bookmark.site.title, `${t.name}: bookmark.title`)
-              requestUtil.deleteCategory(proto.categories.BOOKMARKS)
-                .then((_response) => {
-                  testCanLimitResponse(t)
-                })
-                .catch((error) => { t.fail(error) })
-            })
-            .catch((error) => { t.fail(error) })
-        }, 90000)
       })
     }