When consistent modify/delete operations get chunked, the additional …

…chunks will be flagged with with "isAdditionalChunk" so that consistent operations are never applied multiple times on the sync server. (#2000)

For delete operations, this is only an optimisation, since executing the deletions several times based on given criteria is not an error.

For modify operations, this was not an issue until we added PropModifications.

PropModifications can make mathematical addition and subtraction and those operations must not be done twice if consistency shall be waterproof.

If a query such as: db.people.where('name').startsWith('A').modify({
  age: add(1)
});
...and there happens to be 1000 local people matching the criteria, those people would end up getting their age added not with 1 but with 5 if modifyChunkSize is 200 since the operation would result in 5 mutations of 200 each - all with criteria and changeSpec in them. When reaching the server, it would ignore the changes that the client computed and instead run the criteria on its database and execute the addition 5 times - one for each chunk. With this commit, all but the first chunk will be flagged with isAdditionalChunk=true, making the server only execute the consistent operation on the initial chunk and ignore the rest. However, the keys of the remaining chunks are still important information for server, so are the local results that came out from it

addons/dexie-cloud/src/middlewares/createMutationTrackingMiddleware.ts

@@ -265,6 +265,10 @@ export function createMutationTrackingMiddleware({
                       txid,
                       userId,
                     };
+
+              if ('isAdditionalChunk' in req && req.isAdditionalChunk) {
+                mut.isAdditionalChunk = true;
+              }
               return keys.length > 0 || ('criteria' in req && req.criteria)
                 ? mutsTable
                     .mutate({ type: 'add', trans, values: [mut] }) // Log entry

libs/dexie-cloud-common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dexie-cloud-common",
-  "version": "1.0.33",
+  "version": "1.0.34",
   "description": "Library for shared code between dexie-cloud-addon, dexie-cloud (CLI) and dexie-cloud-server",
   "type": "module",
   "module": "dist/index.js",

libs/dexie-cloud-common/src/DBOperation.ts

@@ -34,6 +34,7 @@ export interface DBOperationCommon<PK=DBOpPrimaryKey> {
   txid?: string | null;
   userId?: string | null;
   opNo?: number;
+  isAdditionalChunk?: boolean;
 }
 export interface DBInsertOperation<PK=DBOpPrimaryKey> extends DBOperationCommon<PK> {
   type: "insert";

src/classes/collection/collection.ts

@@ -74,7 +74,7 @@ export class Collection implements ICollection {
    * https://dexie.org/docs/Collection/Collection.clone()
    * 
    **/
-  clone(props?) {
+  clone(props?): this {
     var rv = Object.create(this.constructor.prototype),
       ctx = Object.create(this._ctx);
     if (props) extend(ctx, props);
@@ -87,7 +87,7 @@ export class Collection implements ICollection {
    * https://dexie.org/docs/Collection/Collection.raw()
    * 
    **/
-  raw() {
+  raw(): this {
     this._ctx.valueMapper = null;
     return this;
   }
@@ -503,6 +503,12 @@ export class Collection implements ICollection {
         }
       }
       return this.clone().primaryKeys().then(keys => {
+        const criteria = isPlainKeyRange(ctx) &&
+        ctx.limit === Infinity &&
+        (typeof changes !== 'function' || changes === deleteCallback) && {
+          index: ctx.index,
+          range: ctx.range
+        };
 
         const nextChunk = (offset: number) => {
           const count = Math.min(limit, keys.length - offset);
@@ -539,12 +545,6 @@ export class Collection implements ICollection {
                 }
               }
             }
-            const criteria = isPlainKeyRange(ctx) &&
-              ctx.limit === Infinity &&
-              (typeof changes !== 'function' || changes === deleteCallback) && {
-                index: ctx.index,
-                range: ctx.range
-              };
 
             return Promise.resolve(addValues.length > 0 &&
               coreTable.mutate({trans, type: 'add', values: addValues})
@@ -563,14 +563,16 @@ export class Collection implements ICollection {
                   values: putValues,
                   criteria,
                   changeSpec: typeof changes !== 'function'
-                    && changes
+                    && changes,
+                  isAdditionalChunk: offset > 0
                 }).then(res=>applyMutateResult(putValues.length, res))
             ).then(()=>(deleteKeys.length > 0 || (criteria && changes === deleteCallback)) &&
                 coreTable.mutate({
                   trans,
                   type: 'delete',
                   keys: deleteKeys,
-                  criteria
+                  criteria,
+                  isAdditionalChunk: offset > 0
                 }).then(res=>applyMutateResult(deleteKeys.length, res))
             ).then(()=>{
               return keys.length > offset + count && nextChunk(offset + limit);

src/public/types/dbcore.d.ts

@@ -57,6 +57,7 @@ export interface DBCorePutRequest {
     range: DBCoreKeyRange;
   };
   changeSpec?: {[keyPath: string]: any}; // Common changeSpec for each key
+  isAdditionalChunk?: boolean;
   updates?: {
     keys: any[],
     changeSpecs: {[keyPath: string]: any}[]; // changeSpec per key.  
@@ -74,6 +75,7 @@ export interface DBCoreDeleteRequest {
     index: string | null;
     range: DBCoreKeyRange;
   };
+  isAdditionalChunk?: boolean;
 }
 
 export interface DBCoreDeleteRangeRequest {