From 076dc15b108dcfb916251183e8265cb0b1d67fec Mon Sep 17 00:00:00 2001 From: David Fahlander Date: Sun, 26 May 2024 15:28:01 +0200 Subject: [PATCH] When consistent modify/delete operations get chunked, the additional chunks will be flagged with with "isAdditionalChunk" so that consistent operations are never applied multiple times on the sync server. (#2000) For delete operations, this is only an optimisation, since executing the deletions several times based on given criteria is not an error. For modify operations, this was not an issue until we added PropModifications. PropModifications can make mathematical addition and subtraction and those operations must not be done twice if consistency shall be waterproof. If a query such as: db.people.where('name').startsWith('A').modify({ age: add(1) }); ...and there happens to be 1000 local people matching the criteria, those people would end up getting their age added not with 1 but with 5 if modifyChunkSize is 200 since the operation would result in 5 mutations of 200 each - all with criteria and changeSpec in them. When reaching the server, it would ignore the changes that the client computed and instead run the criteria on its database and execute the addition 5 times - one for each chunk. With this commit, all but the first chunk will be flagged with isAdditionalChunk=true, making the server only execute the consistent operation on the initial chunk and ignore the rest. However, the keys of the remaining chunks are still important information for server, so are the local results that came out from it --- .../createMutationTrackingMiddleware.ts | 4 ++++ libs/dexie-cloud-common/package.json | 2 +- libs/dexie-cloud-common/src/DBOperation.ts | 1 + src/classes/collection/collection.ts | 22 ++++++++++--------- src/public/types/dbcore.d.ts | 2 ++ 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/addons/dexie-cloud/src/middlewares/createMutationTrackingMiddleware.ts b/addons/dexie-cloud/src/middlewares/createMutationTrackingMiddleware.ts index f7802a712..e6f49dfd9 100644 --- a/addons/dexie-cloud/src/middlewares/createMutationTrackingMiddleware.ts +++ b/addons/dexie-cloud/src/middlewares/createMutationTrackingMiddleware.ts @@ -265,6 +265,10 @@ export function createMutationTrackingMiddleware({ txid, userId, }; + + if ('isAdditionalChunk' in req && req.isAdditionalChunk) { + mut.isAdditionalChunk = true; + } return keys.length > 0 || ('criteria' in req && req.criteria) ? mutsTable .mutate({ type: 'add', trans, values: [mut] }) // Log entry diff --git a/libs/dexie-cloud-common/package.json b/libs/dexie-cloud-common/package.json index 112059e97..aeb8a7004 100644 --- a/libs/dexie-cloud-common/package.json +++ b/libs/dexie-cloud-common/package.json @@ -1,6 +1,6 @@ { "name": "dexie-cloud-common", - "version": "1.0.33", + "version": "1.0.34", "description": "Library for shared code between dexie-cloud-addon, dexie-cloud (CLI) and dexie-cloud-server", "type": "module", "module": "dist/index.js", diff --git a/libs/dexie-cloud-common/src/DBOperation.ts b/libs/dexie-cloud-common/src/DBOperation.ts index 6bee062ba..0c49c5a17 100644 --- a/libs/dexie-cloud-common/src/DBOperation.ts +++ b/libs/dexie-cloud-common/src/DBOperation.ts @@ -34,6 +34,7 @@ export interface DBOperationCommon { txid?: string | null; userId?: string | null; opNo?: number; + isAdditionalChunk?: boolean; } export interface DBInsertOperation extends DBOperationCommon { type: "insert"; diff --git a/src/classes/collection/collection.ts b/src/classes/collection/collection.ts index f14b94e2d..f56204c8e 100644 --- a/src/classes/collection/collection.ts +++ b/src/classes/collection/collection.ts @@ -74,7 +74,7 @@ export class Collection implements ICollection { * https://dexie.org/docs/Collection/Collection.clone() * **/ - clone(props?) { + clone(props?): this { var rv = Object.create(this.constructor.prototype), ctx = Object.create(this._ctx); if (props) extend(ctx, props); @@ -87,7 +87,7 @@ export class Collection implements ICollection { * https://dexie.org/docs/Collection/Collection.raw() * **/ - raw() { + raw(): this { this._ctx.valueMapper = null; return this; } @@ -503,6 +503,12 @@ export class Collection implements ICollection { } } return this.clone().primaryKeys().then(keys => { + const criteria = isPlainKeyRange(ctx) && + ctx.limit === Infinity && + (typeof changes !== 'function' || changes === deleteCallback) && { + index: ctx.index, + range: ctx.range + }; const nextChunk = (offset: number) => { const count = Math.min(limit, keys.length - offset); @@ -539,12 +545,6 @@ export class Collection implements ICollection { } } } - const criteria = isPlainKeyRange(ctx) && - ctx.limit === Infinity && - (typeof changes !== 'function' || changes === deleteCallback) && { - index: ctx.index, - range: ctx.range - }; return Promise.resolve(addValues.length > 0 && coreTable.mutate({trans, type: 'add', values: addValues}) @@ -563,14 +563,16 @@ export class Collection implements ICollection { values: putValues, criteria, changeSpec: typeof changes !== 'function' - && changes + && changes, + isAdditionalChunk: offset > 0 }).then(res=>applyMutateResult(putValues.length, res)) ).then(()=>(deleteKeys.length > 0 || (criteria && changes === deleteCallback)) && coreTable.mutate({ trans, type: 'delete', keys: deleteKeys, - criteria + criteria, + isAdditionalChunk: offset > 0 }).then(res=>applyMutateResult(deleteKeys.length, res)) ).then(()=>{ return keys.length > offset + count && nextChunk(offset + limit); diff --git a/src/public/types/dbcore.d.ts b/src/public/types/dbcore.d.ts index 9501454f5..a2ce66fce 100644 --- a/src/public/types/dbcore.d.ts +++ b/src/public/types/dbcore.d.ts @@ -57,6 +57,7 @@ export interface DBCorePutRequest { range: DBCoreKeyRange; }; changeSpec?: {[keyPath: string]: any}; // Common changeSpec for each key + isAdditionalChunk?: boolean; updates?: { keys: any[], changeSpecs: {[keyPath: string]: any}[]; // changeSpec per key. @@ -74,6 +75,7 @@ export interface DBCoreDeleteRequest { index: string | null; range: DBCoreKeyRange; }; + isAdditionalChunk?: boolean; } export interface DBCoreDeleteRangeRequest {