From 7324968ba4ed4f6e14329c3a5a8866b50eec20cf Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 3 Sep 2024 16:06:02 +0200 Subject: [PATCH 01/33] initial sync rules syntax for events --- packages/sync-rules/.gitignore | 1 + packages/sync-rules/package.json | 2 + packages/sync-rules/scripts/compile-schema.js | 11 +++ packages/sync-rules/src/SqlEventDescriptor.ts | 72 +++++++++++++++++++ packages/sync-rules/src/SqlSyncRules.ts | 51 ++++++++++++- packages/sync-rules/src/json_schema.ts | 23 +++++- 6 files changed, 156 insertions(+), 4 deletions(-) create mode 100644 packages/sync-rules/.gitignore create mode 100644 packages/sync-rules/scripts/compile-schema.js create mode 100644 packages/sync-rules/src/SqlEventDescriptor.ts diff --git a/packages/sync-rules/.gitignore b/packages/sync-rules/.gitignore new file mode 100644 index 000000000..b73759d7b --- /dev/null +++ b/packages/sync-rules/.gitignore @@ -0,0 +1 @@ +schema/ \ No newline at end of file diff --git a/packages/sync-rules/package.json b/packages/sync-rules/package.json index 8a117ce02..9a9dc85bf 100644 --- a/packages/sync-rules/package.json +++ b/packages/sync-rules/package.json @@ -16,6 +16,8 @@ "clean": "rm -r ./dist && tsc -b --clean", "build": "tsc -b", "build:tests": "tsc -b test/tsconfig.json", + "compile:schema": "pnpm build && node scripts/compile-schema.js", + "postversion": "pnpm compile:schema", "test": "vitest" }, "dependencies": { diff --git a/packages/sync-rules/scripts/compile-schema.js b/packages/sync-rules/scripts/compile-schema.js new file mode 100644 index 000000000..ad9217e49 --- /dev/null +++ b/packages/sync-rules/scripts/compile-schema.js @@ -0,0 +1,11 @@ +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { syncRulesSchema } from '../dist/json_schema.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const schemaDir = path.join(__dirname, '../schema'); + +fs.mkdirSync(schemaDir); + +fs.writeFileSync(path.join(schemaDir, 'sync_rules.json'), JSON.stringify(syncRulesSchema, null, '\t')); diff --git a/packages/sync-rules/src/SqlEventDescriptor.ts b/packages/sync-rules/src/SqlEventDescriptor.ts new file mode 100644 index 000000000..2ba350f02 --- /dev/null +++ b/packages/sync-rules/src/SqlEventDescriptor.ts @@ -0,0 +1,72 @@ +import { IdSequence } from './IdSequence.js'; +import { SourceTableInterface } from './SourceTableInterface.js'; +import { SqlParameterQuery } from './SqlParameterQuery.js'; +import { TablePattern } from './TablePattern.js'; +import { SqlRuleError } from './errors.js'; +import { EvaluatedParametersResult, QueryParseOptions, SourceSchema, SqliteRow } from './types.js'; + +export interface QueryParseResult { + /** + * True if parsed in some form, even if there are errors. + */ + parsed: boolean; + + errors: SqlRuleError[]; +} + +/** + * A sync rules event which is triggered from a SQL table change. + */ +export class SqlEventDescriptor { + name: string; + bucket_parameters?: string[]; + + constructor(name: string, public idSequence: IdSequence) { + this.name = name; + } + + parameter_queries: SqlParameterQuery[] = []; + parameterIdSequence = new IdSequence(); + + addParameterQuery(sql: string, schema: SourceSchema | undefined, options: QueryParseOptions): QueryParseResult { + const parameterQuery = SqlParameterQuery.fromSql(this.name, sql, schema, options); + parameterQuery.id = this.parameterIdSequence.nextId(); + if (false == parameterQuery instanceof SqlParameterQuery) { + throw new Error('Parameter queries for events can not be global'); + } + + this.parameter_queries.push(parameterQuery as SqlParameterQuery); + + return { + parsed: true, + errors: parameterQuery.errors + }; + } + + evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] { + let results: EvaluatedParametersResult[] = []; + for (let query of this.parameter_queries) { + if (query.applies(sourceTable)) { + results.push(...query.evaluateParameterRow(row)); + } + } + return results; + } + + getSourceTables(): Set { + let result = new Set(); + for (let query of this.parameter_queries) { + result.add(query.sourceTable!); + } + return result; + } + + tableTriggersEvent(table: SourceTableInterface): boolean { + for (let query of this.parameter_queries) { + if (query.applies(table)) { + return true; + } + } + return false; + } +} diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 15b84adc6..be4bbc3f2 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -1,9 +1,10 @@ -import { LineCounter, parseDocument, Scalar, YAMLMap, YAMLSeq } from 'yaml'; +import { isScalar, LineCounter, parseDocument, Scalar, YAMLMap, YAMLSeq } from 'yaml'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; import { IdSequence } from './IdSequence.js'; import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { QueryParseResult, SqlBucketDescriptor } from './SqlBucketDescriptor.js'; +import { SqlEventDescriptor } from './SqlEventDescriptor.js'; import { TablePattern } from './TablePattern.js'; import { EvaluatedParameters, @@ -27,6 +28,7 @@ const ACCEPT_POTENTIALLY_DANGEROUS_QUERIES = Symbol('ACCEPT_POTENTIALLY_DANGEROU export class SqlSyncRules implements SyncRules { bucket_descriptors: SqlBucketDescriptor[] = []; + event_descriptors: SqlEventDescriptor[] = []; idSequence = new IdSequence(); content: string; @@ -132,6 +134,34 @@ export class SqlSyncRules implements SyncRules { rules.bucket_descriptors.push(descriptor); } + const eventMap = parsed.get('event_definitions') as YAMLMap; + for (const event of eventMap?.items ?? []) { + const { key, value } = event as { key: Scalar; value: Scalar | YAMLSeq }; + const eventDescriptor = new SqlEventDescriptor(key.toString(), rules.idSequence); + + if (value instanceof Scalar) { + rules.withScalar(value, (q) => { + return eventDescriptor.addParameterQuery(q, schema, { + accept_potentially_dangerous_queries: false + }); + }); + } else if (value instanceof YAMLSeq) { + for (let item of value.items) { + if (!isScalar(item)) { + // TODO position + rules.errors.push(new YamlError(new Error(`Parameters for events must be scalar.`))); + continue; + } + rules.withScalar(item, (q) => { + return eventDescriptor.addParameterQuery(q, schema, { + accept_potentially_dangerous_queries: false + }); + }); + } + } + rules.event_descriptors.push(eventDescriptor); + } + // Validate that there are no additional properties. // Since these errors don't contain line numbers, do this last. const valid = validateSyncRulesSchema(parsed.toJSON()); @@ -277,16 +307,31 @@ export class SqlSyncRules implements SyncRules { } getSourceTables(): TablePattern[] { - let sourceTables = new Map(); - for (let bucket of this.bucket_descriptors) { + const sourceTables = new Map(); + for (const bucket of this.bucket_descriptors) { for (let r of bucket.getSourceTables()) { const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; sourceTables.set(key, r); } } + for (const event of this.event_descriptors) { + for (let r of event.getSourceTables()) { + const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; + sourceTables.set(key, r); + } + } return [...sourceTables.values()]; } + tableTriggersEvent(table: SourceTableInterface): boolean { + for (let bucket of this.event_descriptors) { + if (bucket.tableTriggersEvent(table)) { + return true; + } + } + return false; + } + tableSyncsData(table: SourceTableInterface): boolean { for (let bucket of this.bucket_descriptors) { if (bucket.tableSyncsData(table)) { diff --git a/packages/sync-rules/src/json_schema.ts b/packages/sync-rules/src/json_schema.ts index dfe614825..311edec7a 100644 --- a/packages/sync-rules/src/json_schema.ts +++ b/packages/sync-rules/src/json_schema.ts @@ -1,7 +1,7 @@ import ajvModule from 'ajv'; // Hack to make this work both in NodeJS and a browser const Ajv = ajvModule.default ?? ajvModule; -const ajv = new Ajv({ allErrors: true, verbose: true }); +const ajv = new Ajv({ allErrors: true, verbose: true, allowUnionTypes: true }); export const syncRulesSchema: ajvModule.Schema = { type: 'object', @@ -44,6 +44,27 @@ export const syncRulesSchema: ajvModule.Schema = { additionalProperties: false } } + }, + event_definitions: { + type: 'object', + description: 'Record of sync replication event definitions', + examples: [ + { write_checkpoint: 'select user_id, client_id, checkpoint from write_checkpoints' }, + , + { + write_checkpoint: ['select user_id, client_id, checkpoint from write_checkpoints'] + } + ], + patternProperties: { + '.*': { + type: ['string', 'array'], + items: { + type: 'string' + }, + minItems: 1, + uniqueItems: true + } + } } }, required: ['bucket_definitions'], From eae63d07bbd054543621b7390d392b551d163ae9 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 4 Sep 2024 11:50:06 +0200 Subject: [PATCH 02/33] initial implementation of replication events --- .../src/module/PostgresModule.ts | 7 +- .../src/replication/WalStream.ts | 71 +++++++++++++++++-- .../replication/WalStreamReplicationJob.ts | 8 ++- .../src/replication/WalStreamReplicator.ts | 8 ++- .../test/src/slow_tests.test.ts | 13 ++-- .../test/src/wal_stream_utils.ts | 11 +-- .../src/replication/ReplicationEngine.ts | 9 ++- .../replication/ReplicationEventManager.ts | 51 +++++++++++++ .../src/replication/replication-index.ts | 3 +- .../service-core/src/storage/SourceTable.ts | 9 +++ .../storage/mongo/MongoSyncBucketStorage.ts | 1 + packages/sync-rules/src/SqlSyncRules.ts | 11 ++- packages/sync-rules/src/index.ts | 23 +++--- 13 files changed, 188 insertions(+), 37 deletions(-) create mode 100644 packages/service-core/src/replication/ReplicationEventManager.ts diff --git a/modules/module-postgres/src/module/PostgresModule.ts b/modules/module-postgres/src/module/PostgresModule.ts index 0323b06b1..dd6deab0a 100644 --- a/modules/module-postgres/src/module/PostgresModule.ts +++ b/modules/module-postgres/src/module/PostgresModule.ts @@ -1,11 +1,11 @@ import { api, auth, ConfigurationFileSyncRulesProvider, replication, system } from '@powersync/service-core'; import * as jpgwire from '@powersync/service-jpgwire'; -import * as types from '../types/types.js'; import { PostgresRouteAPIAdapter } from '../api/PostgresRouteAPIAdapter.js'; import { SupabaseKeyCollector } from '../auth/SupabaseKeyCollector.js'; -import { WalStreamReplicator } from '../replication/WalStreamReplicator.js'; import { ConnectionManagerFactory } from '../replication/ConnectionManagerFactory.js'; import { PostgresErrorRateLimiter } from '../replication/PostgresErrorRateLimiter.js'; +import { WalStreamReplicator } from '../replication/WalStreamReplicator.js'; +import * as types from '../types/types.js'; export class PostgresModule extends replication.ReplicationModule { constructor() { @@ -48,7 +48,8 @@ export class PostgresModule extends replication.ReplicationModule(); private startedStreaming = false; + private event_cache = new Map(); + constructor(options: WalStreamOptions) { this.storage = options.storage; this.sync_rules = options.storage.sync_rules; this.group_id = options.storage.group_id; this.slot_name = options.storage.slot_name; this.connections = options.connections; + this.event_manager = options.event_manager; this.abort_signal = options.abort_signal; this.abort_signal.addEventListener( @@ -506,6 +519,48 @@ WHERE oid = $1::regclass`, return null; } + async writeEvent(msg: pgwire.PgoutputMessage) { + if (msg.tag == 'insert' || msg.tag == 'update' || msg.tag == 'delete') { + const table = this.getTable(getRelId(msg.relation)); + if (!table.triggerEvents) { + return null; + } + const relevantEventDescriptions = this.sync_rules.event_descriptors.filter( + (evt) => !!Array.from(evt.getSourceTables()).find((sourceTable) => sourceTable.matches(table)) + ); + + const before = msg.tag == 'update' ? util.constructBeforeRecord(msg) : undefined; + const after = msg.tag !== 'delete' ? util.constructAfterRecord(msg) : undefined; + + const dataOp = { + op: msg.tag as replication.EventOp, + after, + before + }; + + for (const eventDescription of relevantEventDescriptions) { + // eventDescription.evaluateParameterRow; + if (!this.event_cache.has(eventDescription)) { + const dataMap: replication.ReplicationEventData = new Map(); + dataMap.set(table, [dataOp]); + this.event_cache.set(eventDescription, { + event: eventDescription, + storage: this.storage, + data: dataMap + }); + continue; + } + const cached = this.event_cache.get(eventDescription)!; + if (!cached.data.has(table)) { + cached.data.set(table, [dataOp]); + continue; + } + + cached.data.get(table)!.push(dataOp); + } + } + } + async replicate() { try { // If anything errors here, the entire replication process is halted, and @@ -567,13 +622,19 @@ WHERE oid = $1::regclass`, inTx = false; await batch.commit(msg.lsn!); await this.ack(msg.lsn!, replicationStream); + // Flush cached event operations + for (const event of this.event_cache.values()) { + await this.event_manager.fireEvent(event); + } + this.event_cache.clear(); } else { if (count % 100 == 0) { logger.info(`${this.slot_name} replicating op ${count} ${msg.lsn}`); } count += 1; - const result = await this.writeChange(batch, msg); + await this.writeChange(batch, msg); + await this.writeEvent(msg); } } diff --git a/modules/module-postgres/src/replication/WalStreamReplicationJob.ts b/modules/module-postgres/src/replication/WalStreamReplicationJob.ts index 3c5731729..a21b1a453 100644 --- a/modules/module-postgres/src/replication/WalStreamReplicationJob.ts +++ b/modules/module-postgres/src/replication/WalStreamReplicationJob.ts @@ -1,17 +1,19 @@ -import { MissingReplicationSlotError, WalStream } from './WalStream.js'; import { container } from '@powersync/lib-services-framework'; import { PgManager } from './PgManager.js'; +import { MissingReplicationSlotError, WalStream } from './WalStream.js'; import { replication } from '@powersync/service-core'; import { ConnectionManagerFactory } from './ConnectionManagerFactory.js'; export interface WalStreamReplicationJobOptions extends replication.AbstractReplicationJobOptions { connectionFactory: ConnectionManagerFactory; + eventManager: replication.ReplicationEventManager; } export class WalStreamReplicationJob extends replication.AbstractReplicationJob { private connectionFactory: ConnectionManagerFactory; private connectionManager: PgManager; + private readonly eventManager: replication.ReplicationEventManager; constructor(options: WalStreamReplicationJobOptions) { super(options); @@ -21,6 +23,7 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob idleTimeout: 30_000, maxSize: 2 }); + this.eventManager = options.eventManager; } async cleanUp(): Promise { @@ -108,7 +111,8 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob const stream = new WalStream({ abort_signal: this.abortController.signal, storage: this.options.storage, - connections: this.connectionManager + connections: this.connectionManager, + event_manager: this.eventManager }); await stream.replicate(); } catch (e) { diff --git a/modules/module-postgres/src/replication/WalStreamReplicator.ts b/modules/module-postgres/src/replication/WalStreamReplicator.ts index 8002b7806..12bf29bd3 100644 --- a/modules/module-postgres/src/replication/WalStreamReplicator.ts +++ b/modules/module-postgres/src/replication/WalStreamReplicator.ts @@ -1,17 +1,20 @@ import { AbstractReplicatorOptions, replication } from '@powersync/service-core'; -import { WalStreamReplicationJob } from './WalStreamReplicationJob.js'; import { ConnectionManagerFactory } from './ConnectionManagerFactory.js'; +import { WalStreamReplicationJob } from './WalStreamReplicationJob.js'; export interface WalStreamReplicatorOptions extends AbstractReplicatorOptions { connectionFactory: ConnectionManagerFactory; + eventManager: replication.ReplicationEventManager; } export class WalStreamReplicator extends replication.AbstractReplicator { private readonly connectionFactory: ConnectionManagerFactory; + private readonly eventManager: replication.ReplicationEventManager; constructor(options: WalStreamReplicatorOptions) { super(options); this.connectionFactory = options.connectionFactory; + this.eventManager = options.eventManager; } createJob(options: replication.CreateJobOptions): WalStreamReplicationJob { @@ -19,7 +22,8 @@ export class WalStreamReplicator extends replication.AbstractReplicator = new Map(); + readonly eventManager: ReplicationEventManager; + + constructor() { + this.eventManager = new ReplicationEventManager(); + } + /** * Register a Replicator with the engine * diff --git a/packages/service-core/src/replication/ReplicationEventManager.ts b/packages/service-core/src/replication/ReplicationEventManager.ts new file mode 100644 index 000000000..e52469930 --- /dev/null +++ b/packages/service-core/src/replication/ReplicationEventManager.ts @@ -0,0 +1,51 @@ +import * as sync_rules from '@powersync/service-sync-rules'; +import * as storage from '../storage/storage-index.js'; + +export enum EventOp { + INSERT = 'insert', + UPDATE = 'update', + DELETE = 'delete' +} + +export type EventData = { + op: EventOp; + before?: sync_rules.SqliteRow; + after?: sync_rules.SqliteRow; +}; + +export type ReplicationEventData = Map; + +export type ReplicationEventPayload = { + event: sync_rules.SqlEventDescriptor; + data: ReplicationEventData; + storage: storage.SyncRulesBucketStorage; +}; + +export interface ReplicationEventHandler { + event_name: string; + handle(event: ReplicationEventPayload): Promise; +} + +export class ReplicationEventManager { + handlers: Map>; + + constructor() { + this.handlers = new Map(); + } + + async fireEvent(payload: ReplicationEventPayload): Promise { + const handlers = this.handlers.get(payload.event.name); + + for (const handler of handlers?.values() ?? []) { + await handler.handle(payload); + } + } + + registerHandler(handler: ReplicationEventHandler) { + const { event_name } = handler; + if (!this.handlers.has(event_name)) { + this.handlers.set(event_name, new Set()); + } + this.handlers.get(event_name)?.add(handler); + } +} diff --git a/packages/service-core/src/replication/replication-index.ts b/packages/service-core/src/replication/replication-index.ts index 455ca913b..456fd1dea 100644 --- a/packages/service-core/src/replication/replication-index.ts +++ b/packages/service-core/src/replication/replication-index.ts @@ -1,5 +1,6 @@ -export * from './ErrorRateLimiter.js'; export * from './AbstractReplicationJob.js'; export * from './AbstractReplicator.js'; +export * from './ErrorRateLimiter.js'; export * from './ReplicationEngine.js'; +export * from './ReplicationEventManager.js'; export * from './ReplicationModule.js'; diff --git a/packages/service-core/src/storage/SourceTable.ts b/packages/service-core/src/storage/SourceTable.ts index 6379732f5..839bd42c6 100644 --- a/packages/service-core/src/storage/SourceTable.ts +++ b/packages/service-core/src/storage/SourceTable.ts @@ -24,6 +24,15 @@ export class SourceTable { */ public syncParameters = true; + /** + * True if the table is used in sync rules for events. + * + * This value is resolved externally, and cached here. + * + * Defaults to true for tests. + */ + public triggerEvents = true; + constructor( public readonly id: any, public readonly connectionTag: string, diff --git a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts index fdd292206..5d537eee4 100644 --- a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts +++ b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts @@ -136,6 +136,7 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { replicationColumns, doc.snapshot_done ?? true ); + sourceTable.triggerEvents = options.sync_rules.tableTriggersEvent(sourceTable); sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable); diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index be4bbc3f2..9e96b5c13 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -314,13 +314,20 @@ export class SqlSyncRules implements SyncRules { sourceTables.set(key, r); } } + + return [...sourceTables.values()]; + } + + getEventTables(): TablePattern[] { + const eventTables = new Map(); + for (const event of this.event_descriptors) { for (let r of event.getSourceTables()) { const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; - sourceTables.set(key, r); + eventTables.set(key, r); } } - return [...sourceTables.values()]; + return [...eventTables.values()]; } tableTriggersEvent(table: SourceTableInterface): boolean { diff --git a/packages/sync-rules/src/index.ts b/packages/sync-rules/src/index.ts index d72a1e58c..574c72081 100644 --- a/packages/sync-rules/src/index.ts +++ b/packages/sync-rules/src/index.ts @@ -1,20 +1,21 @@ +export * from './DartSchemaGenerator.js'; export * from './errors.js'; +export * from './ExpressionType.js'; +export * from './generators.js'; export * from './IdSequence.js'; +export * from './JsLegacySchemaGenerator.js'; +export * from './json_schema.js'; +export * from './request_functions.js'; +export * from './SchemaGenerator.js'; export * from './SourceTableInterface.js'; export * from './sql_filters.js'; export * from './sql_functions.js'; +export * from './SqlDataQuery.js'; +export * from './SqlEventDescriptor.js'; +export * from './SqlParameterQuery.js'; export * from './SqlSyncRules.js'; +export * from './StaticSchema.js'; export * from './TablePattern.js'; +export * from './TsSchemaGenerator.js'; export * from './types.js'; export * from './utils.js'; -export * from './SqlParameterQuery.js'; -export * from './json_schema.js'; -export * from './StaticSchema.js'; -export * from './ExpressionType.js'; -export * from './SchemaGenerator.js'; -export * from './DartSchemaGenerator.js'; -export * from './JsLegacySchemaGenerator.js'; -export * from './TsSchemaGenerator.js'; -export * from './generators.js'; -export * from './SqlDataQuery.js'; -export * from './request_functions.js'; From d6ad8c1fdc37d60aa24c2b6a79c76f3e7eb943d9 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 4 Sep 2024 12:14:19 +0200 Subject: [PATCH 03/33] evaluate parameter queries --- modules/module-postgres/src/replication/WalStream.ts | 12 +++++------- .../src/replication/ReplicationEventManager.ts | 4 ++-- packages/sync-rules/src/json_schema.ts | 1 - 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index af5b8c261..b312e22c5 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -532,14 +532,12 @@ WHERE oid = $1::regclass`, const before = msg.tag == 'update' ? util.constructBeforeRecord(msg) : undefined; const after = msg.tag !== 'delete' ? util.constructAfterRecord(msg) : undefined; - const dataOp = { - op: msg.tag as replication.EventOp, - after, - before - }; - for (const eventDescription of relevantEventDescriptions) { - // eventDescription.evaluateParameterRow; + const dataOp = { + op: msg.tag as replication.EventOp, + after: after ? eventDescription.evaluateParameterRow(table, after) : undefined, + before: before ? eventDescription.evaluateParameterRow(table, before) : undefined + }; if (!this.event_cache.has(eventDescription)) { const dataMap: replication.ReplicationEventData = new Map(); dataMap.set(table, [dataOp]); diff --git a/packages/service-core/src/replication/ReplicationEventManager.ts b/packages/service-core/src/replication/ReplicationEventManager.ts index e52469930..0029f2f7b 100644 --- a/packages/service-core/src/replication/ReplicationEventManager.ts +++ b/packages/service-core/src/replication/ReplicationEventManager.ts @@ -9,8 +9,8 @@ export enum EventOp { export type EventData = { op: EventOp; - before?: sync_rules.SqliteRow; - after?: sync_rules.SqliteRow; + before?: sync_rules.EvaluatedParametersResult[]; + after?: sync_rules.EvaluatedParametersResult[]; }; export type ReplicationEventData = Map; diff --git a/packages/sync-rules/src/json_schema.ts b/packages/sync-rules/src/json_schema.ts index 311edec7a..dddc55370 100644 --- a/packages/sync-rules/src/json_schema.ts +++ b/packages/sync-rules/src/json_schema.ts @@ -50,7 +50,6 @@ export const syncRulesSchema: ajvModule.Schema = { description: 'Record of sync replication event definitions', examples: [ { write_checkpoint: 'select user_id, client_id, checkpoint from write_checkpoints' }, - , { write_checkpoint: ['select user_id, client_id, checkpoint from write_checkpoints'] } From 744c8f6e12540ea906558a8cd0b36f3a76ec4e81 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 10 Sep 2024 09:40:23 +0200 Subject: [PATCH 04/33] add event batching --- .../src/replication/WalStream.ts | 116 ++++++++++++------ .../src/replication/ReplicationEventBatch.ts | 98 +++++++++++++++ .../src/replication/replication-index.ts | 1 + 3 files changed, 180 insertions(+), 35 deletions(-) create mode 100644 packages/service-core/src/replication/ReplicationEventBatch.ts diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index b312e22c5..16610e8c9 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -345,13 +345,21 @@ WHERE oid = $1::regclass`, } async initialReplication(db: pgwire.PgConnection, lsn: string) { - const sourceTables = this.sync_rules.getSourceTables(); + const replicationTables = [...this.sync_rules.getSourceTables(), ...this.sync_rules.getEventTables()]; + await this.storage.startBatch({}, async (batch) => { - for (let tablePattern of sourceTables) { + const eventBatch = new replication.ReplicationEventBatch({ + manager: this.event_manager, + storage: this.storage + }); + + for (let tablePattern of replicationTables) { const tables = await this.getQualifiedTableNames(batch, db, tablePattern); for (let table of tables) { - await this.snapshotTable(batch, db, table); - await batch.markSnapshotDone([table], lsn); + await this.snapshotTable(batch, db, table, eventBatch); + if (table.syncAny) { + await batch.markSnapshotDone([table], lsn); + } await touch(); } @@ -366,7 +374,12 @@ WHERE oid = $1::regclass`, } } - private async snapshotTable(batch: storage.BucketStorageBatch, db: pgwire.PgConnection, table: storage.SourceTable) { + private async snapshotTable( + batch: storage.BucketStorageBatch, + db: pgwire.PgConnection, + table: storage.SourceTable, + eventBatch?: replication.ReplicationEventBatch + ) { logger.info(`${this.slot_name} Replicating ${table.qualifiedName}`); const estimatedCount = await this.estimatedCount(db, table); let at = 0; @@ -400,10 +413,21 @@ WHERE oid = $1::regclass`, throw new Error(`Aborted initial replication of ${this.slot_name}`); } - for (let record of WalStream.getQueryData(rows)) { + for (const record of WalStream.getQueryData(rows)) { // This auto-flushes when the batch reaches its size limit - await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record }); + if (table.syncAny) { + await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record }); + } + // Event batch is only provided in the initial snapshot + if (eventBatch) { + // The method checks internally if an event should be emitted + await this.writeSnapshotEvent(eventBatch, { + table, + data: record + }); + } } + at += rows.length; Metrics.getInstance().rows_replicated_total.add(rows.length); @@ -519,42 +543,51 @@ WHERE oid = $1::regclass`, return null; } - async writeEvent(msg: pgwire.PgoutputMessage) { + protected async writeSnapshotEvent( + eventBatch: replication.ReplicationEventBatch, + params: { table: storage.SourceTable; data: SqliteRow } + ) { + const { table, data } = params; + + if (!table.triggerEvents) { + return null; + } + + for (const eventDescription of this.getTableEvents(table)) { + await eventBatch.save({ + event: eventDescription, + table, + data: { + op: replication.EventOp.INSERT, + after: eventDescription.evaluateParameterRow(table, data) + } + }); + } + } + + /** + * Writes a replication event which is triggered from the replication stream. + */ + protected async writeReplicationEvent(batch: replication.ReplicationEventBatch, msg: pgwire.PgoutputMessage) { if (msg.tag == 'insert' || msg.tag == 'update' || msg.tag == 'delete') { const table = this.getTable(getRelId(msg.relation)); if (!table.triggerEvents) { return null; } - const relevantEventDescriptions = this.sync_rules.event_descriptors.filter( - (evt) => !!Array.from(evt.getSourceTables()).find((sourceTable) => sourceTable.matches(table)) - ); const before = msg.tag == 'update' ? util.constructBeforeRecord(msg) : undefined; const after = msg.tag !== 'delete' ? util.constructAfterRecord(msg) : undefined; - for (const eventDescription of relevantEventDescriptions) { - const dataOp = { - op: msg.tag as replication.EventOp, - after: after ? eventDescription.evaluateParameterRow(table, after) : undefined, - before: before ? eventDescription.evaluateParameterRow(table, before) : undefined - }; - if (!this.event_cache.has(eventDescription)) { - const dataMap: replication.ReplicationEventData = new Map(); - dataMap.set(table, [dataOp]); - this.event_cache.set(eventDescription, { - event: eventDescription, - storage: this.storage, - data: dataMap - }); - continue; - } - const cached = this.event_cache.get(eventDescription)!; - if (!cached.data.has(table)) { - cached.data.set(table, [dataOp]); - continue; - } - - cached.data.get(table)!.push(dataOp); + for (const eventDescription of this.getTableEvents(table)) { + await batch.save({ + event: eventDescription, + table, + data: { + op: msg.tag as replication.EventOp, + after: after ? eventDescription.evaluateParameterRow(table, after) : undefined, + before: before ? eventDescription.evaluateParameterRow(table, before) : undefined + } + }); } } } @@ -598,6 +631,10 @@ WHERE oid = $1::regclass`, // Replication never starts in the middle of a transaction let inTx = false; let count = 0; + const eventBatch = new replication.ReplicationEventBatch({ + manager: this.event_manager, + storage: this.storage + }); for await (const chunk of replicationStream.pgoutputDecode()) { await touch(); @@ -632,7 +669,7 @@ WHERE oid = $1::regclass`, count += 1; await this.writeChange(batch, msg); - await this.writeEvent(msg); + await this.writeReplicationEvent(eventBatch, msg); } } @@ -659,6 +696,15 @@ WHERE oid = $1::regclass`, replicationStream.ack(lsn); } + + /** + * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable} + */ + protected getTableEvents(table: storage.SourceTable): SqlEventDescriptor[] { + return this.sync_rules.event_descriptors.filter((evt) => + [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table)) + ); + } } async function touch() { diff --git a/packages/service-core/src/replication/ReplicationEventBatch.ts b/packages/service-core/src/replication/ReplicationEventBatch.ts new file mode 100644 index 000000000..30535e6e1 --- /dev/null +++ b/packages/service-core/src/replication/ReplicationEventBatch.ts @@ -0,0 +1,98 @@ +import * as sync_rules from '@powersync/service-sync-rules'; +import * as storage from '../storage/storage-index.js'; +import { EventData, ReplicationEventData, ReplicationEventManager } from './ReplicationEventManager.js'; + +export type ReplicationEventBatchOptions = { + manager: ReplicationEventManager; + storage: storage.SyncRulesBucketStorage; + max_batch_size?: number; +}; + +export type ReplicationEventWriteParams = { + event: sync_rules.SqlEventDescriptor; + table: storage.SourceTable; + data: EventData; +}; + +const MAX_BATCH_SIZE = 1000; + +export class ReplicationEventBatch { + readonly manager: ReplicationEventManager; + readonly storage: storage.SyncRulesBucketStorage; + + protected event_cache: Map; + protected readonly maxBatchSize: number; + + /** + * Keeps track of the number of rows in the cache. + * This avoids having to calculate the size on demand. + */ + private _eventCacheRowCount: number; + + constructor(options: ReplicationEventBatchOptions) { + this.event_cache = new Map(); + this.manager = options.manager; + this.maxBatchSize = options.max_batch_size || MAX_BATCH_SIZE; + this.storage = options.storage; + this._eventCacheRowCount = 0; + } + + /** + * Returns the number of rows/events in the cache + */ + get cacheSize() { + return this._eventCacheRowCount; + } + + dispose() { + this.event_cache.clear(); + // This is not required, but cleans things up a bit. + this._eventCacheRowCount = 0; + } + + /** + * Queues a replication event. The cache is automatically flushed + * if it exceeds {@link ReplicationEventBatchOptions['max_batch_size']}. + * + */ + async save(params: ReplicationEventWriteParams) { + const { data, event, table } = params; + if (!this.event_cache.has(event)) { + this.event_cache.set(event, new Map()); + } + + const eventEntry = this.event_cache.get(event)!; + + if (!eventEntry.has(table)) { + eventEntry.set(table, []); + } + + const tableEntry = eventEntry.get(table)!; + tableEntry.push(data); + this._eventCacheRowCount++; + + if (this.cacheSize >= this.maxBatchSize) { + await this.flush(); + } + } + + /** + * Flushes cached changes. Events will be emitted to the + * {@link ReplicationEventManager}. + */ + async flush() { + try { + for (const [eventDescription, eventData] of this.event_cache) { + // TODO: how should errors be dealt with here + await this.manager.fireEvent({ + event: eventDescription, + storage: this.storage, + data: eventData + }); + } + } finally { + this.event_cache.clear(); + this._eventCacheRowCount = 0; + } + } +} diff --git a/packages/service-core/src/replication/replication-index.ts b/packages/service-core/src/replication/replication-index.ts index 456fd1dea..f9f6fdda6 100644 --- a/packages/service-core/src/replication/replication-index.ts +++ b/packages/service-core/src/replication/replication-index.ts @@ -2,5 +2,6 @@ export * from './AbstractReplicationJob.js'; export * from './AbstractReplicator.js'; export * from './ErrorRateLimiter.js'; export * from './ReplicationEngine.js'; +export * from './ReplicationEventBatch.js'; export * from './ReplicationEventManager.js'; export * from './ReplicationModule.js'; From fda1a246230924aab856976588adb8f930394540 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 10 Sep 2024 09:48:20 +0200 Subject: [PATCH 05/33] add sync rules id to checkpoints --- .../src/replication/ReplicationEventBatch.ts | 3 +-- .../service-core/src/storage/BucketStorage.ts | 20 +++++++++++++++---- .../service-core/src/storage/mongo/models.ts | 3 ++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/packages/service-core/src/replication/ReplicationEventBatch.ts b/packages/service-core/src/replication/ReplicationEventBatch.ts index 30535e6e1..8bdbc9842 100644 --- a/packages/service-core/src/replication/ReplicationEventBatch.ts +++ b/packages/service-core/src/replication/ReplicationEventBatch.ts @@ -18,10 +18,10 @@ const MAX_BATCH_SIZE = 1000; export class ReplicationEventBatch { readonly manager: ReplicationEventManager; + readonly maxBatchSize: number; readonly storage: storage.SyncRulesBucketStorage; protected event_cache: Map; - protected readonly maxBatchSize: number; /** * Keeps track of the number of rows in the cache. @@ -53,7 +53,6 @@ export class ReplicationEventBatch { /** * Queues a replication event. The cache is automatically flushed * if it exceeds {@link ReplicationEventBatchOptions['max_batch_size']}. - * */ async save(params: ReplicationEventWriteParams) { const { data, event, table } = params; diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index 887bc1bee..9c75e3ffa 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -8,8 +8,12 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules'; import * as util from '../util/util-index.js'; -import { SourceTable } from './SourceTable.js'; import { SourceEntityDescriptor } from './SourceEntity.js'; +import { SourceTable } from './SourceTable.js'; + +export interface UserCheckpointOptions { + sync_rules_id?: number; +} export interface BucketStorageFactory { /** @@ -80,11 +84,19 @@ export interface BucketStorageFactory { */ getActiveCheckpoint(): Promise; - createWriteCheckpoint(user_id: string, lsns: Record): Promise; + createWriteCheckpoint( + user_id: string, + lsns: Record, + options?: UserCheckpointOptions + ): Promise; - lastWriteCheckpoint(user_id: string, lsn: string): Promise; + lastWriteCheckpoint(user_id: string, lsn: string, options?: UserCheckpointOptions): Promise; - watchWriteCheckpoint(user_id: string, signal: AbortSignal): AsyncIterable; + watchWriteCheckpoint( + user_id: string, + signal: AbortSignal, + options?: UserCheckpointOptions + ): AsyncIterable; /** * Get storage size of active sync rules. diff --git a/packages/service-core/src/storage/mongo/models.ts b/packages/service-core/src/storage/mongo/models.ts index ef26564bc..c0b710a3f 100644 --- a/packages/service-core/src/storage/mongo/models.ts +++ b/packages/service-core/src/storage/mongo/models.ts @@ -1,5 +1,5 @@ -import * as bson from 'bson'; import { SqliteJsonValue } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; export interface SourceKey { /** group_id */ @@ -155,6 +155,7 @@ export interface WriteCheckpointDocument { user_id: string; lsns: Record; client_id: bigint; + sync_rules_id?: number; } export interface InstanceDocument { From 527a7cdc27ab8b6cb42c43b7e2f2dcaa64552655 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 12 Sep 2024 14:45:26 +0200 Subject: [PATCH 06/33] wip: cleanup event emission --- .../src/replication/WalStream.ts | 24 ++++++++++--------- .../replication/ReplicationEventManager.ts | 5 ++++ packages/sync-rules/scripts/compile-schema.js | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index 16610e8c9..5287c2cb9 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -345,7 +345,7 @@ WHERE oid = $1::regclass`, } async initialReplication(db: pgwire.PgConnection, lsn: string) { - const replicationTables = [...this.sync_rules.getSourceTables(), ...this.sync_rules.getEventTables()]; + const replicationTables = [...this.sync_rules.getSourceTables()]; await this.storage.startBatch({}, async (batch) => { const eventBatch = new replication.ReplicationEventBatch({ @@ -356,14 +356,12 @@ WHERE oid = $1::regclass`, for (let tablePattern of replicationTables) { const tables = await this.getQualifiedTableNames(batch, db, tablePattern); for (let table of tables) { - await this.snapshotTable(batch, db, table, eventBatch); - if (table.syncAny) { - await batch.markSnapshotDone([table], lsn); - } - + await this.snapshotTable(batch, db, table, eventBatch, lsn); + await batch.markSnapshotDone([table], lsn); await touch(); } } + await eventBatch.flush(); await batch.commit(lsn); }); } @@ -378,7 +376,8 @@ WHERE oid = $1::regclass`, batch: storage.BucketStorageBatch, db: pgwire.PgConnection, table: storage.SourceTable, - eventBatch?: replication.ReplicationEventBatch + eventBatch?: replication.ReplicationEventBatch, + lsn?: string ) { logger.info(`${this.slot_name} Replicating ${table.qualifiedName}`); const estimatedCount = await this.estimatedCount(db, table); @@ -423,6 +422,7 @@ WHERE oid = $1::regclass`, // The method checks internally if an event should be emitted await this.writeSnapshotEvent(eventBatch, { table, + lsn: lsn!, data: record }); } @@ -545,9 +545,9 @@ WHERE oid = $1::regclass`, protected async writeSnapshotEvent( eventBatch: replication.ReplicationEventBatch, - params: { table: storage.SourceTable; data: SqliteRow } + params: { table: storage.SourceTable; data: SqliteRow; lsn: string } ) { - const { table, data } = params; + const { data, lsn, table } = params; if (!table.triggerEvents) { return null; @@ -558,6 +558,7 @@ WHERE oid = $1::regclass`, event: eventDescription, table, data: { + head: lsn, op: replication.EventOp.INSERT, after: eventDescription.evaluateParameterRow(table, data) } @@ -583,6 +584,7 @@ WHERE oid = $1::regclass`, event: eventDescription, table, data: { + head: msg.lsn!, op: msg.tag as replication.EventOp, after: after ? eventDescription.evaluateParameterRow(table, after) : undefined, before: before ? eventDescription.evaluateParameterRow(table, before) : undefined @@ -646,7 +648,6 @@ WHERE oid = $1::regclass`, // chunkLastLsn may come from normal messages in the chunk, // or from a PrimaryKeepalive message. const { messages, lastLsn: chunkLastLsn } = chunk; - for (const msg of messages) { if (msg.tag == 'relation') { await this.handleRelation(batch, getPgOutputRelation(msg), true); @@ -655,6 +656,7 @@ WHERE oid = $1::regclass`, } else if (msg.tag == 'commit') { Metrics.getInstance().transactions_replicated_total.add(1); inTx = false; + await eventBatch.flush(); await batch.commit(msg.lsn!); await this.ack(msg.lsn!, replicationStream); // Flush cached event operations @@ -668,8 +670,8 @@ WHERE oid = $1::regclass`, } count += 1; - await this.writeChange(batch, msg); await this.writeReplicationEvent(eventBatch, msg); + await this.writeChange(batch, msg); } } diff --git a/packages/service-core/src/replication/ReplicationEventManager.ts b/packages/service-core/src/replication/ReplicationEventManager.ts index 0029f2f7b..868123cbe 100644 --- a/packages/service-core/src/replication/ReplicationEventManager.ts +++ b/packages/service-core/src/replication/ReplicationEventManager.ts @@ -9,6 +9,11 @@ export enum EventOp { export type EventData = { op: EventOp; + /** + * The replication HEAD at the moment where this event ocurred. + * For Postgres this is the LSN. + */ + head: string; before?: sync_rules.EvaluatedParametersResult[]; after?: sync_rules.EvaluatedParametersResult[]; }; diff --git a/packages/sync-rules/scripts/compile-schema.js b/packages/sync-rules/scripts/compile-schema.js index ad9217e49..267de50d5 100644 --- a/packages/sync-rules/scripts/compile-schema.js +++ b/packages/sync-rules/scripts/compile-schema.js @@ -6,6 +6,6 @@ import { syncRulesSchema } from '../dist/json_schema.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const schemaDir = path.join(__dirname, '../schema'); -fs.mkdirSync(schemaDir); +fs.mkdirSync(schemaDir, { recursive: true }); fs.writeFileSync(path.join(schemaDir, 'sync_rules.json'), JSON.stringify(syncRulesSchema, null, '\t')); From 9cb58f8d76d567b1b3c458d7c479fde93eab6498 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 16 Sep 2024 09:21:42 +0200 Subject: [PATCH 07/33] wip --- .../src/replication/ReplicationEventBatch.ts | 2 +- .../service-core/src/storage/BucketStorage.ts | 36 +++++++++++-------- .../src/storage/MongoBucketStorage.ts | 10 ++++-- .../src/storage/mongo/Checkpoint.ts | 23 ++++++++++++ 4 files changed, 52 insertions(+), 19 deletions(-) create mode 100644 packages/service-core/src/storage/mongo/Checkpoint.ts diff --git a/packages/service-core/src/replication/ReplicationEventBatch.ts b/packages/service-core/src/replication/ReplicationEventBatch.ts index 8bdbc9842..463bf6844 100644 --- a/packages/service-core/src/replication/ReplicationEventBatch.ts +++ b/packages/service-core/src/replication/ReplicationEventBatch.ts @@ -82,7 +82,7 @@ export class ReplicationEventBatch { async flush() { try { for (const [eventDescription, eventData] of this.event_cache) { - // TODO: how should errors be dealt with here + // TODO: Handle errors with hooks await this.manager.fireEvent({ event: eventDescription, storage: this.storage, diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index 9c75e3ffa..b37dc5924 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -10,10 +10,7 @@ import { import * as util from '../util/util-index.js'; import { SourceEntityDescriptor } from './SourceEntity.js'; import { SourceTable } from './SourceTable.js'; - -export interface UserCheckpointOptions { - sync_rules_id?: number; -} +import { WriteCheckpointFilters, WriteCheckpointOptions } from './mongo/Checkpoint.js'; export interface BucketStorageFactory { /** @@ -84,19 +81,28 @@ export interface BucketStorageFactory { */ getActiveCheckpoint(): Promise; - createWriteCheckpoint( - user_id: string, - lsns: Record, - options?: UserCheckpointOptions - ): Promise; + /** + * Creates a raw write checkpoint given primitive values. + */ + createRawWriteCheckpoint(checkpoint: WriteCheckpointOptions): Promise; - lastWriteCheckpoint(user_id: string, lsn: string, options?: UserCheckpointOptions): Promise; + /** + * Creates a mapping of user_id + LSN(s) to an + * automatically (managed) incrementing write checkpoint. + */ + createWriteCheckpoint(user_id: string, lsns: Record): Promise; - watchWriteCheckpoint( - user_id: string, - signal: AbortSignal, - options?: UserCheckpointOptions - ): AsyncIterable; + /** + * Gets the last write checkpoint before the specified filters. + * Checkpoint will be before the specified LSN(s) if provided. + * Checkpoint will belong to the specified sync rules if provided. + */ + lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise; + + /** + * Yields the latest write checkpoint whenever the sync checkpoint updates. + */ + watchWriteCheckpoint(filters: WriteCheckpointFilters, signal: AbortSignal): AsyncIterable; /** * Get storage size of active sync rules. diff --git a/packages/service-core/src/storage/MongoBucketStorage.ts b/packages/service-core/src/storage/MongoBucketStorage.ts index d1fd1fab0..5ada1f2e2 100644 --- a/packages/service-core/src/storage/MongoBucketStorage.ts +++ b/packages/service-core/src/storage/MongoBucketStorage.ts @@ -21,6 +21,7 @@ import { UpdateSyncRulesOptions, WriteCheckpoint } from './BucketStorage.js'; +import { WriteCheckpointFilters } from './mongo/Checkpoint.js'; import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js'; import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js'; import { PowerSyncMongo, PowerSyncMongoOptions } from './mongo/db.js'; @@ -272,10 +273,13 @@ export class MongoBucketStorage implements BucketStorageFactory { return doc!.client_id; } - async lastWriteCheckpoint(user_id: string, lsn: string): Promise { + // async lastWriteCheckpoint(user_id: string, lsn: string): Promise { + async lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise { + const { user_id, lsns, sync_rules_id } = filters; const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({ user_id: user_id, - 'lsns.1': { $lte: lsn } + sync_rules_id, + 'lsns.1': { $lte: lsns } }); return lastWriteCheckpoint?.client_id ?? null; } @@ -502,7 +506,7 @@ export class MongoBucketStorage implements BucketStorageFactory { // 1. checkpoint (op_id) changes. // 2. write checkpoint changes for the specific user - const currentWriteCheckpoint = await this.lastWriteCheckpoint(user_id, lsn ?? ''); + const currentWriteCheckpoint = await this.lastWriteCheckpoint(user_id); if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) { // No change - wait for next one diff --git a/packages/service-core/src/storage/mongo/Checkpoint.ts b/packages/service-core/src/storage/mongo/Checkpoint.ts new file mode 100644 index 000000000..8f33a02c8 --- /dev/null +++ b/packages/service-core/src/storage/mongo/Checkpoint.ts @@ -0,0 +1,23 @@ +export interface WriteCheckpointFilters { + /** + * LSN(s) at the creation of the checkpoint. + */ + lsns?: Record; + + /** + * Sync rules which were active when this checkpoint was created. + */ + sync_rules_id?: number; + + /** + * Identifier for User's account. + */ + user_id: string; +} + +export interface WriteCheckpointOptions extends WriteCheckpointFilters { + /** + * Strictly incrementing write checkpoint number + */ + checkpoint: bigint; +} From a05dea6b58c06701652b9c3028a7ad0849c16d59 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 16 Sep 2024 17:35:55 +0200 Subject: [PATCH 08/33] move replication events to storage. Separate implementations for custom write checkpoints --- .../src/module/PostgresModule.ts | 7 +- .../src/replication/WalStream.ts | 101 +----------------- .../replication/WalStreamReplicationJob.ts | 6 +- .../src/replication/WalStreamReplicator.ts | 6 +- .../test/src/slow_tests.test.ts | 7 +- modules/module-postgres/test/src/util.ts | 13 ++- .../test/src/wal_stream_utils.ts | 5 +- .../src/entry/commands/compact-action.ts | 5 +- .../src/replication/ReplicationEngine.ts | 7 -- .../src/replication/replication-index.ts | 2 - .../src/routes/endpoints/checkpointing.ts | 7 +- .../service-core/src/storage/BucketStorage.ts | 46 ++++---- .../src/storage/MongoBucketStorage.ts | 70 ++++++------ .../ReplicationEventBatch.ts | 0 .../ReplicationEventManager.ts | 15 +-- .../service-core/src/storage/StorageEngine.ts | 34 +++++- .../src/storage/StorageProvider.ts | 14 ++- .../src/storage/mongo/Checkpoint.ts | 23 ---- .../src/storage/mongo/MongoBucketBatch.ts | 33 +++++- .../mongo/MongoCustomWriteCheckpointAPI.ts | 43 ++++++++ .../mongo/MongoManagedWriteCheckpointAPI.ts | 44 ++++++++ .../src/storage/mongo/MongoStorageProvider.ts | 10 +- .../storage/mongo/MongoSyncBucketStorage.ts | 7 +- packages/service-core/src/storage/mongo/db.ts | 6 +- .../service-core/src/storage/mongo/models.ts | 9 +- .../service-core/src/storage/storage-index.ts | 2 + .../src/storage/write-checkpoint.ts | 46 ++++++++ packages/service-core/src/sync/sync.ts | 7 +- packages/service-core/test/src/util.ts | 3 +- 29 files changed, 341 insertions(+), 237 deletions(-) rename packages/service-core/src/{replication => storage}/ReplicationEventBatch.ts (100%) rename packages/service-core/src/{replication => storage}/ReplicationEventManager.ts (81%) delete mode 100644 packages/service-core/src/storage/mongo/Checkpoint.ts create mode 100644 packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts create mode 100644 packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts create mode 100644 packages/service-core/src/storage/write-checkpoint.ts diff --git a/modules/module-postgres/src/module/PostgresModule.ts b/modules/module-postgres/src/module/PostgresModule.ts index 7e1b1ab37..eabffca0a 100644 --- a/modules/module-postgres/src/module/PostgresModule.ts +++ b/modules/module-postgres/src/module/PostgresModule.ts @@ -1,4 +1,4 @@ -import { api, auth, ConfigurationFileSyncRulesProvider, replication, system } from '@powersync/service-core'; +import { api, auth, ConfigurationFileSyncRulesProvider, modules, replication, system } from '@powersync/service-core'; import * as jpgwire from '@powersync/service-jpgwire'; import { PostgresRouteAPIAdapter } from '../api/PostgresRouteAPIAdapter.js'; import { SupabaseKeyCollector } from '../auth/SupabaseKeyCollector.js'; @@ -49,8 +49,7 @@ export class PostgresModule extends replication.ReplicationModule { + async teardown(options: modules.TearDownOptions): Promise { const normalisedConfig = this.resolveConfig(this.decodedConfig!); const connectionManager = new PgManager(normalisedConfig, { idleTimeout: 30_000, diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index ed6fd7509..5767d4a3c 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -1,14 +1,7 @@ import { container, errors, logger } from '@powersync/lib-services-framework'; -import { Metrics, replication, SourceEntityDescriptor, storage } from '@powersync/service-core'; +import { Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; -import { - DatabaseInputRow, - SqlEventDescriptor, - SqliteRow, - SqlSyncRules, - TablePattern, - toSyncRulesRow -} from '@powersync/service-sync-rules'; +import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules'; import * as util from '../utils/pgwire_utils.js'; import { PgManager } from './PgManager.js'; import { getPgOutputRelation, getRelId } from './PgRelation.js'; @@ -21,7 +14,6 @@ export interface WalStreamOptions { connections: PgManager; storage: storage.SyncRulesBucketStorage; abort_signal: AbortSignal; - event_manager: replication.ReplicationEventManager; } interface InitResult { @@ -48,21 +40,16 @@ export class WalStream { private abort_signal: AbortSignal; - private event_manager: replication.ReplicationEventManager; - private relation_cache = new Map(); private startedStreaming = false; - private event_cache = new Map(); - constructor(options: WalStreamOptions) { this.storage = options.storage; this.sync_rules = options.storage.sync_rules; this.group_id = options.storage.group_id; this.slot_name = options.storage.slot_name; this.connections = options.connections; - this.event_manager = options.event_manager; this.abort_signal = options.abort_signal; this.abort_signal.addEventListener( @@ -349,12 +336,11 @@ WHERE oid = $1::regclass`, for (let tablePattern of this.sync_rules.getSourceTables()) { const tables = await this.getQualifiedTableNames(batch, db, tablePattern); for (let table of tables) { - await this.snapshotTable(batch, db, table, eventBatch, lsn); + await this.snapshotTable(batch, db, table, lsn); await batch.markSnapshotDone([table], lsn); await touch(); } } - await eventBatch.flush(); await batch.commit(lsn); }); } @@ -369,7 +355,6 @@ WHERE oid = $1::regclass`, batch: storage.BucketStorageBatch, db: pgwire.PgConnection, table: storage.SourceTable, - eventBatch?: replication.ReplicationEventBatch, lsn?: string ) { logger.info(`${this.slot_name} Replicating ${table.qualifiedName}`); @@ -410,15 +395,6 @@ WHERE oid = $1::regclass`, if (table.syncAny) { await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record }); } - // Event batch is only provided in the initial snapshot - if (eventBatch) { - // The method checks internally if an event should be emitted - await this.writeSnapshotEvent(eventBatch, { - table, - lsn: lsn!, - data: record - }); - } } at += rows.length; @@ -536,57 +512,6 @@ WHERE oid = $1::regclass`, return null; } - protected async writeSnapshotEvent( - eventBatch: replication.ReplicationEventBatch, - params: { table: storage.SourceTable; data: SqliteRow; lsn: string } - ) { - const { data, lsn, table } = params; - - if (!table.triggerEvents) { - return null; - } - - for (const eventDescription of this.getTableEvents(table)) { - await eventBatch.save({ - event: eventDescription, - table, - data: { - head: lsn, - op: replication.EventOp.INSERT, - after: eventDescription.evaluateParameterRow(table, data) - } - }); - } - } - - /** - * Writes a replication event which is triggered from the replication stream. - */ - protected async writeReplicationEvent(batch: replication.ReplicationEventBatch, msg: pgwire.PgoutputMessage) { - if (msg.tag == 'insert' || msg.tag == 'update' || msg.tag == 'delete') { - const table = this.getTable(getRelId(msg.relation)); - if (!table.triggerEvents) { - return null; - } - - const before = msg.tag == 'update' ? util.constructBeforeRecord(msg) : undefined; - const after = msg.tag !== 'delete' ? util.constructAfterRecord(msg) : undefined; - - for (const eventDescription of this.getTableEvents(table)) { - await batch.save({ - event: eventDescription, - table, - data: { - head: msg.lsn!, - op: msg.tag as replication.EventOp, - after: after ? eventDescription.evaluateParameterRow(table, after) : undefined, - before: before ? eventDescription.evaluateParameterRow(table, before) : undefined - } - }); - } - } - } - async replicate() { try { // If anything errors here, the entire replication process is halted, and @@ -626,10 +551,6 @@ WHERE oid = $1::regclass`, // Replication never starts in the middle of a transaction let inTx = false; let count = 0; - const eventBatch = new replication.ReplicationEventBatch({ - manager: this.event_manager, - storage: this.storage - }); for await (const chunk of replicationStream.pgoutputDecode()) { await touch(); @@ -649,21 +570,14 @@ WHERE oid = $1::regclass`, } else if (msg.tag == 'commit') { Metrics.getInstance().transactions_replicated_total.add(1); inTx = false; - await eventBatch.flush(); await batch.commit(msg.lsn!); await this.ack(msg.lsn!, replicationStream); - // Flush cached event operations - for (const event of this.event_cache.values()) { - await this.event_manager.fireEvent(event); - } - this.event_cache.clear(); } else { if (count % 100 == 0) { logger.info(`${this.slot_name} replicating op ${count} ${msg.lsn}`); } count += 1; - await this.writeReplicationEvent(eventBatch, msg); await this.writeChange(batch, msg); } } @@ -691,15 +605,6 @@ WHERE oid = $1::regclass`, replicationStream.ack(lsn); } - - /** - * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable} - */ - protected getTableEvents(table: storage.SourceTable): SqlEventDescriptor[] { - return this.sync_rules.event_descriptors.filter((evt) => - [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table)) - ); - } } async function touch() { diff --git a/modules/module-postgres/src/replication/WalStreamReplicationJob.ts b/modules/module-postgres/src/replication/WalStreamReplicationJob.ts index 3bfb0029b..1f7b1cebe 100644 --- a/modules/module-postgres/src/replication/WalStreamReplicationJob.ts +++ b/modules/module-postgres/src/replication/WalStreamReplicationJob.ts @@ -8,13 +8,11 @@ import { cleanUpReplicationSlot } from './replication-utils.js'; export interface WalStreamReplicationJobOptions extends replication.AbstractReplicationJobOptions { connectionFactory: ConnectionManagerFactory; - eventManager: replication.ReplicationEventManager; } export class WalStreamReplicationJob extends replication.AbstractReplicationJob { private connectionFactory: ConnectionManagerFactory; private readonly connectionManager: PgManager; - private readonly eventManager: replication.ReplicationEventManager; constructor(options: WalStreamReplicationJobOptions) { super(options); @@ -24,7 +22,6 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob idleTimeout: 30_000, maxSize: 2 }); - this.eventManager = options.eventManager; } async cleanUp(): Promise { @@ -111,8 +108,7 @@ export class WalStreamReplicationJob extends replication.AbstractReplicationJob const stream = new WalStream({ abort_signal: this.abortController.signal, storage: this.options.storage, - connections: connectionManager, - event_manager: this.eventManager + connections: connectionManager }); await stream.replicate(); } catch (e) { diff --git a/modules/module-postgres/src/replication/WalStreamReplicator.ts b/modules/module-postgres/src/replication/WalStreamReplicator.ts index 2f194c94d..329c78607 100644 --- a/modules/module-postgres/src/replication/WalStreamReplicator.ts +++ b/modules/module-postgres/src/replication/WalStreamReplicator.ts @@ -5,17 +5,14 @@ import { WalStreamReplicationJob } from './WalStreamReplicationJob.js'; export interface WalStreamReplicatorOptions extends replication.AbstractReplicatorOptions { connectionFactory: ConnectionManagerFactory; - eventManager: replication.ReplicationEventManager; } export class WalStreamReplicator extends replication.AbstractReplicator { private readonly connectionFactory: ConnectionManagerFactory; - private readonly eventManager: replication.ReplicationEventManager; constructor(options: WalStreamReplicatorOptions) { super(options); this.connectionFactory = options.connectionFactory; - this.eventManager = options.eventManager; } createJob(options: replication.CreateJobOptions): WalStreamReplicationJob { @@ -23,8 +20,7 @@ export class WalStreamReplicator extends replication.AbstractReplicator { await db.clear(); - return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }); + return new MongoBucketStorage(db, { + slot_name_prefix: 'test_', + event_manager: new storage.ReplicationEventManager() + }); }; export async function clearTestDb(db: pgwire.PgClient) { diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index 6f376dfe0..43bf8b268 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -1,7 +1,7 @@ import { fromAsync } from '@core-tests/stream_utils.js'; import { PgManager } from '@module/replication/PgManager.js'; import { PUBLICATION_NAME, WalStream, WalStreamOptions } from '@module/replication/WalStream.js'; -import { BucketStorageFactory, replication, SyncRulesBucketStorage } from '@powersync/service-core'; +import { BucketStorageFactory, SyncRulesBucketStorage } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js'; @@ -72,8 +72,7 @@ export class WalStreamTestContext { const options: WalStreamOptions = { storage: this.storage, connections: this.connectionManager, - abort_signal: this.abortController.signal, - event_manager: new replication.ReplicationEventManager() + abort_signal: this.abortController.signal }; this._walStream = new WalStream(options); return this._walStream!; diff --git a/packages/service-core/src/entry/commands/compact-action.ts b/packages/service-core/src/entry/commands/compact-action.ts index 6016644e5..7a9f0e8b6 100644 --- a/packages/service-core/src/entry/commands/compact-action.ts +++ b/packages/service-core/src/entry/commands/compact-action.ts @@ -33,7 +33,10 @@ export function registerCompactAction(program: Command) { const client = psdb.client; await client.connect(); try { - const bucketStorage = new storage.MongoBucketStorage(psdb, { slot_name_prefix: configuration.slot_name_prefix }); + const bucketStorage = new storage.MongoBucketStorage(psdb, { + event_manager: new storage.ReplicationEventManager(), + slot_name_prefix: configuration.slot_name_prefix + }); const active = await bucketStorage.getActiveSyncRules(); if (active == null) { logger.info('No active instance to compact'); diff --git a/packages/service-core/src/replication/ReplicationEngine.ts b/packages/service-core/src/replication/ReplicationEngine.ts index fd035be96..12d52f50a 100644 --- a/packages/service-core/src/replication/ReplicationEngine.ts +++ b/packages/service-core/src/replication/ReplicationEngine.ts @@ -1,16 +1,9 @@ import { logger } from '@powersync/lib-services-framework'; import { AbstractReplicator } from './AbstractReplicator.js'; -import { ReplicationEventManager } from './ReplicationEventManager.js'; export class ReplicationEngine { private readonly replicators: Map = new Map(); - readonly eventManager: ReplicationEventManager; - - constructor() { - this.eventManager = new ReplicationEventManager(); - } - /** * Register a Replicator with the engine * diff --git a/packages/service-core/src/replication/replication-index.ts b/packages/service-core/src/replication/replication-index.ts index f9f6fdda6..0b37534c9 100644 --- a/packages/service-core/src/replication/replication-index.ts +++ b/packages/service-core/src/replication/replication-index.ts @@ -2,6 +2,4 @@ export * from './AbstractReplicationJob.js'; export * from './AbstractReplicator.js'; export * from './ErrorRateLimiter.js'; export * from './ReplicationEngine.js'; -export * from './ReplicationEventBatch.js'; -export * from './ReplicationEventManager.js'; export * from './ReplicationModule.js'; diff --git a/packages/service-core/src/routes/endpoints/checkpointing.ts b/packages/service-core/src/routes/endpoints/checkpointing.ts index c60c3b233..ae45973a2 100644 --- a/packages/service-core/src/routes/endpoints/checkpointing.ts +++ b/packages/service-core/src/routes/endpoints/checkpointing.ts @@ -1,5 +1,5 @@ -import * as t from 'ts-codec'; import { logger, router, schema } from '@powersync/lib-services-framework'; +import * as t from 'ts-codec'; import * as util from '../../util/util-index.js'; import { authUser } from '../auth.js'; @@ -63,7 +63,10 @@ export const writeCheckpoint2 = routeDefinition({ storageEngine: { activeBucketStorage } } = service_context; - const writeCheckpoint = await activeBucketStorage.createWriteCheckpoint(full_user_id, { '1': currentCheckpoint }); + const writeCheckpoint = await activeBucketStorage.createWriteCheckpoint({ + user_id: full_user_id, + heads: { '1': currentCheckpoint } + }); logger.info(`Write checkpoint 2: ${JSON.stringify({ currentCheckpoint, id: String(full_user_id) })}`); return { diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index f4c3ce716..bec2f8ddb 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -8,11 +8,26 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules'; import * as util from '../util/util-index.js'; +import { ReplicationEventManager } from './ReplicationEventManager.js'; import { SourceEntityDescriptor } from './SourceEntity.js'; import { SourceTable } from './SourceTable.js'; -import { WriteCheckpointFilters, WriteCheckpointOptions } from './mongo/Checkpoint.js'; +import { WriteCheckpointAPI, WriteCheckpointFilters } from './write-checkpoint.js'; -export interface BucketStorageFactory { +// Checkpoints + +/** + * Checkpoints + * Need to be either custom or manual + * + * Need to use a different collection for different types + * Need indexes on a new collection + * + * Manual + * Need to specify the write checkpoint value + * Need to specify the sync rules for the checkpoint + */ + +export interface BucketStorageFactory extends WriteCheckpointAPI { /** * Update sync rules from configuration, if changed. */ @@ -21,6 +36,11 @@ export interface BucketStorageFactory { options?: { lock?: boolean } ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }>; + /** + * Manager which enables handling events for replicated data. + */ + events: ReplicationEventManager; + /** * Get a storage instance to query sync data for specific sync rules. */ @@ -82,25 +102,7 @@ export interface BucketStorageFactory { getActiveCheckpoint(): Promise; /** - * Creates a raw write checkpoint given primitive values. - */ - createRawWriteCheckpoint(checkpoint: WriteCheckpointOptions): Promise; - - /** - * Creates a mapping of user_id + LSN(s) to an - * automatically (managed) incrementing write checkpoint. - */ - createWriteCheckpoint(user_id: string, lsns: Record): Promise; - - /** - * Gets the last write checkpoint before the specified filters. - * Checkpoint will be before the specified LSN(s) if provided. - * Checkpoint will belong to the specified sync rules if provided. - */ - lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise; - - /** - * Yields the latest write checkpoint whenever the sync checkpoint updates. + * Yields the latest user write checkpoint whenever the sync checkpoint updates. */ watchWriteCheckpoint(filters: WriteCheckpointFilters, signal: AbortSignal): AsyncIterable; @@ -369,6 +371,8 @@ export interface SaveBucketData { evaluated: EvaluatedRow[]; } +export type SaveOp = 'insert' | 'update' | 'delete'; + export type SaveOptions = SaveInsert | SaveUpdate | SaveDelete; export interface SaveInsert { diff --git a/packages/service-core/src/storage/MongoBucketStorage.ts b/packages/service-core/src/storage/MongoBucketStorage.ts index 6b90b684e..c0847deb3 100644 --- a/packages/service-core/src/storage/MongoBucketStorage.ts +++ b/packages/service-core/src/storage/MongoBucketStorage.ts @@ -19,12 +19,21 @@ import { UpdateSyncRulesOptions, WriteCheckpoint } from './BucketStorage.js'; -import { WriteCheckpointFilters } from './mongo/Checkpoint.js'; -import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js'; -import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js'; import { PowerSyncMongo, PowerSyncMongoOptions } from './mongo/db.js'; import { SyncRuleDocument, SyncRuleState } from './mongo/models.js'; +import { MongoCustomWriteCheckpointAPI } from './mongo/MongoCustomWriteCheckpointAPI.js'; +import { MongoManagedWriteCheckpointAPI } from './mongo/MongoManagedWriteCheckpointAPI.js'; +import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js'; +import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js'; import { generateSlotName } from './mongo/util.js'; +import { ReplicationEventManager } from './ReplicationEventManager.js'; +import { + DEFAULT_WRITE_CHECKPOINT_MODE, + WriteCheckpointAPI, + WriteCheckpointFilters, + WriteCheckpointMode, + WriteCheckpointOptions +} from './write-checkpoint.js'; export interface MongoBucketStorageOptions extends PowerSyncMongoOptions {} @@ -34,6 +43,11 @@ export class MongoBucketStorage implements BucketStorageFactory { // TODO: This is still Postgres specific and needs to be reworked public readonly slot_name_prefix: string; + readonly events: ReplicationEventManager; + readonly write_checkpoint_mode: WriteCheckpointMode; + + protected readonly writeCheckpointAPI: WriteCheckpointAPI; + private readonly storageCache = new LRUCache({ max: 3, fetchMethod: async (id) => { @@ -54,11 +68,24 @@ export class MongoBucketStorage implements BucketStorageFactory { public readonly db: PowerSyncMongo; - constructor(db: PowerSyncMongo, options: { slot_name_prefix: string }) { + constructor( + db: PowerSyncMongo, + options: { + slot_name_prefix: string; + event_manager: ReplicationEventManager; + write_checkpoint_mode?: WriteCheckpointMode; + } + ) { this.client = db.client; this.db = db; this.session = this.client.startSession(); this.slot_name_prefix = options.slot_name_prefix; + this.events = options.event_manager; + this.write_checkpoint_mode = options.write_checkpoint_mode ?? DEFAULT_WRITE_CHECKPOINT_MODE; + this.writeCheckpointAPI = + this.write_checkpoint_mode == WriteCheckpointMode.MANAGED + ? new MongoManagedWriteCheckpointAPI(db) + : new MongoCustomWriteCheckpointAPI(db); } getInstance(options: PersistedSyncRules): MongoSyncBucketStorage { @@ -252,33 +279,12 @@ export class MongoBucketStorage implements BucketStorageFactory { }); } - async createWriteCheckpoint(user_id: string, lsns: Record): Promise { - const doc = await this.db.write_checkpoints.findOneAndUpdate( - { - user_id: user_id - }, - { - $set: { - lsns: lsns - }, - $inc: { - client_id: 1n - } - }, - { upsert: true, returnDocument: 'after' } - ); - return doc!.client_id; + async createWriteCheckpoint(options: WriteCheckpointOptions): Promise { + return this.writeCheckpointAPI.createWriteCheckpoint(options); } - // async lastWriteCheckpoint(user_id: string, lsn: string): Promise { async lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise { - const { user_id, lsns, sync_rules_id } = filters; - const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({ - user_id: user_id, - sync_rules_id, - 'lsns.1': { $lte: lsns } - }); - return lastWriteCheckpoint?.client_id ?? null; + return this.writeCheckpointAPI.lastWriteCheckpoint(filters); } async getActiveCheckpoint(): Promise { @@ -490,7 +496,7 @@ export class MongoBucketStorage implements BucketStorageFactory { /** * User-specific watch on the latest checkpoint and/or write checkpoint. */ - async *watchWriteCheckpoint(user_id: string, signal: AbortSignal): AsyncIterable { + async *watchWriteCheckpoint(filters: WriteCheckpointFilters, signal: AbortSignal): AsyncIterable { let lastCheckpoint: util.OpId | null = null; let lastWriteCheckpoint: bigint | null = null; @@ -503,7 +509,11 @@ export class MongoBucketStorage implements BucketStorageFactory { // 1. checkpoint (op_id) changes. // 2. write checkpoint changes for the specific user - const currentWriteCheckpoint = await this.lastWriteCheckpoint(user_id); + const bucketStorage = await cp.getBucketStorage(); // TODO validate and optimize + const currentWriteCheckpoint = await this.lastWriteCheckpoint({ + sync_rules_id: bucketStorage?.group_id, + ...filters + }); if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) { // No change - wait for next one diff --git a/packages/service-core/src/replication/ReplicationEventBatch.ts b/packages/service-core/src/storage/ReplicationEventBatch.ts similarity index 100% rename from packages/service-core/src/replication/ReplicationEventBatch.ts rename to packages/service-core/src/storage/ReplicationEventBatch.ts diff --git a/packages/service-core/src/replication/ReplicationEventManager.ts b/packages/service-core/src/storage/ReplicationEventManager.ts similarity index 81% rename from packages/service-core/src/replication/ReplicationEventManager.ts rename to packages/service-core/src/storage/ReplicationEventManager.ts index 868123cbe..c7fb4a984 100644 --- a/packages/service-core/src/replication/ReplicationEventManager.ts +++ b/packages/service-core/src/storage/ReplicationEventManager.ts @@ -1,14 +1,9 @@ import * as sync_rules from '@powersync/service-sync-rules'; -import * as storage from '../storage/storage-index.js'; - -export enum EventOp { - INSERT = 'insert', - UPDATE = 'update', - DELETE = 'delete' -} +import { SaveOp, SyncRulesBucketStorage } from './BucketStorage.js'; +import { SourceTable } from './SourceTable.js'; export type EventData = { - op: EventOp; + op: SaveOp; /** * The replication HEAD at the moment where this event ocurred. * For Postgres this is the LSN. @@ -18,12 +13,12 @@ export type EventData = { after?: sync_rules.EvaluatedParametersResult[]; }; -export type ReplicationEventData = Map; +export type ReplicationEventData = Map; export type ReplicationEventPayload = { event: sync_rules.SqlEventDescriptor; data: ReplicationEventData; - storage: storage.SyncRulesBucketStorage; + storage: SyncRulesBucketStorage; }; export interface ReplicationEventHandler { diff --git a/packages/service-core/src/storage/StorageEngine.ts b/packages/service-core/src/storage/StorageEngine.ts index 966851a26..3a8dce9c4 100644 --- a/packages/service-core/src/storage/StorageEngine.ts +++ b/packages/service-core/src/storage/StorageEngine.ts @@ -1,17 +1,29 @@ import { ResolvedPowerSyncConfig } from '../util/util-index.js'; import { BucketStorageFactory } from './BucketStorage.js'; -import { BucketStorageProvider, ActiveStorage } from './StorageProvider.js'; +import { ReplicationEventManager } from './ReplicationEventManager.js'; +import { ActiveStorage, BucketStorageProvider, StorageSettings } from './StorageProvider.js'; +import { DEFAULT_WRITE_CHECKPOINT_MODE } from './write-checkpoint.js'; export type StorageEngineOptions = { configuration: ResolvedPowerSyncConfig; }; +export const DEFAULT_STORAGE_SETTINGS: StorageSettings = { + writeCheckpointMode: DEFAULT_WRITE_CHECKPOINT_MODE +}; + export class StorageEngine { // TODO: This will need to revisited when we actually support multiple storage providers. private storageProviders: Map = new Map(); private currentActiveStorage: ActiveStorage | null = null; + readonly events: ReplicationEventManager; - constructor(private options: StorageEngineOptions) {} + private _activeSettings: StorageSettings; + + constructor(private options: StorageEngineOptions) { + this.events = new ReplicationEventManager(); + this._activeSettings = DEFAULT_STORAGE_SETTINGS; + } get activeBucketStorage(): BucketStorageFactory { return this.activeStorage.storage; @@ -25,6 +37,20 @@ export class StorageEngine { return this.currentActiveStorage; } + get activeSettings(): StorageSettings { + return { ...this._activeSettings }; + } + + updateSettings(settings: Partial) { + if (this.currentActiveStorage) { + throw new Error(`Storage is already active, settings cannot be modified.`); + } + this._activeSettings = { + ...this._activeSettings, + ...settings + }; + } + /** * Register a provider which generates a {@link BucketStorageFactory} * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig} @@ -36,7 +62,9 @@ export class StorageEngine { public async start(): Promise { const { configuration } = this.options; this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({ - resolvedConfig: configuration + resolvedConfig: configuration, + eventManager: this.events, + ...this.activeSettings }); } diff --git a/packages/service-core/src/storage/StorageProvider.ts b/packages/service-core/src/storage/StorageProvider.ts index 9e6078874..4e79ecbdb 100644 --- a/packages/service-core/src/storage/StorageProvider.ts +++ b/packages/service-core/src/storage/StorageProvider.ts @@ -1,5 +1,7 @@ -import { BucketStorageFactory } from './BucketStorage.js'; import * as util from '../util/util-index.js'; +import { BucketStorageFactory } from './BucketStorage.js'; +import { ReplicationEventManager } from './ReplicationEventManager.js'; +import { WriteCheckpointMode } from './write-checkpoint.js'; export interface ActiveStorage { storage: BucketStorageFactory; @@ -11,7 +13,15 @@ export interface ActiveStorage { tearDown(): Promise; } -export interface GetStorageOptions { +/** + * Settings which can be modified by various modules in their initialization. + */ +export interface StorageSettings { + writeCheckpointMode: WriteCheckpointMode; +} + +export interface GetStorageOptions extends StorageSettings { + eventManager: ReplicationEventManager; // TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage resolvedConfig: util.ResolvedPowerSyncConfig; } diff --git a/packages/service-core/src/storage/mongo/Checkpoint.ts b/packages/service-core/src/storage/mongo/Checkpoint.ts deleted file mode 100644 index 8f33a02c8..000000000 --- a/packages/service-core/src/storage/mongo/Checkpoint.ts +++ /dev/null @@ -1,23 +0,0 @@ -export interface WriteCheckpointFilters { - /** - * LSN(s) at the creation of the checkpoint. - */ - lsns?: Record; - - /** - * Sync rules which were active when this checkpoint was created. - */ - sync_rules_id?: number; - - /** - * Identifier for User's account. - */ - user_id: string; -} - -export interface WriteCheckpointOptions extends WriteCheckpointFilters { - /** - * Strictly incrementing write checkpoint number - */ - checkpoint: bigint; -} diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index 973f6a28a..a62bd84f3 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -1,10 +1,11 @@ -import { SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules'; +import { SqlEventDescriptor, SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import * as mongo from 'mongodb'; import { container, errors, logger } from '@powersync/lib-services-framework'; import * as util from '../../util/util-index.js'; import { BucketStorageBatch, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js'; +import { ReplicationEventBatch } from '../ReplicationEventBatch.js'; import { SourceTable } from '../SourceTable.js'; import { PowerSyncMongo } from './db.js'; import { CurrentBucket, CurrentDataDocument, SourceKey } from './models.js'; @@ -50,6 +51,8 @@ export class MongoBucketBatch implements BucketStorageBatch { private persisted_op: bigint | null = null; + private event_batch: ReplicationEventBatch; + /** * For tests only - not for persistence logic. */ @@ -61,7 +64,8 @@ export class MongoBucketBatch implements BucketStorageBatch { group_id: number, slot_name: string, last_checkpoint_lsn: string | null, - no_checkpoint_before_lsn: string + no_checkpoint_before_lsn: string, + event_batch: ReplicationEventBatch ) { this.db = db; this.client = db.client; @@ -71,6 +75,7 @@ export class MongoBucketBatch implements BucketStorageBatch { this.session = this.client.startSession(); this.last_checkpoint_lsn = last_checkpoint_lsn; this.no_checkpoint_before_lsn = no_checkpoint_before_lsn; + this.event_batch = event_batch; } async flush(): Promise { @@ -83,6 +88,7 @@ export class MongoBucketBatch implements BucketStorageBatch { result = r; } } + await this.event_batch.flush(); return result; } @@ -611,6 +617,20 @@ export class MongoBucketBatch implements BucketStorageBatch { this.batch ??= new OperationBatch(); this.batch.push(new RecordOperation(record)); + const { after, before, sourceTable, tag } = record; + for (const event of this.getTableEvents(sourceTable)) { + await this.event_batch.save({ + table: sourceTable, + data: { + head: 'TODO', + op: tag, + after: after && util.isCompleteRow(after) ? event.evaluateParameterRow(sourceTable, after) : undefined, + before: before && util.isCompleteRow(before) ? event.evaluateParameterRow(sourceTable, before) : undefined + }, + event + }); + } + if (this.batch.shouldFlush()) { const r = await this.flush(); // HACK: Give other streams a chance to also flush @@ -754,6 +774,15 @@ export class MongoBucketBatch implements BucketStorageBatch { return copy; }); } + + /** + * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable} + */ + protected getTableEvents(table: SourceTable): SqlEventDescriptor[] { + return this.sync_rules.event_descriptors.filter((evt) => + [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table)) + ); + } } export function currentBucketKey(b: CurrentBucket) { diff --git a/packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts b/packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts new file mode 100644 index 000000000..c0c27bd84 --- /dev/null +++ b/packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts @@ -0,0 +1,43 @@ +import { WriteCheckpointAPI, WriteCheckpointFilters, WriteCheckpointOptions } from '../write-checkpoint.js'; +import { PowerSyncMongo } from './db.js'; + +/** + * Implements a write checkpoint API which manages a mapping of + * `user_id` to a provided incrementing `write_checkpoint` `bigint`. + */ +export class MongoCustomWriteCheckpointAPI implements WriteCheckpointAPI { + constructor(protected db: PowerSyncMongo) {} + + async createWriteCheckpoint(options: WriteCheckpointOptions): Promise { + const { checkpoint, user_id, heads, sync_rules_id } = options; + const doc = await this.db.custom_write_checkpoints.findOneAndUpdate( + { + user_id: user_id + }, + { + $set: { + heads, // HEADs are technically not relevant, by we can store them if provided + checkpoint, + sync_rules_id + } + }, + { upsert: true, returnDocument: 'after' } + ); + return doc!.checkpoint; + } + + async lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise { + const { user_id, heads = {} } = filters; + const lsnFilter = Object.fromEntries( + Object.entries(heads).map(([connectionKey, lsn]) => { + return [`heads.${connectionKey}`, { $lte: lsn }]; + }) + ); + + const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({ + user_id: user_id, + ...lsnFilter + }); + return lastWriteCheckpoint?.checkpoint ?? null; + } +} diff --git a/packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts b/packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts new file mode 100644 index 000000000..7a3c5652d --- /dev/null +++ b/packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts @@ -0,0 +1,44 @@ +import { WriteCheckpointAPI, WriteCheckpointFilters, WriteCheckpointOptions } from '../write-checkpoint.js'; +import { PowerSyncMongo } from './db.js'; + +/** + * Implements a write checkpoint API which manages a mapping of + * Replication HEAD + `user_id` to a managed incrementing `write_checkpoint` `bigint`. + */ +export class MongoManagedWriteCheckpointAPI implements WriteCheckpointAPI { + constructor(protected db: PowerSyncMongo) {} + + async createWriteCheckpoint(options: WriteCheckpointOptions): Promise { + const { user_id, heads: lsns } = options; + const doc = await this.db.write_checkpoints.findOneAndUpdate( + { + user_id: user_id + }, + { + $set: { + lsns: lsns + }, + $inc: { + client_id: 1n + } + }, + { upsert: true, returnDocument: 'after' } + ); + return doc!.client_id; + } + + async lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise { + const { user_id, heads: lsns } = filters; + const lsnFilter = Object.fromEntries( + Object.entries(lsns!).map(([connectionKey, lsn]) => { + return [`lsns.${connectionKey}`, { $lte: lsn }]; + }) + ); + + const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({ + user_id: user_id, + ...lsnFilter + }); + return lastWriteCheckpoint?.client_id ?? null; + } +} diff --git a/packages/service-core/src/storage/mongo/MongoStorageProvider.ts b/packages/service-core/src/storage/mongo/MongoStorageProvider.ts index e5af38922..ebea0eb6e 100644 --- a/packages/service-core/src/storage/mongo/MongoStorageProvider.ts +++ b/packages/service-core/src/storage/mongo/MongoStorageProvider.ts @@ -1,8 +1,8 @@ +import { logger } from '@powersync/lib-services-framework'; import * as db from '../../db/db-index.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; -import { BucketStorageProvider, ActiveStorage, GetStorageOptions } from '../StorageProvider.js'; +import { ActiveStorage, BucketStorageProvider, GetStorageOptions } from '../StorageProvider.js'; import { PowerSyncMongo } from './db.js'; -import { logger } from '@powersync/lib-services-framework'; export class MongoStorageProvider implements BucketStorageProvider { get type() { @@ -10,7 +10,7 @@ export class MongoStorageProvider implements BucketStorageProvider { } async getStorage(options: GetStorageOptions): Promise { - const { resolvedConfig } = options; + const { eventManager, resolvedConfig } = options; const client = db.mongo.createMongoClient(resolvedConfig.storage); @@ -19,7 +19,9 @@ export class MongoStorageProvider implements BucketStorageProvider { return { storage: new MongoBucketStorage(database, { // TODO currently need the entire resolved config due to this - slot_name_prefix: resolvedConfig.slot_name_prefix + slot_name_prefix: resolvedConfig.slot_name_prefix, + event_manager: eventManager, + write_checkpoint_mode: options.writeCheckpointMode }), shutDown: () => client.close(), tearDown: () => { diff --git a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts index d20a1401e..caca7690f 100644 --- a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts +++ b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts @@ -21,6 +21,7 @@ import { } from '../BucketStorage.js'; import { ChecksumCache, FetchPartialBucketChecksum } from '../ChecksumCache.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; +import { ReplicationEventBatch } from '../ReplicationEventBatch.js'; import { SourceTable } from '../SourceTable.js'; import { PowerSyncMongo } from './db.js'; import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './models.js'; @@ -75,7 +76,11 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { this.group_id, this.slot_name, checkpoint_lsn, - doc?.no_checkpoint_before ?? options.zeroLSN + doc?.no_checkpoint_before ?? options.zeroLSN, + new ReplicationEventBatch({ + manager: this.factory.events, + storage: this + }) ); try { await callback(batch); diff --git a/packages/service-core/src/storage/mongo/db.ts b/packages/service-core/src/storage/mongo/db.ts index 1cc3f8471..e4e32ac18 100644 --- a/packages/service-core/src/storage/mongo/db.ts +++ b/packages/service-core/src/storage/mongo/db.ts @@ -1,11 +1,13 @@ import * as mongo from 'mongodb'; +import { configFile } from '@powersync/service-types'; import * as db from '../../db/db-index.js'; import * as locks from '../../locks/locks-index.js'; import { BucketDataDocument, BucketParameterDocument, CurrentDataDocument, + CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, SourceTableDocument, @@ -13,7 +15,6 @@ import { WriteCheckpointDocument } from './models.js'; import { BSON_DESERIALIZE_OPTIONS } from './util.js'; -import { configFile } from '@powersync/service-types'; export interface PowerSyncMongoOptions { /** @@ -33,6 +34,7 @@ export class PowerSyncMongo { readonly op_id_sequence: mongo.Collection; readonly sync_rules: mongo.Collection; readonly source_tables: mongo.Collection; + readonly custom_write_checkpoints: mongo.Collection; readonly write_checkpoints: mongo.Collection; readonly instance: mongo.Collection; readonly locks: mongo.Collection; @@ -54,6 +56,8 @@ export class PowerSyncMongo { this.op_id_sequence = db.collection('op_id_sequence'); this.sync_rules = db.collection('sync_rules'); this.source_tables = db.collection('source_tables'); + // TODO add indexes + this.custom_write_checkpoints = db.collection('custom_write_checkpoints'); this.write_checkpoints = db.collection('write_checkpoints'); this.instance = db.collection('instance'); this.locks = this.db.collection('locks'); diff --git a/packages/service-core/src/storage/mongo/models.ts b/packages/service-core/src/storage/mongo/models.ts index c0b710a3f..2956f1202 100644 --- a/packages/service-core/src/storage/mongo/models.ts +++ b/packages/service-core/src/storage/mongo/models.ts @@ -150,12 +150,19 @@ export interface SyncRuleDocument { content: string; } +export interface CustomWriteCheckpointDocument { + _id: bson.ObjectId; + user_id: string; + heads?: Record; + checkpoint: bigint; + sync_rules_id: number; +} + export interface WriteCheckpointDocument { _id: bson.ObjectId; user_id: string; lsns: Record; client_id: bigint; - sync_rules_id?: number; } export interface InstanceDocument { diff --git a/packages/service-core/src/storage/storage-index.ts b/packages/service-core/src/storage/storage-index.ts index 3c137b8d1..50c88c314 100644 --- a/packages/service-core/src/storage/storage-index.ts +++ b/packages/service-core/src/storage/storage-index.ts @@ -1,5 +1,6 @@ export * from './BucketStorage.js'; export * from './MongoBucketStorage.js'; +export * from './ReplicationEventManager.js'; export * from './SourceEntity.js'; export * from './SourceTable.js'; export * from './StorageEngine.js'; @@ -16,3 +17,4 @@ export * from './mongo/MongoSyncRulesLock.js'; export * from './mongo/OperationBatch.js'; export * from './mongo/PersistedBatch.js'; export * from './mongo/util.js'; +export * from './write-checkpoint.js'; diff --git a/packages/service-core/src/storage/write-checkpoint.ts b/packages/service-core/src/storage/write-checkpoint.ts new file mode 100644 index 000000000..2d43bc8a6 --- /dev/null +++ b/packages/service-core/src/storage/write-checkpoint.ts @@ -0,0 +1,46 @@ +export enum WriteCheckpointMode { + /** + * Raw mappings of `user_id` to `write_checkpoint`s should + * be supplied for each set of sync rules. + */ + CUSTOM = 'manual', + /** + * Write checkpoints are stored as a mapping of `user_id` plus + * replication HEAD (lsn in Postgres) to an automatically generated + * incrementing `write_checkpoint` (stored as`client_id`). + */ + MANAGED = 'managed' +} + +export interface WriteCheckpointFilters { + /** + * Replication HEAD(s) at the creation of the checkpoint. + */ + heads?: Record; + + /** + * Sync rules which were active when this checkpoint was created. + */ + sync_rules_id?: number; + + /** + * Identifier for User's account. + */ + user_id: string; +} + +export interface WriteCheckpointOptions extends WriteCheckpointFilters { + /** + * Strictly incrementing write checkpoint number. + * Defaults to an automatically incrementing operation. + */ + checkpoint?: bigint; +} + +export interface WriteCheckpointAPI { + createWriteCheckpoint(options: WriteCheckpointOptions): Promise; + + lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise; +} + +export const DEFAULT_WRITE_CHECKPOINT_MODE = WriteCheckpointMode.MANAGED; diff --git a/packages/service-core/src/sync/sync.ts b/packages/service-core/src/sync/sync.ts index e439888f8..9186aa71a 100644 --- a/packages/service-core/src/sync/sync.ts +++ b/packages/service-core/src/sync/sync.ts @@ -93,7 +93,12 @@ async function* streamResponseInner( } const checkpointUserId = util.checkpointUserId(syncParams.token_parameters.user_id as string, params.client_id); - const stream = storage.watchWriteCheckpoint(checkpointUserId, signal); + const stream = storage.watchWriteCheckpoint( + { + user_id: checkpointUserId + }, + signal + ); for await (const next of stream) { const { base, writeCheckpoint } = next; const checkpoint = base.checkpoint; diff --git a/packages/service-core/test/src/util.ts b/packages/service-core/test/src/util.ts index 4d11c3d5b..cfe428c67 100644 --- a/packages/service-core/test/src/util.ts +++ b/packages/service-core/test/src/util.ts @@ -1,6 +1,7 @@ import { Metrics } from '@/metrics/Metrics.js'; import { BucketStorageFactory, SyncBucketDataBatch } from '@/storage/BucketStorage.js'; import { MongoBucketStorage } from '@/storage/MongoBucketStorage.js'; +import { ReplicationEventManager } from '@/storage/ReplicationEventManager.js'; import { SourceTable } from '@/storage/SourceTable.js'; import { PowerSyncMongo } from '@/storage/mongo/db.js'; import { SyncBucketData } from '@/util/protocol-types.js'; @@ -22,7 +23,7 @@ export type StorageFactory = () => Promise; export const MONGO_STORAGE_FACTORY: StorageFactory = async () => { const db = await connectMongo(); await db.clear(); - return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }); + return new MongoBucketStorage(db, { slot_name_prefix: 'test_', event_manager: new ReplicationEventManager() }); }; export const ZERO_LSN = '0/0'; From 00393d53ddbaab54294834bf57719eabeedee11d Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 17 Sep 2024 13:17:27 +0200 Subject: [PATCH 09/33] cleanup --- .../src/storage/ReplicationEventManager.ts | 5 ----- .../src/storage/mongo/MongoBucketBatch.ts | 1 - packages/sync-rules/package.json | 3 ++- packages/sync-rules/src/SqlSyncRules.ts | 11 +++-------- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/packages/service-core/src/storage/ReplicationEventManager.ts b/packages/service-core/src/storage/ReplicationEventManager.ts index c7fb4a984..d39267146 100644 --- a/packages/service-core/src/storage/ReplicationEventManager.ts +++ b/packages/service-core/src/storage/ReplicationEventManager.ts @@ -4,11 +4,6 @@ import { SourceTable } from './SourceTable.js'; export type EventData = { op: SaveOp; - /** - * The replication HEAD at the moment where this event ocurred. - * For Postgres this is the LSN. - */ - head: string; before?: sync_rules.EvaluatedParametersResult[]; after?: sync_rules.EvaluatedParametersResult[]; }; diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index a62bd84f3..64d0e71a9 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -622,7 +622,6 @@ export class MongoBucketBatch implements BucketStorageBatch { await this.event_batch.save({ table: sourceTable, data: { - head: 'TODO', op: tag, after: after && util.isCompleteRow(after) ? event.evaluateParameterRow(sourceTable, after) : undefined, before: before && util.isCompleteRow(before) ? event.evaluateParameterRow(sourceTable, before) : undefined diff --git a/packages/sync-rules/package.json b/packages/sync-rules/package.json index 9a9dc85bf..ad2b58d2b 100644 --- a/packages/sync-rules/package.json +++ b/packages/sync-rules/package.json @@ -9,7 +9,8 @@ "access": "public" }, "files": [ - "dist/**/*" + "dist/**/*", + "schema/*" ], "type": "module", "scripts": { diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 9e96b5c13..fa5ec6f83 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -322,7 +322,7 @@ export class SqlSyncRules implements SyncRules { const eventTables = new Map(); for (const event of this.event_descriptors) { - for (let r of event.getSourceTables()) { + for (const r of event.getSourceTables()) { const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; eventTables.set(key, r); } @@ -331,16 +331,11 @@ export class SqlSyncRules implements SyncRules { } tableTriggersEvent(table: SourceTableInterface): boolean { - for (let bucket of this.event_descriptors) { - if (bucket.tableTriggersEvent(table)) { - return true; - } - } - return false; + return this.event_descriptors.some((bucket) => bucket.tableTriggersEvent(table)); } tableSyncsData(table: SourceTableInterface): boolean { - for (let bucket of this.bucket_descriptors) { + for (const bucket of this.bucket_descriptors) { if (bucket.tableSyncsData(table)) { return true; } From 6e749c27a76cdb6f2d094cf295ad90dd2b36edd0 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 17 Sep 2024 17:51:18 +0200 Subject: [PATCH 10/33] use event source definitions for event sync rules --- .../src/storage/ReplicationEventManager.ts | 4 +- .../src/storage/mongo/MongoBucketBatch.ts | 4 +- packages/sync-rules/src/SqlDataQuery.ts | 4 +- packages/sync-rules/src/SqlEventDescriptor.ts | 72 ----- packages/sync-rules/src/SqlSyncRules.ts | 44 +-- .../src/events/SqlEventDescriptor.ts | 64 +++++ .../src/events/SqlEventSourceQuery.ts | 253 ++++++++++++++++++ packages/sync-rules/src/index.ts | 3 +- packages/sync-rules/src/json_schema.ts | 25 +- 9 files changed, 365 insertions(+), 108 deletions(-) delete mode 100644 packages/sync-rules/src/SqlEventDescriptor.ts create mode 100644 packages/sync-rules/src/events/SqlEventDescriptor.ts create mode 100644 packages/sync-rules/src/events/SqlEventSourceQuery.ts diff --git a/packages/service-core/src/storage/ReplicationEventManager.ts b/packages/service-core/src/storage/ReplicationEventManager.ts index d39267146..e339c8016 100644 --- a/packages/service-core/src/storage/ReplicationEventManager.ts +++ b/packages/service-core/src/storage/ReplicationEventManager.ts @@ -4,8 +4,8 @@ import { SourceTable } from './SourceTable.js'; export type EventData = { op: SaveOp; - before?: sync_rules.EvaluatedParametersResult[]; - after?: sync_rules.EvaluatedParametersResult[]; + before?: sync_rules.SqliteRow; + after?: sync_rules.SqliteRow; }; export type ReplicationEventData = Map; diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index 64d0e71a9..c305803d4 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -623,8 +623,8 @@ export class MongoBucketBatch implements BucketStorageBatch { table: sourceTable, data: { op: tag, - after: after && util.isCompleteRow(after) ? event.evaluateParameterRow(sourceTable, after) : undefined, - before: before && util.isCompleteRow(before) ? event.evaluateParameterRow(sourceTable, before) : undefined + after: after && util.isCompleteRow(after) ? after : undefined, + before: before && util.isCompleteRow(before) ? before : undefined }, event }); diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index c1176bc9b..ca204f4be 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -7,6 +7,7 @@ import { SqlTools } from './sql_filters.js'; import { castAsText } from './sql_functions.js'; import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; import { TablePattern } from './TablePattern.js'; +import { TableQuerySchema } from './TableQuerySchema.js'; import { EvaluationResult, ParameterMatchClause, @@ -18,9 +19,8 @@ import { SqliteRow } from './types.js'; import { filterJsonRow, getBucketId, isSelectStatement } from './utils.js'; -import { TableQuerySchema } from './TableQuerySchema.js'; -interface RowValueExtractor { +export interface RowValueExtractor { extract(tables: QueryParameters, into: SqliteRow): void; getTypes(schema: QuerySchema, into: Record): void; } diff --git a/packages/sync-rules/src/SqlEventDescriptor.ts b/packages/sync-rules/src/SqlEventDescriptor.ts deleted file mode 100644 index 2ba350f02..000000000 --- a/packages/sync-rules/src/SqlEventDescriptor.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { IdSequence } from './IdSequence.js'; -import { SourceTableInterface } from './SourceTableInterface.js'; -import { SqlParameterQuery } from './SqlParameterQuery.js'; -import { TablePattern } from './TablePattern.js'; -import { SqlRuleError } from './errors.js'; -import { EvaluatedParametersResult, QueryParseOptions, SourceSchema, SqliteRow } from './types.js'; - -export interface QueryParseResult { - /** - * True if parsed in some form, even if there are errors. - */ - parsed: boolean; - - errors: SqlRuleError[]; -} - -/** - * A sync rules event which is triggered from a SQL table change. - */ -export class SqlEventDescriptor { - name: string; - bucket_parameters?: string[]; - - constructor(name: string, public idSequence: IdSequence) { - this.name = name; - } - - parameter_queries: SqlParameterQuery[] = []; - parameterIdSequence = new IdSequence(); - - addParameterQuery(sql: string, schema: SourceSchema | undefined, options: QueryParseOptions): QueryParseResult { - const parameterQuery = SqlParameterQuery.fromSql(this.name, sql, schema, options); - parameterQuery.id = this.parameterIdSequence.nextId(); - if (false == parameterQuery instanceof SqlParameterQuery) { - throw new Error('Parameter queries for events can not be global'); - } - - this.parameter_queries.push(parameterQuery as SqlParameterQuery); - - return { - parsed: true, - errors: parameterQuery.errors - }; - } - - evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[] { - let results: EvaluatedParametersResult[] = []; - for (let query of this.parameter_queries) { - if (query.applies(sourceTable)) { - results.push(...query.evaluateParameterRow(row)); - } - } - return results; - } - - getSourceTables(): Set { - let result = new Set(); - for (let query of this.parameter_queries) { - result.add(query.sourceTable!); - } - return result; - } - - tableTriggersEvent(table: SourceTableInterface): boolean { - for (let query of this.parameter_queries) { - if (query.applies(table)) { - return true; - } - } - return false; - } -} diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index fa5ec6f83..401d2a7e5 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -1,10 +1,10 @@ import { isScalar, LineCounter, parseDocument, Scalar, YAMLMap, YAMLSeq } from 'yaml'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; +import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; import { IdSequence } from './IdSequence.js'; import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { QueryParseResult, SqlBucketDescriptor } from './SqlBucketDescriptor.js'; -import { SqlEventDescriptor } from './SqlEventDescriptor.js'; import { TablePattern } from './TablePattern.js'; import { EvaluatedParameters, @@ -136,29 +136,31 @@ export class SqlSyncRules implements SyncRules { const eventMap = parsed.get('event_definitions') as YAMLMap; for (const event of eventMap?.items ?? []) { - const { key, value } = event as { key: Scalar; value: Scalar | YAMLSeq }; - const eventDescriptor = new SqlEventDescriptor(key.toString(), rules.idSequence); + const { key, value } = event as { key: Scalar; value: YAMLSeq }; - if (value instanceof Scalar) { - rules.withScalar(value, (q) => { - return eventDescriptor.addParameterQuery(q, schema, { - accept_potentially_dangerous_queries: false - }); - }); - } else if (value instanceof YAMLSeq) { - for (let item of value.items) { - if (!isScalar(item)) { - // TODO position - rules.errors.push(new YamlError(new Error(`Parameters for events must be scalar.`))); - continue; - } - rules.withScalar(item, (q) => { - return eventDescriptor.addParameterQuery(q, schema, { - accept_potentially_dangerous_queries: false - }); - }); + if (false == value instanceof YAMLMap) { + rules.errors.push(new YamlError(new Error(`Event definitions must be objects.`))); + continue; + } + + const payloads = value.get('payloads') as YAMLSeq; + if (false == payloads instanceof YAMLSeq) { + rules.errors.push(new YamlError(new Error(`Event definition payloads must be an array.`))); + continue; + } + + const eventDescriptor = new SqlEventDescriptor(key.toString(), rules.idSequence); + for (let item of payloads.items) { + if (!isScalar(item)) { + // TODO position + rules.errors.push(new YamlError(new Error(`Payload queries for events must be scalar.`))); + continue; } + rules.withScalar(item, (q) => { + return eventDescriptor.addSourceQuery(q, schema); + }); } + rules.event_descriptors.push(eventDescriptor); } diff --git a/packages/sync-rules/src/events/SqlEventDescriptor.ts b/packages/sync-rules/src/events/SqlEventDescriptor.ts new file mode 100644 index 000000000..6f4a03a9c --- /dev/null +++ b/packages/sync-rules/src/events/SqlEventDescriptor.ts @@ -0,0 +1,64 @@ +import { SqlRuleError } from '../errors.js'; +import { IdSequence } from '../IdSequence.js'; +import { SourceTableInterface } from '../SourceTableInterface.js'; +import { QueryParseResult } from '../SqlBucketDescriptor.js'; +import { TablePattern } from '../TablePattern.js'; +import { EvaluateRowOptions, SourceSchema } from '../types.js'; +import { EvaluatedEventRowWithErrors, SqlEventSourceQuery } from './SqlEventSourceQuery.js'; + +/** + * A sync rules event which is triggered from a SQL table change. + */ +export class SqlEventDescriptor { + name: string; + source_queries: SqlEventSourceQuery[] = []; + + constructor(name: string, public idSequence: IdSequence) { + this.name = name; + } + + addSourceQuery(sql: string, schema?: SourceSchema): QueryParseResult { + const source = SqlEventSourceQuery.fromSql(this.name, sql, schema); + + // Each source query should be for a unique table + const existingSourceQuery = this.source_queries.find((q) => q.table == source.table); + if (existingSourceQuery) { + return { + parsed: false, + errors: [new SqlRuleError('Each payload query should query a unique table', sql)] + }; + } + + source.ruleId = this.idSequence.nextId(); + this.source_queries.push(source); + + return { + parsed: true, + errors: source.errors + }; + } + + evaluateRowWithErrors(options: EvaluateRowOptions): EvaluatedEventRowWithErrors { + // There should only be 1 payload result per source query + const matchingQuery = this.source_queries.find((q) => q.applies(options.sourceTable)); + if (!matchingQuery) { + return { + errors: [{ error: `No marching source query found for table ${options.sourceTable.table}` }] + }; + } + + return matchingQuery.evaluateRowWithErrors(options.sourceTable, options.record); + } + + getSourceTables(): Set { + let result = new Set(); + for (let query of this.source_queries) { + result.add(query.sourceTable!); + } + return result; + } + + tableTriggersEvent(table: SourceTableInterface): boolean { + return this.source_queries.some((query) => query.applies(table)); + } +} diff --git a/packages/sync-rules/src/events/SqlEventSourceQuery.ts b/packages/sync-rules/src/events/SqlEventSourceQuery.ts new file mode 100644 index 000000000..055ecb916 --- /dev/null +++ b/packages/sync-rules/src/events/SqlEventSourceQuery.ts @@ -0,0 +1,253 @@ +import { parse, SelectedColumn } from 'pgsql-ast-parser'; +import { SqlRuleError } from '../errors.js'; +import { ColumnDefinition, ExpressionType } from '../ExpressionType.js'; +import { SourceTableInterface } from '../SourceTableInterface.js'; +import { SqlTools } from '../sql_filters.js'; +import { checkUnsupportedFeatures, isClauseError } from '../sql_support.js'; +import { RowValueExtractor } from '../SqlDataQuery.js'; +import { TablePattern } from '../TablePattern.js'; +import { TableQuerySchema } from '../TableQuerySchema.js'; +import { + EvaluationError, + ParameterMatchClause, + QueryParameters, + QuerySchema, + SourceSchema, + SourceSchemaTable, + SqliteJsonRow, + SqliteRow +} from '../types.js'; +import { filterJsonRow, isSelectStatement } from '../utils.js'; + +export type EvaluatedEventSourceRow = { + data: SqliteJsonRow; + ruleId?: string; +}; + +export type EvaluatedEventRowWithErrors = { + result?: EvaluatedEventSourceRow; + errors: EvaluationError[]; +}; + +/** + * Defines how a Replicated Row is mapped to source parameters for events. + * TODO cleanup duplication + */ +export class SqlEventSourceQuery { + static fromSql(descriptor_name: string, sql: string, schema?: SourceSchema) { + const parsed = parse(sql, { locationTracking: true }); + const rows = new SqlEventSourceQuery(); + + if (parsed.length > 1) { + throw new SqlRuleError('Only a single SELECT statement is supported', sql, parsed[1]?._location); + } + const q = parsed[0]; + if (!isSelectStatement(q)) { + throw new SqlRuleError('Only SELECT statements are supported', sql, q._location); + } + + rows.errors.push(...checkUnsupportedFeatures(sql, q)); + + if (q.from == null || q.from.length != 1 || q.from[0].type != 'table') { + throw new SqlRuleError('Must SELECT from a single table', sql, q.from?.[0]._location); + } + + const tableRef = q.from?.[0].name; + if (tableRef?.name == null) { + throw new SqlRuleError('Must SELECT from a single table', sql, q.from?.[0]._location); + } + const alias: string = tableRef.alias ?? tableRef.name; + + const sourceTable = new TablePattern(tableRef.schema, tableRef.name); + let querySchema: QuerySchema | undefined = undefined; + if (schema) { + const tables = schema.getTables(sourceTable); + if (tables.length == 0) { + const e = new SqlRuleError( + `Table ${sourceTable.schema}.${sourceTable.tablePattern} not found`, + sql, + q.from?.[0]?._location + ); + e.type = 'warning'; + + rows.errors.push(e); + } else { + querySchema = new TableQuerySchema(tables, alias); + } + } + + const where = q.where; + const tools = new SqlTools({ + table: alias, + parameter_tables: ['bucket'], + value_tables: [alias], + sql, + schema: querySchema + }); + const filter = tools.compileWhereClause(where); + + rows.sourceTable = sourceTable; + rows.table = alias; + rows.sql = sql; + rows.filter = filter; + rows.descriptor_name = descriptor_name; + rows.columns = q.columns ?? []; + rows.tools = tools; + + for (let column of q.columns ?? []) { + const name = tools.getOutputName(column); + if (name != '*') { + const clause = tools.compileRowValueExtractor(column.expr); + if (isClauseError(clause)) { + // Error logged already + continue; + } + rows.extractors.push({ + extract: (tables, output) => { + output[name] = clause.evaluate(tables); + }, + getTypes(schema, into) { + into[name] = { name, type: clause.getType(schema) }; + } + }); + } else { + rows.extractors.push({ + extract: (tables, output) => { + const row = tables[alias]; + for (let key in row) { + if (key.startsWith('_')) { + continue; + } + output[key] ??= row[key]; + } + }, + getTypes(schema, into) { + for (let column of schema.getColumns(alias)) { + into[column.name] ??= column; + } + } + }); + } + } + rows.errors.push(...tools.errors); + return rows; + } + + sourceTable?: TablePattern; + table?: string; + sql?: string; + columns?: SelectedColumn[]; + extractors: RowValueExtractor[] = []; + filter?: ParameterMatchClause; + descriptor_name?: string; + tools?: SqlTools; + + ruleId?: string; + + errors: SqlRuleError[] = []; + + constructor() {} + + applies(table: SourceTableInterface) { + return this.sourceTable?.matches(table); + } + + addSpecialParameters(table: SourceTableInterface, row: SqliteRow) { + if (this.sourceTable!.isWildcard) { + return { + ...row, + _table_suffix: this.sourceTable!.suffix(table.table) + }; + } else { + return row; + } + } + + isUnaliasedWildcard() { + return this.sourceTable!.isWildcard && this.table == this.sourceTable!.tablePattern; + } + + evaluateRowWithErrors(table: SourceTableInterface, row: SqliteRow): EvaluatedEventRowWithErrors { + try { + const tables = { [this.table!]: this.addSpecialParameters(table, row) }; + + const data = this.transformRow(tables); + return { + result: { + data, + ruleId: this.ruleId + }, + errors: [] + }; + } catch (e) { + return { errors: [e.message ?? `Evaluating data query failed`] }; + } + } + + private transformRow(tables: QueryParameters): SqliteJsonRow { + let result: SqliteRow = {}; + for (let extractor of this.extractors) { + extractor.extract(tables, result); + } + return filterJsonRow(result); + } + + columnOutputNames(): string[] { + return this.columns!.map((c) => { + return this.tools!.getOutputName(c); + }); + } + + getColumnOutputs(schema: SourceSchema): { name: string; columns: ColumnDefinition[] }[] { + let result: { name: string; columns: ColumnDefinition[] }[] = []; + + if (this.isUnaliasedWildcard()) { + // Separate results + for (let schemaTable of schema.getTables(this.sourceTable!)) { + let output: Record = {}; + + this.getColumnOutputsFor(schemaTable, output); + + result.push({ + name: schemaTable.table, + columns: Object.values(output) + }); + } + } else { + // Merged results + let output: Record = {}; + for (let schemaTable of schema.getTables(this.sourceTable!)) { + this.getColumnOutputsFor(schemaTable, output); + } + result.push({ + name: this.table!, + columns: Object.values(output) + }); + } + + return result; + } + + private getColumnOutputsFor(schemaTable: SourceSchemaTable, output: Record) { + const querySchema: QuerySchema = { + getType: (table, column) => { + if (table == this.table!) { + return schemaTable.getType(column) ?? ExpressionType.NONE; + } else { + // TODO: bucket parameters? + return ExpressionType.NONE; + } + }, + getColumns: (table) => { + if (table == this.table!) { + return schemaTable.getColumns(); + } else { + return []; + } + } + }; + for (let extractor of this.extractors) { + extractor.getTypes(querySchema, output); + } + } +} diff --git a/packages/sync-rules/src/index.ts b/packages/sync-rules/src/index.ts index 574c72081..877d52c37 100644 --- a/packages/sync-rules/src/index.ts +++ b/packages/sync-rules/src/index.ts @@ -1,5 +1,7 @@ export * from './DartSchemaGenerator.js'; export * from './errors.js'; +export * from './events/SqlEventDescriptor.js'; +export * from './events/SqlEventSourceQuery.js'; export * from './ExpressionType.js'; export * from './generators.js'; export * from './IdSequence.js'; @@ -11,7 +13,6 @@ export * from './SourceTableInterface.js'; export * from './sql_filters.js'; export * from './sql_functions.js'; export * from './SqlDataQuery.js'; -export * from './SqlEventDescriptor.js'; export * from './SqlParameterQuery.js'; export * from './SqlSyncRules.js'; export * from './StaticSchema.js'; diff --git a/packages/sync-rules/src/json_schema.ts b/packages/sync-rules/src/json_schema.ts index dddc55370..2eab66073 100644 --- a/packages/sync-rules/src/json_schema.ts +++ b/packages/sync-rules/src/json_schema.ts @@ -49,19 +49,28 @@ export const syncRulesSchema: ajvModule.Schema = { type: 'object', description: 'Record of sync replication event definitions', examples: [ - { write_checkpoint: 'select user_id, client_id, checkpoint from write_checkpoints' }, { - write_checkpoint: ['select user_id, client_id, checkpoint from write_checkpoints'] + write_checkpoints: { + payloads: ['select user_id, client_id, checkpoint from checkpoints'] + } } ], patternProperties: { '.*': { - type: ['string', 'array'], - items: { - type: 'string' - }, - minItems: 1, - uniqueItems: true + type: ['object'], + required: ['payloads'], + examples: [{ payloads: ['select user_id, client_id, checkpoint from checkpoints'] }], + properties: { + payloads: { + description: 'Queries which extract event payload fields from replicated table rows.', + type: 'array', + items: { + type: 'string' + } + }, + additionalProperties: false, + uniqueItems: true + } } } } From f174f140e0c988fe968242042dcc374c9c9da919 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 18 Sep 2024 17:51:13 +0200 Subject: [PATCH 11/33] support events for tables which aren't used in buckets --- .../src/replication/WalStream.ts | 4 +- .../service-core/src/storage/SourceTable.ts | 4 +- .../src/storage/mongo/MongoBucketBatch.ts | 58 +++++++++++-------- .../storage/mongo/MongoSyncBucketStorage.ts | 2 +- packages/sync-rules/src/SqlSyncRules.ts | 9 ++- 5 files changed, 46 insertions(+), 31 deletions(-) diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index 5767d4a3c..f9e0c7c40 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -392,9 +392,7 @@ WHERE oid = $1::regclass`, for (const record of WalStream.getQueryData(rows)) { // This auto-flushes when the batch reaches its size limit - if (table.syncAny) { - await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record }); - } + await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record }); } at += rows.length; diff --git a/packages/service-core/src/storage/SourceTable.ts b/packages/service-core/src/storage/SourceTable.ts index 839bd42c6..d227e4c33 100644 --- a/packages/service-core/src/storage/SourceTable.ts +++ b/packages/service-core/src/storage/SourceTable.ts @@ -31,7 +31,7 @@ export class SourceTable { * * Defaults to true for tests. */ - public triggerEvents = true; + public syncEvent = true; constructor( public readonly id: any, @@ -63,6 +63,6 @@ export class SourceTable { } get syncAny() { - return this.syncData || this.syncParameters; + return this.syncData || this.syncParameters || this.syncEvent; } } diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index c305803d4..d760778fd 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -8,7 +8,7 @@ import { BucketStorageBatch, FlushedResult, mergeToast, SaveOptions } from '../B import { ReplicationEventBatch } from '../ReplicationEventBatch.js'; import { SourceTable } from '../SourceTable.js'; import { PowerSyncMongo } from './db.js'; -import { CurrentBucket, CurrentDataDocument, SourceKey } from './models.js'; +import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; @@ -552,26 +552,29 @@ export class MongoBucketBatch implements BucketStorageBatch { return false; } + const now = new Date(); + const update: Partial = { + last_checkpoint_lsn: lsn, + last_checkpoint_ts: now, + last_keepalive_ts: now, + snapshot_done: true, + last_fatal_error: null + }; + if (this.persisted_op != null) { - const now = new Date(); - await this.db.sync_rules.updateOne( - { - _id: this.group_id - }, - { - $set: { - last_checkpoint: this.persisted_op, - last_checkpoint_lsn: lsn, - last_checkpoint_ts: now, - last_keepalive_ts: now, - snapshot_done: true, - last_fatal_error: null - } - }, - { session: this.session } - ); - this.persisted_op = null; + update.last_checkpoint = this.persisted_op; } + + await this.db.sync_rules.updateOne( + { + _id: this.group_id + }, + { + $set: update + }, + { session: this.session } + ); + this.persisted_op = null; this.last_checkpoint_lsn = lsn; return true; } @@ -612,11 +615,6 @@ export class MongoBucketBatch implements BucketStorageBatch { } async save(record: SaveOptions): Promise { - logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`); - - this.batch ??= new OperationBatch(); - this.batch.push(new RecordOperation(record)); - const { after, before, sourceTable, tag } = record; for (const event of this.getTableEvents(sourceTable)) { await this.event_batch.save({ @@ -630,6 +628,18 @@ export class MongoBucketBatch implements BucketStorageBatch { }); } + /** + * Handle case where this table is just an event table + */ + if (!sourceTable.syncData && !sourceTable.syncParameters) { + return null; + } + + logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`); + + this.batch ??= new OperationBatch(); + this.batch.push(new RecordOperation(record)); + if (this.batch.shouldFlush()) { const r = await this.flush(); // HACK: Give other streams a chance to also flush diff --git a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts index caca7690f..b4252200a 100644 --- a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts +++ b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts @@ -145,7 +145,7 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { replicationColumns, doc.snapshot_done ?? true ); - sourceTable.triggerEvents = options.sync_rules.tableTriggersEvent(sourceTable); + sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable); sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable); diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 401d2a7e5..2bfd1a315 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -311,7 +311,14 @@ export class SqlSyncRules implements SyncRules { getSourceTables(): TablePattern[] { const sourceTables = new Map(); for (const bucket of this.bucket_descriptors) { - for (let r of bucket.getSourceTables()) { + for (const r of bucket.getSourceTables()) { + const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; + sourceTables.set(key, r); + } + } + + for (const event of this.event_descriptors) { + for (const r of event.getSourceTables()) { const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; sourceTables.set(key, r); } From 8b8905df1043edf340f9ed24e3a01d56652d0643 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 09:56:13 +0200 Subject: [PATCH 12/33] Added batching for events --- .../src/storage/ReplicationEventBatch.ts | 20 ++++------- .../src/storage/ReplicationEventManager.ts | 35 ++++++++++++++++++- .../src/storage/mongo/MongoBucketBatch.ts | 2 +- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/packages/service-core/src/storage/ReplicationEventBatch.ts b/packages/service-core/src/storage/ReplicationEventBatch.ts index 463bf6844..64f17ba0a 100644 --- a/packages/service-core/src/storage/ReplicationEventBatch.ts +++ b/packages/service-core/src/storage/ReplicationEventBatch.ts @@ -80,18 +80,12 @@ export class ReplicationEventBatch { * {@link ReplicationEventManager}. */ async flush() { - try { - for (const [eventDescription, eventData] of this.event_cache) { - // TODO: Handle errors with hooks - await this.manager.fireEvent({ - event: eventDescription, - storage: this.storage, - data: eventData - }); - } - } finally { - this.event_cache.clear(); - this._eventCacheRowCount = 0; - } + await this.manager.fireEvents({ + batch_data: this.event_cache, + storage: this.storage + }); + + this.event_cache.clear(); + this._eventCacheRowCount = 0; } } diff --git a/packages/service-core/src/storage/ReplicationEventManager.ts b/packages/service-core/src/storage/ReplicationEventManager.ts index e339c8016..b847aa4cd 100644 --- a/packages/service-core/src/storage/ReplicationEventManager.ts +++ b/packages/service-core/src/storage/ReplicationEventManager.ts @@ -1,3 +1,4 @@ +import { logger } from '@powersync/lib-services-framework'; import * as sync_rules from '@powersync/service-sync-rules'; import { SaveOp, SyncRulesBucketStorage } from './BucketStorage.js'; import { SourceTable } from './SourceTable.js'; @@ -16,6 +17,11 @@ export type ReplicationEventPayload = { storage: SyncRulesBucketStorage; }; +export type BatchReplicationEventPayload = { + storage: SyncRulesBucketStorage; + batch_data: Map; +}; + export interface ReplicationEventHandler { event_name: string; handle(event: ReplicationEventPayload): Promise; @@ -28,11 +34,38 @@ export class ReplicationEventManager { this.handlers = new Map(); } + /** + * Fires an event, passing the specified payload to all registered handlers. + * This call resolves once all handlers have processed the event. + * Handler exceptions are caught and logged. + */ async fireEvent(payload: ReplicationEventPayload): Promise { const handlers = this.handlers.get(payload.event.name); for (const handler of handlers?.values() ?? []) { - await handler.handle(payload); + try { + await handler.handle(payload); + } catch (ex) { + // Exceptions in handlers don't affect the source. + logger.info(`Caught exception when processing "${handler.event_name}" event.`, ex); + } + } + } + + /** + * Fires a batch of events, passing the specified payload to all registered handlers. + * This call resolves once all handlers have processed the events. + * Handler exceptions are caught and logged by the {@link fireEvent} method. + */ + async fireEvents(batch: BatchReplicationEventPayload) { + const { batch_data, storage } = batch; + + for (const [eventDescription, eventData] of batch_data) { + await this.fireEvent({ + event: eventDescription, + storage: storage, + data: eventData + }); } } diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index d760778fd..1e7183266 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -629,7 +629,7 @@ export class MongoBucketBatch implements BucketStorageBatch { } /** - * Handle case where this table is just an event table + * Return if the table is just an event table */ if (!sourceTable.syncData && !sourceTable.syncParameters) { return null; From 3451ef43cc4732c962a7a44c60efe001c5301789 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 10:48:14 +0200 Subject: [PATCH 13/33] move base sql data query --- .../{SqlDataQuery.ts => BaseSqlDataQuery.ts} | 0 .../src/events/SqlEventSourceQuery.ts | 101 +----------------- 2 files changed, 5 insertions(+), 96 deletions(-) rename packages/sync-rules/src/{SqlDataQuery.ts => BaseSqlDataQuery.ts} (100%) diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts similarity index 100% rename from packages/sync-rules/src/SqlDataQuery.ts rename to packages/sync-rules/src/BaseSqlDataQuery.ts diff --git a/packages/sync-rules/src/events/SqlEventSourceQuery.ts b/packages/sync-rules/src/events/SqlEventSourceQuery.ts index 055ecb916..4b38024ca 100644 --- a/packages/sync-rules/src/events/SqlEventSourceQuery.ts +++ b/packages/sync-rules/src/events/SqlEventSourceQuery.ts @@ -1,23 +1,20 @@ import { parse, SelectedColumn } from 'pgsql-ast-parser'; import { SqlRuleError } from '../errors.js'; -import { ColumnDefinition, ExpressionType } from '../ExpressionType.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { SqlTools } from '../sql_filters.js'; import { checkUnsupportedFeatures, isClauseError } from '../sql_support.js'; -import { RowValueExtractor } from '../SqlDataQuery.js'; +import { RowValueExtractor, SqlDataQuery } from '../SqlDataQuery.js'; import { TablePattern } from '../TablePattern.js'; import { TableQuerySchema } from '../TableQuerySchema.js'; import { EvaluationError, ParameterMatchClause, - QueryParameters, QuerySchema, SourceSchema, - SourceSchemaTable, SqliteJsonRow, SqliteRow } from '../types.js'; -import { filterJsonRow, isSelectStatement } from '../utils.js'; +import { isSelectStatement } from '../utils.js'; export type EvaluatedEventSourceRow = { data: SqliteJsonRow; @@ -31,9 +28,9 @@ export type EvaluatedEventRowWithErrors = { /** * Defines how a Replicated Row is mapped to source parameters for events. - * TODO cleanup duplication + * This shares some implementation with {@link SqlDataQuery} with some subtle differences. */ -export class SqlEventSourceQuery { +export class SqlEventSourceQuery extends SqlDataQuery { static fromSql(descriptor_name: string, sql: string, schema?: SourceSchema) { const parsed = parse(sql, { locationTracking: true }); const rows = new SqlEventSourceQuery(); @@ -79,7 +76,7 @@ export class SqlEventSourceQuery { const where = q.where; const tools = new SqlTools({ table: alias, - parameter_tables: ['bucket'], + parameter_tables: [], value_tables: [alias], sql, schema: querySchema @@ -146,27 +143,6 @@ export class SqlEventSourceQuery { errors: SqlRuleError[] = []; - constructor() {} - - applies(table: SourceTableInterface) { - return this.sourceTable?.matches(table); - } - - addSpecialParameters(table: SourceTableInterface, row: SqliteRow) { - if (this.sourceTable!.isWildcard) { - return { - ...row, - _table_suffix: this.sourceTable!.suffix(table.table) - }; - } else { - return row; - } - } - - isUnaliasedWildcard() { - return this.sourceTable!.isWildcard && this.table == this.sourceTable!.tablePattern; - } - evaluateRowWithErrors(table: SourceTableInterface, row: SqliteRow): EvaluatedEventRowWithErrors { try { const tables = { [this.table!]: this.addSpecialParameters(table, row) }; @@ -183,71 +159,4 @@ export class SqlEventSourceQuery { return { errors: [e.message ?? `Evaluating data query failed`] }; } } - - private transformRow(tables: QueryParameters): SqliteJsonRow { - let result: SqliteRow = {}; - for (let extractor of this.extractors) { - extractor.extract(tables, result); - } - return filterJsonRow(result); - } - - columnOutputNames(): string[] { - return this.columns!.map((c) => { - return this.tools!.getOutputName(c); - }); - } - - getColumnOutputs(schema: SourceSchema): { name: string; columns: ColumnDefinition[] }[] { - let result: { name: string; columns: ColumnDefinition[] }[] = []; - - if (this.isUnaliasedWildcard()) { - // Separate results - for (let schemaTable of schema.getTables(this.sourceTable!)) { - let output: Record = {}; - - this.getColumnOutputsFor(schemaTable, output); - - result.push({ - name: schemaTable.table, - columns: Object.values(output) - }); - } - } else { - // Merged results - let output: Record = {}; - for (let schemaTable of schema.getTables(this.sourceTable!)) { - this.getColumnOutputsFor(schemaTable, output); - } - result.push({ - name: this.table!, - columns: Object.values(output) - }); - } - - return result; - } - - private getColumnOutputsFor(schemaTable: SourceSchemaTable, output: Record) { - const querySchema: QuerySchema = { - getType: (table, column) => { - if (table == this.table!) { - return schemaTable.getType(column) ?? ExpressionType.NONE; - } else { - // TODO: bucket parameters? - return ExpressionType.NONE; - } - }, - getColumns: (table) => { - if (table == this.table!) { - return schemaTable.getColumns(); - } else { - return []; - } - } - }; - for (let extractor of this.extractors) { - extractor.getTypes(querySchema, output); - } - } } From 4a212059fc8fb2b95c68d3d7b98bd4a0c2094c7f Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 10:54:51 +0200 Subject: [PATCH 14/33] splitting data query types --- packages/sync-rules/src/BaseSqlDataQuery.ts | 170 ++---------------- packages/sync-rules/src/SqlDataQuery.ts | 157 ++++++++++++++++ .../src/events/SqlEventSourceQuery.ts | 17 +- 3 files changed, 171 insertions(+), 173 deletions(-) create mode 100644 packages/sync-rules/src/SqlDataQuery.ts diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts index ca204f4be..cf71940c6 100644 --- a/packages/sync-rules/src/BaseSqlDataQuery.ts +++ b/packages/sync-rules/src/BaseSqlDataQuery.ts @@ -1,16 +1,12 @@ -import { JSONBig } from '@powersync/service-jsonbig'; -import { parse, SelectedColumn } from 'pgsql-ast-parser'; +import { SelectedColumn } from 'pgsql-ast-parser'; import { SqlRuleError } from './errors.js'; import { ColumnDefinition, ExpressionType } from './ExpressionType.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlTools } from './sql_filters.js'; import { castAsText } from './sql_functions.js'; -import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; import { TablePattern } from './TablePattern.js'; -import { TableQuerySchema } from './TableQuerySchema.js'; import { EvaluationResult, - ParameterMatchClause, QueryParameters, QuerySchema, SourceSchema, @@ -18,163 +14,19 @@ import { SqliteJsonRow, SqliteRow } from './types.js'; -import { filterJsonRow, getBucketId, isSelectStatement } from './utils.js'; +import { filterJsonRow, getBucketId } from './utils.js'; export interface RowValueExtractor { extract(tables: QueryParameters, into: SqliteRow): void; getTypes(schema: QuerySchema, into: Record): void; } -export class SqlDataQuery { - static fromSql(descriptor_name: string, bucket_parameters: string[], sql: string, schema?: SourceSchema) { - const parsed = parse(sql, { locationTracking: true }); - const rows = new SqlDataQuery(); - - if (parsed.length > 1) { - throw new SqlRuleError('Only a single SELECT statement is supported', sql, parsed[1]?._location); - } - const q = parsed[0]; - if (!isSelectStatement(q)) { - throw new SqlRuleError('Only SELECT statements are supported', sql, q._location); - } - - rows.errors.push(...checkUnsupportedFeatures(sql, q)); - - if (q.from == null || q.from.length != 1 || q.from[0].type != 'table') { - throw new SqlRuleError('Must SELECT from a single table', sql, q.from?.[0]._location); - } - - const tableRef = q.from?.[0].name; - if (tableRef?.name == null) { - throw new SqlRuleError('Must SELECT from a single table', sql, q.from?.[0]._location); - } - const alias: string = tableRef.alias ?? tableRef.name; - - const sourceTable = new TablePattern(tableRef.schema, tableRef.name); - let querySchema: QuerySchema | undefined = undefined; - if (schema) { - const tables = schema.getTables(sourceTable); - if (tables.length == 0) { - const e = new SqlRuleError( - `Table ${sourceTable.schema}.${sourceTable.tablePattern} not found`, - sql, - q.from?.[0]?._location - ); - e.type = 'warning'; - - rows.errors.push(e); - } else { - querySchema = new TableQuerySchema(tables, alias); - } - } - - const where = q.where; - const tools = new SqlTools({ - table: alias, - parameter_tables: ['bucket'], - value_tables: [alias], - sql, - schema: querySchema - }); - const filter = tools.compileWhereClause(where); - - const inputParameterNames = filter.inputParameters!.map((p) => p.key); - const bucketParameterNames = bucket_parameters.map((p) => `bucket.${p}`); - const allParams = new Set([...inputParameterNames, ...bucketParameterNames]); - if ( - (!filter.error && allParams.size != filter.inputParameters!.length) || - allParams.size != bucket_parameters.length - ) { - rows.errors.push( - new SqlRuleError( - `Query must cover all bucket parameters. Expected: ${JSONBig.stringify( - bucketParameterNames - )} Got: ${JSONBig.stringify(inputParameterNames)}`, - sql, - q._location - ) - ); - } - - rows.sourceTable = sourceTable; - rows.table = alias; - rows.sql = sql; - rows.filter = filter; - rows.descriptor_name = descriptor_name; - rows.bucket_parameters = bucket_parameters; - rows.columns = q.columns ?? []; - rows.tools = tools; - - let hasId = false; - let hasWildcard = false; - - for (let column of q.columns ?? []) { - const name = tools.getOutputName(column); - if (name != '*') { - const clause = tools.compileRowValueExtractor(column.expr); - if (isClauseError(clause)) { - // Error logged already - continue; - } - rows.extractors.push({ - extract: (tables, output) => { - output[name] = clause.evaluate(tables); - }, - getTypes(schema, into) { - into[name] = { name, type: clause.getType(schema) }; - } - }); - } else { - rows.extractors.push({ - extract: (tables, output) => { - const row = tables[alias]; - for (let key in row) { - if (key.startsWith('_')) { - continue; - } - output[key] ??= row[key]; - } - }, - getTypes(schema, into) { - for (let column of schema.getColumns(alias)) { - into[column.name] ??= column; - } - } - }); - } - if (name == 'id') { - hasId = true; - } else if (name == '*') { - hasWildcard = true; - if (querySchema == null) { - // Not performing schema-based validation - assume there is an id - hasId = true; - } else { - const idType = querySchema.getType(alias, 'id'); - if (!idType.isNone()) { - hasId = true; - } - } - } - } - if (!hasId) { - const error = new SqlRuleError(`Query must return an "id" column`, sql, q.columns?.[0]._location); - if (hasWildcard) { - // Schema-based validations are always warnings - error.type = 'warning'; - } - rows.errors.push(error); - } - rows.errors.push(...tools.errors); - return rows; - } - +export abstract class BaseSqlDataQuery { sourceTable?: TablePattern; table?: string; sql?: string; columns?: SelectedColumn[]; extractors: RowValueExtractor[] = []; - filter?: ParameterMatchClause; descriptor_name?: string; bucket_parameters?: string[]; tools?: SqlTools; @@ -248,14 +100,6 @@ export class SqlDataQuery { } } - private transformRow(tables: QueryParameters): SqliteJsonRow { - let result: SqliteRow = {}; - for (let extractor of this.extractors) { - extractor.extract(tables, result); - } - return filterJsonRow(result); - } - columnOutputNames(): string[] { return this.columns!.map((c) => { return this.tools!.getOutputName(c); @@ -292,6 +136,14 @@ export class SqlDataQuery { return result; } + protected transformRow(tables: QueryParameters): SqliteJsonRow { + let result: SqliteRow = {}; + for (let extractor of this.extractors) { + extractor.extract(tables, result); + } + return filterJsonRow(result); + } + private getColumnOutputsFor(schemaTable: SourceSchemaTable, output: Record) { const querySchema: QuerySchema = { getType: (table, column) => { diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts new file mode 100644 index 000000000..d5f556277 --- /dev/null +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -0,0 +1,157 @@ +import { JSONBig } from '@powersync/service-jsonbig'; +import { parse } from 'pgsql-ast-parser'; +import { BaseSqlDataQuery } from './BaseSqlDataQuery.js'; +import { SqlRuleError } from './errors.js'; +import { SqlTools } from './sql_filters.js'; +import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; +import { TablePattern } from './TablePattern.js'; +import { TableQuerySchema } from './TableQuerySchema.js'; +import { ParameterMatchClause, QuerySchema, SourceSchema } from './types.js'; +import { isSelectStatement } from './utils.js'; + +export class SqlDataQuery extends BaseSqlDataQuery { + filter?: ParameterMatchClause; + + static fromSql(descriptor_name: string, bucket_parameters: string[], sql: string, schema?: SourceSchema) { + const parsed = parse(sql, { locationTracking: true }); + const rows = new SqlDataQuery(); + + if (parsed.length > 1) { + throw new SqlRuleError('Only a single SELECT statement is supported', sql, parsed[1]?._location); + } + const q = parsed[0]; + if (!isSelectStatement(q)) { + throw new SqlRuleError('Only SELECT statements are supported', sql, q._location); + } + + rows.errors.push(...checkUnsupportedFeatures(sql, q)); + + if (q.from == null || q.from.length != 1 || q.from[0].type != 'table') { + throw new SqlRuleError('Must SELECT from a single table', sql, q.from?.[0]._location); + } + + const tableRef = q.from?.[0].name; + if (tableRef?.name == null) { + throw new SqlRuleError('Must SELECT from a single table', sql, q.from?.[0]._location); + } + const alias: string = tableRef.alias ?? tableRef.name; + + const sourceTable = new TablePattern(tableRef.schema, tableRef.name); + let querySchema: QuerySchema | undefined = undefined; + if (schema) { + const tables = schema.getTables(sourceTable); + if (tables.length == 0) { + const e = new SqlRuleError( + `Table ${sourceTable.schema}.${sourceTable.tablePattern} not found`, + sql, + q.from?.[0]?._location + ); + e.type = 'warning'; + + rows.errors.push(e); + } else { + querySchema = new TableQuerySchema(tables, alias); + } + } + + const where = q.where; + const tools = new SqlTools({ + table: alias, + parameter_tables: ['bucket'], + value_tables: [alias], + sql, + schema: querySchema + }); + const filter = tools.compileWhereClause(where); + + const inputParameterNames = filter.inputParameters!.map((p) => p.key); + const bucketParameterNames = bucket_parameters.map((p) => `bucket.${p}`); + const allParams = new Set([...inputParameterNames, ...bucketParameterNames]); + if ( + (!filter.error && allParams.size != filter.inputParameters!.length) || + allParams.size != bucket_parameters.length + ) { + rows.errors.push( + new SqlRuleError( + `Query must cover all bucket parameters. Expected: ${JSONBig.stringify( + bucketParameterNames + )} Got: ${JSONBig.stringify(inputParameterNames)}`, + sql, + q._location + ) + ); + } + + rows.sourceTable = sourceTable; + rows.table = alias; + rows.sql = sql; + rows.filter = filter; + rows.descriptor_name = descriptor_name; + rows.bucket_parameters = bucket_parameters; + rows.columns = q.columns ?? []; + rows.tools = tools; + + let hasId = false; + let hasWildcard = false; + + for (let column of q.columns ?? []) { + const name = tools.getOutputName(column); + if (name != '*') { + const clause = tools.compileRowValueExtractor(column.expr); + if (isClauseError(clause)) { + // Error logged already + continue; + } + rows.extractors.push({ + extract: (tables, output) => { + output[name] = clause.evaluate(tables); + }, + getTypes(schema, into) { + into[name] = { name, type: clause.getType(schema) }; + } + }); + } else { + rows.extractors.push({ + extract: (tables, output) => { + const row = tables[alias]; + for (let key in row) { + if (key.startsWith('_')) { + continue; + } + output[key] ??= row[key]; + } + }, + getTypes(schema, into) { + for (let column of schema.getColumns(alias)) { + into[column.name] ??= column; + } + } + }); + } + if (name == 'id') { + hasId = true; + } else if (name == '*') { + hasWildcard = true; + if (querySchema == null) { + // Not performing schema-based validation - assume there is an id + hasId = true; + } else { + const idType = querySchema.getType(alias, 'id'); + if (!idType.isNone()) { + hasId = true; + } + } + } + } + if (!hasId) { + const error = new SqlRuleError(`Query must return an "id" column`, sql, q.columns?.[0]._location); + if (hasWildcard) { + // Schema-based validations are always warnings + error.type = 'warning'; + } + rows.errors.push(error); + } + rows.errors.push(...tools.errors); + return rows; + } +} diff --git a/packages/sync-rules/src/events/SqlEventSourceQuery.ts b/packages/sync-rules/src/events/SqlEventSourceQuery.ts index 4b38024ca..a3a473f6a 100644 --- a/packages/sync-rules/src/events/SqlEventSourceQuery.ts +++ b/packages/sync-rules/src/events/SqlEventSourceQuery.ts @@ -1,19 +1,12 @@ import { parse, SelectedColumn } from 'pgsql-ast-parser'; +import { BaseSqlDataQuery, RowValueExtractor } from '../BaseSqlDataQuery.js'; import { SqlRuleError } from '../errors.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { SqlTools } from '../sql_filters.js'; import { checkUnsupportedFeatures, isClauseError } from '../sql_support.js'; -import { RowValueExtractor, SqlDataQuery } from '../SqlDataQuery.js'; import { TablePattern } from '../TablePattern.js'; import { TableQuerySchema } from '../TableQuerySchema.js'; -import { - EvaluationError, - ParameterMatchClause, - QuerySchema, - SourceSchema, - SqliteJsonRow, - SqliteRow -} from '../types.js'; +import { EvaluationError, QuerySchema, SourceSchema, SqliteJsonRow, SqliteRow } from '../types.js'; import { isSelectStatement } from '../utils.js'; export type EvaluatedEventSourceRow = { @@ -28,9 +21,8 @@ export type EvaluatedEventRowWithErrors = { /** * Defines how a Replicated Row is mapped to source parameters for events. - * This shares some implementation with {@link SqlDataQuery} with some subtle differences. */ -export class SqlEventSourceQuery extends SqlDataQuery { +export class SqlEventSourceQuery extends BaseSqlDataQuery { static fromSql(descriptor_name: string, sql: string, schema?: SourceSchema) { const parsed = parse(sql, { locationTracking: true }); const rows = new SqlEventSourceQuery(); @@ -81,12 +73,10 @@ export class SqlEventSourceQuery extends SqlDataQuery { sql, schema: querySchema }); - const filter = tools.compileWhereClause(where); rows.sourceTable = sourceTable; rows.table = alias; rows.sql = sql; - rows.filter = filter; rows.descriptor_name = descriptor_name; rows.columns = q.columns ?? []; rows.tools = tools; @@ -135,7 +125,6 @@ export class SqlEventSourceQuery extends SqlDataQuery { sql?: string; columns?: SelectedColumn[]; extractors: RowValueExtractor[] = []; - filter?: ParameterMatchClause; descriptor_name?: string; tools?: SqlTools; From 684c3f1fc751df91beec1aad1c9cbfc4cb37e115 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 10:55:20 +0200 Subject: [PATCH 15/33] rename abstract sql data query --- .../src/{BaseSqlDataQuery.ts => AbstractSqlDataQuery.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/sync-rules/src/{BaseSqlDataQuery.ts => AbstractSqlDataQuery.ts} (100%) diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/AbstractSqlDataQuery.ts similarity index 100% rename from packages/sync-rules/src/BaseSqlDataQuery.ts rename to packages/sync-rules/src/AbstractSqlDataQuery.ts From 249b5dfa4b915cca1f1dbe5f7f451fa27358e650 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 11:00:51 +0200 Subject: [PATCH 16/33] cleanup data query classes --- .../sync-rules/src/AbstractSqlDataQuery.ts | 50 ++----------------- packages/sync-rules/src/SqlDataQuery.ts | 45 +++++++++++++++-- .../src/events/SqlEventSourceQuery.ts | 19 ++----- 3 files changed, 47 insertions(+), 67 deletions(-) diff --git a/packages/sync-rules/src/AbstractSqlDataQuery.ts b/packages/sync-rules/src/AbstractSqlDataQuery.ts index cf71940c6..1a05cef26 100644 --- a/packages/sync-rules/src/AbstractSqlDataQuery.ts +++ b/packages/sync-rules/src/AbstractSqlDataQuery.ts @@ -3,25 +3,16 @@ import { SqlRuleError } from './errors.js'; import { ColumnDefinition, ExpressionType } from './ExpressionType.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlTools } from './sql_filters.js'; -import { castAsText } from './sql_functions.js'; import { TablePattern } from './TablePattern.js'; -import { - EvaluationResult, - QueryParameters, - QuerySchema, - SourceSchema, - SourceSchemaTable, - SqliteJsonRow, - SqliteRow -} from './types.js'; -import { filterJsonRow, getBucketId } from './utils.js'; +import { QueryParameters, QuerySchema, SourceSchema, SourceSchemaTable, SqliteJsonRow, SqliteRow } from './types.js'; +import { filterJsonRow } from './utils.js'; export interface RowValueExtractor { extract(tables: QueryParameters, into: SqliteRow): void; getTypes(schema: QuerySchema, into: Record): void; } -export abstract class BaseSqlDataQuery { +export abstract class AbstractSqlDataQuery { sourceTable?: TablePattern; table?: string; sql?: string; @@ -65,41 +56,6 @@ export abstract class BaseSqlDataQuery { return this.sourceTable!.isWildcard && this.table == this.sourceTable!.tablePattern; } - evaluateRow(table: SourceTableInterface, row: SqliteRow): EvaluationResult[] { - try { - const tables = { [this.table!]: this.addSpecialParameters(table, row) }; - const bucketParameters = this.filter!.filterRow(tables); - const bucketIds = bucketParameters.map((params) => - getBucketId(this.descriptor_name!, this.bucket_parameters!, params) - ); - - const data = this.transformRow(tables); - let id = data.id; - if (typeof id != 'string') { - // While an explicit cast would be better, this covers against very common - // issues when initially testing out sync, for example when the id column is an - // auto-incrementing integer. - // If there is no id column, we use a blank id. This will result in the user syncing - // a single arbitrary row for this table - better than just not being able to sync - // anything. - id = castAsText(id) ?? ''; - } - const outputTable = this.getOutputName(table.table); - - return bucketIds.map((bucketId) => { - return { - bucket: bucketId, - table: outputTable, - id: id, - data, - ruleId: this.ruleId - } as EvaluationResult; - }); - } catch (e) { - return [{ error: e.message ?? `Evaluating data query failed` }]; - } - } - columnOutputNames(): string[] { return this.columns!.map((c) => { return this.tools!.getOutputName(c); diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index d5f556277..f500746f9 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -1,15 +1,17 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { parse } from 'pgsql-ast-parser'; -import { BaseSqlDataQuery } from './BaseSqlDataQuery.js'; +import { AbstractSqlDataQuery } from './AbstractSqlDataQuery.js'; import { SqlRuleError } from './errors.js'; +import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlTools } from './sql_filters.js'; +import { castAsText } from './sql_functions.js'; import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; import { TablePattern } from './TablePattern.js'; import { TableQuerySchema } from './TableQuerySchema.js'; -import { ParameterMatchClause, QuerySchema, SourceSchema } from './types.js'; -import { isSelectStatement } from './utils.js'; +import { EvaluationResult, ParameterMatchClause, QuerySchema, SourceSchema, SqliteRow } from './types.js'; +import { getBucketId, isSelectStatement } from './utils.js'; -export class SqlDataQuery extends BaseSqlDataQuery { +export class SqlDataQuery extends AbstractSqlDataQuery { filter?: ParameterMatchClause; static fromSql(descriptor_name: string, bucket_parameters: string[], sql: string, schema?: SourceSchema) { @@ -154,4 +156,39 @@ export class SqlDataQuery extends BaseSqlDataQuery { rows.errors.push(...tools.errors); return rows; } + + evaluateRow(table: SourceTableInterface, row: SqliteRow): EvaluationResult[] { + try { + const tables = { [this.table!]: this.addSpecialParameters(table, row) }; + const bucketParameters = this.filter!.filterRow(tables); + const bucketIds = bucketParameters.map((params) => + getBucketId(this.descriptor_name!, this.bucket_parameters!, params) + ); + + const data = this.transformRow(tables); + let id = data.id; + if (typeof id != 'string') { + // While an explicit cast would be better, this covers against very common + // issues when initially testing out sync, for example when the id column is an + // auto-incrementing integer. + // If there is no id column, we use a blank id. This will result in the user syncing + // a single arbitrary row for this table - better than just not being able to sync + // anything. + id = castAsText(id) ?? ''; + } + const outputTable = this.getOutputName(table.table); + + return bucketIds.map((bucketId) => { + return { + bucket: bucketId, + table: outputTable, + id: id, + data, + ruleId: this.ruleId + } as EvaluationResult; + }); + } catch (e) { + return [{ error: e.message ?? `Evaluating data query failed` }]; + } + } } diff --git a/packages/sync-rules/src/events/SqlEventSourceQuery.ts b/packages/sync-rules/src/events/SqlEventSourceQuery.ts index a3a473f6a..26083d56a 100644 --- a/packages/sync-rules/src/events/SqlEventSourceQuery.ts +++ b/packages/sync-rules/src/events/SqlEventSourceQuery.ts @@ -1,5 +1,5 @@ -import { parse, SelectedColumn } from 'pgsql-ast-parser'; -import { BaseSqlDataQuery, RowValueExtractor } from '../BaseSqlDataQuery.js'; +import { parse } from 'pgsql-ast-parser'; +import { AbstractSqlDataQuery } from '../AbstractSqlDataQuery.js'; import { SqlRuleError } from '../errors.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { SqlTools } from '../sql_filters.js'; @@ -22,7 +22,7 @@ export type EvaluatedEventRowWithErrors = { /** * Defines how a Replicated Row is mapped to source parameters for events. */ -export class SqlEventSourceQuery extends BaseSqlDataQuery { +export class SqlEventSourceQuery extends AbstractSqlDataQuery { static fromSql(descriptor_name: string, sql: string, schema?: SourceSchema) { const parsed = parse(sql, { locationTracking: true }); const rows = new SqlEventSourceQuery(); @@ -65,7 +65,6 @@ export class SqlEventSourceQuery extends BaseSqlDataQuery { } } - const where = q.where; const tools = new SqlTools({ table: alias, parameter_tables: [], @@ -120,18 +119,6 @@ export class SqlEventSourceQuery extends BaseSqlDataQuery { return rows; } - sourceTable?: TablePattern; - table?: string; - sql?: string; - columns?: SelectedColumn[]; - extractors: RowValueExtractor[] = []; - descriptor_name?: string; - tools?: SqlTools; - - ruleId?: string; - - errors: SqlRuleError[] = []; - evaluateRowWithErrors(table: SourceTableInterface, row: SqliteRow): EvaluatedEventRowWithErrors { try { const tables = { [this.table!]: this.addSpecialParameters(table, row) }; From ba13a8615505888f2ef5d8bcbc41c320594d63ea Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 12:42:48 +0200 Subject: [PATCH 17/33] fix lsn undefined error --- packages/service-core/src/storage/MongoBucketStorage.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/service-core/src/storage/MongoBucketStorage.ts b/packages/service-core/src/storage/MongoBucketStorage.ts index ab4d06c42..67a4f6159 100644 --- a/packages/service-core/src/storage/MongoBucketStorage.ts +++ b/packages/service-core/src/storage/MongoBucketStorage.ts @@ -508,9 +508,16 @@ export class MongoBucketStorage implements BucketStorageFactory { // 2. write checkpoint changes for the specific user const bucketStorage = await cp.getBucketStorage(); // TODO validate and optimize + + const lsnFilters: Record = lsn ? {1: lsn} : {}; + const currentWriteCheckpoint = await this.lastWriteCheckpoint({ sync_rules_id: bucketStorage?.group_id, - ...filters + ...filters, + heads: { + ...(filters.heads ?? {}), + ...lsnFilters + } }); if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) { From d32421c998ea8c636735df921bdb5e7d18b65bd6 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 15:27:08 +0200 Subject: [PATCH 18/33] Added batch checkpoint method --- .../src/storage/MongoBucketStorage.ts | 6 +++- .../mongo/MongoCustomWriteCheckpointAPI.ts | 30 ++++++++++++------- .../mongo/MongoManagedWriteCheckpointAPI.ts | 19 ++++++++++++ .../src/storage/write-checkpoint.ts | 4 ++- 4 files changed, 47 insertions(+), 12 deletions(-) diff --git a/packages/service-core/src/storage/MongoBucketStorage.ts b/packages/service-core/src/storage/MongoBucketStorage.ts index 67a4f6159..87ccd33f7 100644 --- a/packages/service-core/src/storage/MongoBucketStorage.ts +++ b/packages/service-core/src/storage/MongoBucketStorage.ts @@ -285,6 +285,10 @@ export class MongoBucketStorage implements BucketStorageFactory { }); } + async batchCreateWriteCheckpoints(checkpoints: WriteCheckpointOptions[]): Promise { + return this.writeCheckpointAPI.batchCreateWriteCheckpoints(checkpoints); + } + async createWriteCheckpoint(options: WriteCheckpointOptions): Promise { return this.writeCheckpointAPI.createWriteCheckpoint(options); } @@ -509,7 +513,7 @@ export class MongoBucketStorage implements BucketStorageFactory { const bucketStorage = await cp.getBucketStorage(); // TODO validate and optimize - const lsnFilters: Record = lsn ? {1: lsn} : {}; + const lsnFilters: Record = lsn ? { 1: lsn } : {}; const currentWriteCheckpoint = await this.lastWriteCheckpoint({ sync_rules_id: bucketStorage?.group_id, diff --git a/packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts b/packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts index c0c27bd84..871e07c49 100644 --- a/packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts +++ b/packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts @@ -8,15 +8,31 @@ import { PowerSyncMongo } from './db.js'; export class MongoCustomWriteCheckpointAPI implements WriteCheckpointAPI { constructor(protected db: PowerSyncMongo) {} + async batchCreateWriteCheckpoints(checkpoints: WriteCheckpointOptions[]): Promise { + await this.db.custom_write_checkpoints.bulkWrite( + checkpoints.map((checkpoint) => ({ + updateOne: { + filter: { user_id: checkpoint.user_id }, + update: { + $set: { + checkpoint: checkpoint.checkpoint, + sync_rules_id: checkpoint.sync_rules_id + } + }, + upsert: true + } + })) + ); + } + async createWriteCheckpoint(options: WriteCheckpointOptions): Promise { - const { checkpoint, user_id, heads, sync_rules_id } = options; + const { checkpoint, user_id, sync_rules_id } = options; const doc = await this.db.custom_write_checkpoints.findOneAndUpdate( { user_id: user_id }, { $set: { - heads, // HEADs are technically not relevant, by we can store them if provided checkpoint, sync_rules_id } @@ -27,16 +43,10 @@ export class MongoCustomWriteCheckpointAPI implements WriteCheckpointAPI { } async lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise { - const { user_id, heads = {} } = filters; - const lsnFilter = Object.fromEntries( - Object.entries(heads).map(([connectionKey, lsn]) => { - return [`heads.${connectionKey}`, { $lte: lsn }]; - }) - ); + const { user_id } = filters; const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({ - user_id: user_id, - ...lsnFilter + user_id: user_id }); return lastWriteCheckpoint?.checkpoint ?? null; } diff --git a/packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts b/packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts index 7a3c5652d..1f83730dd 100644 --- a/packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts +++ b/packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts @@ -8,6 +8,25 @@ import { PowerSyncMongo } from './db.js'; export class MongoManagedWriteCheckpointAPI implements WriteCheckpointAPI { constructor(protected db: PowerSyncMongo) {} + async batchCreateWriteCheckpoints(checkpoints: WriteCheckpointOptions[]): Promise { + await this.db.write_checkpoints.bulkWrite( + checkpoints.map((checkpoint) => ({ + updateOne: { + filter: { user_id: checkpoint.user_id }, + update: { + $set: { + lsns: checkpoint.heads + }, + $inc: { + client_id: 1n + } + }, + upsert: true + } + })) + ); + } + async createWriteCheckpoint(options: WriteCheckpointOptions): Promise { const { user_id, heads: lsns } = options; const doc = await this.db.write_checkpoints.findOneAndUpdate( diff --git a/packages/service-core/src/storage/write-checkpoint.ts b/packages/service-core/src/storage/write-checkpoint.ts index 2d43bc8a6..b44c15fa3 100644 --- a/packages/service-core/src/storage/write-checkpoint.ts +++ b/packages/service-core/src/storage/write-checkpoint.ts @@ -38,7 +38,9 @@ export interface WriteCheckpointOptions extends WriteCheckpointFilters { } export interface WriteCheckpointAPI { - createWriteCheckpoint(options: WriteCheckpointOptions): Promise; + batchCreateWriteCheckpoints(checkpoints: WriteCheckpointOptions[]): Promise; + + createWriteCheckpoint(checkpoint: WriteCheckpointOptions): Promise; lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise; } From 0fc61f878e75fdb2234adf168c0cbd48f64eb55c Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 15:31:00 +0200 Subject: [PATCH 19/33] rename basesql query --- .../src/{AbstractSqlDataQuery.ts => BaseSqlDataQuery.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/sync-rules/src/{AbstractSqlDataQuery.ts => BaseSqlDataQuery.ts} (100%) diff --git a/packages/sync-rules/src/AbstractSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts similarity index 100% rename from packages/sync-rules/src/AbstractSqlDataQuery.ts rename to packages/sync-rules/src/BaseSqlDataQuery.ts From 27c8b0835524ca71c807132a5328374585cb3283 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 15:32:46 +0200 Subject: [PATCH 20/33] rename base data query --- packages/sync-rules/src/BaseSqlDataQuery.ts | 2 +- packages/sync-rules/src/SqlDataQuery.ts | 4 ++-- packages/sync-rules/src/events/SqlEventSourceQuery.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts index 1a05cef26..5d892a641 100644 --- a/packages/sync-rules/src/BaseSqlDataQuery.ts +++ b/packages/sync-rules/src/BaseSqlDataQuery.ts @@ -12,7 +12,7 @@ export interface RowValueExtractor { getTypes(schema: QuerySchema, into: Record): void; } -export abstract class AbstractSqlDataQuery { +export class BaseSqlDataQuery { sourceTable?: TablePattern; table?: string; sql?: string; diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index 44a1caf29..e3abe1220 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -1,6 +1,6 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { parse } from 'pgsql-ast-parser'; -import { AbstractSqlDataQuery } from './AbstractSqlDataQuery.js'; +import { BaseSqlDataQuery } from './BaseSqlDataQuery.js'; import { SqlRuleError } from './errors.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlTools } from './sql_filters.js'; @@ -12,7 +12,7 @@ import { TableQuerySchema } from './TableQuerySchema.js'; import { EvaluationResult, ParameterMatchClause, QuerySchema, SqliteRow } from './types.js'; import { getBucketId, isSelectStatement } from './utils.js'; -export class SqlDataQuery extends AbstractSqlDataQuery { +export class SqlDataQuery extends BaseSqlDataQuery { filter?: ParameterMatchClause; static fromSql(descriptor_name: string, bucket_parameters: string[], sql: string, options: SyncRulesOptions) { diff --git a/packages/sync-rules/src/events/SqlEventSourceQuery.ts b/packages/sync-rules/src/events/SqlEventSourceQuery.ts index b51fc546c..24601fa8f 100644 --- a/packages/sync-rules/src/events/SqlEventSourceQuery.ts +++ b/packages/sync-rules/src/events/SqlEventSourceQuery.ts @@ -1,5 +1,5 @@ import { parse } from 'pgsql-ast-parser'; -import { AbstractSqlDataQuery } from '../AbstractSqlDataQuery.js'; +import { BaseSqlDataQuery } from '../BaseSqlDataQuery.js'; import { SqlRuleError } from '../errors.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { SqlTools } from '../sql_filters.js'; @@ -23,7 +23,7 @@ export type EvaluatedEventRowWithErrors = { /** * Defines how a Replicated Row is mapped to source parameters for events. */ -export class SqlEventSourceQuery extends AbstractSqlDataQuery { +export class SqlEventSourceQuery extends BaseSqlDataQuery { static fromSql(descriptor_name: string, sql: string, options: SyncRulesOptions) { const parsed = parse(sql, { locationTracking: true }); const rows = new SqlEventSourceQuery(); From 7a584e32353985dc90296109def16447dbde5d17 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 15:51:30 +0200 Subject: [PATCH 21/33] cleanup --- modules/module-postgres/src/replication/WalStream.ts | 9 ++------- packages/service-core/src/storage/MongoBucketStorage.ts | 3 +-- packages/service-core/src/storage/mongo/models.ts | 1 - packages/sync-rules/src/SqlSyncRules.ts | 1 - packages/sync-rules/src/json_schema.ts | 2 +- 5 files changed, 4 insertions(+), 12 deletions(-) diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index 79f5c6c9b..39219640e 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -338,7 +338,7 @@ WHERE oid = $1::regclass`, for (let tablePattern of sourceTables) { const tables = await this.getQualifiedTableNames(batch, db, tablePattern); for (let table of tables) { - await this.snapshotTable(batch, db, table, lsn); + await this.snapshotTable(batch, db, table); await batch.markSnapshotDone([table], lsn); await touch(); } @@ -353,12 +353,7 @@ WHERE oid = $1::regclass`, } } - private async snapshotTable( - batch: storage.BucketStorageBatch, - db: pgwire.PgConnection, - table: storage.SourceTable, - lsn?: string - ) { + private async snapshotTable(batch: storage.BucketStorageBatch, db: pgwire.PgConnection, table: storage.SourceTable) { logger.info(`${this.slot_name} Replicating ${table.qualifiedName}`); const estimatedCount = await this.estimatedCount(db, table); let at = 0; diff --git a/packages/service-core/src/storage/MongoBucketStorage.ts b/packages/service-core/src/storage/MongoBucketStorage.ts index 87ccd33f7..2d98ea864 100644 --- a/packages/service-core/src/storage/MongoBucketStorage.ts +++ b/packages/service-core/src/storage/MongoBucketStorage.ts @@ -510,8 +510,7 @@ export class MongoBucketStorage implements BucketStorageFactory { // What is important is: // 1. checkpoint (op_id) changes. // 2. write checkpoint changes for the specific user - - const bucketStorage = await cp.getBucketStorage(); // TODO validate and optimize + const bucketStorage = await cp.getBucketStorage(); const lsnFilters: Record = lsn ? { 1: lsn } : {}; diff --git a/packages/service-core/src/storage/mongo/models.ts b/packages/service-core/src/storage/mongo/models.ts index af764b87d..a85886c4e 100644 --- a/packages/service-core/src/storage/mongo/models.ts +++ b/packages/service-core/src/storage/mongo/models.ts @@ -162,7 +162,6 @@ export interface SyncRuleDocument { export interface CustomWriteCheckpointDocument { _id: bson.ObjectId; user_id: string; - heads?: Record; checkpoint: bigint; sync_rules_id: number; } diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 63def0e0d..ca5f05d85 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -165,7 +165,6 @@ export class SqlSyncRules implements SyncRules { const eventDescriptor = new SqlEventDescriptor(key.toString(), rules.idSequence); for (let item of payloads.items) { if (!isScalar(item)) { - // TODO position rules.errors.push(new YamlError(new Error(`Payload queries for events must be scalar.`))); continue; } diff --git a/packages/sync-rules/src/json_schema.ts b/packages/sync-rules/src/json_schema.ts index 2eab66073..b2f9367a9 100644 --- a/packages/sync-rules/src/json_schema.ts +++ b/packages/sync-rules/src/json_schema.ts @@ -1,7 +1,7 @@ import ajvModule from 'ajv'; // Hack to make this work both in NodeJS and a browser const Ajv = ajvModule.default ?? ajvModule; -const ajv = new Ajv({ allErrors: true, verbose: true, allowUnionTypes: true }); +const ajv = new Ajv({ allErrors: true, verbose: true }); export const syncRulesSchema: ajvModule.Schema = { type: 'object', From a65749d178774d60b7b8ef5a5be73c15389ac788 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 15:53:59 +0200 Subject: [PATCH 22/33] add index for custom write checkpoints --- ...099539247-custom-write-checkpoint-index.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 packages/service-core/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts diff --git a/packages/service-core/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts b/packages/service-core/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts new file mode 100644 index 000000000..7a4973faf --- /dev/null +++ b/packages/service-core/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts @@ -0,0 +1,34 @@ +import * as storage from '../../../storage/storage-index.js'; +import * as utils from '../../../util/util-index.js'; + +export const up = async (context: utils.MigrationContext) => { + const { runner_config } = context; + const config = await utils.loadConfig(runner_config); + const db = storage.createPowerSyncMongo(config.storage); + + try { + await db.custom_write_checkpoints.createIndex( + { + user_id: 1 + }, + { name: 'user_id' } + ); + } finally { + await db.client.close(); + } +}; + +export const down = async (context: utils.MigrationContext) => { + const { runner_config } = context; + const config = await utils.loadConfig(runner_config); + + const db = storage.createPowerSyncMongo(config.storage); + + try { + if (await db.custom_write_checkpoints.indexExists('user_id')) { + await db.custom_write_checkpoints.dropIndex('user_id'); + } + } finally { + await db.client.close(); + } +}; From e9a63740faf3cb13ffc366ea92243d9717e7b228 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 23 Sep 2024 16:24:58 +0200 Subject: [PATCH 23/33] cleanup --- packages/service-core/src/storage/BucketStorage.ts | 4 ++-- packages/service-core/src/storage/MongoBucketStorage.ts | 5 ++--- packages/service-core/src/sync/sync.ts | 7 +------ 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index cda1f07fd..9caba263e 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -11,7 +11,7 @@ import * as util from '../util/util-index.js'; import { ReplicationEventManager } from './ReplicationEventManager.js'; import { SourceEntityDescriptor } from './SourceEntity.js'; import { SourceTable } from './SourceTable.js'; -import { ReplicaId, WriteCheckpointAPI, WriteCheckpointFilters } from './storage-index.js'; +import { ReplicaId, WriteCheckpointAPI } from './storage-index.js'; export interface BucketStorageFactory extends WriteCheckpointAPI { /** @@ -90,7 +90,7 @@ export interface BucketStorageFactory extends WriteCheckpointAPI { /** * Yields the latest user write checkpoint whenever the sync checkpoint updates. */ - watchWriteCheckpoint(filters: WriteCheckpointFilters, signal: AbortSignal): AsyncIterable; + watchWriteCheckpoint(user_id: string, signal: AbortSignal): AsyncIterable; /** * Get storage size of active sync rules. diff --git a/packages/service-core/src/storage/MongoBucketStorage.ts b/packages/service-core/src/storage/MongoBucketStorage.ts index 2d98ea864..fe818c2c8 100644 --- a/packages/service-core/src/storage/MongoBucketStorage.ts +++ b/packages/service-core/src/storage/MongoBucketStorage.ts @@ -498,7 +498,7 @@ export class MongoBucketStorage implements BucketStorageFactory { /** * User-specific watch on the latest checkpoint and/or write checkpoint. */ - async *watchWriteCheckpoint(filters: WriteCheckpointFilters, signal: AbortSignal): AsyncIterable { + async *watchWriteCheckpoint(user_id: string, signal: AbortSignal): AsyncIterable { let lastCheckpoint: util.OpId | null = null; let lastWriteCheckpoint: bigint | null = null; @@ -515,10 +515,9 @@ export class MongoBucketStorage implements BucketStorageFactory { const lsnFilters: Record = lsn ? { 1: lsn } : {}; const currentWriteCheckpoint = await this.lastWriteCheckpoint({ + user_id, sync_rules_id: bucketStorage?.group_id, - ...filters, heads: { - ...(filters.heads ?? {}), ...lsnFilters } }); diff --git a/packages/service-core/src/sync/sync.ts b/packages/service-core/src/sync/sync.ts index 5d2211690..8f2f900a0 100644 --- a/packages/service-core/src/sync/sync.ts +++ b/packages/service-core/src/sync/sync.ts @@ -108,12 +108,7 @@ async function* streamResponseInner( } const checkpointUserId = util.checkpointUserId(syncParams.token_parameters.user_id as string, params.client_id); - const stream = storage.watchWriteCheckpoint( - { - user_id: checkpointUserId - }, - signal - ); + const stream = storage.watchWriteCheckpoint(checkpointUserId, signal); for await (const next of stream) { const { base, writeCheckpoint } = next; const checkpoint = base.checkpoint; From 7a6f08b5194259e409f9ec26284e2763e021223a Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 26 Sep 2024 17:55:24 +0200 Subject: [PATCH 24/33] wip: Fire individual replication events. Spread Write Checkpoint API. Add batch Write Checkpoint functionality to storage batch. --- .../src/routes/endpoints/checkpointing.ts | 2 +- .../service-core/src/storage/BucketStorage.ts | 7 +- .../src/storage/MongoBucketStorage.ts | 32 +++-- .../src/storage/ReplicationEventBatch.ts | 91 ------------ .../src/storage/ReplicationEventManager.ts | 31 +--- .../src/storage/mongo/MongoBucketBatch.ts | 32 +++-- .../mongo/MongoCustomWriteCheckpointAPI.ts | 53 ------- .../mongo/MongoManagedWriteCheckpointAPI.ts | 63 --------- .../storage/mongo/MongoSyncBucketStorage.ts | 10 +- .../storage/mongo/MongoWriteCheckpointAPI.ts | 132 ++++++++++++++++++ packages/service-core/src/storage/mongo/db.ts | 1 - .../src/storage/write-checkpoint.ts | 45 ++++-- 12 files changed, 219 insertions(+), 280 deletions(-) delete mode 100644 packages/service-core/src/storage/ReplicationEventBatch.ts delete mode 100644 packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts delete mode 100644 packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts create mode 100644 packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts diff --git a/packages/service-core/src/routes/endpoints/checkpointing.ts b/packages/service-core/src/routes/endpoints/checkpointing.ts index ae45973a2..0cfd2dc12 100644 --- a/packages/service-core/src/routes/endpoints/checkpointing.ts +++ b/packages/service-core/src/routes/endpoints/checkpointing.ts @@ -63,7 +63,7 @@ export const writeCheckpoint2 = routeDefinition({ storageEngine: { activeBucketStorage } } = service_context; - const writeCheckpoint = await activeBucketStorage.createWriteCheckpoint({ + const writeCheckpoint = await activeBucketStorage.createManagedWriteCheckpoint({ user_id: full_user_id, heads: { '1': currentCheckpoint } }); diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index 9caba263e..73d28fc4f 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -11,7 +11,7 @@ import * as util from '../util/util-index.js'; import { ReplicationEventManager } from './ReplicationEventManager.js'; import { SourceEntityDescriptor } from './SourceEntity.js'; import { SourceTable } from './SourceTable.js'; -import { ReplicaId, WriteCheckpointAPI } from './storage-index.js'; +import { BatchedCustomWriteCheckpointOptions, ReplicaId, WriteCheckpointAPI } from './storage-index.js'; export interface BucketStorageFactory extends WriteCheckpointAPI { /** @@ -345,6 +345,11 @@ export interface BucketStorageBatch { keepalive(lsn: string): Promise; markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise; + + /** + * Queues the creation of a custom Write Checkpoint. This will be persisted after operations are flushed. + */ + addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void; } export interface SaveParameterData { diff --git a/packages/service-core/src/storage/MongoBucketStorage.ts b/packages/service-core/src/storage/MongoBucketStorage.ts index fe818c2c8..5f07375ea 100644 --- a/packages/service-core/src/storage/MongoBucketStorage.ts +++ b/packages/service-core/src/storage/MongoBucketStorage.ts @@ -22,18 +22,18 @@ import { } from './BucketStorage.js'; import { PowerSyncMongo, PowerSyncMongoOptions } from './mongo/db.js'; import { SyncRuleDocument, SyncRuleState } from './mongo/models.js'; -import { MongoCustomWriteCheckpointAPI } from './mongo/MongoCustomWriteCheckpointAPI.js'; -import { MongoManagedWriteCheckpointAPI } from './mongo/MongoManagedWriteCheckpointAPI.js'; import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js'; import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js'; +import { MongoWriteCheckpointAPI } from './mongo/MongoWriteCheckpointAPI.js'; import { generateSlotName } from './mongo/util.js'; import { ReplicationEventManager } from './ReplicationEventManager.js'; import { + CustomWriteCheckpointOptions, DEFAULT_WRITE_CHECKPOINT_MODE, + LastWriteCheckpointFilters, + ManagedWriteCheckpointOptions, WriteCheckpointAPI, - WriteCheckpointFilters, - WriteCheckpointMode, - WriteCheckpointOptions + WriteCheckpointMode } from './write-checkpoint.js'; export interface MongoBucketStorageOptions extends PowerSyncMongoOptions {} @@ -83,10 +83,10 @@ export class MongoBucketStorage implements BucketStorageFactory { this.slot_name_prefix = options.slot_name_prefix; this.events = options.event_manager; this.write_checkpoint_mode = options.write_checkpoint_mode ?? DEFAULT_WRITE_CHECKPOINT_MODE; - this.writeCheckpointAPI = - this.write_checkpoint_mode == WriteCheckpointMode.MANAGED - ? new MongoManagedWriteCheckpointAPI(db) - : new MongoCustomWriteCheckpointAPI(db); + this.writeCheckpointAPI = new MongoWriteCheckpointAPI({ + db, + mode: this.write_checkpoint_mode + }); } getInstance(options: PersistedSyncRulesContent): MongoSyncBucketStorage { @@ -285,15 +285,19 @@ export class MongoBucketStorage implements BucketStorageFactory { }); } - async batchCreateWriteCheckpoints(checkpoints: WriteCheckpointOptions[]): Promise { - return this.writeCheckpointAPI.batchCreateWriteCheckpoints(checkpoints); + async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise { + return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints(checkpoints); + } + + async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise { + return this.writeCheckpointAPI.createCustomWriteCheckpoint(options); } - async createWriteCheckpoint(options: WriteCheckpointOptions): Promise { - return this.writeCheckpointAPI.createWriteCheckpoint(options); + async createManagedWriteCheckpoint(options: ManagedWriteCheckpointOptions): Promise { + return this.writeCheckpointAPI.createManagedWriteCheckpoint(options); } - async lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise { + async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise { return this.writeCheckpointAPI.lastWriteCheckpoint(filters); } diff --git a/packages/service-core/src/storage/ReplicationEventBatch.ts b/packages/service-core/src/storage/ReplicationEventBatch.ts deleted file mode 100644 index 64f17ba0a..000000000 --- a/packages/service-core/src/storage/ReplicationEventBatch.ts +++ /dev/null @@ -1,91 +0,0 @@ -import * as sync_rules from '@powersync/service-sync-rules'; -import * as storage from '../storage/storage-index.js'; -import { EventData, ReplicationEventData, ReplicationEventManager } from './ReplicationEventManager.js'; - -export type ReplicationEventBatchOptions = { - manager: ReplicationEventManager; - storage: storage.SyncRulesBucketStorage; - max_batch_size?: number; -}; - -export type ReplicationEventWriteParams = { - event: sync_rules.SqlEventDescriptor; - table: storage.SourceTable; - data: EventData; -}; - -const MAX_BATCH_SIZE = 1000; - -export class ReplicationEventBatch { - readonly manager: ReplicationEventManager; - readonly maxBatchSize: number; - readonly storage: storage.SyncRulesBucketStorage; - - protected event_cache: Map; - - /** - * Keeps track of the number of rows in the cache. - * This avoids having to calculate the size on demand. - */ - private _eventCacheRowCount: number; - - constructor(options: ReplicationEventBatchOptions) { - this.event_cache = new Map(); - this.manager = options.manager; - this.maxBatchSize = options.max_batch_size || MAX_BATCH_SIZE; - this.storage = options.storage; - this._eventCacheRowCount = 0; - } - - /** - * Returns the number of rows/events in the cache - */ - get cacheSize() { - return this._eventCacheRowCount; - } - - dispose() { - this.event_cache.clear(); - // This is not required, but cleans things up a bit. - this._eventCacheRowCount = 0; - } - - /** - * Queues a replication event. The cache is automatically flushed - * if it exceeds {@link ReplicationEventBatchOptions['max_batch_size']}. - */ - async save(params: ReplicationEventWriteParams) { - const { data, event, table } = params; - if (!this.event_cache.has(event)) { - this.event_cache.set(event, new Map()); - } - - const eventEntry = this.event_cache.get(event)!; - - if (!eventEntry.has(table)) { - eventEntry.set(table, []); - } - - const tableEntry = eventEntry.get(table)!; - tableEntry.push(data); - this._eventCacheRowCount++; - - if (this.cacheSize >= this.maxBatchSize) { - await this.flush(); - } - } - - /** - * Flushes cached changes. Events will be emitted to the - * {@link ReplicationEventManager}. - */ - async flush() { - await this.manager.fireEvents({ - batch_data: this.event_cache, - storage: this.storage - }); - - this.event_cache.clear(); - this._eventCacheRowCount = 0; - } -} diff --git a/packages/service-core/src/storage/ReplicationEventManager.ts b/packages/service-core/src/storage/ReplicationEventManager.ts index b847aa4cd..dd1a7e1a7 100644 --- a/packages/service-core/src/storage/ReplicationEventManager.ts +++ b/packages/service-core/src/storage/ReplicationEventManager.ts @@ -1,6 +1,6 @@ import { logger } from '@powersync/lib-services-framework'; import * as sync_rules from '@powersync/service-sync-rules'; -import { SaveOp, SyncRulesBucketStorage } from './BucketStorage.js'; +import { BucketStorageBatch, SaveOp } from './BucketStorage.js'; import { SourceTable } from './SourceTable.js'; export type EventData = { @@ -9,17 +9,11 @@ export type EventData = { after?: sync_rules.SqliteRow; }; -export type ReplicationEventData = Map; - export type ReplicationEventPayload = { + batch: BucketStorageBatch; + data: EventData; event: sync_rules.SqlEventDescriptor; - data: ReplicationEventData; - storage: SyncRulesBucketStorage; -}; - -export type BatchReplicationEventPayload = { - storage: SyncRulesBucketStorage; - batch_data: Map; + table: SourceTable; }; export interface ReplicationEventHandler { @@ -52,23 +46,6 @@ export class ReplicationEventManager { } } - /** - * Fires a batch of events, passing the specified payload to all registered handlers. - * This call resolves once all handlers have processed the events. - * Handler exceptions are caught and logged by the {@link fireEvent} method. - */ - async fireEvents(batch: BatchReplicationEventPayload) { - const { batch_data, storage } = batch; - - for (const [eventDescription, eventData] of batch_data) { - await this.fireEvent({ - event: eventDescription, - storage: storage, - data: eventData - }); - } - } - registerHandler(handler: ReplicationEventHandler) { const { event_name } = handler; if (!this.handlers.has(event_name)) { diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index 4d3e1a880..3260bc3dc 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -5,11 +5,13 @@ import * as mongo from 'mongodb'; import { container, errors, logger } from '@powersync/lib-services-framework'; import * as util from '../../util/util-index.js'; import { BucketStorageBatch, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js'; -import { ReplicationEventBatch } from '../ReplicationEventBatch.js'; +import { ReplicationEventManager } from '../ReplicationEventManager.js'; import { SourceTable } from '../SourceTable.js'; +import { BatchedCustomWriteCheckpointOptions, CustomWriteCheckpointOptions } from '../write-checkpoint.js'; import { PowerSyncMongo } from './db.js'; import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js'; import { MongoIdSequence } from './MongoIdSequence.js'; +import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, replicaIdEquals, serializeLookup } from './util.js'; @@ -37,6 +39,7 @@ export class MongoBucketBatch implements BucketStorageBatch { private readonly slot_name: string; private batch: OperationBatch | null = null; + private write_checkpoint_batch: CustomWriteCheckpointOptions[] = []; /** * Last LSN received associated with a checkpoint. @@ -51,7 +54,7 @@ export class MongoBucketBatch implements BucketStorageBatch { private persisted_op: bigint | null = null; - private event_batch: ReplicationEventBatch; + private events: ReplicationEventManager; /** * For tests only - not for persistence logic. @@ -65,17 +68,24 @@ export class MongoBucketBatch implements BucketStorageBatch { slot_name: string, last_checkpoint_lsn: string | null, no_checkpoint_before_lsn: string, - event_batch: ReplicationEventBatch + events: ReplicationEventManager ) { - this.db = db; this.client = db.client; - this.sync_rules = sync_rules; + this.db = db; + this.events = events; this.group_id = group_id; - this.slot_name = slot_name; - this.session = this.client.startSession(); this.last_checkpoint_lsn = last_checkpoint_lsn; this.no_checkpoint_before_lsn = no_checkpoint_before_lsn; - this.event_batch = event_batch; + this.session = this.client.startSession(); + this.slot_name = slot_name; + this.sync_rules = sync_rules; + } + + addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void { + this.write_checkpoint_batch.push({ + ...checkpoint, + sync_rules_id: this.group_id + }); } async flush(): Promise { @@ -88,7 +98,8 @@ export class MongoBucketBatch implements BucketStorageBatch { result = r; } } - await this.event_batch.flush(); + await batchCreateCustomWriteCheckpoints(this.db, this.write_checkpoint_batch); + this.write_checkpoint_batch = []; return result; } @@ -617,7 +628,8 @@ export class MongoBucketBatch implements BucketStorageBatch { async save(record: SaveOptions): Promise { const { after, before, sourceTable, tag } = record; for (const event of this.getTableEvents(sourceTable)) { - await this.event_batch.save({ + await this.events.fireEvent({ + batch: this, table: sourceTable, data: { op: tag, diff --git a/packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts b/packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts deleted file mode 100644 index 871e07c49..000000000 --- a/packages/service-core/src/storage/mongo/MongoCustomWriteCheckpointAPI.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { WriteCheckpointAPI, WriteCheckpointFilters, WriteCheckpointOptions } from '../write-checkpoint.js'; -import { PowerSyncMongo } from './db.js'; - -/** - * Implements a write checkpoint API which manages a mapping of - * `user_id` to a provided incrementing `write_checkpoint` `bigint`. - */ -export class MongoCustomWriteCheckpointAPI implements WriteCheckpointAPI { - constructor(protected db: PowerSyncMongo) {} - - async batchCreateWriteCheckpoints(checkpoints: WriteCheckpointOptions[]): Promise { - await this.db.custom_write_checkpoints.bulkWrite( - checkpoints.map((checkpoint) => ({ - updateOne: { - filter: { user_id: checkpoint.user_id }, - update: { - $set: { - checkpoint: checkpoint.checkpoint, - sync_rules_id: checkpoint.sync_rules_id - } - }, - upsert: true - } - })) - ); - } - - async createWriteCheckpoint(options: WriteCheckpointOptions): Promise { - const { checkpoint, user_id, sync_rules_id } = options; - const doc = await this.db.custom_write_checkpoints.findOneAndUpdate( - { - user_id: user_id - }, - { - $set: { - checkpoint, - sync_rules_id - } - }, - { upsert: true, returnDocument: 'after' } - ); - return doc!.checkpoint; - } - - async lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise { - const { user_id } = filters; - - const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({ - user_id: user_id - }); - return lastWriteCheckpoint?.checkpoint ?? null; - } -} diff --git a/packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts b/packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts deleted file mode 100644 index 1f83730dd..000000000 --- a/packages/service-core/src/storage/mongo/MongoManagedWriteCheckpointAPI.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { WriteCheckpointAPI, WriteCheckpointFilters, WriteCheckpointOptions } from '../write-checkpoint.js'; -import { PowerSyncMongo } from './db.js'; - -/** - * Implements a write checkpoint API which manages a mapping of - * Replication HEAD + `user_id` to a managed incrementing `write_checkpoint` `bigint`. - */ -export class MongoManagedWriteCheckpointAPI implements WriteCheckpointAPI { - constructor(protected db: PowerSyncMongo) {} - - async batchCreateWriteCheckpoints(checkpoints: WriteCheckpointOptions[]): Promise { - await this.db.write_checkpoints.bulkWrite( - checkpoints.map((checkpoint) => ({ - updateOne: { - filter: { user_id: checkpoint.user_id }, - update: { - $set: { - lsns: checkpoint.heads - }, - $inc: { - client_id: 1n - } - }, - upsert: true - } - })) - ); - } - - async createWriteCheckpoint(options: WriteCheckpointOptions): Promise { - const { user_id, heads: lsns } = options; - const doc = await this.db.write_checkpoints.findOneAndUpdate( - { - user_id: user_id - }, - { - $set: { - lsns: lsns - }, - $inc: { - client_id: 1n - } - }, - { upsert: true, returnDocument: 'after' } - ); - return doc!.client_id; - } - - async lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise { - const { user_id, heads: lsns } = filters; - const lsnFilter = Object.fromEntries( - Object.entries(lsns!).map(([connectionKey, lsn]) => { - return [`lsns.${connectionKey}`, { $lte: lsn }]; - }) - ); - - const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({ - user_id: user_id, - ...lsnFilter - }); - return lastWriteCheckpoint?.client_id ?? null; - } -} diff --git a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts index e15fe59e9..7cdcfee02 100644 --- a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts +++ b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts @@ -12,7 +12,6 @@ import { DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES, FlushedResult, ParseSyncRulesOptions, - PersistedSyncRules, PersistedSyncRulesContent, ResolveTableOptions, ResolveTableResult, @@ -24,7 +23,7 @@ import { } from '../BucketStorage.js'; import { ChecksumCache, FetchPartialBucketChecksum } from '../ChecksumCache.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; -import { ReplicationEventBatch } from '../ReplicationEventBatch.js'; +import { ReplicationEventManager } from '../ReplicationEventManager.js'; import { SourceTable } from '../SourceTable.js'; import { PowerSyncMongo } from './db.js'; import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './models.js'; @@ -41,6 +40,7 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { }); private parsedSyncRulesCache: SqlSyncRules | undefined; + readonly events: ReplicationEventManager; constructor( public readonly factory: MongoBucketStorage, @@ -49,6 +49,7 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { public readonly slot_name: string ) { this.db = factory.db; + this.events = factory.events; } getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules { @@ -87,10 +88,7 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { this.slot_name, checkpoint_lsn, doc?.no_checkpoint_before ?? options.zeroLSN, - new ReplicationEventBatch({ - manager: this.factory.events, - storage: this - }) + this.events ); try { await callback(batch); diff --git a/packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts b/packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts new file mode 100644 index 000000000..9a66a961f --- /dev/null +++ b/packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts @@ -0,0 +1,132 @@ +import { logger } from '@powersync/lib-services-framework'; +import { + CustomWriteCheckpointFilters, + CustomWriteCheckpointOptions, + LastWriteCheckpointFilters, + ManagedWriteCheckpointFilters, + ManagedWriteCheckpointOptions, + WriteCheckpointAPI, + WriteCheckpointMode +} from '../write-checkpoint.js'; +import { PowerSyncMongo } from './db.js'; + +export type MongoCheckpointAPIOptions = { + db: PowerSyncMongo; + mode: WriteCheckpointMode; +}; + +export class MongoWriteCheckpointAPI implements WriteCheckpointAPI { + readonly db: PowerSyncMongo; + readonly mode: WriteCheckpointMode; + + constructor(options: MongoCheckpointAPIOptions) { + this.db = options.db; + this.mode = options.mode; + } + + async batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise { + return batchCreateCustomWriteCheckpoints(this.db, checkpoints); + } + + async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise { + if (this.mode !== WriteCheckpointMode.CUSTOM) { + logger.warn(`Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`); + } + + const { checkpoint, user_id, sync_rules_id } = options; + const doc = await this.db.custom_write_checkpoints.findOneAndUpdate( + { + user_id: user_id, + sync_rules_id + }, + { + $set: { + checkpoint + } + }, + { upsert: true, returnDocument: 'after' } + ); + return doc!.checkpoint; + } + + async createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise { + if (this.mode !== WriteCheckpointMode.CUSTOM) { + logger.warn( + `Creating a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"` + ); + } + + const { user_id, heads: lsns } = checkpoint; + const doc = await this.db.write_checkpoints.findOneAndUpdate( + { + user_id: user_id + }, + { + $set: { + lsns + }, + $inc: { + client_id: 1n + } + }, + { upsert: true, returnDocument: 'after' } + ); + return doc!.client_id; + } + + async lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise { + switch (this.mode) { + case WriteCheckpointMode.CUSTOM: + if (false == 'sync_rules_id' in filters) { + throw new Error(`Sync rules ID is required for custom Write Checkpoint filtering`); + } + return this.lastCustomWriteCheckpoint(filters); + case WriteCheckpointMode.MANAGED: + if (false == 'heads' in filters) { + throw new Error(`Replication HEAD is required for managed Write Checkpoint filtering`); + } + return this.lastManagedWriteCheckpoint(filters); + } + } + + protected async lastCustomWriteCheckpoint(filters: CustomWriteCheckpointFilters) { + const { user_id, sync_rules_id } = filters; + const lastWriteCheckpoint = await this.db.custom_write_checkpoints.findOne({ + user_id, + sync_rules_id + }); + return lastWriteCheckpoint?.checkpoint ?? null; + } + + protected async lastManagedWriteCheckpoint(filters: ManagedWriteCheckpointFilters) { + const { user_id } = filters; + const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({ + user_id: user_id + }); + return lastWriteCheckpoint?.client_id ?? null; + } +} + +export async function batchCreateCustomWriteCheckpoints( + db: PowerSyncMongo, + checkpoints: CustomWriteCheckpointOptions[] +): Promise { + if (!checkpoints.length) { + return; + } + + await db.custom_write_checkpoints.bulkWrite( + checkpoints.map((checkpoint) => ({ + updateOne: { + filter: { user_id: checkpoint.user_id, sync_rules_id: checkpoint.sync_rules_id }, + update: { + $set: { + checkpoint: checkpoint.checkpoint, + sync_rules_id: checkpoint.sync_rules_id + } + }, + upsert: true + } + })) + ); +} diff --git a/packages/service-core/src/storage/mongo/db.ts b/packages/service-core/src/storage/mongo/db.ts index e4e32ac18..dddfdf918 100644 --- a/packages/service-core/src/storage/mongo/db.ts +++ b/packages/service-core/src/storage/mongo/db.ts @@ -56,7 +56,6 @@ export class PowerSyncMongo { this.op_id_sequence = db.collection('op_id_sequence'); this.sync_rules = db.collection('sync_rules'); this.source_tables = db.collection('source_tables'); - // TODO add indexes this.custom_write_checkpoints = db.collection('custom_write_checkpoints'); this.write_checkpoints = db.collection('write_checkpoints'); this.instance = db.collection('instance'); diff --git a/packages/service-core/src/storage/write-checkpoint.ts b/packages/service-core/src/storage/write-checkpoint.ts index b44c15fa3..e4e335175 100644 --- a/packages/service-core/src/storage/write-checkpoint.ts +++ b/packages/service-core/src/storage/write-checkpoint.ts @@ -12,37 +12,56 @@ export enum WriteCheckpointMode { MANAGED = 'managed' } -export interface WriteCheckpointFilters { +export interface BaseWriteCheckpointIdentifier { /** - * Replication HEAD(s) at the creation of the checkpoint. + * Identifier for User's account. */ - heads?: Record; + user_id: string; +} +export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier { /** * Sync rules which were active when this checkpoint was created. */ - sync_rules_id?: number; + sync_rules_id: number; +} +export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpointIdentifier { /** - * Identifier for User's account. + * A supplied incrementing Write Checkpoint number */ - user_id: string; + checkpoint: bigint; } -export interface WriteCheckpointOptions extends WriteCheckpointFilters { +export type CustomWriteCheckpointOptions = BatchedCustomWriteCheckpointOptions & CustomWriteCheckpointFilters; + +/** + * Managed Write Checkpoints are a mapping of User ID to replication HEAD + */ +export interface ManagedWriteCheckpointFilters { /** - * Strictly incrementing write checkpoint number. - * Defaults to an automatically incrementing operation. + * Replication HEAD(s) at the creation of the checkpoint. */ - checkpoint?: bigint; + heads: Record; + + /** + * Identifier for User's account. + */ + user_id: string; } +export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters; + +export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters; + export interface WriteCheckpointAPI { - batchCreateWriteCheckpoints(checkpoints: WriteCheckpointOptions[]): Promise; + batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise; + + createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise; - createWriteCheckpoint(checkpoint: WriteCheckpointOptions): Promise; + createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise; - lastWriteCheckpoint(filters: WriteCheckpointFilters): Promise; + lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise; } export const DEFAULT_WRITE_CHECKPOINT_MODE = WriteCheckpointMode.MANAGED; From d7db921f0411317a9383596a0e774d5b18ec9af4 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 8 Oct 2024 11:34:05 +0200 Subject: [PATCH 25/33] Update index to be unique. Cleanup. --- ...099539247-custom-write-checkpoint-index.ts | 11 ++- .../storage/mongo/MongoWriteCheckpointAPI.ts | 10 +-- packages/sync-rules/src/BaseSqlDataQuery.ts | 10 +-- packages/sync-rules/src/SqlDataQuery.ts | 82 +------------------ .../src/events/SqlEventSourceQuery.ts | 4 +- 5 files changed, 23 insertions(+), 94 deletions(-) diff --git a/packages/service-core/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts b/packages/service-core/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts index 7a4973faf..2bac37fcc 100644 --- a/packages/service-core/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts +++ b/packages/service-core/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts @@ -1,6 +1,8 @@ import * as storage from '../../../storage/storage-index.js'; import * as utils from '../../../util/util-index.js'; +const INDEX_NAME = 'user_sync_rule_unique'; + export const up = async (context: utils.MigrationContext) => { const { runner_config } = context; const config = await utils.loadConfig(runner_config); @@ -9,9 +11,10 @@ export const up = async (context: utils.MigrationContext) => { try { await db.custom_write_checkpoints.createIndex( { - user_id: 1 + user_id: 1, + sync_rules_id: 1 }, - { name: 'user_id' } + { name: INDEX_NAME, unique: true } ); } finally { await db.client.close(); @@ -25,8 +28,8 @@ export const down = async (context: utils.MigrationContext) => { const db = storage.createPowerSyncMongo(config.storage); try { - if (await db.custom_write_checkpoints.indexExists('user_id')) { - await db.custom_write_checkpoints.dropIndex('user_id'); + if (await db.custom_write_checkpoints.indexExists(INDEX_NAME)) { + await db.custom_write_checkpoints.dropIndex(INDEX_NAME); } } finally { await db.client.close(); diff --git a/packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts b/packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts index 9a66a961f..c41322fa1 100644 --- a/packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts +++ b/packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts @@ -50,7 +50,7 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI { } async createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise { - if (this.mode !== WriteCheckpointMode.CUSTOM) { + if (this.mode !== WriteCheckpointMode.MANAGED) { logger.warn( `Creating a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"` ); @@ -116,13 +116,13 @@ export async function batchCreateCustomWriteCheckpoints( } await db.custom_write_checkpoints.bulkWrite( - checkpoints.map((checkpoint) => ({ + checkpoints.map((checkpointOptions) => ({ updateOne: { - filter: { user_id: checkpoint.user_id, sync_rules_id: checkpoint.sync_rules_id }, + filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id }, update: { $set: { - checkpoint: checkpoint.checkpoint, - sync_rules_id: checkpoint.sync_rules_id + checkpoint: checkpointOptions.checkpoint, + sync_rules_id: checkpointOptions.sync_rules_id } }, upsert: true diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts index 5d892a641..342314d9b 100644 --- a/packages/sync-rules/src/BaseSqlDataQuery.ts +++ b/packages/sync-rules/src/BaseSqlDataQuery.ts @@ -1,6 +1,6 @@ import { SelectedColumn } from 'pgsql-ast-parser'; import { SqlRuleError } from './errors.js'; -import { ColumnDefinition, ExpressionType } from './ExpressionType.js'; +import { ColumnDefinition } from './ExpressionType.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlTools } from './sql_filters.js'; import { TablePattern } from './TablePattern.js'; @@ -100,14 +100,14 @@ export class BaseSqlDataQuery { return filterJsonRow(result); } - private getColumnOutputsFor(schemaTable: SourceSchemaTable, output: Record) { + protected getColumnOutputsFor(schemaTable: SourceSchemaTable, output: Record) { const querySchema: QuerySchema = { - getType: (table, column) => { + getColumn: (table, column) => { if (table == this.table!) { - return schemaTable.getType(column) ?? ExpressionType.NONE; + return schemaTable.getColumn(column); } else { // TODO: bucket parameters? - return ExpressionType.NONE; + return undefined; } }, getColumns: (table) => { diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index 4e7e6ddd3..6f082a96a 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -2,7 +2,7 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { parse } from 'pgsql-ast-parser'; import { BaseSqlDataQuery } from './BaseSqlDataQuery.js'; import { SqlRuleError } from './errors.js'; -import { ColumnDefinition, ExpressionType } from './ExpressionType.js'; +import { ExpressionType } from './ExpressionType.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlTools } from './sql_filters.js'; import { castAsText } from './sql_functions.js'; @@ -10,17 +10,8 @@ import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; import { SyncRulesOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; import { TableQuerySchema } from './TableQuerySchema.js'; -import { - EvaluationResult, - ParameterMatchClause, - QueryParameters, - QuerySchema, - SourceSchema, - SourceSchemaTable, - SqliteJsonRow, - SqliteRow -} from './types.js'; -import { filterJsonRow, getBucketId, isSelectStatement } from './utils.js'; +import { EvaluationResult, ParameterMatchClause, QuerySchema, SqliteRow } from './types.js'; +import { getBucketId, isSelectStatement } from './utils.js'; export class SqlDataQuery extends BaseSqlDataQuery { filter?: ParameterMatchClause; @@ -206,71 +197,4 @@ export class SqlDataQuery extends BaseSqlDataQuery { return [{ error: e.message ?? `Evaluating data query failed` }]; } } - - protected transformRow(tables: QueryParameters): SqliteJsonRow { - let result: SqliteRow = {}; - for (let extractor of this.extractors) { - extractor.extract(tables, result); - } - return filterJsonRow(result); - } - - columnOutputNames(): string[] { - return this.columns!.map((c) => { - return this.tools!.getOutputName(c); - }); - } - - getColumnOutputs(schema: SourceSchema): { name: string; columns: ColumnDefinition[] }[] { - let result: { name: string; columns: ColumnDefinition[] }[] = []; - - if (this.isUnaliasedWildcard()) { - // Separate results - for (let schemaTable of schema.getTables(this.sourceTable!)) { - let output: Record = {}; - - this.getColumnOutputsFor(schemaTable, output); - - result.push({ - name: this.getOutputName(schemaTable.table), - columns: Object.values(output) - }); - } - } else { - // Merged results - let output: Record = {}; - for (let schemaTable of schema.getTables(this.sourceTable!)) { - this.getColumnOutputsFor(schemaTable, output); - } - result.push({ - name: this.table!, - columns: Object.values(output) - }); - } - - return result; - } - - private getColumnOutputsFor(schemaTable: SourceSchemaTable, output: Record) { - const querySchema: QuerySchema = { - getColumn: (table, column) => { - if (table == this.table!) { - return schemaTable.getColumn(column); - } else { - // TODO: bucket parameters? - return undefined; - } - }, - getColumns: (table) => { - if (table == this.table!) { - return schemaTable.getColumns(); - } else { - return []; - } - } - }; - for (let extractor of this.extractors) { - extractor.getTypes(querySchema, output); - } - } } diff --git a/packages/sync-rules/src/events/SqlEventSourceQuery.ts b/packages/sync-rules/src/events/SqlEventSourceQuery.ts index 24601fa8f..c5ca1e1ab 100644 --- a/packages/sync-rules/src/events/SqlEventSourceQuery.ts +++ b/packages/sync-rules/src/events/SqlEventSourceQuery.ts @@ -1,6 +1,7 @@ import { parse } from 'pgsql-ast-parser'; import { BaseSqlDataQuery } from '../BaseSqlDataQuery.js'; import { SqlRuleError } from '../errors.js'; +import { ExpressionType } from '../ExpressionType.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { SqlTools } from '../sql_filters.js'; import { checkUnsupportedFeatures, isClauseError } from '../sql_support.js'; @@ -95,7 +96,8 @@ export class SqlEventSourceQuery extends BaseSqlDataQuery { output[name] = clause.evaluate(tables); }, getTypes(schema, into) { - into[name] = { name, type: clause.getType(schema) }; + const def = clause.getColumnDefinition(schema); + into[name] = { name, type: def?.type ?? ExpressionType.NONE, originalType: def?.originalType }; } }); } else { From 6b42be2e8d995a8ca4677136d5781f36f5cd26c6 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 8 Oct 2024 11:41:28 +0200 Subject: [PATCH 26/33] update custom write checkpoint types --- packages/service-core/src/storage/BucketStorage.ts | 4 ++-- .../src/storage/mongo/MongoBucketBatch.ts | 4 ++-- packages/service-core/src/storage/write-checkpoint.ts | 11 ++--------- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index 73d28fc4f..ab7e4a0a9 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -11,7 +11,7 @@ import * as util from '../util/util-index.js'; import { ReplicationEventManager } from './ReplicationEventManager.js'; import { SourceEntityDescriptor } from './SourceEntity.js'; import { SourceTable } from './SourceTable.js'; -import { BatchedCustomWriteCheckpointOptions, ReplicaId, WriteCheckpointAPI } from './storage-index.js'; +import { CustomWriteCheckpointOptions, ReplicaId, WriteCheckpointAPI } from './storage-index.js'; export interface BucketStorageFactory extends WriteCheckpointAPI { /** @@ -349,7 +349,7 @@ export interface BucketStorageBatch { /** * Queues the creation of a custom Write Checkpoint. This will be persisted after operations are flushed. */ - addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void; + addCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): void; } export interface SaveParameterData { diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index 3260bc3dc..1e16c0896 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -7,7 +7,7 @@ import * as util from '../../util/util-index.js'; import { BucketStorageBatch, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js'; import { ReplicationEventManager } from '../ReplicationEventManager.js'; import { SourceTable } from '../SourceTable.js'; -import { BatchedCustomWriteCheckpointOptions, CustomWriteCheckpointOptions } from '../write-checkpoint.js'; +import { CustomWriteCheckpointOptions } from '../write-checkpoint.js'; import { PowerSyncMongo } from './db.js'; import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js'; import { MongoIdSequence } from './MongoIdSequence.js'; @@ -81,7 +81,7 @@ export class MongoBucketBatch implements BucketStorageBatch { this.sync_rules = sync_rules; } - addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void { + addCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): void { this.write_checkpoint_batch.push({ ...checkpoint, sync_rules_id: this.group_id diff --git a/packages/service-core/src/storage/write-checkpoint.ts b/packages/service-core/src/storage/write-checkpoint.ts index e4e335175..821e437b2 100644 --- a/packages/service-core/src/storage/write-checkpoint.ts +++ b/packages/service-core/src/storage/write-checkpoint.ts @@ -26,28 +26,21 @@ export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentif sync_rules_id: number; } -export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpointIdentifier { +export interface CustomWriteCheckpointOptions extends CustomWriteCheckpointFilters { /** * A supplied incrementing Write Checkpoint number */ checkpoint: bigint; } -export type CustomWriteCheckpointOptions = BatchedCustomWriteCheckpointOptions & CustomWriteCheckpointFilters; - /** * Managed Write Checkpoints are a mapping of User ID to replication HEAD */ -export interface ManagedWriteCheckpointFilters { +export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdentifier { /** * Replication HEAD(s) at the creation of the checkpoint. */ heads: Record; - - /** - * Identifier for User's account. - */ - user_id: string; } export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters; From 89b16f701c7ce7ba66f302acd39eaa1d09f2e915 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 8 Oct 2024 12:10:06 +0200 Subject: [PATCH 27/33] cleanup --- packages/service-core/src/storage/BucketStorage.ts | 4 ++-- packages/service-core/src/storage/write-checkpoint.ts | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index ab7e4a0a9..73d28fc4f 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -11,7 +11,7 @@ import * as util from '../util/util-index.js'; import { ReplicationEventManager } from './ReplicationEventManager.js'; import { SourceEntityDescriptor } from './SourceEntity.js'; import { SourceTable } from './SourceTable.js'; -import { CustomWriteCheckpointOptions, ReplicaId, WriteCheckpointAPI } from './storage-index.js'; +import { BatchedCustomWriteCheckpointOptions, ReplicaId, WriteCheckpointAPI } from './storage-index.js'; export interface BucketStorageFactory extends WriteCheckpointAPI { /** @@ -349,7 +349,7 @@ export interface BucketStorageBatch { /** * Queues the creation of a custom Write Checkpoint. This will be persisted after operations are flushed. */ - addCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): void; + addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void; } export interface SaveParameterData { diff --git a/packages/service-core/src/storage/write-checkpoint.ts b/packages/service-core/src/storage/write-checkpoint.ts index 821e437b2..0b61fe0c1 100644 --- a/packages/service-core/src/storage/write-checkpoint.ts +++ b/packages/service-core/src/storage/write-checkpoint.ts @@ -33,6 +33,13 @@ export interface CustomWriteCheckpointOptions extends CustomWriteCheckpointFilte checkpoint: bigint; } +/** + * Options for creating a custom Write Checkpoint in a batch. + * A {@link BucketStorageBatch} is already associated with a Sync Rules instance. + * The `sync_rules_id` is not required here. + */ +export type BatchedCustomWriteCheckpointOptions = Omit; + /** * Managed Write Checkpoints are a mapping of User ID to replication HEAD */ From 0d45657cf66370c34bad4c1055a7dba94ec19f24 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 8 Oct 2024 13:42:25 +0200 Subject: [PATCH 28/33] fix tests. work around circular dependencies in sync rules package --- packages/sync-rules/src/sql_filters.ts | 10 ++++---- packages/sync-rules/src/sql_functions.ts | 14 ++++++++++- packages/sync-rules/src/sql_support.ts | 24 +++++-------------- .../sync-rules/test/src/sync_rules.test.ts | 9 +------ 4 files changed, 25 insertions(+), 32 deletions(-) diff --git a/packages/sync-rules/src/sql_filters.ts b/packages/sync-rules/src/sql_filters.ts index 9313ffd28..5cf9961f3 100644 --- a/packages/sync-rules/src/sql_filters.ts +++ b/packages/sync-rules/src/sql_filters.ts @@ -1,7 +1,9 @@ -import { Expr, ExprRef, Name, NodeLocation, QName, QNameAliased, SelectedColumn, parse } from 'pgsql-ast-parser'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { Expr, ExprRef, Name, NodeLocation, QName, QNameAliased, SelectedColumn } from 'pgsql-ast-parser'; import { nil } from 'pgsql-ast-parser/src/utils.js'; -import { ExpressionType, TYPE_NONE } from './ExpressionType.js'; +import { ExpressionType } from './ExpressionType.js'; import { SqlRuleError } from './errors.js'; +import { REQUEST_FUNCTIONS } from './request_functions.js'; import { BASIC_OPERATORS, OPERATOR_IN, @@ -13,6 +15,7 @@ import { SQL_FUNCTIONS, SqlFunction, castOperator, + getOperatorFunction, sqliteTypeOf } from './sql_functions.js'; import { @@ -20,7 +23,6 @@ import { SQLITE_TRUE, andFilters, compileStaticOperator, - getOperatorFunction, isClauseError, isParameterMatchClause, isParameterValueClause, @@ -44,8 +46,6 @@ import { TrueIfParametersMatch } from './types.js'; import { isJsonValue } from './utils.js'; -import { JSONBig } from '@powersync/service-jsonbig'; -import { REQUEST_FUNCTIONS } from './request_functions.js'; export const MATCH_CONST_FALSE: TrueIfParametersMatch = []; export const MATCH_CONST_TRUE: TrueIfParametersMatch = [{}]; diff --git a/packages/sync-rules/src/sql_functions.ts b/packages/sync-rules/src/sql_functions.ts index 715f1494f..ce419d49a 100644 --- a/packages/sync-rules/src/sql_functions.ts +++ b/packages/sync-rules/src/sql_functions.ts @@ -1,5 +1,5 @@ import { JSONBig } from '@powersync/service-jsonbig'; -import { getOperatorFunction, SQLITE_FALSE, SQLITE_TRUE, sqliteBool, sqliteNot } from './sql_support.js'; +import { SQLITE_FALSE, SQLITE_TRUE, sqliteBool, sqliteNot } from './sql_support.js'; import { SqliteValue } from './types.js'; import { jsonValueToSqlite } from './utils.js'; // Declares @syncpoint/wkx module @@ -44,6 +44,18 @@ export interface DocumentedSqlFunction extends SqlFunction { documentation?: string; } +export function getOperatorFunction(op: string): SqlFunction { + return { + debugName: `operator${op}`, + call(...args: SqliteValue[]) { + return evaluateOperator(op, args[0], args[1]); + }, + getReturnType(args) { + return getOperatorReturnType(op, args[0], args[1]); + } + }; +} + const upper: DocumentedSqlFunction = { debugName: 'upper', call(value: SqliteValue) { diff --git a/packages/sync-rules/src/sql_support.ts b/packages/sync-rules/src/sql_support.ts index e2b074c55..b64aacd4f 100644 --- a/packages/sync-rules/src/sql_support.ts +++ b/packages/sync-rules/src/sql_support.ts @@ -1,3 +1,8 @@ +import { SelectFromStatement } from 'pgsql-ast-parser'; +import { SqlRuleError } from './errors.js'; +import { ExpressionType } from './ExpressionType.js'; +import { MATCH_CONST_FALSE, MATCH_CONST_TRUE } from './sql_filters.js'; +import { evaluateOperator, getOperatorReturnType } from './sql_functions.js'; import { ClauseError, CompiledClause, @@ -6,16 +11,11 @@ import { ParameterMatchClause, ParameterValueClause, QueryParameters, - SqliteValue, RowValueClause, + SqliteValue, StaticValueClause, TrueIfParametersMatch } from './types.js'; -import { MATCH_CONST_FALSE, MATCH_CONST_TRUE } from './sql_filters.js'; -import { SqlFunction, evaluateOperator, getOperatorReturnType } from './sql_functions.js'; -import { SelectFromStatement } from 'pgsql-ast-parser'; -import { SqlRuleError } from './errors.js'; -import { ExpressionType } from './ExpressionType.js'; export function isParameterMatchClause(clause: CompiledClause): clause is ParameterMatchClause { return Array.isArray((clause as ParameterMatchClause).inputParameters); @@ -78,18 +78,6 @@ export function compileStaticOperator(op: string, left: RowValueClause, right: R }; } -export function getOperatorFunction(op: string): SqlFunction { - return { - debugName: `operator${op}`, - call(...args: SqliteValue[]) { - return evaluateOperator(op, args[0], args[1]); - }, - getReturnType(args) { - return getOperatorReturnType(op, args[0], args[1]); - } - }; -} - export function andFilters(a: CompiledClause, b: CompiledClause): CompiledClause { if (isRowValueClause(a) && isRowValueClause(b)) { // Optimization diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 26ff3adb3..54d507440 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -1,12 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { - DEFAULT_TAG, - DartSchemaGenerator, - JsLegacySchemaGenerator, - SqlSyncRules, - StaticSchema, - TsSchemaGenerator -} from '../../src/index.js'; +import { SqlSyncRules } from '../../src/index.js'; import { ASSETS, BASIC_SCHEMA, PARSE_OPTIONS, TestSourceTable, USERS, normalizeTokenParameters } from './util.js'; From b10c6647b53a37d0e0f151eea1e5802da79f0747 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 8 Oct 2024 16:59:58 +0200 Subject: [PATCH 29/33] throw if incorrect type of checkpoint is created --- .../src/storage/mongo/MongoWriteCheckpointAPI.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts b/packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts index c41322fa1..230db3153 100644 --- a/packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts +++ b/packages/service-core/src/storage/mongo/MongoWriteCheckpointAPI.ts @@ -1,4 +1,4 @@ -import { logger } from '@powersync/lib-services-framework'; +import * as framework from '@powersync/lib-services-framework'; import { CustomWriteCheckpointFilters, CustomWriteCheckpointOptions, @@ -30,7 +30,9 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI { async createCustomWriteCheckpoint(options: CustomWriteCheckpointOptions): Promise { if (this.mode !== WriteCheckpointMode.CUSTOM) { - logger.warn(`Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"`); + throw new framework.errors.ValidationError( + `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"` + ); } const { checkpoint, user_id, sync_rules_id } = options; @@ -51,7 +53,7 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI { async createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise { if (this.mode !== WriteCheckpointMode.MANAGED) { - logger.warn( + throw new framework.errors.ValidationError( `Creating a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.mode}"` ); } @@ -78,12 +80,14 @@ export class MongoWriteCheckpointAPI implements WriteCheckpointAPI { switch (this.mode) { case WriteCheckpointMode.CUSTOM: if (false == 'sync_rules_id' in filters) { - throw new Error(`Sync rules ID is required for custom Write Checkpoint filtering`); + throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`); } return this.lastCustomWriteCheckpoint(filters); case WriteCheckpointMode.MANAGED: if (false == 'heads' in filters) { - throw new Error(`Replication HEAD is required for managed Write Checkpoint filtering`); + throw new framework.errors.ValidationError( + `Replication HEAD is required for managed Write Checkpoint filtering` + ); } return this.lastManagedWriteCheckpoint(filters); } From 85e1d4aec9c6059d66e3054b1af696c9290b9a27 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 9 Oct 2024 17:26:20 +0200 Subject: [PATCH 30/33] Replace Replication Event manager with Listeners --- libs/lib-services/src/utils/BaseObserver.ts | 6 +- .../src/utils/DisposableObserver.ts | 18 ++++++ libs/lib-services/src/utils/utils-index.ts | 1 + .../test/src/slow_tests.test.ts | 4 +- modules/module-postgres/test/src/util.ts | 5 +- .../test/src/wal_stream_utils.ts | 13 ++--- packages/service-core/src/api/diagnostics.ts | 2 +- .../src/entry/commands/compact-action.ts | 5 +- .../src/replication/AbstractReplicationJob.ts | 4 +- .../src/replication/AbstractReplicator.ts | 9 +-- packages/service-core/src/runner/teardown.ts | 6 +- .../service-core/src/storage/BucketStorage.ts | 29 +++++++--- .../src/storage/MongoBucketStorage.ts | 27 ++++++--- .../src/storage/ReplicationEventManager.ts | 56 ------------------- .../src/storage/ReplicationEventPayload.ts | 16 ++++++ .../service-core/src/storage/StorageEngine.ts | 15 ++--- .../src/storage/StorageProvider.ts | 2 - .../src/storage/mongo/MongoBucketBatch.ts | 45 ++++++++------- .../src/storage/mongo/MongoStorageProvider.ts | 3 +- .../storage/mongo/MongoSyncBucketStorage.ts | 36 ++++++------ .../service-core/src/storage/storage-index.ts | 2 +- packages/service-core/test/src/util.ts | 6 +- service/src/runners/server.ts | 1 + 23 files changed, 157 insertions(+), 154 deletions(-) create mode 100644 libs/lib-services/src/utils/DisposableObserver.ts delete mode 100644 packages/service-core/src/storage/ReplicationEventManager.ts create mode 100644 packages/service-core/src/storage/ReplicationEventPayload.ts diff --git a/libs/lib-services/src/utils/BaseObserver.ts b/libs/lib-services/src/utils/BaseObserver.ts index 4c5b689ce..937fde59a 100644 --- a/libs/lib-services/src/utils/BaseObserver.ts +++ b/libs/lib-services/src/utils/BaseObserver.ts @@ -1,6 +1,10 @@ import { v4 as uuid } from 'uuid'; -export class BaseObserver { +export interface ObserverClient { + registerListener(listener: Partial): () => void; +} + +export class BaseObserver implements ObserverClient { protected listeners: { [id: string]: Partial }; constructor() { diff --git a/libs/lib-services/src/utils/DisposableObserver.ts b/libs/lib-services/src/utils/DisposableObserver.ts new file mode 100644 index 000000000..4c51630eb --- /dev/null +++ b/libs/lib-services/src/utils/DisposableObserver.ts @@ -0,0 +1,18 @@ +import { BaseObserver, ObserverClient } from './BaseObserver.js'; + +export interface DisposableListener { + disposed: () => void; +} + +export interface DisposableObserverClient extends ObserverClient, Disposable {} + +export class DisposableObserver + extends BaseObserver + implements DisposableObserverClient +{ + [Symbol.dispose]() { + this.iterateListeners((cb) => cb.disposed?.()); + // Delete all callbacks + Object.keys(this.listeners).forEach((key) => delete this.listeners[key]); + } +} diff --git a/libs/lib-services/src/utils/utils-index.ts b/libs/lib-services/src/utils/utils-index.ts index ee42d4057..59b89d274 100644 --- a/libs/lib-services/src/utils/utils-index.ts +++ b/libs/lib-services/src/utils/utils-index.ts @@ -1,2 +1,3 @@ export * from './BaseObserver.js'; +export * from './DisposableObserver.js'; export * from './environment-variables.js'; diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index c214e921a..7c5bad017 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -82,7 +82,7 @@ bucket_definitions: - SELECT * FROM "test_data" `; const syncRules = await f.updateSyncRules({ content: syncRuleContent }); - const storage = f.getInstance(syncRules); + using storage = f.getInstance(syncRules); abortController = new AbortController(); const options: WalStreamOptions = { abort_signal: abortController.signal, @@ -234,7 +234,7 @@ bucket_definitions: - SELECT id, description FROM "test_data" `; const syncRules = await f.updateSyncRules({ content: syncRuleContent }); - const storage = f.getInstance(syncRules); + using storage = f.getInstance(syncRules); // 1. Setup some base data that will be replicated in initial replication await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`); diff --git a/modules/module-postgres/test/src/util.ts b/modules/module-postgres/test/src/util.ts index 8ae07c4f5..c8142739d 100644 --- a/modules/module-postgres/test/src/util.ts +++ b/modules/module-postgres/test/src/util.ts @@ -2,7 +2,7 @@ import { connectMongo } from '@core-tests/util.js'; import * as types from '@module/types/types.js'; import * as pg_utils from '@module/utils/pgwire_utils.js'; import { logger } from '@powersync/lib-services-framework'; -import { BucketStorageFactory, Metrics, MongoBucketStorage, OpId, storage } from '@powersync/service-core'; +import { BucketStorageFactory, Metrics, MongoBucketStorage, OpId } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import { pgwireRows } from '@powersync/service-jpgwire'; import { env } from './env.js'; @@ -36,8 +36,7 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY: StorageFactory = async () => { await db.clear(); return new MongoBucketStorage(db, { - slot_name_prefix: 'test_', - event_manager: new storage.ReplicationEventManager() + slot_name_prefix: 'test_' }); }; diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index ee53c551a..23eced2e7 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -20,16 +20,12 @@ export function walStreamTest( const connectionManager = new PgManager(TEST_CONNECTION_OPTIONS, {}); await clearTestDb(connectionManager.pool); - const context = new WalStreamTestContext(f, connectionManager); - try { - await test(context); - } finally { - await context.dispose(); - } + await using context = new WalStreamTestContext(f, connectionManager); + await test(context); }; } -export class WalStreamTestContext { +export class WalStreamTestContext implements AsyncDisposable { private _walStream?: WalStream; private abortController = new AbortController(); private streamPromise?: Promise; @@ -41,10 +37,11 @@ export class WalStreamTestContext { public connectionManager: PgManager ) {} - async dispose() { + async [Symbol.asyncDispose]() { this.abortController.abort(); await this.streamPromise; await this.connectionManager.destroy(); + this.storage?.[Symbol.dispose](); } get pool() { diff --git a/packages/service-core/src/api/diagnostics.ts b/packages/service-core/src/api/diagnostics.ts index 2ebf5ada5..d323fcf81 100644 --- a/packages/service-core/src/api/diagnostics.ts +++ b/packages/service-core/src/api/diagnostics.ts @@ -57,7 +57,7 @@ export async function getSyncRulesStatus( // This method can run under some situations if no connection is configured yet. // It will return a default tag in such a case. This default tag is not module specific. const tag = sourceConfig.tag ?? DEFAULT_TAG; - const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined; + using systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined; const status = await systemStorage?.getStatus(); let replication_lag_bytes: number | undefined = undefined; diff --git a/packages/service-core/src/entry/commands/compact-action.ts b/packages/service-core/src/entry/commands/compact-action.ts index 5e3aab745..128c54841 100644 --- a/packages/service-core/src/entry/commands/compact-action.ts +++ b/packages/service-core/src/entry/commands/compact-action.ts @@ -37,15 +37,14 @@ export function registerCompactAction(program: Command) { await client.connect(); try { const bucketStorage = new storage.MongoBucketStorage(psdb, { - slot_name_prefix: configuration.slot_name_prefix, - event_manager: new storage.ReplicationEventManager() + slot_name_prefix: configuration.slot_name_prefix }); const active = await bucketStorage.getActiveSyncRulesContent(); if (active == null) { logger.info('No active instance to compact'); return; } - const p = bucketStorage.getInstance(active); + using p = bucketStorage.getInstance(active); logger.info('Performing compaction...'); await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB }); logger.info('Successfully compacted storage.'); diff --git a/packages/service-core/src/replication/AbstractReplicationJob.ts b/packages/service-core/src/replication/AbstractReplicationJob.ts index 913146b83..dc27c2eda 100644 --- a/packages/service-core/src/replication/AbstractReplicationJob.ts +++ b/packages/service-core/src/replication/AbstractReplicationJob.ts @@ -1,7 +1,7 @@ -import * as storage from '../storage/storage-index.js'; -import { ErrorRateLimiter } from './ErrorRateLimiter.js'; import { container, logger } from '@powersync/lib-services-framework'; import winston from 'winston'; +import * as storage from '../storage/storage-index.js'; +import { ErrorRateLimiter } from './ErrorRateLimiter.js'; export interface AbstractReplicationJobOptions { id: string; diff --git a/packages/service-core/src/replication/AbstractReplicator.ts b/packages/service-core/src/replication/AbstractReplicator.ts index 6c5a2a934..fcc3fa0ec 100644 --- a/packages/service-core/src/replication/AbstractReplicator.ts +++ b/packages/service-core/src/replication/AbstractReplicator.ts @@ -1,10 +1,10 @@ +import { container, logger } from '@powersync/lib-services-framework'; import { hrtime } from 'node:process'; +import winston from 'winston'; import * as storage from '../storage/storage-index.js'; -import { container, logger } from '@powersync/lib-services-framework'; +import { StorageEngine } from '../storage/storage-index.js'; import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider.js'; -import winston from 'winston'; import { AbstractReplicationJob } from './AbstractReplicationJob.js'; -import { StorageEngine } from '../storage/storage-index.js'; import { ErrorRateLimiter } from './ErrorRateLimiter.js'; // 5 minutes @@ -192,6 +192,7 @@ export abstract class AbstractReplicator void; + replicationEvent: (event: ReplicationEventPayload) => void; +} + +export interface BucketStorageFactory + extends DisposableObserverClient, + WriteCheckpointAPI { /** * Update sync rules from configuration, if changed. */ @@ -22,11 +30,6 @@ export interface BucketStorageFactory extends WriteCheckpointAPI { options?: { lock?: boolean } ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }>; - /** - * Manager which enables handling events for replicated data. - */ - events: ReplicationEventManager; - /** * Get a storage instance to query sync data for specific sync rules. */ @@ -199,7 +202,11 @@ export interface StartBatchOptions extends ParseSyncRulesOptions { zeroLSN: string; } -export interface SyncRulesBucketStorage { +export interface SyncRulesBucketStorageListener extends DisposableListener { + batchStarted: (batch: BucketStorageBatch) => void; +} + +export interface SyncRulesBucketStorage extends DisposableObserverClient { readonly group_id: number; readonly slot_name: string; @@ -298,7 +305,11 @@ export interface FlushedResult { flushed_op: string; } -export interface BucketStorageBatch { +export interface BucketBatchStorageListener extends DisposableListener { + replicationEvent: (payload: ReplicationEventPayload) => void; +} + +export interface BucketStorageBatch extends DisposableObserverClient { /** * Save an op, and potentially flush. * diff --git a/packages/service-core/src/storage/MongoBucketStorage.ts b/packages/service-core/src/storage/MongoBucketStorage.ts index 5f07375ea..1548e461f 100644 --- a/packages/service-core/src/storage/MongoBucketStorage.ts +++ b/packages/service-core/src/storage/MongoBucketStorage.ts @@ -8,11 +8,12 @@ import * as locks from '../locks/locks-index.js'; import * as sync from '../sync/sync-index.js'; import * as util from '../util/util-index.js'; -import { logger } from '@powersync/lib-services-framework'; +import { DisposableObserver, logger } from '@powersync/lib-services-framework'; import { v4 as uuid } from 'uuid'; import { ActiveCheckpoint, BucketStorageFactory, + BucketStorageFactoryListener, ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent, @@ -26,7 +27,6 @@ import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesC import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js'; import { MongoWriteCheckpointAPI } from './mongo/MongoWriteCheckpointAPI.js'; import { generateSlotName } from './mongo/util.js'; -import { ReplicationEventManager } from './ReplicationEventManager.js'; import { CustomWriteCheckpointOptions, DEFAULT_WRITE_CHECKPOINT_MODE, @@ -38,13 +38,15 @@ import { export interface MongoBucketStorageOptions extends PowerSyncMongoOptions {} -export class MongoBucketStorage implements BucketStorageFactory { +export class MongoBucketStorage + extends DisposableObserver + implements BucketStorageFactory +{ private readonly client: mongo.MongoClient; private readonly session: mongo.ClientSession; // TODO: This is still Postgres specific and needs to be reworked public readonly slot_name_prefix: string; - readonly events: ReplicationEventManager; readonly write_checkpoint_mode: WriteCheckpointMode; protected readonly writeCheckpointAPI: WriteCheckpointAPI; @@ -64,6 +66,9 @@ export class MongoBucketStorage implements BucketStorageFactory { } const rules = new MongoPersistedSyncRulesContent(this.db, doc2); return this.getInstance(rules); + }, + dispose: (storage) => { + storage[Symbol.dispose](); } }); @@ -73,15 +78,14 @@ export class MongoBucketStorage implements BucketStorageFactory { db: PowerSyncMongo, options: { slot_name_prefix: string; - event_manager: ReplicationEventManager; write_checkpoint_mode?: WriteCheckpointMode; } ) { + super(); this.client = db.client; this.db = db; this.session = this.client.startSession(); this.slot_name_prefix = options.slot_name_prefix; - this.events = options.event_manager; this.write_checkpoint_mode = options.write_checkpoint_mode ?? DEFAULT_WRITE_CHECKPOINT_MODE; this.writeCheckpointAPI = new MongoWriteCheckpointAPI({ db, @@ -94,7 +98,16 @@ export class MongoBucketStorage implements BucketStorageFactory { if ((typeof id as any) == 'bigint') { id = Number(id); } - return new MongoSyncBucketStorage(this, id, options, slot_name); + const storage = new MongoSyncBucketStorage(this, id, options, slot_name); + this.iterateListeners((cb) => cb.syncStorageCreated?.(storage)); + storage.registerListener({ + batchStarted: (batch) => { + batch.registerListener({ + replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload)) + }); + } + }); + return storage; } async configureSyncRules(sync_rules: string, options?: { lock?: boolean }) { diff --git a/packages/service-core/src/storage/ReplicationEventManager.ts b/packages/service-core/src/storage/ReplicationEventManager.ts deleted file mode 100644 index dd1a7e1a7..000000000 --- a/packages/service-core/src/storage/ReplicationEventManager.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { logger } from '@powersync/lib-services-framework'; -import * as sync_rules from '@powersync/service-sync-rules'; -import { BucketStorageBatch, SaveOp } from './BucketStorage.js'; -import { SourceTable } from './SourceTable.js'; - -export type EventData = { - op: SaveOp; - before?: sync_rules.SqliteRow; - after?: sync_rules.SqliteRow; -}; - -export type ReplicationEventPayload = { - batch: BucketStorageBatch; - data: EventData; - event: sync_rules.SqlEventDescriptor; - table: SourceTable; -}; - -export interface ReplicationEventHandler { - event_name: string; - handle(event: ReplicationEventPayload): Promise; -} - -export class ReplicationEventManager { - handlers: Map>; - - constructor() { - this.handlers = new Map(); - } - - /** - * Fires an event, passing the specified payload to all registered handlers. - * This call resolves once all handlers have processed the event. - * Handler exceptions are caught and logged. - */ - async fireEvent(payload: ReplicationEventPayload): Promise { - const handlers = this.handlers.get(payload.event.name); - - for (const handler of handlers?.values() ?? []) { - try { - await handler.handle(payload); - } catch (ex) { - // Exceptions in handlers don't affect the source. - logger.info(`Caught exception when processing "${handler.event_name}" event.`, ex); - } - } - } - - registerHandler(handler: ReplicationEventHandler) { - const { event_name } = handler; - if (!this.handlers.has(event_name)) { - this.handlers.set(event_name, new Set()); - } - this.handlers.get(event_name)?.add(handler); - } -} diff --git a/packages/service-core/src/storage/ReplicationEventPayload.ts b/packages/service-core/src/storage/ReplicationEventPayload.ts new file mode 100644 index 000000000..c2fe0aa84 --- /dev/null +++ b/packages/service-core/src/storage/ReplicationEventPayload.ts @@ -0,0 +1,16 @@ +import * as sync_rules from '@powersync/service-sync-rules'; +import { BucketStorageBatch, SaveOp } from './BucketStorage.js'; +import { SourceTable } from './SourceTable.js'; + +export type EventData = { + op: SaveOp; + before?: sync_rules.SqliteRow; + after?: sync_rules.SqliteRow; +}; + +export type ReplicationEventPayload = { + batch: BucketStorageBatch; + data: EventData; + event: sync_rules.SqlEventDescriptor; + table: SourceTable; +}; diff --git a/packages/service-core/src/storage/StorageEngine.ts b/packages/service-core/src/storage/StorageEngine.ts index b62f0ff25..bdafeb240 100644 --- a/packages/service-core/src/storage/StorageEngine.ts +++ b/packages/service-core/src/storage/StorageEngine.ts @@ -1,7 +1,6 @@ -import { logger } from '@powersync/lib-services-framework'; +import { DisposableListener, DisposableObserver, logger } from '@powersync/lib-services-framework'; import { ResolvedPowerSyncConfig } from '../util/util-index.js'; import { BucketStorageFactory } from './BucketStorage.js'; -import { ReplicationEventManager } from './ReplicationEventManager.js'; import { ActiveStorage, BucketStorageProvider, StorageSettings } from './StorageProvider.js'; import { DEFAULT_WRITE_CHECKPOINT_MODE } from './write-checkpoint.js'; @@ -13,16 +12,18 @@ export const DEFAULT_STORAGE_SETTINGS: StorageSettings = { writeCheckpointMode: DEFAULT_WRITE_CHECKPOINT_MODE }; -export class StorageEngine { +export interface StorageEngineListener extends DisposableListener { + storageActivated: (storage: BucketStorageFactory) => void; +} + +export class StorageEngine extends DisposableObserver { // TODO: This will need to revisited when we actually support multiple storage providers. private storageProviders: Map = new Map(); private currentActiveStorage: ActiveStorage | null = null; - readonly events: ReplicationEventManager; - private _activeSettings: StorageSettings; constructor(private options: StorageEngineOptions) { - this.events = new ReplicationEventManager(); + super(); this._activeSettings = DEFAULT_STORAGE_SETTINGS; } @@ -65,9 +66,9 @@ export class StorageEngine { const { configuration } = this.options; this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({ resolvedConfig: configuration, - eventManager: this.events, ...this.activeSettings }); + this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage)); logger.info(`Successfully activated storage: ${configuration.storage.type}.`); logger.info('Successfully started Storage Engine.'); } diff --git a/packages/service-core/src/storage/StorageProvider.ts b/packages/service-core/src/storage/StorageProvider.ts index 4e79ecbdb..7c730fb4b 100644 --- a/packages/service-core/src/storage/StorageProvider.ts +++ b/packages/service-core/src/storage/StorageProvider.ts @@ -1,6 +1,5 @@ import * as util from '../util/util-index.js'; import { BucketStorageFactory } from './BucketStorage.js'; -import { ReplicationEventManager } from './ReplicationEventManager.js'; import { WriteCheckpointMode } from './write-checkpoint.js'; export interface ActiveStorage { @@ -21,7 +20,6 @@ export interface StorageSettings { } export interface GetStorageOptions extends StorageSettings { - eventManager: ReplicationEventManager; // TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage resolvedConfig: util.ResolvedPowerSyncConfig; } diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index 1e16c0896..ed6ee62cd 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -2,10 +2,15 @@ import { SqlEventDescriptor, SqliteRow, SqlSyncRules } from '@powersync/service- import * as bson from 'bson'; import * as mongo from 'mongodb'; -import { container, errors, logger } from '@powersync/lib-services-framework'; +import { container, DisposableObserver, errors, logger } from '@powersync/lib-services-framework'; import * as util from '../../util/util-index.js'; -import { BucketStorageBatch, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js'; -import { ReplicationEventManager } from '../ReplicationEventManager.js'; +import { + BucketBatchStorageListener, + BucketStorageBatch, + FlushedResult, + mergeToast, + SaveOptions +} from '../BucketStorage.js'; import { SourceTable } from '../SourceTable.js'; import { CustomWriteCheckpointOptions } from '../write-checkpoint.js'; import { PowerSyncMongo } from './db.js'; @@ -28,7 +33,7 @@ const MAX_ROW_SIZE = 15 * 1024 * 1024; // In the future, we can investigate allowing multiple replication streams operating independently. const replicationMutex = new util.Mutex(); -export class MongoBucketBatch implements BucketStorageBatch { +export class MongoBucketBatch extends DisposableObserver implements BucketStorageBatch { private readonly client: mongo.MongoClient; public readonly db: PowerSyncMongo; public readonly session: mongo.ClientSession; @@ -54,8 +59,6 @@ export class MongoBucketBatch implements BucketStorageBatch { private persisted_op: bigint | null = null; - private events: ReplicationEventManager; - /** * For tests only - not for persistence logic. */ @@ -67,12 +70,11 @@ export class MongoBucketBatch implements BucketStorageBatch { group_id: number, slot_name: string, last_checkpoint_lsn: string | null, - no_checkpoint_before_lsn: string, - events: ReplicationEventManager + no_checkpoint_before_lsn: string ) { + super(); this.client = db.client; this.db = db; - this.events = events; this.group_id = group_id; this.last_checkpoint_lsn = last_checkpoint_lsn; this.no_checkpoint_before_lsn = no_checkpoint_before_lsn; @@ -545,8 +547,9 @@ export class MongoBucketBatch implements BucketStorageBatch { }); } - async abort() { + async [Symbol.asyncDispose]() { await this.session.endSession(); + super[Symbol.dispose]; } async commit(lsn: string): Promise { @@ -628,16 +631,18 @@ export class MongoBucketBatch implements BucketStorageBatch { async save(record: SaveOptions): Promise { const { after, before, sourceTable, tag } = record; for (const event of this.getTableEvents(sourceTable)) { - await this.events.fireEvent({ - batch: this, - table: sourceTable, - data: { - op: tag, - after: after && util.isCompleteRow(after) ? after : undefined, - before: before && util.isCompleteRow(before) ? before : undefined - }, - event - }); + this.iterateListeners((cb) => + cb.replicationEvent?.({ + batch: this, + table: sourceTable, + data: { + op: tag, + after: after && util.isCompleteRow(after) ? after : undefined, + before: before && util.isCompleteRow(before) ? before : undefined + }, + event + }) + ); } /** diff --git a/packages/service-core/src/storage/mongo/MongoStorageProvider.ts b/packages/service-core/src/storage/mongo/MongoStorageProvider.ts index ebea0eb6e..ef16900eb 100644 --- a/packages/service-core/src/storage/mongo/MongoStorageProvider.ts +++ b/packages/service-core/src/storage/mongo/MongoStorageProvider.ts @@ -10,7 +10,7 @@ export class MongoStorageProvider implements BucketStorageProvider { } async getStorage(options: GetStorageOptions): Promise { - const { eventManager, resolvedConfig } = options; + const { resolvedConfig } = options; const client = db.mongo.createMongoClient(resolvedConfig.storage); @@ -20,7 +20,6 @@ export class MongoStorageProvider implements BucketStorageProvider { storage: new MongoBucketStorage(database, { // TODO currently need the entire resolved config due to this slot_name_prefix: resolvedConfig.slot_name_prefix, - event_manager: eventManager, write_checkpoint_mode: options.writeCheckpointMode }), shutDown: () => client.close(), diff --git a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts index 13f1e442b..71ab4a531 100644 --- a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts +++ b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts @@ -2,6 +2,7 @@ import { SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service import * as bson from 'bson'; import * as mongo from 'mongodb'; +import { DisposableObserver } from '@powersync/lib-services-framework'; import * as db from '../../db/db-index.js'; import * as util from '../../util/util-index.js'; import { @@ -18,12 +19,12 @@ import { StartBatchOptions, SyncBucketDataBatch, SyncRulesBucketStorage, + SyncRulesBucketStorageListener, SyncRuleStatus, TerminateOptions } from '../BucketStorage.js'; import { ChecksumCache, FetchPartialBucketChecksum, PartialChecksum, PartialChecksumMap } from '../ChecksumCache.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; -import { ReplicationEventManager } from '../ReplicationEventManager.js'; import { SourceTable } from '../SourceTable.js'; import { PowerSyncMongo } from './db.js'; import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './models.js'; @@ -31,7 +32,10 @@ import { MongoBucketBatch } from './MongoBucketBatch.js'; import { MongoCompactor } from './MongoCompactor.js'; import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, mapOpEntry, readSingleBatch, serializeLookup } from './util.js'; -export class MongoSyncBucketStorage implements SyncRulesBucketStorage { +export class MongoSyncBucketStorage + extends DisposableObserver + implements SyncRulesBucketStorage +{ private readonly db: PowerSyncMongo; private checksumCache = new ChecksumCache({ fetchChecksums: (batch) => { @@ -40,7 +44,6 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { }); private parsedSyncRulesCache: SqlSyncRules | undefined; - readonly events: ReplicationEventManager; constructor( public readonly factory: MongoBucketStorage, @@ -48,8 +51,8 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { private readonly sync_rules: PersistedSyncRulesContent, public readonly slot_name: string ) { + super(); this.db = factory.db; - this.events = factory.events; } getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules { @@ -81,27 +84,22 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { ); const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null; - const batch = new MongoBucketBatch( + await using batch = new MongoBucketBatch( this.db, this.sync_rules.parsed(options).sync_rules, this.group_id, this.slot_name, checkpoint_lsn, - doc?.no_checkpoint_before ?? options.zeroLSN, - this.events + doc?.no_checkpoint_before ?? options.zeroLSN ); - try { - await callback(batch); - await batch.flush(); - await batch.abort(); - if (batch.last_flushed_op) { - return { flushed_op: String(batch.last_flushed_op) }; - } else { - return null; - } - } catch (e) { - await batch.abort(); - throw e; + this.iterateListeners((cb) => cb.batchStarted?.(batch)); + + await callback(batch); + await batch.flush(); + if (batch.last_flushed_op) { + return { flushed_op: String(batch.last_flushed_op) }; + } else { + return null; } } diff --git a/packages/service-core/src/storage/storage-index.ts b/packages/service-core/src/storage/storage-index.ts index 50c88c314..076248882 100644 --- a/packages/service-core/src/storage/storage-index.ts +++ b/packages/service-core/src/storage/storage-index.ts @@ -1,6 +1,6 @@ export * from './BucketStorage.js'; export * from './MongoBucketStorage.js'; -export * from './ReplicationEventManager.js'; +export * from './ReplicationEventPayload.js'; export * from './SourceEntity.js'; export * from './SourceTable.js'; export * from './StorageEngine.js'; diff --git a/packages/service-core/test/src/util.ts b/packages/service-core/test/src/util.ts index f1db09442..cd8c06f2c 100644 --- a/packages/service-core/test/src/util.ts +++ b/packages/service-core/test/src/util.ts @@ -7,16 +7,14 @@ import { SyncBucketDataBatch } from '@/storage/BucketStorage.js'; import { MongoBucketStorage } from '@/storage/MongoBucketStorage.js'; -import { ReplicationEventManager } from '@/storage/ReplicationEventManager.js'; import { SourceTable } from '@/storage/SourceTable.js'; import { PowerSyncMongo } from '@/storage/mongo/db.js'; import { SyncBucketData } from '@/util/protocol-types.js'; import { getUuidReplicaIdentityBson, hashData } from '@/util/utils.js'; +import { SqlSyncRules } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import * as mongo from 'mongodb'; import { env } from './env.js'; -import { SqlSyncRules } from '@powersync/service-sync-rules'; -import { ReplicaId } from '@/storage/storage-index.js'; // The metrics need to be initialised before they can be used await Metrics.initialise({ @@ -31,7 +29,7 @@ export type StorageFactory = () => Promise; export const MONGO_STORAGE_FACTORY: StorageFactory = async () => { const db = await connectMongo(); await db.clear(); - return new MongoBucketStorage(db, { slot_name_prefix: 'test_', event_manager: new ReplicationEventManager() }); + return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }); }; export const ZERO_LSN = '0/0'; diff --git a/service/src/runners/server.ts b/service/src/runners/server.ts index c91c507be..28939464b 100644 --- a/service/src/runners/server.ts +++ b/service/src/runners/server.ts @@ -78,6 +78,7 @@ export async function startServer(runnerConfig: core.utils.RunnerConfig) { await moduleManager.initialize(serviceContext); logger.info('Starting service...'); + await serviceContext.lifeCycleEngine.start(); logger.info('Service started.'); From 056de1a8c1c62a3a961032ed0deb9787e675b63e Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 10 Oct 2024 09:21:24 +0200 Subject: [PATCH 31/33] Add functionality for nested disposed listeners --- .../src/utils/DisposableObserver.ts | 21 ++++++- .../test/src/DisposeableObserver.test.ts | 58 +++++++++++++++++++ .../src/storage/MongoBucketStorage.ts | 3 +- 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 libs/lib-services/test/src/DisposeableObserver.test.ts diff --git a/libs/lib-services/src/utils/DisposableObserver.ts b/libs/lib-services/src/utils/DisposableObserver.ts index 4c51630eb..1440d57e7 100644 --- a/libs/lib-services/src/utils/DisposableObserver.ts +++ b/libs/lib-services/src/utils/DisposableObserver.ts @@ -1,15 +1,34 @@ import { BaseObserver, ObserverClient } from './BaseObserver.js'; export interface DisposableListener { + /** + * Event which is fired when the `[Symbol.disposed]` method is called. + */ disposed: () => void; } -export interface DisposableObserverClient extends ObserverClient, Disposable {} +export interface DisposableObserverClient extends ObserverClient, Disposable { + /** + * Registers a listener that is automatically disposed when the parent is disposed. + * This is useful for disposing nested listeners. + */ + registerManagedListener: (parent: DisposableObserverClient, cb: Partial) => () => void; +} export class DisposableObserver extends BaseObserver implements DisposableObserverClient { + registerManagedListener(parent: DisposableObserverClient, cb: Partial) { + const disposer = this.registerListener(cb); + parent.registerListener({ + disposed: () => { + disposer(); + } + }); + return disposer; + } + [Symbol.dispose]() { this.iterateListeners((cb) => cb.disposed?.()); // Delete all callbacks diff --git a/libs/lib-services/test/src/DisposeableObserver.test.ts b/libs/lib-services/test/src/DisposeableObserver.test.ts new file mode 100644 index 000000000..1cde6a58b --- /dev/null +++ b/libs/lib-services/test/src/DisposeableObserver.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, test } from 'vitest'; + +import { DisposableListener, DisposableObserver } from '../../src/utils/DisposableObserver.js'; + +describe('DisposableObserver', () => { + test('it should dispose all listeners on dispose', () => { + const listener = new DisposableObserver(); + + let wasDisposed = false; + listener.registerListener({ + disposed: () => { + wasDisposed = true; + } + }); + + listener[Symbol.dispose](); + + expect(wasDisposed).equals(true); + expect(Object.keys(listener['listeners']).length).equals(0); + }); + + test('it should dispose nested listeners for managed listeners', () => { + interface ParentListener extends DisposableListener { + childCreated: (child: DisposableObserver) => void; + } + class ParentObserver extends DisposableObserver { + createChild() { + const child = new DisposableObserver(); + this.iterateListeners((cb) => cb.childCreated?.(child)); + } + } + + const parent = new ParentObserver(); + let aChild: DisposableObserver | null = null; + + parent.registerListener({ + childCreated: (child) => { + aChild = child; + child.registerManagedListener(parent, { + test: () => { + // this does nothing + } + }); + } + }); + + parent.createChild(); + + // The managed listener should add a `disposed` listener + expect(Object.keys(parent['listeners']).length).equals(2); + expect(Object.keys(aChild!['listeners']).length).equals(1); + + parent[Symbol.dispose](); + expect(Object.keys(parent['listeners']).length).equals(0); + // The listener attached to the child should be disposed when the parent was disposed + expect(Object.keys(aChild!['listeners']).length).equals(0); + }); +}); diff --git a/packages/service-core/src/storage/MongoBucketStorage.ts b/packages/service-core/src/storage/MongoBucketStorage.ts index 1548e461f..c0254da71 100644 --- a/packages/service-core/src/storage/MongoBucketStorage.ts +++ b/packages/service-core/src/storage/MongoBucketStorage.ts @@ -102,7 +102,8 @@ export class MongoBucketStorage this.iterateListeners((cb) => cb.syncStorageCreated?.(storage)); storage.registerListener({ batchStarted: (batch) => { - batch.registerListener({ + // This nested listener will be automatically disposed when the storage is disposed + batch.registerManagedListener(storage, { replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload)) }); } From c2283a15b5cae44a59801af71b1a7e5750501b49 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 10 Oct 2024 09:51:32 +0200 Subject: [PATCH 32/33] Add tests --- .../src/storage/mongo/MongoBucketBatch.ts | 2 +- .../test/src/data_storage.test.ts | 52 +++++++++++++++---- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index ed6ee62cd..30f9121bf 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -549,7 +549,7 @@ export class MongoBucketBatch extends DisposableObserver { diff --git a/packages/service-core/test/src/data_storage.test.ts b/packages/service-core/test/src/data_storage.test.ts index 103ff4b4d..8eceacf40 100644 --- a/packages/service-core/test/src/data_storage.test.ts +++ b/packages/service-core/test/src/data_storage.test.ts @@ -1,10 +1,6 @@ -import { - BucketDataBatchOptions, - ParseSyncRulesOptions, - PersistedSyncRulesContent, - StartBatchOptions -} from '@/storage/BucketStorage.js'; -import { RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules'; +import { BucketDataBatchOptions } from '@/storage/BucketStorage.js'; +import { getUuidReplicaIdentityBson } from '@/util/util-index.js'; +import { RequestParameters } from '@powersync/service-sync-rules'; import { describe, expect, test } from 'vitest'; import { fromAsync, oneFromAsync } from './stream_utils.js'; import { @@ -16,10 +12,8 @@ import { PARSE_OPTIONS, rid, StorageFactory, - testRules, - ZERO_LSN + testRules } from './util.js'; -import { getUuidReplicaIdentityBson } from '@/util/util-index.js'; const TEST_TABLE = makeTestTable('test', ['id']); @@ -1406,4 +1400,42 @@ bucket_definitions: expect(getBatchMeta(batch3)).toEqual(null); }); + + test('batch should be disposed automatically', async () => { + const sync_rules = testRules(` + bucket_definitions: + global: + data: [] + `); + + const storage = (await factory()).getInstance(sync_rules); + + let isDisposed = false; + await storage.startBatch(BATCH_OPTIONS, async (batch) => { + batch.registerListener({ + disposed: () => { + isDisposed = true; + } + }); + }); + expect(isDisposed).true; + + isDisposed = false; + let errorCaught = false; + try { + await storage.startBatch(BATCH_OPTIONS, async (batch) => { + batch.registerListener({ + disposed: () => { + isDisposed = true; + } + }); + throw new Error(`Testing exceptions`); + }); + } catch (ex) { + errorCaught = true; + expect(ex.message.includes('Testing')).true; + } + expect(errorCaught).true; + expect(isDisposed).true; + }); } From d2524a0fe0d58bad72aeaf62ccd4a09a6d8de54f Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 10 Oct 2024 15:20:22 +0200 Subject: [PATCH 33/33] added changesets --- .changeset/orange-eagles-tap.md | 5 +++++ .changeset/popular-snails-cough.md | 6 ++++++ .changeset/sour-turkeys-collect.md | 7 +++++++ 3 files changed, 18 insertions(+) create mode 100644 .changeset/orange-eagles-tap.md create mode 100644 .changeset/popular-snails-cough.md create mode 100644 .changeset/sour-turkeys-collect.md diff --git a/.changeset/orange-eagles-tap.md b/.changeset/orange-eagles-tap.md new file mode 100644 index 000000000..7a21eec33 --- /dev/null +++ b/.changeset/orange-eagles-tap.md @@ -0,0 +1,5 @@ +--- +'@powersync/lib-services-framework': minor +--- + +Added disposable listeners and observers diff --git a/.changeset/popular-snails-cough.md b/.changeset/popular-snails-cough.md new file mode 100644 index 000000000..5dc9e2d4d --- /dev/null +++ b/.changeset/popular-snails-cough.md @@ -0,0 +1,6 @@ +--- +'@powersync/service-core': minor +'@powersync/service-sync-rules': minor +--- + +Added ability to emit data replication events diff --git a/.changeset/sour-turkeys-collect.md b/.changeset/sour-turkeys-collect.md new file mode 100644 index 000000000..d9bc279fb --- /dev/null +++ b/.changeset/sour-turkeys-collect.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-module-postgres': patch +'@powersync/service-rsocket-router': patch +'@powersync/service-types': patch +--- + +Updates from Replication events changes