From a82e02e82211f42a26d4ee330d8078194e6eb730 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 8 Jan 2025 13:48:47 +0200 Subject: [PATCH 01/50] postgres bucket storage initial --- .env.template | 3 +- .prettierrc | 6 +- modules/module-postgres-storage/CHANGELOG.md | 1 + modules/module-postgres-storage/LICENSE | 67 ++ modules/module-postgres-storage/README.md | 84 ++ modules/module-postgres-storage/package.json | 47 + modules/module-postgres-storage/src/index.ts | 6 + .../src/locks/PostgresLockManager.ts | 127 +++ .../src/migrations/PostgresMigrationAgent.ts | 41 + .../src/migrations/PostgresMigrationStore.ts | 69 ++ .../migrations/scripts/1684951997326-init.ts | 143 +++ .../src/module/PostgresStorageModule.ts | 27 + .../storage/PostgresBucketStorageFactory.ts | 513 ++++++++++ .../src/storage/PostgresCompactor.ts | 365 ++++++++ .../src/storage/PostgresStorageProvider.ts | 42 + .../src/storage/PostgresSyncRulesStorage.ts | 642 +++++++++++++ .../src/storage/batch/OperationBatch.ts | 129 +++ .../src/storage/batch/PostgresBucketBatch.ts | 883 ++++++++++++++++++ .../storage/batch/PostgresPersistedBatch.ts | 438 +++++++++ .../checkpoints/PostgresWriteCheckpointAPI.ts | 179 ++++ .../PostgresPersistedSyncRulesContent.ts | 67 ++ .../src/types/codecs.ts | 32 + .../src/types/models/ActiveCheckpoint.ts | 15 + .../models/ActiveCheckpointNotification.ts | 14 + .../src/types/models/BucketData.ts | 27 + .../src/types/models/BucketParameters.ts | 16 + .../src/types/models/CurrentData.ts | 24 + .../src/types/models/Instance.ts | 8 + .../src/types/models/SQLiteJSONValue.ts | 10 + .../src/types/models/SourceTable.ts | 28 + .../src/types/models/SyncRules.ts | 50 + .../src/types/models/WriteCheckpoint.ts | 20 + .../src/types/models/models-index.ts | 10 + .../src/types/types.ts | 69 ++ .../module-postgres-storage/src/utils/bson.ts | 17 + .../src/utils/bucket-data.ts | 25 + .../connection/AbstractPostgresConnection.ts | 108 +++ .../src/utils/connection/ConnectionSlot.ts | 144 +++ .../src/utils/connection/DatabaseClient.ts | 193 ++++ .../src/utils/connection/WrappedConnection.ts | 11 + .../module-postgres-storage/src/utils/db.ts | 18 + .../src/utils/ts-codec.ts | 14 + .../__snapshots__/storage_sync.test.ts.snap | 332 +++++++ .../module-postgres-storage/test/src/env.ts | 6 + .../test/src/migrations.test.ts | 24 + .../module-postgres-storage/test/src/setup.ts | 16 + .../test/src/storage.test.ts | 131 +++ .../test/src/storage_compacting.test.ts | 5 + .../test/src/storage_sync.test.ts | 12 + .../module-postgres-storage/test/src/util.ts | 63 ++ .../test/tsconfig.json | 23 + modules/module-postgres-storage/tsconfig.json | 14 + .../module-postgres-storage/vitest.config.ts | 13 + package.json | 4 +- pnpm-lock.yaml | 177 +++- service/Dockerfile | 2 + service/package.json | 1 + service/src/entry.ts | 9 +- tsconfig.json | 3 + 59 files changed, 5556 insertions(+), 11 deletions(-) create mode 100644 modules/module-postgres-storage/CHANGELOG.md create mode 100644 modules/module-postgres-storage/LICENSE create mode 100644 modules/module-postgres-storage/README.md create mode 100644 modules/module-postgres-storage/package.json create mode 100644 modules/module-postgres-storage/src/index.ts create mode 100644 modules/module-postgres-storage/src/locks/PostgresLockManager.ts create mode 100644 modules/module-postgres-storage/src/migrations/PostgresMigrationAgent.ts create mode 100644 modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts create mode 100644 modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts create mode 100644 modules/module-postgres-storage/src/module/PostgresStorageModule.ts create mode 100644 modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts create mode 100644 modules/module-postgres-storage/src/storage/PostgresCompactor.ts create mode 100644 modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts create mode 100644 modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts create mode 100644 modules/module-postgres-storage/src/storage/batch/OperationBatch.ts create mode 100644 modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts create mode 100644 modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts create mode 100644 modules/module-postgres-storage/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts create mode 100644 modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts create mode 100644 modules/module-postgres-storage/src/types/codecs.ts create mode 100644 modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts create mode 100644 modules/module-postgres-storage/src/types/models/ActiveCheckpointNotification.ts create mode 100644 modules/module-postgres-storage/src/types/models/BucketData.ts create mode 100644 modules/module-postgres-storage/src/types/models/BucketParameters.ts create mode 100644 modules/module-postgres-storage/src/types/models/CurrentData.ts create mode 100644 modules/module-postgres-storage/src/types/models/Instance.ts create mode 100644 modules/module-postgres-storage/src/types/models/SQLiteJSONValue.ts create mode 100644 modules/module-postgres-storage/src/types/models/SourceTable.ts create mode 100644 modules/module-postgres-storage/src/types/models/SyncRules.ts create mode 100644 modules/module-postgres-storage/src/types/models/WriteCheckpoint.ts create mode 100644 modules/module-postgres-storage/src/types/models/models-index.ts create mode 100644 modules/module-postgres-storage/src/types/types.ts create mode 100644 modules/module-postgres-storage/src/utils/bson.ts create mode 100644 modules/module-postgres-storage/src/utils/bucket-data.ts create mode 100644 modules/module-postgres-storage/src/utils/connection/AbstractPostgresConnection.ts create mode 100644 modules/module-postgres-storage/src/utils/connection/ConnectionSlot.ts create mode 100644 modules/module-postgres-storage/src/utils/connection/DatabaseClient.ts create mode 100644 modules/module-postgres-storage/src/utils/connection/WrappedConnection.ts create mode 100644 modules/module-postgres-storage/src/utils/db.ts create mode 100644 modules/module-postgres-storage/src/utils/ts-codec.ts create mode 100644 modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap create mode 100644 modules/module-postgres-storage/test/src/env.ts create mode 100644 modules/module-postgres-storage/test/src/migrations.test.ts create mode 100644 modules/module-postgres-storage/test/src/setup.ts create mode 100644 modules/module-postgres-storage/test/src/storage.test.ts create mode 100644 modules/module-postgres-storage/test/src/storage_compacting.test.ts create mode 100644 modules/module-postgres-storage/test/src/storage_sync.test.ts create mode 100644 modules/module-postgres-storage/test/src/util.ts create mode 100644 modules/module-postgres-storage/test/tsconfig.json create mode 100644 modules/module-postgres-storage/tsconfig.json create mode 100644 modules/module-postgres-storage/vitest.config.ts diff --git a/.env.template b/.env.template index a3c5fd5d0..3082d8ec7 100644 --- a/.env.template +++ b/.env.template @@ -1,3 +1,4 @@ # Connections for tests MONGO_TEST_URL="mongodb://localhost:27017/powersync_test" -PG_TEST_URL="postgres://postgres:postgres@localhost:5432/powersync_test" \ No newline at end of file +PG_TEST_URL="postgres://postgres:postgres@localhost:5432/powersync_test" +PG_STORAGE_TEST_URL="postgres://postgres:postgres@localhost:5431/powersync_storage_test" \ No newline at end of file diff --git a/.prettierrc b/.prettierrc index dd651a512..2b98ddbe1 100644 --- a/.prettierrc +++ b/.prettierrc @@ -4,5 +4,9 @@ "tabWidth": 2, "useTabs": false, "printWidth": 120, - "trailingComma": "none" + "trailingComma": "none", + "plugins": ["prettier-plugin-embed", "prettier-plugin-sql"], + "embeddedSqlTags": ["sql", "db.sql", "this.db.sql"], + "language": "postgresql", + "keywordCase": "upper" } diff --git a/modules/module-postgres-storage/CHANGELOG.md b/modules/module-postgres-storage/CHANGELOG.md new file mode 100644 index 000000000..def897232 --- /dev/null +++ b/modules/module-postgres-storage/CHANGELOG.md @@ -0,0 +1 @@ +# @powersync/service-module-postgres-storage diff --git a/modules/module-postgres-storage/LICENSE b/modules/module-postgres-storage/LICENSE new file mode 100644 index 000000000..c8efd46cc --- /dev/null +++ b/modules/module-postgres-storage/LICENSE @@ -0,0 +1,67 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2023-2024 Journey Mobile, Inc. + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under these Terms and Conditions, as indicated by our inclusion of these Terms and Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, Redistribution and Trademark clauses below, we hereby grant you the right to use, copy, modify, create derivative works, publicly perform, publicly display and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use means making the Software available to others in a commercial product or service that: + +1. substitutes for the Software; +2. substitutes for any other product or service we offer using the Software that exists as of the date we make the Software available; or +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; +2. for non-commercial education; +3. for non-commercial research; and +4. in connection with professional services that you provide to a licensee using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our patents, the license grant above includes a license under our patents. If you make a claim against any party that the Software infringes or contributes to the infringement of any patent, then your patent license to the Software ends immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of the Software. +If you redistribute any copies, modifications or derivatives of the Software, you must include a copy of or a link to these Terms and Conditions and not remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of the Software, you have no right under these Terms and Conditions to use our trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under the Apache License, Version 2.0 that is effective on the second anniversary of the date we make the Software available. On or after that date, you may use the Software under the Apache License, Version 2.0, in which case the following will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/modules/module-postgres-storage/README.md b/modules/module-postgres-storage/README.md new file mode 100644 index 000000000..566a55500 --- /dev/null +++ b/modules/module-postgres-storage/README.md @@ -0,0 +1,84 @@ +# Postgres Sync Bucket Storage + +This module provides a `BucketStorageProvider` which uses a Postgres database for persistence. + +## Configuration + +Postgres storage can be enabled by selecting the appropriate storage `type` and providing connection details for a Postgres server. + +The storage connection configuration extends the configuration for a Postgres replication source, thus it accepts and supports the same configurations fields. + +A sample YAML configuration could look like + +```yaml +replication: + # Specify database connection details + # Note only 1 connection is currently supported + # Multiple connection support is on the roadmap + connections: + - type: postgresql + uri: !env PS_DATA_SOURCE_URI + +# Connection settings for sync bucket storage +storage: + type: postgresql + # This accepts the same parameters as a Postgres replication source connection + uri: !env PS_STORAGE_SOURCE_URI +``` + +A separate Postgres server is currently required for replication connections (if using Postgres for replication) and storage. Using the same server might cause unexpected results. + +### Connection credentials + +The Postgres bucket storage implementation requires write access to the provided Postgres database. The module will create a `powersync` schema in the provided database which will contain all the tables and data used for bucket storage. Ensure that the provided credentials specified in the `uri` or `username`, `password` configuration fields has the appropriate write access. + +A sample user could be created with: + +```sql +-- Create the user with a password +CREATE USER powersync_storage_user +WITH + PASSWORD 'secure_password'; + +-- Optionally create a PowerSync schema and make the user its owner +CREATE SCHEMA IF NOT EXISTS powersync AUTHORIZATION powersync_storage_user; + +-- OR: Allow PowerSync to create schemas in the database +GRANT CREATE ON DATABASE example_database TO powersync_storage_user; + +-- Set default privileges for objects created by powersync_storage_user in the database +-- (Ensures the user gets full access to tables they create in any schema) +ALTER DEFAULT PRIVILEGES FOR ROLE powersync_storage_user +GRANT ALL PRIVILEGES ON TABLES TO powersync_storage_user; + +-- [if the schema was pre-created] Grant usage and privileges on the powersync schema +GRANT USAGE ON SCHEMA powersync TO powersync_storage_user; + +-- [if the schema was pre-created] +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA powersync TO powersync_storage_user; +``` + +### Batching + +Replication data is persisted via batch operations. Batching ensures performant, memory optimized writes. Batches are limited in size. Increasing batch size limits can reduce the amount of server round-trips which increases performance, but will result in higher memory usage and potential server issues. + +Batch size limits are defaulted and can optionally be configured in the configuration. + +```yaml +# Connection settings for sync bucket storage +storage: + type: postgresql + # This accepts the same parameters as a Postgres replication source connection + uri: !env PS_STORAGE_SOURCE_URI + batch_limits: + # Maximum estimated byte size of operations in a single batch. + # Defaults to 5 megabytes. + max_estimated_size: 5000000 + # Maximum number of records present in a single batch. + # Defaults to 2000 records. + # Increasing this limit can improve replication times for large volumes of data. + max_record_count: 2000 + # Maximum byte size of size of current_data documents we lookup at a time. + # Defaults to 50 megabytes. + max_current_data_batch_size: 50000000 +``` diff --git a/modules/module-postgres-storage/package.json b/modules/module-postgres-storage/package.json new file mode 100644 index 000000000..134b4627d --- /dev/null +++ b/modules/module-postgres-storage/package.json @@ -0,0 +1,47 @@ +{ + "name": "@powersync/service-module-postgres-storage", + "repository": "https://github.com/powersync-ja/powersync-service", + "types": "dist/@types/index.d.ts", + "version": "0.0.1", + "main": "dist/index.js", + "type": "module", + "scripts": { + "build": "tsc -b", + "build:tests": "tsc -b test/tsconfig.json", + "clean": "rm -rf ./lib && tsc -b --clean", + "test": "vitest" + }, + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "default": "./dist/index.js", + "types": "./dist/@types/index.d.ts" + }, + "./types": { + "import": "./dist/types/types.js", + "require": "./dist/types/types.js", + "default": "./dist/types/types.js", + "types": "./dist/@types/index.d.ts" + } + }, + "dependencies": { + "@powersync/lib-services-framework": "workspace:*", + "@powersync/service-core": "workspace:*", + "@powersync/service-core-tests": "workspace:*", + "@powersync/service-jpgwire": "workspace:*", + "@powersync/service-jsonbig": "^0.17.10", + "@powersync/service-module-postgres": "workspace:*", + "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", + "ix": "^5.0.0", + "lru-cache": "^10.2.2", + "p-defer": "^4.0.1", + "ts-codec": "^1.3.0", + "uuid": "^9.0.1" + }, + "devDependencies": { + "@types/uuid": "^9.0.4", + "typescript": "^5.2.2" + } +} diff --git a/modules/module-postgres-storage/src/index.ts b/modules/module-postgres-storage/src/index.ts new file mode 100644 index 000000000..c89905886 --- /dev/null +++ b/modules/module-postgres-storage/src/index.ts @@ -0,0 +1,6 @@ +export * from './module/PostgresStorageModule.js'; +export * from './storage/PostgresBucketStorageFactory.js'; + +export * from './migrations/PostgresMigrationAgent.js'; + +export * from './types/types.js'; diff --git a/modules/module-postgres-storage/src/locks/PostgresLockManager.ts b/modules/module-postgres-storage/src/locks/PostgresLockManager.ts new file mode 100644 index 000000000..b8aa23d51 --- /dev/null +++ b/modules/module-postgres-storage/src/locks/PostgresLockManager.ts @@ -0,0 +1,127 @@ +import { framework } from '@powersync/service-core'; +import { v4 as uuidv4 } from 'uuid'; +import { sql } from '../utils/connection/AbstractPostgresConnection.js'; +import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; + +const DEFAULT_LOCK_TIMEOUT = 60_000; // 1 minute + +export interface PostgresLockManagerParams extends framework.locks.LockManagerParams { + db: DatabaseClient; +} + +export class PostgresLockManager extends framework.locks.AbstractLockManager { + constructor(protected params: PostgresLockManagerParams) { + super(params); + } + + protected get db() { + return this.params.db; + } + + get timeout() { + return this.params.timeout ?? DEFAULT_LOCK_TIMEOUT; + } + + get name() { + return this.params.name; + } + + async init() { + /** + * Locks are required for migrations, which means this table can't be + * created inside a migration. This ensures the locks table is present. + */ + await this.db.query(sql` + CREATE TABLE IF NOT EXISTS locks ( + name TEXT PRIMARY KEY, + lock_id UUID NOT NULL, + ts TIMESTAMPTZ NOT NULL + ); + `); + } + + protected async acquireHandle(): Promise { + const id = await this._acquireId(); + if (!id) { + return null; + } + return { + refresh: () => this.refreshHandle(id), + release: () => this.releaseHandle(id) + }; + } + + protected async _acquireId(): Promise { + const now = new Date(); + const expiredTs = new Date(now.getTime() - this.timeout); + const lockId = uuidv4(); + + try { + // Attempt to acquire or refresh the lock + const res = await this.db.query(sql` + INSERT INTO + locks (name, lock_id, ts) + VALUES + ( + ${{ type: 'varchar', value: this.name }}, + ${{ type: 'uuid', value: lockId }}, + ${{ type: 1184, value: now.toISOString() }} + ) + ON CONFLICT (name) DO UPDATE + SET + lock_id = CASE + WHEN locks.ts <= $4 THEN $2 + ELSE locks.lock_id + END, + ts = CASE + WHEN locks.ts <= $4 THEN $3 + ELSE locks.ts + END + WHERE + locks.ts <= ${{ type: 1184, value: expiredTs.toISOString() }} + RETURNING + lock_id; + `); + + if (res.rows.length === 0) { + // Lock is active and could not be acquired + return null; + } + + return lockId; + } catch (err) { + console.error('Error acquiring lock:', err); + throw err; + } + } + + protected async refreshHandle(lockId: string) { + const res = await this.db.query(sql` + UPDATE locks + SET + ts = ${{ type: 1184, value: new Date().toISOString() }} + WHERE + lock_id = ${{ type: 'uuid', value: lockId }} + RETURNING + lock_id; + `); + + if (res.rows.length === 0) { + throw new Error('Lock not found, could not refresh'); + } + } + + protected async releaseHandle(lockId: string) { + const res = await this.db.query(sql` + DELETE FROM locks + WHERE + lock_id = ${{ type: 'uuid', value: lockId }} + RETURNING + lock_id; + `); + + if (res.rows.length == 0) { + throw new Error('Lock not found, could not release'); + } + } +} diff --git a/modules/module-postgres-storage/src/migrations/PostgresMigrationAgent.ts b/modules/module-postgres-storage/src/migrations/PostgresMigrationAgent.ts new file mode 100644 index 000000000..07ed3a94a --- /dev/null +++ b/modules/module-postgres-storage/src/migrations/PostgresMigrationAgent.ts @@ -0,0 +1,41 @@ +import * as framework from '@powersync/lib-services-framework'; +import { migrations } from '@powersync/service-core'; +import * as pg_types from '@powersync/service-module-postgres/types'; +import * as path from 'path'; +import { fileURLToPath } from 'url'; +import { PostgresLockManager } from '../locks/PostgresLockManager.js'; +import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; +import { PostgresMigrationStore } from './PostgresMigrationStore.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const MIGRATIONS_DIR = path.join(__dirname, 'scripts'); + +export class PostgresMigrationAgent extends migrations.AbstractPowerSyncMigrationAgent { + store: framework.MigrationStore; + locks: framework.LockManager; + + protected db: DatabaseClient; + + constructor(config: pg_types.PostgresConnectionConfig) { + super(); + + this.db = new DatabaseClient(pg_types.normalizeConnectionConfig(config)); + this.store = new PostgresMigrationStore({ + db: this.db + }); + this.locks = new PostgresLockManager({ + name: 'migrations', + db: this.db + }); + } + + getInternalScriptsDir(): string { + return MIGRATIONS_DIR; + } + + async [Symbol.asyncDispose](): Promise { + await this.db[Symbol.asyncDispose](); + } +} diff --git a/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts b/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts new file mode 100644 index 000000000..6923d6a5d --- /dev/null +++ b/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts @@ -0,0 +1,69 @@ +import { migrations } from '@powersync/lib-services-framework'; +import { sql } from '../utils/connection/AbstractPostgresConnection.js'; +import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; + +export type PostgresMigrationStoreOptions = { + db: DatabaseClient; +}; + +export class PostgresMigrationStore implements migrations.MigrationStore { + constructor(protected options: PostgresMigrationStoreOptions) {} + + protected get db() { + return this.options.db; + } + + async init() { + await this.db.query(sql` + CREATE TABLE IF NOT EXISTS migrations ( + id SERIAL PRIMARY KEY, + last_run TEXT, + LOG JSONB NOT NULL + ); + `); + } + + async clear() { + await this.db.query(sql`DELETE FROM migrations;`); + } + + async load(): Promise { + const res = await this.db.queryRows<{ last_run: string; log: string }>(sql` + SELECT + last_run, + LOG + FROM + migrations + LIMIT + 1 + `); + + if (res.length === 0) { + return; + } + + const { last_run, log } = res[0]; + + return { + last_run: last_run, + log: log ? JSON.parse(log) : [] + }; + } + + async save(state: migrations.MigrationState): Promise { + await this.db.query(sql` + INSERT INTO + migrations (last_run, LOG) + VALUES + ( + ${{ type: 'varchar', value: state.last_run }}, + ${{ type: 'jsonb', value: state.log }} + ) + ON CONFLICT (id) DO + UPDATE + SET + last_run = EXCLUDED.last_run, + LOG = EXCLUDED.log; + `); + } +} diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts new file mode 100644 index 000000000..3e10324d9 --- /dev/null +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -0,0 +1,143 @@ +import { migrations } from '@powersync/service-core'; +import { PostgresConnectionConfig, normalizeConnectionConfig } from '@powersync/service-module-postgres/types'; +import { DatabaseClient } from '../../utils/connection/DatabaseClient.js'; +import { dropTables } from '../../utils/db.js'; + +export const up: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + await using client = new DatabaseClient(normalizeConnectionConfig(configuration.storage as PostgresConnectionConfig)); + + /** + * Request an explicit connection which will automatically set the search + * path to the powersync schema + */ + await client.lockConnection(async (db) => { + await db.sql` + CREATE SEQUENCE op_id_sequence AS int8 START + WITH + 1 + `.execute(); + + await db.sql` + CREATE SEQUENCE sync_rules_id_sequence AS int START + WITH + 1 + `.execute(); + + await db.sql` + CREATE TABLE bucket_data ( + group_id integer NOT NULL, + bucket_name TEXT NOT NULL, + op_id bigint NOT NULL, + CONSTRAINT unique_id UNIQUE (group_id, bucket_name, op_id), + op text NOT NULL, + source_table TEXT, + source_key bytea, + table_name TEXT, + row_id TEXT, + checksum bigint NOT NULL, + data TEXT, + target_op bigint + ) + `.execute(); + + await db.sql`CREATE TABLE instance (id TEXT PRIMARY KEY) `.execute(); + + await db.sql` + CREATE TABLE sync_rules ( + id BIGSERIAL PRIMARY KEY, + state TEXT NOT NULL, + snapshot_done BOOLEAN NOT NULL DEFAULT FALSE, + last_checkpoint BIGINT, + last_checkpoint_lsn TEXT, + no_checkpoint_before TEXT, + slot_name TEXT, + last_checkpoint_ts TIMESTAMP WITH TIME ZONE, + last_keepalive_ts TIMESTAMP WITH TIME ZONE, + keepalive_op TEXT, + last_fatal_error TEXT, + content TEXT NOT NULL + ); + `.execute(); + + await db.sql` + CREATE TABLE bucket_parameters ( + id BIGINT DEFAULT nextval('op_id_sequence') PRIMARY KEY, + group_id integer NOT NULL, + source_table TEXT NOT NULL, + source_key bytea NOT NULL, + lookup bytea NOT NULL, + bucket_parameters jsonb NOT NULL + ); + `.execute(); + + await db.sql` + CREATE INDEX bucket_parameters_lookup_index ON bucket_parameters (group_id ASC, lookup ASC, id DESC) + `.execute(); + + await db.sql` + CREATE INDEX bucket_parameters_source_index ON bucket_parameters (group_id, source_table, source_key) + `.execute(); + + await db.sql` + CREATE TABLE current_data ( + group_id integer NOT NULL, + source_table TEXT NOT NULL, + source_key bytea NOT NULL, + CONSTRAINT unique_current_data_id UNIQUE (group_id, source_table, source_key), + buckets jsonb NOT NULL, + data bytea NOT NULL, + lookups bytea[] NOT NULL + ); + `.execute(); + + await db.sql`CREATE INDEX current_data_lookup ON current_data (group_id, source_table, source_key)`.execute(); + + await db.sql` + CREATE TABLE source_tables ( + --- This is currently a TEXT column to make the (shared) tests easier to integrate + --- we could improve this if necessary + id TEXT PRIMARY KEY, + group_id integer NOT NULL, + connection_id integer NOT NULL, + relation_id integer, + schema_name text NOT NULL, + table_name text NOT NULL, + replica_id_columns jsonb, + snapshot_done BOOLEAN NOT NULL DEFAULT FALSE + ) + `.execute(); + + await db.sql`CREATE INDEX source_table_lookup ON source_tables (group_id, table_name)`.execute(); + + await db.sql` + CREATE TABLE write_checkpoints ( + user_id text PRIMARY KEY, + lsns jsonb NOT NULL, + write_checkpoint BIGINT NOT NULL + ) + `.execute(); + + await db.sql`CREATE INDEX write_checkpoint_by_user ON write_checkpoints (user_id)`.execute(); + + await db.sql` + CREATE TABLE custom_write_checkpoints ( + user_id text NOT NULL, + write_checkpoint BIGINT NOT NULL, + sync_rules_id integer NOT NULL, + CONSTRAINT unique_user_sync UNIQUE (user_id, sync_rules_id) + ); + `.execute(); + }); +}; + +export const down: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + await using db = new DatabaseClient(normalizeConnectionConfig(configuration.storage as PostgresConnectionConfig)); + + await dropTables(db); +}; diff --git a/modules/module-postgres-storage/src/module/PostgresStorageModule.ts b/modules/module-postgres-storage/src/module/PostgresStorageModule.ts new file mode 100644 index 000000000..2dc4b18db --- /dev/null +++ b/modules/module-postgres-storage/src/module/PostgresStorageModule.ts @@ -0,0 +1,27 @@ +import { modules, system } from '@powersync/service-core'; +import * as pg_types from '@powersync/service-module-postgres/types'; +import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'; +import { PostgresStorageProvider } from '../storage/PostgresStorageProvider.js'; + +export class PostgresStorageModule extends modules.AbstractModule { + constructor() { + super({ + name: 'Postgres Bucket Storage' + }); + } + + async initialize(context: system.ServiceContextContainer): Promise { + const { storageEngine } = context; + + // Register the ability to use Postgres as a BucketStorage + storageEngine.registerProvider(new PostgresStorageProvider()); + + if (pg_types.isPostgresConfig(context.configuration.storage)) { + context.migrations.registerMigrationAgent(new PostgresMigrationAgent(context.configuration.storage)); + } + } + + async teardown(): Promise { + // Teardown for this module is implemented in the storage engine + } +} diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts new file mode 100644 index 000000000..d34cff7ff --- /dev/null +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -0,0 +1,513 @@ +import * as framework from '@powersync/lib-services-framework'; +import { storage, sync, utils } from '@powersync/service-core'; +import * as pg_wire from '@powersync/service-jpgwire'; +import * as sync_rules from '@powersync/service-sync-rules'; +import crypto from 'crypto'; +import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js'; +import { LRUCache } from 'lru-cache/min'; +import * as timers from 'timers/promises'; +import * as uuid from 'uuid'; +import { PostgresLockManager } from '../locks/PostgresLockManager.js'; +import { models, NormalizedPostgresStorageConfig } from '../types/types.js'; +import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; +import { notifySyncRulesUpdate } from './batch/PostgresBucketBatch.js'; +import { PostgresSyncRulesStorage } from './PostgresSyncRulesStorage.js'; +import { PostgresPersistedSyncRulesContent } from './sync-rules/PostgresPersistedSyncRulesContent.js'; + +export type PostgresBucketStorageOptions = { + config: NormalizedPostgresStorageConfig; + slot_name_prefix: string; +}; + +export class PostgresBucketStorageFactory + extends framework.DisposableObserver + implements storage.BucketStorageFactory +{ + readonly db: DatabaseClient; + public readonly slot_name_prefix: string; + + protected notificationConnection: pg_wire.PgConnection | null; + private sharedIterator = new sync.BroadcastIterable((signal) => this.watchActiveCheckpoint(signal)); + + // TODO we might be able to share this + private readonly storageCache = new LRUCache({ + max: 3, + fetchMethod: async (id) => { + const syncRulesRow = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + id = ${{ value: id, type: 'int4' }} + ` + .decoded(models.SyncRules) + .first(); + if (syncRulesRow == null) { + // Deleted in the meantime? + return undefined; + } + const rules = new PostgresPersistedSyncRulesContent(this.db, syncRulesRow); + return this.getInstance(rules); + }, + dispose: (storage) => { + storage[Symbol.dispose](); + } + }); + + constructor(protected options: PostgresBucketStorageOptions) { + super(); + this.db = new DatabaseClient(options.config); + this.slot_name_prefix = options.slot_name_prefix; + + this.notificationConnection = null; + } + + async [Symbol.dispose]() { + await this.notificationConnection?.end(); + await this.db[Symbol.asyncDispose](); + } + + getInstance(syncRules: storage.PersistedSyncRulesContent): storage.SyncRulesBucketStorage { + const storage = new PostgresSyncRulesStorage({ + factory: this, + db: this.db, + sync_rules: syncRules, + batchLimits: this.options.config.batch_limits + }); + this.iterateListeners((cb) => cb.syncStorageCreated?.(storage)); + storage.registerListener({ + batchStarted: (batch) => { + // This nested listener will be automatically disposed when the storage is disposed + batch.registerManagedListener(storage, { + replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload)) + }); + } + }); + return storage; + } + + async getStorageMetrics(): Promise { + const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' }); + if (active_sync_rules == null) { + return { + operations_size_bytes: 0, + parameters_size_bytes: 0, + replication_size_bytes: 0 + }; + } + + const bucketData = await this.db.sql` + SELECT + --- This can differ from the octet_length + sum(pg_column_size(data)) AS operations_size_bytes + FROM + bucket_data + WHERE + group_id = ${{ type: 'int8', value: active_sync_rules.id }} + `.first<{ operations_size_bytes: bigint }>(); + + const parameterData = await this.db.sql` + SELECT + --- This can differ from the octet_length + ( + sum(pg_column_size(bucket_parameters)) + sum(pg_column_size(lookup)) + sum(pg_column_size(source_key)) + ) AS parameter_size_bytes + FROM + bucket_parameters + WHERE + group_id = ${{ type: 'int8', value: active_sync_rules.id }} + `.first<{ parameter_size_bytes: bigint }>(); + + const currentData = await this.db.sql` + SELECT + --- This can differ from the octet_length + ( + sum(pg_column_size(data)) + sum(pg_column_size(lookups)) + sum(pg_column_size(source_key)) + sum(pg_column_size(buckets)) + ) AS current_size_bytes + FROM + current_data + WHERE + group_id = ${{ type: 'int8', value: active_sync_rules.id }} + `.first<{ current_size_bytes: bigint }>(); + + return { + operations_size_bytes: Number(bucketData!.operations_size_bytes), + parameters_size_bytes: Number(parameterData!.parameter_size_bytes), + replication_size_bytes: Number(currentData!.current_size_bytes) + }; + } + + async getPowerSyncInstanceId(): Promise { + const instanceRow = await this.db.sql` + SELECT + id + FROM + instance + ` + .decoded(models.Instance) + .first(); + if (instanceRow) { + return instanceRow.id; + } + const lockManager = new PostgresLockManager({ + db: this.db, + name: `instance-id-insertion-lock` + }); + await lockManager.lock(async () => { + await this.db.sql` + INSERT INTO + instance (id) + VALUES + (${{ type: 'varchar', value: uuid.v4() }}) + `.execute(); + }); + const newInstanceRow = await this.db.sql` + SELECT + id + FROM + instance + ` + .decoded(models.Instance) + .first(); + return newInstanceRow!.id; + } + + // TODO possibly share implementation in abstract class + async configureSyncRules( + sync_rules: string, + options?: { lock?: boolean } + ): Promise<{ + updated: boolean; + persisted_sync_rules?: storage.PersistedSyncRulesContent; + lock?: storage.ReplicationLock; + }> { + const next = await this.getNextSyncRulesContent(); + const active = await this.getActiveSyncRulesContent(); + + if (next?.sync_rules_content == sync_rules) { + framework.logger.info('Sync rules from configuration unchanged'); + return { updated: false }; + } else if (next == null && active?.sync_rules_content == sync_rules) { + framework.logger.info('Sync rules from configuration unchanged'); + return { updated: false }; + } else { + framework.logger.info('Sync rules updated from configuration'); + const persisted_sync_rules = await this.updateSyncRules({ + content: sync_rules, + lock: options?.lock + }); + return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined }; + } + } + + async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise { + // TODO some shared implementation for this might be nice + // Parse and validate before applying any changes + sync_rules.SqlSyncRules.fromYaml(options.content, { + // No schema-based validation at this point + schema: undefined, + defaultSchema: 'not_applicable', // Not needed for validation + throwOnError: true + }); + + return this.db.transaction(async (db) => { + await db.sql` + UPDATE sync_rules + SET + state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }} + WHERE + state = ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }} + `.execute(); + + const newSyncRulesRow = await db.sql` + WITH + next_id AS ( + SELECT + nextval('sync_rules_id_sequence') AS id + ) + INSERT INTO + sync_rules (id, content, state, slot_name) + VALUES + ( + ( + SELECT + id + FROM + next_id + ), + ${{ type: 'varchar', value: options.content }}, + ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }}, + CONCAT( + ${{ type: 'varchar', value: this.slot_name_prefix }}, + ( + SELECT + id + FROM + next_id + ), + '_', + ${{ type: 'varchar', value: crypto.randomBytes(2).toString('hex') }} + ) + ) + RETURNING + * + ` + .decoded(models.SyncRules) + .first(); + + await notifySyncRulesUpdate(this.db, newSyncRulesRow!); + + return new PostgresPersistedSyncRulesContent(this.db, newSyncRulesRow!); + }); + } + + async slotRemoved(slot_name: string): Promise { + const next = await this.getNextSyncRulesContent(); + const active = await this.getActiveSyncRulesContent(); + + // In both the below cases, we create a new sync rules instance. + // The current one will continue erroring until the next one has finished processing. + // TODO: Update + if (next != null && next.slot_name == slot_name) { + // We need to redo the "next" sync rules + await this.updateSyncRules({ + content: next.sync_rules_content + }); + // Pro-actively stop replicating + await this.db.sql` + UPDATE sync_rules + SET + state = ${{ value: storage.SyncRuleState.STOP, type: 'varchar' }} + WHERE + id = ${{ value: next.id, type: 'int4' }} + AND state = ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }} + `.execute(); + } else if (next == null && active?.slot_name == slot_name) { + // Slot removed for "active" sync rules, while there is no "next" one. + await this.updateSyncRules({ + content: active.sync_rules_content + }); + + // Pro-actively stop replicating + await this.db.sql` + UPDATE sync_rules + SET + state = ${{ value: storage.SyncRuleState.STOP, type: 'varchar' }} + WHERE + id = ${{ value: active.id, type: 'int4' }} + AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }} + `.execute(); + } + } + + // TODO possibly share via abstract class + async getActiveSyncRules(options: storage.ParseSyncRulesOptions): Promise { + const content = await this.getActiveSyncRulesContent(); + return content?.parsed(options) ?? null; + } + + async getActiveSyncRulesContent(): Promise { + const activeRow = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }} + ORDER BY + id DESC + LIMIT + 1 + ` + .decoded(models.SyncRules) + .first(); + if (!activeRow) { + return null; + } + + return new PostgresPersistedSyncRulesContent(this.db, activeRow); + } + + // TODO possibly share via abstract class + async getNextSyncRules(options: storage.ParseSyncRulesOptions): Promise { + const content = await this.getNextSyncRulesContent(); + return content?.parsed(options) ?? null; + } + + async getNextSyncRulesContent(): Promise { + const nextRow = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + state = ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }} + ORDER BY + id DESC + LIMIT + 1 + ` + .decoded(models.SyncRules) + .first(); + if (!nextRow) { + return null; + } + + return new PostgresPersistedSyncRulesContent(this.db, nextRow); + } + + async getReplicatingSyncRules(): Promise { + const rows = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }} + OR state = ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }} + ` + .decoded(models.SyncRules) + .rows(); + + return rows.map((row) => new PostgresPersistedSyncRulesContent(this.db, row)); + } + + async getStoppedSyncRules(): Promise { + const rows = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + state = ${{ value: storage.SyncRuleState.STOP, type: 'varchar' }} + ` + .decoded(models.SyncRules) + .rows(); + + return rows.map((row) => new PostgresPersistedSyncRulesContent(this.db, row)); + } + + async getActiveCheckpoint(): Promise { + const activeCheckpoint = await this.db.sql` + SELECT + id, + last_checkpoint, + last_checkpoint_lsn + FROM + sync_rules + WHERE + state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }} + ORDER BY + id DESC + LIMIT + 1 + ` + .decoded(models.ActiveCheckpoint) + .first(); + + return this.makeActiveCheckpoint(activeCheckpoint); + } + + async *watchWriteCheckpoint(user_id: string, signal: AbortSignal): AsyncIterable { + let lastCheckpoint: utils.OpId | null = null; + let lastWriteCheckpoint: bigint | null = null; + + const iter = wrapWithAbort(this.sharedIterator, signal); + for await (const cp of iter) { + const { checkpoint, lsn } = cp; + + // lsn changes are not important by itself. + // What is important is: + // 1. checkpoint (op_id) changes. + // 2. write checkpoint changes for the specific user + const bucketStorage = await cp.getBucketStorage(); + if (!bucketStorage) { + continue; + } + + const lsnFilters: Record = lsn ? { 1: lsn } : {}; + + const currentWriteCheckpoint = await bucketStorage.lastWriteCheckpoint({ + user_id, + heads: { + ...lsnFilters + } + }); + + if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) { + // No change - wait for next one + // In some cases, many LSNs may be produced in a short time. + // Add a delay to throttle the write checkpoint lookup a bit. + await timers.setTimeout(20 + 10 * Math.random()); + continue; + } + + lastWriteCheckpoint = currentWriteCheckpoint; + lastCheckpoint = checkpoint; + + yield { base: cp, writeCheckpoint: currentWriteCheckpoint }; + } + } + + protected async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable { + const doc = await this.db.sql` + SELECT + id, + last_checkpoint, + last_checkpoint_lsn + FROM + sync_rules + WHERE + state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }} + LIMIT + 1 + ` + .decoded(models.ActiveCheckpoint) + .first(); + + const sink = new sync.LastValueSink(undefined); + + const disposeListener = this.db.registerListener({ + notification: (notification) => sink.next(notification.payload) + }); + + signal.addEventListener('aborted', async () => { + disposeListener(); + sink.complete(); + }); + + yield this.makeActiveCheckpoint(doc); + + let lastOp: storage.ActiveCheckpoint | null = null; + for await (const payload of sink.withSignal(signal)) { + if (signal.aborted) { + return; + } + + const notification = models.ActiveCheckpointNotification.decode(payload); + const activeCheckpoint = this.makeActiveCheckpoint(notification.active_checkpoint); + + if (lastOp == null || activeCheckpoint.lsn != lastOp.lsn || activeCheckpoint.checkpoint != lastOp.checkpoint) { + lastOp = activeCheckpoint; + yield activeCheckpoint; + } + } + } + + private makeActiveCheckpoint(row: models.ActiveCheckpointDecoded | null) { + return { + checkpoint: utils.timestampToOpId(row?.last_checkpoint ?? 0n), + lsn: row?.last_checkpoint_lsn ?? null, + hasSyncRules() { + return row != null; + }, + getBucketStorage: async () => { + if (row == null) { + return null; + } + return (await this.storageCache.fetch(Number(row.id))) ?? null; + } + } satisfies storage.ActiveCheckpoint; + } +} diff --git a/modules/module-postgres-storage/src/storage/PostgresCompactor.ts b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts new file mode 100644 index 000000000..103a8f165 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts @@ -0,0 +1,365 @@ +import { logger } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import * as pgwire from '@powersync/service-jpgwire'; +import * as t from 'ts-codec'; +import { models } from '../types/types.js'; +import { sql } from '../utils/connection/AbstractPostgresConnection.js'; +import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; +import { pick } from '../utils/ts-codec.js'; +import { encodedCacheKey } from './batch/OperationBatch.js'; + +interface CurrentBucketState { + /** Bucket name */ + bucket: string; + /** + * Rows seen in the bucket, with the last op_id of each. + */ + seen: Map; + /** + * Estimated memory usage of the seen Map. + */ + trackingSize: number; + + /** + * Last (lowest) seen op_id that is not a PUT. + */ + lastNotPut: bigint | null; + + /** + * Number of REMOVE/MOVE operations seen since lastNotPut. + */ + opsSincePut: number; +} + +/** + * Additional options, primarily for testing. + */ +export interface PostgresCompactOptions extends storage.CompactOptions { + /** Minimum of 2 */ + clearBatchLimit?: number; + /** Minimum of 1 */ + moveBatchLimit?: number; + /** Minimum of 1 */ + moveBatchQueryLimit?: number; +} + +const DEFAULT_CLEAR_BATCH_LIMIT = 5000; +const DEFAULT_MOVE_BATCH_LIMIT = 2000; +const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; + +/** This default is primarily for tests. */ +const DEFAULT_MEMORY_LIMIT_MB = 64; + +export class PostgresCompactor { + private updates: pgwire.Statement[] = []; + + private idLimitBytes: number; + private moveBatchLimit: number; + private moveBatchQueryLimit: number; + private clearBatchLimit: number; + private maxOpId: bigint | undefined; + private buckets: string[] | undefined; + + constructor( + private db: DatabaseClient, + private group_id: number, + options?: PostgresCompactOptions + ) { + this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024; + this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT; + this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT; + this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT; + this.maxOpId = options?.maxOpId; + this.buckets = options?.compactBuckets; + } + + /** + * Compact buckets by converting operations into MOVE and/or CLEAR operations. + * + * See /docs/compacting-operations.md for details. + */ + async compact() { + if (this.buckets) { + for (let bucket of this.buckets) { + // We can make this more efficient later on by iterating + // through the buckets in a single query. + // That makes batching more tricky, so we leave for later. + await this.compactInternal(bucket); + } + } else { + await this.compactInternal(undefined); + } + } + + async compactInternal(bucket: string | undefined) { + const idLimitBytes = this.idLimitBytes; + + let currentState: CurrentBucketState | null = null; + + let bucketLower: string | null = null; + let bucketUpper: string | null = null; + + if (bucket?.includes('[')) { + // Exact bucket name + bucketLower = bucket; + bucketUpper = bucket; + } else if (bucket) { + // Bucket definition name + bucketLower = `${bucket}[`; + bucketUpper = `${bucket}[\uFFFF`; + } + + let upperOpIdLimit = BigInt('9223372036854775807'); // 2^63 - 1 + + while (true) { + const batch = await this.db.sql` + SELECT + op, + op_id, + source_table, + table_name, + row_id, + source_key, + bucket_name + FROM + bucket_data + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + AND bucket_name LIKE COALESCE(${{ type: 'varchar', value: bucketLower }}, '%') + AND op_id < ${{ type: 'int8', value: upperOpIdLimit }} + ORDER BY + bucket_name, + op_id DESC + LIMIT + ${{ type: 'int4', value: this.moveBatchQueryLimit }} + ` + .decoded( + // TODO maybe a subtype + pick(models.BucketData, ['op', 'source_table', 'table_name', 'source_key', 'row_id', 'op_id', 'bucket_name']) + ) + .rows(); + + if (batch.length == 0) { + // We've reached the end + break; + } + + // Set upperBound for the next batch + upperOpIdLimit = batch[batch.length - 1].op_id; + + for (const doc of batch) { + if (currentState == null || doc.bucket_name != currentState.bucket) { + if (currentState != null && currentState.lastNotPut != null && currentState.opsSincePut >= 1) { + // Important to flush before clearBucket() + await this.flush(); + logger.info( + `Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + + const bucket = currentState.bucket; + const clearOp = currentState.lastNotPut; + // Free memory before clearing bucket + currentState = null; + await this.clearBucket(bucket, clearOp); + } + currentState = { + bucket: doc.bucket_name, + seen: new Map(), + trackingSize: 0, + lastNotPut: null, + opsSincePut: 0 + }; + } + + if (this.maxOpId != null && doc.op_id > this.maxOpId) { + continue; + } + + let isPersistentPut = doc.op == 'PUT'; + + if (doc.op == 'REMOVE' || doc.op == 'PUT') { + const key = `${doc.table_name}/${doc.row_id}/${encodedCacheKey(doc.source_table!, doc.source_key!)}`; + const targetOp = currentState.seen.get(utils.flatstr(key)); + if (targetOp) { + // Will convert to MOVE, so don't count as PUT + isPersistentPut = false; + + this.updates.push(sql` + UPDATE bucket_data + SET + op = 'MOVE', + target_op = ${{ type: 'int8', value: targetOp }}, + table_name = NULL, + row_id = NULL, + data = NULL, + source_table = NULL, + source_key = NULL + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + AND bucket_name = ${{ type: 'varchar', value: doc.bucket_name }} + AND op_id = ${{ type: 'int8', value: doc.op_id }} + `); + } else { + if (currentState.trackingSize >= idLimitBytes) { + // Reached memory limit. + // Keep the highest seen values in this case. + } else { + // flatstr reduces the memory usage by flattening the string + currentState.seen.set(utils.flatstr(key), doc.op_id); + // length + 16 for the string + // 24 for the bigint + // 50 for map overhead + // 50 for additional overhead + currentState.trackingSize += key.length + 140; + } + } + } + + if (isPersistentPut) { + currentState.lastNotPut = null; + currentState.opsSincePut = 0; + } else if (doc.op != 'CLEAR') { + if (currentState.lastNotPut == null) { + currentState.lastNotPut = doc.op_id; + } + currentState.opsSincePut += 1; + } + + if (this.updates.length >= this.moveBatchLimit) { + await this.flush(); + } + } + } + + await this.flush(); + currentState?.seen.clear(); + if (currentState?.lastNotPut != null && currentState?.opsSincePut > 1) { + logger.info( + `Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + const bucket = currentState.bucket; + const clearOp = currentState.lastNotPut; + // Free memory before clearing bucket + currentState = null; + await this.clearBucket(bucket, clearOp); + } + } + + private async flush() { + if (this.updates.length > 0) { + logger.info(`Compacting ${this.updates.length} ops`); + await this.db.query(...this.updates); + this.updates = []; + } + } + + /** + * Perform a CLEAR compact for a bucket. + * + * @param bucket bucket name + * @param op op_id of the last non-PUT operation, which will be converted to CLEAR. + */ + private async clearBucket(bucket: string, op: bigint) { + /** + * This entire method could be implemented as a Postgres function, but this might make debugging + * a bit more challenging. + */ + let done = false; + while (!done) { + await this.db.lockConnection(async (db) => { + /** + * Start a transaction where each read returns the state at the start of the transaction,. + * Similar to the MongoDB readConcern: { level: 'snapshot' } mode. + */ + await db.sql`BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ`.execute(); + + try { + let checksum = 0; + let lastOpId: bigint | null = null; + let targetOp: bigint | null = null; + let gotAnOp = false; + + const codec = pick(models.BucketData, ['op', 'source_table', 'source_key', 'op_id', 'checksum', 'target_op']); + for await (const operations of db.streamRows>(sql` + SELECT + source_table, + source_key, + op, + op_id, + checksum, + target_op + FROM + bucket_data + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + AND bucket_name = ${{ type: 'varchar', value: bucket }} + AND op_id <= ${{ type: 'int8', value: op }} + ORDER BY + op_id + LIMIT + ${{ type: 'int4', value: this.clearBatchLimit }} + `)) { + const decodedOps = operations.map((o) => codec.decode(o)); + for (const op of decodedOps) { + if ([models.OpType.MOVE, models.OpType.REMOVE, models.OpType.CLEAR].includes(op.op)) { + checksum = utils.addChecksums(checksum, Number(op.checksum)); + lastOpId = op.op_id; + if (op.op != models.OpType.CLEAR) { + gotAnOp = true; + } + if (op.target_op != null) { + if (targetOp == null || op.target_op > targetOp) { + targetOp = op.target_op; + } + } + } else { + throw new Error(`Unexpected ${op.op} operation at ${this.group_id}:${bucket}:${op.op_id}`); + } + } + } + + if (!gotAnOp) { + done = true; + return; + } + + logger.info(`Flushing CLEAR at ${lastOpId}`); + + await db.sql` + DELETE FROM bucket_data + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + AND bucket_name = ${{ type: 'varchar', value: bucket }} + AND op_id <= ${{ type: 'int8', value: lastOpId }} + `.execute(); + + await db.sql` + INSERT INTO + bucket_data ( + group_id, + bucket_name, + op_id, + op, + checksum, + target_op + ) + VALUES + ( + ${{ type: 'int4', value: this.group_id }}, + ${{ type: 'varchar', value: bucket }}, + ${{ type: 'int8', value: lastOpId }}, + ${{ type: 'varchar', value: models.OpType.CLEAR }}, + ${{ type: 'int8', value: checksum }}, + ${{ type: 'int8', value: targetOp }} + ) + `.execute(); + + await db.sql`COMMIT`.execute(); + } catch (ex) { + await db.sql`ROLLBACK`.execute(); + throw ex; + } + }); + } + } +} diff --git a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts new file mode 100644 index 000000000..276ba1064 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts @@ -0,0 +1,42 @@ +import { logger } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { POSTGRES_CONNECTION_TYPE } from '@powersync/service-module-postgres/types'; + +import { normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js'; +import { dropTables } from '../utils/db.js'; +import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; + +export class PostgresStorageProvider implements storage.BucketStorageProvider { + get type() { + return POSTGRES_CONNECTION_TYPE; + } + + async getStorage(options: storage.GetStorageOptions): Promise { + const { resolvedConfig } = options; + + const { storage } = resolvedConfig; + if (storage.type != POSTGRES_CONNECTION_TYPE) { + // This should not be reached since the generation should be managed externally. + throw new Error( + `Cannot create Postgres bucket storage with provided config ${storage.type} !== ${POSTGRES_CONNECTION_TYPE}` + ); + } + + const decodedConfig = PostgresStorageConfig.decode(storage as any); + const normalizedConfig = normalizePostgresStorageConfig(decodedConfig); + const storageFactory = new PostgresBucketStorageFactory({ + config: normalizedConfig, + slot_name_prefix: options.resolvedConfig.slot_name_prefix + }); + return { + storage: storageFactory, + shutDown: async () => storageFactory.db[Symbol.asyncDispose](), + tearDown: async () => { + logger.info(`Tearing down Postgres storage: ${normalizedConfig.database}...`); + await dropTables(storage.db); + await storageFactory.db[Symbol.asyncDispose](); + return true; + } + } satisfies storage.ActiveStorage; + } +} diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts new file mode 100644 index 000000000..e75a55e3e --- /dev/null +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -0,0 +1,642 @@ +import { DisposableObserver } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as t from 'ts-codec'; +import * as uuid from 'uuid'; +import { bigint } from '../types/codecs.js'; +import { models, RequiredOperationBatchLimits } from '../types/types.js'; +import { replicaIdToSubkey } from '../utils/bson.js'; +import { mapOpEntry } from '../utils/bucket-data.js'; +import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; +import { pick } from '../utils/ts-codec.js'; +import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js'; +import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js'; +import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; +import { PostgresCompactor } from './PostgresCompactor.js'; + +export type PostgresSyncRulesStorageOptions = { + factory: PostgresBucketStorageFactory; + db: DatabaseClient; + sync_rules: storage.PersistedSyncRulesContent; + write_checkpoint_mode?: storage.WriteCheckpointMode; + batchLimits: RequiredOperationBatchLimits; +}; + +export class PostgresSyncRulesStorage + extends DisposableObserver + implements storage.SyncRulesBucketStorage +{ + public readonly group_id: number; + public readonly sync_rules: storage.PersistedSyncRulesContent; + public readonly slot_name: string; + public readonly factory: PostgresBucketStorageFactory; + + protected db: DatabaseClient; + protected writeCheckpointAPI: PostgresWriteCheckpointAPI; + + // TODO we might be able to share this in an abstract class + private parsedSyncRulesCache: { parsed: sync_rules.SqlSyncRules; options: storage.ParseSyncRulesOptions } | undefined; + private checksumCache = new storage.ChecksumCache({ + fetchChecksums: (batch) => { + return this.getChecksumsInternal(batch); + } + }); + + constructor(protected options: PostgresSyncRulesStorageOptions) { + super(); + this.group_id = options.sync_rules.id; + this.db = options.db; + this.sync_rules = options.sync_rules; + this.slot_name = options.sync_rules.slot_name; + this.factory = options.factory; + + this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({ + db: this.db, + mode: options.write_checkpoint_mode ?? storage.WriteCheckpointMode.MANAGED + }); + } + + get writeCheckpointMode(): storage.WriteCheckpointMode { + return this.writeCheckpointAPI.writeCheckpointMode; + } + + // TODO we might be able to share this in an abstract class + getParsedSyncRules(options: storage.ParseSyncRulesOptions): sync_rules.SqlSyncRules { + const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; + /** + * Check if the cached sync rules, if present, had the same options. + * Parse sync rules if the options are different or if there is no cached value. + */ + if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { + this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).sync_rules, options }; + } + + return this.parsedSyncRulesCache!.parsed; + } + + async reportError(e: any): Promise { + const message = String(e.message ?? 'Replication failure'); + await this.db.sql` + UPDATE sync_rules + SET + last_fatal_error = ${{ type: 'varchar', value: message }} + WHERE + id = ${{ type: 'int4', value: this.group_id }}; + `.execute(); + } + + compact(options?: storage.CompactOptions): Promise { + return new PostgresCompactor(this.db, this.group_id, options).compact(); + } + + batchCreateCustomWriteCheckpoints(checkpoints: storage.BatchedCustomWriteCheckpointOptions[]): Promise { + return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints( + checkpoints.map((c) => ({ ...c, sync_rules_id: this.group_id })) + ); + } + + createCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): Promise { + return this.writeCheckpointAPI.createCustomWriteCheckpoint({ + ...checkpoint, + sync_rules_id: this.group_id + }); + } + + lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise { + return this.writeCheckpointAPI.lastWriteCheckpoint({ + ...filters, + sync_rules_id: this.group_id + }); + } + + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { + return this.writeCheckpointAPI.setWriteCheckpointMode(mode); + } + + createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { + return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint); + } + + async getCheckpoint(): Promise { + const checkpointRow = await this.db.sql` + SELECT + last_checkpoint, + last_checkpoint_lsn + FROM + sync_rules + WHERE + id = ${{ type: 'int8', value: this.group_id }} + ` + .decoded(pick(models.SyncRules, ['last_checkpoint', 'last_checkpoint_lsn'])) + .first(); + + return { + checkpoint: utils.timestampToOpId(checkpointRow?.last_checkpoint ?? 0n), + lsn: checkpointRow?.last_checkpoint_lsn ?? null + }; + } + + async resolveTable(options: storage.ResolveTableOptions): Promise { + const { group_id, connection_id, connection_tag, entity_descriptor } = options; + + const { schema, name: table, objectId, replicationColumns } = entity_descriptor; + + const columns = replicationColumns.map((column) => ({ + name: column.name, + type: column.type, + // The PGWire returns this as a BigInt. We want to store this as JSONB + type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId + })); + return this.db.transaction(async (db) => { + let sourceTableRow = await db.sql` + SELECT + * + FROM + source_tables + WHERE + group_id = ${{ type: 'int4', value: group_id }} + AND connection_id = ${{ type: 'int4', value: connection_id }} + AND relation_id = ${{ type: 'int4', value: objectId }} + AND schema_name = ${{ type: 'varchar', value: schema }} + AND table_name = ${{ type: 'varchar', value: table }} + AND replica_id_columns = ${{ type: 'jsonb', value: columns }} + ` + .decoded(models.SourceTable) + .first(); + + if (sourceTableRow == null) { + const row = await db.sql` + INSERT INTO + source_tables ( + id, + group_id, + connection_id, + relation_id, + schema_name, + table_name, + replica_id_columns + ) + VALUES + ( + ${{ type: 'varchar', value: uuid.v4() }}, + ${{ type: 'int4', value: group_id }}, + ${{ type: 'int4', value: connection_id }}, + ${{ type: 'int4', value: objectId }}, + ${{ type: 'varchar', value: schema }}, + ${{ type: 'varchar', value: table }}, + ${{ type: 'jsonb', value: columns }} + ) + RETURNING + * + ` + .decoded(models.SourceTable) + .first(); + sourceTableRow = row; + } + + const sourceTable = new storage.SourceTable( + sourceTableRow!.id, + connection_tag, + objectId, + schema, + table, + replicationColumns, + sourceTableRow!.snapshot_done ?? true + ); + sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable); + sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); + sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable); + + const truncatedTables = await db.sql` + SELECT + * + FROM + source_tables + WHERE + group_id = ${{ type: 'int4', value: group_id }} + AND connection_id = ${{ type: 'int4', value: connection_id }} + AND id != ${{ type: 'varchar', value: sourceTableRow!.id }} + AND ( + relation_id = ${{ type: 'int4', value: objectId }} + OR ( + schema_name = ${{ type: 'varchar', value: schema }} + AND table_name = ${{ type: 'varchar', value: table }} + ) + ) + ` + .decoded(models.SourceTable) + .rows(); + + return { + table: sourceTable, + dropTables: truncatedTables.map( + (doc) => + new storage.SourceTable( + doc.id, + connection_tag, + Number(doc.relation_id ?? 0), + doc.schema_name, + doc.table_name, + doc.replica_id_columns?.map((c) => ({ + name: c.name, + typeOid: c.typeId, + type: c.type + })) ?? [], + doc.snapshot_done ?? true + ) + ) + }; + }); + } + + async startBatch( + options: storage.StartBatchOptions, + callback: (batch: storage.BucketStorageBatch) => Promise + ): Promise { + const syncRules = await this.db.sql` + SELECT + last_checkpoint_lsn, + no_checkpoint_before, + keepalive_op + FROM + sync_rules + WHERE + id = ${{ type: 'int4', value: this.group_id }} + ` + .decoded(pick(models.SyncRules, ['last_checkpoint_lsn', 'no_checkpoint_before', 'keepalive_op'])) + .first(); + + const checkpoint_lsn = syncRules?.last_checkpoint_lsn ?? null; + + await using batch = new PostgresBucketBatch({ + db: this.db, + sync_rules: this.sync_rules.parsed(options).sync_rules, + group_id: this.group_id, + slot_name: this.slot_name, + last_checkpoint_lsn: checkpoint_lsn, + keep_alive_op: syncRules?.keepalive_op, + no_checkpoint_before_lsn: syncRules?.no_checkpoint_before ?? options.zeroLSN, + store_current_data: options.storeCurrentData, + skip_existing_rows: options.skipExistingRows ?? false, + batch_limits: this.options.batchLimits + }); + this.iterateListeners((cb) => cb.batchStarted?.(batch)); + + await callback(batch); + await batch.flush(); + if (batch.last_flushed_op) { + return { flushed_op: String(batch.last_flushed_op) }; + } else { + return null; + } + } + + async getParameterSets( + checkpoint: utils.OpId, + lookups: sync_rules.SqliteJsonValue[][] + ): Promise { + const rows = await this.db.sql` + SELECT DISTINCT + ON (lookup, source_table, source_key) lookup, + source_table, + source_key, + id, + bucket_parameters + FROM + bucket_parameters + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + AND lookup = ANY ( + SELECT + decode((FILTER ->> 0)::text, 'hex') -- Decode the hex string to bytea + FROM + jsonb_array_elements(${{ + type: 'jsonb', + value: lookups.map((l) => storage.serializeLookupBuffer(l).toString('hex')) + }}) AS FILTER + ) + AND id <= ${{ type: 'int8', value: BigInt(checkpoint) }} + ORDER BY + lookup, + source_table, + source_key, + id DESC + ` + .decoded(pick(models.BucketParameters, ['bucket_parameters'])) + .rows(); + + const groupedParameters = rows.map((row) => { + return row.bucket_parameters; + }); + return groupedParameters.flat(); + } + + async *getBucketDataBatch( + checkpoint: utils.OpId, + dataBuckets: Map, + options?: storage.BucketDataBatchOptions + ): AsyncIterable { + if (dataBuckets.size == 0) { + return; + } + + const end = checkpoint ? BigInt(checkpoint) : BigInt(2) ** BigInt(64) - BigInt(1); + const filters = Array.from(dataBuckets.entries()).map(([name, start]) => ({ + bucket_name: name, + start: start + })); + + const rowLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; + const sizeLimit = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; + + let batchSize = 0; + let currentBatch: utils.SyncBucketData | null = null; + let targetOp: bigint | null = null; + let rowCount = 0; + + for await (const rawRows of this.db.streamRows({ + statement: /* sql */ ` + WITH + filter_data AS ( + SELECT + FILTER ->> 'bucket_name' AS bucket_name, + (FILTER ->> 'start')::BIGINT AS start_op_id + FROM + jsonb_array_elements($1::jsonb) AS FILTER + ) + SELECT + b.*, + octet_length(b.data) AS data_size + FROM + bucket_data b + JOIN filter_data f ON b.bucket_name = f.bucket_name + AND b.op_id > f.start_op_id + AND b.op_id <= $2 + WHERE + b.group_id = $3 + ORDER BY + b.bucket_name ASC, + b.op_id ASC + LIMIT + $4 + `, + params: [ + { type: 'jsonb', value: filters }, + { type: 'int8', value: end }, + { type: 'int8', value: this.group_id }, + { type: 'int4', value: rowLimit + 1 } // Increase the row limit by 1 in order to detect hasMore + ] + })) { + const rows = rawRows.map((q) => { + return models.BucketData.and( + t.object({ + data_size: t.Null.or(bigint) + }) + ).decode(q as any); + }); + + for (const row of rows) { + const { bucket_name } = row; + const rowSize = row.data_size ? Number(row.data_size) : 0; + + if ( + currentBatch == null || + currentBatch.bucket != bucket_name || + batchSize >= sizeLimit || + (currentBatch?.data.length && batchSize + rowSize > sizeLimit) || + currentBatch.data.length >= rowLimit + ) { + let start: string | undefined = undefined; + if (currentBatch != null) { + if (currentBatch.bucket == bucket_name) { + currentBatch.has_more = true; + } + + const yieldBatch = currentBatch; + start = currentBatch.after; + currentBatch = null; + batchSize = 0; + yield { batch: yieldBatch, targetOp: targetOp }; + targetOp = null; + if (rowCount >= rowLimit) { + // We've yielded all the requested rows + break; + } + } + + start ??= dataBuckets.get(bucket_name); + if (start == null) { + throw new Error(`data for unexpected bucket: ${bucket_name}`); + } + currentBatch = { + bucket: bucket_name, + after: start, + has_more: false, + data: [], + next_after: start + }; + targetOp = null; + } + + const entry = mapOpEntry(row); + + if (row.source_table && row.source_key) { + entry.subkey = replicaIdToSubkey(row.source_table, storage.deserializeReplicaId(row.source_key)); + } + + if (row.target_op != null) { + // MOVE, CLEAR + const rowTargetOp = row.target_op; + if (targetOp == null || rowTargetOp > targetOp) { + targetOp = rowTargetOp; + } + } + + currentBatch.data.push(entry); + currentBatch.next_after = entry.op_id; + + // Obtained from pg_column_size(data) AS data_size + // We could optimize this and persist the column instead of calculating + // on query. + batchSize += row.data_size ? Number(row.data_size) : 0; + + // Manually track the total rows yielded + rowCount++; + } + } + if (currentBatch != null) { + const yieldBatch = currentBatch; + currentBatch = null; + yield { batch: yieldBatch, targetOp: targetOp }; + targetOp = null; + } + } + + async getChecksums(checkpoint: utils.OpId, buckets: string[]): Promise { + return this.checksumCache.getChecksumMap(checkpoint, buckets); + } + + async terminate(options?: storage.TerminateOptions) { + if (!options || options?.clearStorage) { + await this.clear(); + } + await this.db.sql` + UPDATE sync_rules + SET + state = ${{ type: 'varchar', value: storage.SyncRuleState.TERMINATED }}, + snapshot_done = ${{ type: 'bool', value: false }} + `.execute(); + } + + async getStatus(): Promise { + const syncRulesRow = await this.db.sql` + SELECT + snapshot_done, + last_checkpoint_lsn, + state + FROM + sync_rules + WHERE + id = ${{ type: 'int4', value: this.group_id }} + ` + .decoded(pick(models.SyncRules, ['snapshot_done', 'last_checkpoint_lsn', 'state'])) + .first(); + + if (syncRulesRow == null) { + throw new Error('Cannot find sync rules status'); + } + + return { + snapshot_done: syncRulesRow.snapshot_done, + active: syncRulesRow.state == storage.SyncRuleState.ACTIVE, + checkpoint_lsn: syncRulesRow.last_checkpoint_lsn ?? null + }; + } + + async clear(): Promise { + await this.db.sql` + UPDATE sync_rules + SET + snapshot_done = FALSE, + last_checkpoint_lsn = NULL, + last_checkpoint = NULL, + no_checkpoint_before = NULL + WHERE + id = ${{ type: 'int8', value: this.group_id }} + `.execute(); + + await this.db.sql` + DELETE FROM bucket_data + WHERE + group_id = ${{ type: 'int8', value: this.group_id }} + `.execute(); + + await this.db.sql` + DELETE FROM bucket_parameters + WHERE + group_id = ${{ type: 'int8', value: this.group_id }} + `.execute(); + + await this.db.sql` + DELETE FROM current_data + WHERE + group_id = ${{ type: 'int8', value: this.group_id }} + `.execute(); + + await this.db.sql` + DELETE FROM source_tables + WHERE + group_id = ${{ type: 'int8', value: this.group_id }} + `.execute(); + } + + async autoActivate(): Promise { + await this.db.transaction(async (db) => { + const syncRulesRow = await db.sql` + SELECT + state + FROM + sync_rules + WHERE + id = ${{ type: 'int4', value: this.group_id }} + ` + .decoded(pick(models.SyncRules, ['state'])) + .first(); + + if (syncRulesRow && syncRulesRow.state == storage.SyncRuleState.PROCESSING) { + await db.sql` + UPDATE sync_rules + SET + state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }} + WHERE + id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + } + + await db.sql` + UPDATE sync_rules + SET + state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }} + WHERE + state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }} + AND id != ${{ type: 'int4', value: this.group_id }} + `.execute(); + }); + } + + private async getChecksumsInternal(batch: storage.FetchPartialBucketChecksum[]): Promise { + if (batch.length == 0) { + return new Map(); + } + + const rangedBatch = batch.map((b) => ({ + ...b, + start: b.start ?? 0 + })); + + const results = await this.db.sql` + WITH + filter_data AS ( + SELECT + FILTER ->> 'bucket' AS bucket_name, + (FILTER ->> 'start')::BIGINT AS start_op_id, + (FILTER ->> 'end')::BIGINT AS end_op_id + FROM + jsonb_array_elements(${{ type: 'jsonb', value: rangedBatch }}::jsonb) AS FILTER + ) + SELECT + b.bucket_name AS bucket, + SUM(b.checksum) AS checksum_total, + COUNT(*) AS total, + MAX( + CASE + WHEN b.op = 'CLEAR' THEN 1 + ELSE 0 + END + ) AS has_clear_op + FROM + bucket_data b + JOIN filter_data f ON b.bucket_name = f.bucket_name + AND b.op_id > f.start_op_id + AND b.op_id <= f.end_op_id + WHERE + b.group_id = ${{ type: 'int8', value: this.group_id }} + GROUP BY + b.bucket_name; + `.rows<{ bucket: string; checksum_total: bigint; total: bigint; has_clear_op: number }>(); + + return new Map( + results.map((doc) => { + return [ + doc.bucket, + { + bucket: doc.bucket, + partialCount: Number(doc.total), + partialChecksum: Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff, + isFullChecksum: doc.has_clear_op == 1 + } satisfies storage.PartialChecksum + ]; + }) + ); + } +} diff --git a/modules/module-postgres-storage/src/storage/batch/OperationBatch.ts b/modules/module-postgres-storage/src/storage/batch/OperationBatch.ts new file mode 100644 index 000000000..f00139031 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/batch/OperationBatch.ts @@ -0,0 +1,129 @@ +/** + * TODO share this implementation better in the core package. + * There are some subtle differences in this implementation. + */ + +import { ToastableSqliteRow } from '@powersync/service-sync-rules'; + +import { storage } from '@powersync/service-core'; +import { RequiredOperationBatchLimits } from '../../types/types.js'; + +/** + * Batch of input operations. + * + * We accumulate operations up to MAX_RECORD_BATCH_SIZE, + * then further split into sub-batches if MAX_CURRENT_DATA_BATCH_SIZE is exceeded. + */ +export class OperationBatch { + batch: RecordOperation[] = []; + currentSize: number = 0; + + readonly maxBatchCount: number; + readonly maxRecordSize: number; + readonly maxCurrentDataBatchSize: number; + + get length() { + return this.batch.length; + } + + constructor(protected options: RequiredOperationBatchLimits) { + this.maxBatchCount = options.max_record_count; + this.maxRecordSize = options.max_estimated_size; + this.maxCurrentDataBatchSize = options.max_current_data_batch_size; + } + + push(op: RecordOperation) { + this.batch.push(op); + this.currentSize += op.estimatedSize; + } + + shouldFlush() { + return this.batch.length >= this.maxBatchCount || this.currentSize > this.maxCurrentDataBatchSize; + } + + /** + * + * @param sizes Map of source key to estimated size of the current_data document, or undefined if current_data is not persisted. + * + */ + *batched(sizes: Map | undefined): Generator { + if (sizes == null) { + yield this.batch; + return; + } + let currentBatch: RecordOperation[] = []; + let currentBatchSize = 0; + for (let op of this.batch) { + const key = op.internalBeforeKey; + const size = sizes.get(key) ?? 0; + if (currentBatchSize + size > this.maxCurrentDataBatchSize && currentBatch.length > 0) { + yield currentBatch; + currentBatch = []; + currentBatchSize = 0; + } + currentBatchSize += size; + currentBatch.push(op); + } + if (currentBatch.length > 0) { + yield currentBatch; + } + } +} + +export class RecordOperation { + public readonly afterId: storage.ReplicaId | null; + public readonly beforeId: storage.ReplicaId; + public readonly internalBeforeKey: string; + public readonly internalAfterKey: string | null; + public readonly estimatedSize: number; + + constructor(public readonly record: storage.SaveOptions) { + const afterId = record.afterReplicaId ?? null; + const beforeId = record.beforeReplicaId ?? record.afterReplicaId; + this.afterId = afterId; + this.beforeId = beforeId; + this.internalBeforeKey = cacheKey(record.sourceTable.id, beforeId); + this.internalAfterKey = afterId ? cacheKey(record.sourceTable.id, afterId) : null; + this.estimatedSize = estimateRowSize(record.before) + estimateRowSize(record.after); + } +} + +/** + * In-memory cache key - must not be persisted. + */ +export function cacheKey(sourceTableId: string, id: storage.ReplicaId) { + return encodedCacheKey(sourceTableId, storage.serializeReplicaId(id)); +} + +/** + * Calculates a cache key for a stored ReplicaId. This is usually stored as a bytea/Buffer. + */ +export function encodedCacheKey(sourceTableId: string, storedKey: Buffer) { + return `${sourceTableId}.${storedKey.toString('base64')}`; +} + +/** + * Estimate in-memory size of row. + */ +function estimateRowSize(record: ToastableSqliteRow | undefined) { + if (record == null) { + return 12; + } + let size = 0; + for (let [key, value] of Object.entries(record)) { + size += 12 + key.length; + // number | string | null | bigint | Uint8Array + if (value == null) { + size += 4; + } else if (typeof value == 'number') { + size += 8; + } else if (typeof value == 'bigint') { + size += 8; + } else if (typeof value == 'string') { + size += value.length; + } else if (value instanceof Uint8Array) { + size += value.byteLength; + } + } + return size; +} diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts new file mode 100644 index 000000000..3ac629319 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -0,0 +1,883 @@ +import { container, DisposableObserver, errors, logger } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as timers from 'timers/promises'; +import * as t from 'ts-codec'; +import { CurrentBucket, CurrentData, CurrentDataDecoded } from '../../types/models/CurrentData.js'; +import { models, RequiredOperationBatchLimits } from '../../types/types.js'; +import { sql } from '../../utils/connection/AbstractPostgresConnection.js'; +import { NOTIFICATION_CHANNEL } from '../../utils/connection/ConnectionSlot.js'; +import { DatabaseClient } from '../../utils/connection/DatabaseClient.js'; +import { WrappedConnection } from '../../utils/connection/WrappedConnection.js'; +import { pick } from '../../utils/ts-codec.js'; +import { batchCreateCustomWriteCheckpoints } from '../checkpoints/PostgresWriteCheckpointAPI.js'; +import { cacheKey, encodedCacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; +import { PostgresPersistedBatch } from './PostgresPersistedBatch.js'; + +export interface PostgresBucketBatchOptions { + db: DatabaseClient; + sync_rules: sync_rules.SqlSyncRules; + group_id: number; + slot_name: string; + last_checkpoint_lsn: string | null; + no_checkpoint_before_lsn: string; + store_current_data: boolean; + keep_alive_op?: string | null; + /** + * Set to true for initial replication. + */ + skip_existing_rows: boolean; + batch_limits: RequiredOperationBatchLimits; +} + +/** + * Intermediate type which helps for only watching the active sync rules + * via the Postgres NOTIFY protocol. + */ +const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(storage.SyncRuleState) })); +type StatefulCheckpointDecoded = t.Decoded; + +// The limits here are not as strict as MongoDB +const MAX_ROW_SIZE = 40_000_000; + +export class PostgresBucketBatch + extends DisposableObserver + implements storage.BucketStorageBatch +{ + public last_flushed_op: bigint | null = null; + + protected db: DatabaseClient; + protected group_id: number; + protected last_checkpoint_lsn: string | null; + protected no_checkpoint_before_lsn: string; + + protected persisted_op: bigint | null; + + protected write_checkpoint_batch: storage.CustomWriteCheckpointOptions[]; + protected readonly sync_rules: sync_rules.SqlSyncRules; + protected batch: OperationBatch | null; + private lastWaitingLogThrottled = 0; + + constructor(protected options: PostgresBucketBatchOptions) { + super(); + this.db = options.db; + this.group_id = options.group_id; + this.last_checkpoint_lsn = options.last_checkpoint_lsn; + this.no_checkpoint_before_lsn = options.no_checkpoint_before_lsn; + this.write_checkpoint_batch = []; + this.sync_rules = options.sync_rules; + this.batch = null; + this.persisted_op = null; + if (options.keep_alive_op) { + this.persisted_op = BigInt(options.keep_alive_op); + } + } + + get lastCheckpointLsn() { + return this.last_checkpoint_lsn; + } + + async save(record: storage.SaveOptions): Promise { + // TODO maybe share with abstract class + const { after, afterReplicaId, before, beforeReplicaId, sourceTable, tag } = record; + for (const event of this.getTableEvents(sourceTable)) { + this.iterateListeners((cb) => + cb.replicationEvent?.({ + batch: this, + table: sourceTable, + data: { + op: tag, + after: after && utils.isCompleteRow(this.options.store_current_data, after) ? after : undefined, + before: before && utils.isCompleteRow(this.options.store_current_data, before) ? before : undefined + }, + event + }) + ); + } + /** + * Return if the table is just an event table + */ + if (!sourceTable.syncData && !sourceTable.syncParameters) { + return null; + } + + logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`); + + if (!sourceTable.syncData && !sourceTable.syncParameters) { + return null; + } + + logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`); + + this.batch ??= new OperationBatch(this.options.batch_limits); + this.batch.push(new RecordOperation(record)); + + if (this.batch.shouldFlush()) { + const r = await this.flush(); + // HACK: Give other streams a chance to also flush + await timers.setTimeout(5); + return r; + } + return null; + } + + async truncate(sourceTables: storage.SourceTable[]): Promise { + await this.flush(); + + let last_op: bigint | null = null; + for (let table of sourceTables) { + last_op = await this.truncateSingle(table); + } + + if (last_op) { + this.persisted_op = last_op; + } + + return { + flushed_op: String(last_op!) + }; + } + + protected async truncateSingle(sourceTable: storage.SourceTable) { + // To avoid too large transactions, we limit the amount of data we delete per transaction. + // Since we don't use the record data here, we don't have explicit size limits per batch. + const BATCH_LIMIT = 2000; + let lastBatchCount = BATCH_LIMIT; + let processedCount = 0; + const codec = pick(models.CurrentData, ['buckets', 'lookups', 'source_key']); + + while (lastBatchCount == BATCH_LIMIT) { + lastBatchCount = 0; + for await (const rows of this.db.streamRows>(sql` + SELECT + buckets, + lookups, + source_key + FROM + current_data + WHERE + group_id = ${{ type: 'int8', value: this.group_id }} + AND source_table = ${{ type: 'varchar', value: sourceTable.id }} + LIMIT + ${{ type: 'int4', value: BATCH_LIMIT }} + `)) { + lastBatchCount += rows.length; + processedCount += rows.length; + await this.withReplicationTransaction(async (db) => { + const persistedBatch = new PostgresPersistedBatch({ + group_id: this.group_id, + ...this.options.batch_limits + }); + + const decodedRows = rows.map((row) => codec.decode(row)); + for (const value of decodedRows) { + persistedBatch.saveBucketData({ + before_buckets: value.buckets, + evaluated: [], + table: sourceTable, + source_key: value.source_key + }); + persistedBatch.saveParameterData({ + existing_lookups: value.lookups, + evaluated: [], + table: sourceTable, + source_key: value.source_key + }); + persistedBatch.deleteCurrentData({ + source_key: value.source_key, + source_table_id: sourceTable.id + }); + } + await persistedBatch.flush(db); + }); + } + } + if (processedCount == 0) { + // The op sequence should not have progressed + return null; + } + + const currentSequence = await this.db.sql` + SELECT + LAST_VALUE AS value + FROM + op_id_sequence; + `.first<{ value: bigint }>(); + return currentSequence!.value; + } + + async drop(sourceTables: storage.SourceTable[]): Promise { + await this.truncate(sourceTables); + const result = await this.flush(); + + await this.db.transaction(async (db) => { + for (const table of sourceTables) { + await db.sql` + DELETE FROM source_tables + WHERE + id = ${{ type: 'varchar', value: table.id }} + `.execute(); + } + }); + return result; + } + + async flush(): Promise { + let result: storage.FlushedResult | null = null; + // One flush may be split over multiple transactions. + // Each flushInner() is one transaction. + while (this.batch != null) { + let r = await this.flushInner(); + if (r) { + result = r; + } + } + await batchCreateCustomWriteCheckpoints(this.db, this.write_checkpoint_batch); + this.write_checkpoint_batch = []; + return result; + } + + private async flushInner(): Promise { + const batch = this.batch; + if (batch == null) { + return null; + } + + let resumeBatch: OperationBatch | null = null; + + const lastOp = await this.withReplicationTransaction(async (db) => { + resumeBatch = await this.replicateBatch(db, batch); + + const sequence = await db.sql` + SELECT + LAST_VALUE AS value + FROM + op_id_sequence; + `.first<{ value: bigint }>(); + return sequence!.value; + }); + + // null if done, set if we need another flush + this.batch = resumeBatch; + + if (lastOp == null) { + throw new Error('Unexpected last_op == null'); + } + + this.persisted_op = lastOp; + this.last_flushed_op = lastOp; + return { flushed_op: String(lastOp) }; + } + + async commit(lsn: string): Promise { + await this.flush(); + + if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) { + // When re-applying transactions, don't create a new checkpoint until + // we are past the last transaction. + logger.info(`Re-applied transaction ${lsn} - skipping checkpoint`); + return false; + } + + if (lsn < this.no_checkpoint_before_lsn) { + if (Date.now() - this.lastWaitingLogThrottled > 5_000) { + logger.info( + `Waiting until ${this.no_checkpoint_before_lsn} before creating checkpoint, currently at ${lsn}. Persisted op: ${this.persisted_op}` + ); + this.lastWaitingLogThrottled = Date.now(); + } + + // Edge case: During initial replication, we have a no_checkpoint_before_lsn set, + // and don't actually commit the snapshot. + // The first commit can happen from an implicit keepalive message. + // That needs the persisted_op to get an accurate checkpoint, so + // we persist that in keepalive_op. + + await this.db.sql` + UPDATE sync_rules + SET + keepalive_op = ${{ type: 'varchar', value: this.persisted_op == null ? null : String(this.persisted_op) }} + WHERE + id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + + return false; + } + const now = new Date().toISOString(); + const update: Partial = { + last_checkpoint_lsn: lsn, + last_checkpoint_ts: now, + last_keepalive_ts: now, + snapshot_done: true, + last_fatal_error: null, + keepalive_op: null + }; + + if (this.persisted_op != null) { + update.last_checkpoint = this.persisted_op.toString(); + } + + const doc = await this.db.sql` + UPDATE sync_rules + SET + keepalive_op = ${{ type: 'varchar', value: update.keepalive_op }}, + last_fatal_error = ${{ type: 'varchar', value: update.last_fatal_error }}, + snapshot_done = ${{ type: 'bool', value: update.snapshot_done }}, + last_keepalive_ts = ${{ type: 1184, value: update.last_keepalive_ts }}, + last_checkpoint = ${{ type: 'int8', value: update.last_checkpoint }}, + last_checkpoint_ts = ${{ type: 1184, value: update.last_checkpoint_ts }}, + last_checkpoint_lsn = ${{ type: 'varchar', value: update.last_checkpoint_lsn }} + WHERE + id = ${{ type: 'int4', value: this.group_id }} + RETURNING + id, + state, + last_checkpoint, + last_checkpoint_lsn + ` + .decoded(StatefulCheckpoint) + .first(); + + await notifySyncRulesUpdate(this.db, doc!); + + this.persisted_op = null; + this.last_checkpoint_lsn = lsn; + return true; + } + + async keepalive(lsn: string): Promise { + if (this.last_checkpoint_lsn != null && lsn <= this.last_checkpoint_lsn) { + // No-op + return false; + } + + if (lsn < this.no_checkpoint_before_lsn) { + return false; + } + + if (this.persisted_op != null) { + // The commit may have been skipped due to "no_checkpoint_before_lsn". + // Apply it now if relevant + logger.info(`Commit due to keepalive at ${lsn} / ${this.persisted_op}`); + return await this.commit(lsn); + } + + const updated = await this.db.sql` + UPDATE sync_rules + SET + snapshot_done = ${{ type: 'bool', value: true }}, + last_checkpoint_lsn = ${{ type: 'varchar', value: lsn }}, + last_fatal_error = ${{ type: 'varchar', value: null }}, + last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }} + WHERE + id = ${{ type: 'int8', value: this.group_id }} + RETURNING + id, + state, + last_checkpoint, + last_checkpoint_lsn + ` + .decoded(StatefulCheckpoint) + .first(); + + await notifySyncRulesUpdate(this.db, updated!); + + this.last_checkpoint_lsn = lsn; + return true; + } + + async markSnapshotDone( + tables: storage.SourceTable[], + no_checkpoint_before_lsn: string + ): Promise { + const ids = tables.map((table) => table.id.toString()); + + await this.db.transaction(async (db) => { + await db.sql` + UPDATE source_tables + SET + snapshot_done = ${{ type: 'bool', value: true }} + WHERE + id IN ( + SELECT + (value ->> 0)::TEXT + FROM + jsonb_array_elements(${{ type: 'jsonb', value: ids }}) AS value + ); + `.execute(); + + if (no_checkpoint_before_lsn > this.no_checkpoint_before_lsn) { + this.no_checkpoint_before_lsn = no_checkpoint_before_lsn; + + await db.sql` + UPDATE sync_rules + SET + no_checkpoint_before = ${{ type: 'varchar', value: no_checkpoint_before_lsn }}, + last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }} + WHERE + id = ${{ type: 'int8', value: this.group_id }} + `.execute(); + } + }); + return tables.map((table) => { + const copy = new storage.SourceTable( + table.id, + table.connectionTag, + table.objectId, + table.schema, + table.table, + table.replicaIdColumns, + table.snapshotComplete + ); + copy.syncData = table.syncData; + copy.syncParameters = table.syncParameters; + return copy; + }); + } + + addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void { + this.write_checkpoint_batch.push({ + ...checkpoint, + sync_rules_id: this.group_id + }); + } + + protected async replicateBatch(db: WrappedConnection, batch: OperationBatch) { + let sizes: Map | undefined = undefined; + if (this.options.store_current_data && !this.options.skip_existing_rows) { + // We skip this step if we don't store current_data, since the sizes will + // always be small in that case. + + // With skipExistingRows, we don't load the full documents into memory, + // so we can also skip the size lookup step. + + // Find sizes of current_data documents, to assist in intelligent batching without + // exceeding memory limits. + const sizeLookups = batch.batch.map((r) => { + return { + source_table: r.record.sourceTable.id.toString(), + /** + * Encode to hex in order to pass a jsonb + */ + source_key: storage.serializeReplicaId(r.beforeId).toString('hex') + }; + }); + + sizes = new Map(); + + for await (const rows of db.streamRows<{ + source_table: string; + source_key: storage.ReplicaId; + data_size: number; + }>(sql` + WITH + filter_data AS ( + SELECT + decode(FILTER ->> 'source_key', 'hex') AS source_key, -- Decoding from hex to bytea + (FILTER ->> 'source_table') AS source_table_id + FROM + jsonb_array_elements(${{ type: 'jsonb', value: sizeLookups }}::jsonb) AS FILTER + ) + SELECT + pg_column_size(c.data) AS data_size, + c.source_table, + c.source_key + FROM + current_data c + JOIN filter_data f ON c.source_table = f.source_table_id + AND c.source_key = f.source_key + WHERE + c.group_id = ${{ type: 'int4', value: this.group_id }} + `)) { + for (const row of rows) { + const key = cacheKey(row.source_table, row.source_key); + sizes.set(key, row.data_size); + } + } + } + + // If set, we need to start a new transaction with this batch. + let resumeBatch: OperationBatch | null = null; + + // Now batch according to the sizes + // This is a single batch if storeCurrentData == false + for await (const b of batch.batched(sizes)) { + if (resumeBatch) { + // These operations need to be completed in a new transaction. + for (let op of b) { + resumeBatch.push(op); + } + continue; + } + + const lookups = b.map((r) => { + return { + source_table: r.record.sourceTable.id, + source_key: storage.serializeReplicaId(r.beforeId).toString('hex') + }; + }); + + const current_data_lookup = new Map(); + for await (const currentDataRows of db.streamRows({ + statement: /* sql */ ` + WITH + filter_data AS ( + SELECT + decode(FILTER ->> 'source_key', 'hex') AS source_key, -- Decoding from hex to bytea + (FILTER ->> 'source_table') AS source_table_id + FROM + jsonb_array_elements($1::jsonb) AS FILTER + ) + SELECT + --- With skipExistingRows, we only need to know whether or not the row exists. + ${this.options.skip_existing_rows ? `c.source_table, c.source_key` : 'c.*'} + FROM + current_data c + JOIN filter_data f ON c.source_table = f.source_table_id + AND c.source_key = f.source_key + WHERE + c.group_id = $2 + `, + params: [ + { + type: 'jsonb', + value: lookups + }, + { + type: 'int8', + value: this.group_id + } + ] + })) { + for (const row of currentDataRows) { + const decoded = this.options.skip_existing_rows + ? pick(CurrentData, ['source_key', 'source_table']).decode(row) + : CurrentData.decode(row); + current_data_lookup.set( + encodedCacheKey(decoded.source_table, decoded.source_key), + decoded as CurrentDataDecoded + ); + } + } + + let persistedBatch: PostgresPersistedBatch | null = new PostgresPersistedBatch({ + group_id: this.group_id, + ...this.options.batch_limits + }); + + for (const op of b) { + // These operations need to be completed in a new transaction + if (resumeBatch) { + resumeBatch.push(op); + continue; + } + + const currentData = current_data_lookup.get(op.internalBeforeKey) ?? null; + if (currentData != null) { + // If it will be used again later, it will be set again using nextData below + current_data_lookup.delete(op.internalBeforeKey); + } + const nextData = await this.saveOperation(persistedBatch!, op, currentData); + if (nextData != null) { + // Update our current_data and size cache + current_data_lookup.set(op.internalAfterKey!, nextData); + sizes?.set(op.internalAfterKey!, nextData.data.byteLength); + } + + if (persistedBatch!.shouldFlushTransaction()) { + await persistedBatch!.flush(db); + // The operations stored in this batch will be processed in the `resumeBatch` + persistedBatch = null; + // Return the remaining entries for the next resume transaction + resumeBatch = new OperationBatch(this.options.batch_limits); + } + } + + if (persistedBatch) { + /** + * The operations were less than the max size if here. Flush now. + * `persistedBatch` will be `null` if the operations should be flushed in a new transaction. + */ + await persistedBatch.flush(db); + } + } + return resumeBatch; + } + + protected async saveOperation( + persistedBatch: PostgresPersistedBatch, + operation: RecordOperation, + currentData?: CurrentDataDecoded | null + ) { + const record = operation.record; + // We store bytea colums for source keys + const beforeId = operation.beforeId; + const afterId = operation.afterId; + let after = record.after; + const sourceTable = record.sourceTable; + + let existingBuckets: CurrentBucket[] = []; + let newBuckets: CurrentBucket[] = []; + let existingLookups: Buffer[] = []; + let newLookups: Buffer[] = []; + + if (this.options.skip_existing_rows) { + if (record.tag == storage.SaveOperationTag.INSERT) { + if (currentData != null) { + // Initial replication, and we already have the record. + // This may be a different version of the record, but streaming replication + // will take care of that. + // Skip the insert here. + return null; + } + } else { + throw new Error(`${record.tag} not supported with skipExistingRows: true`); + } + } + + if (record.tag == storage.SaveOperationTag.UPDATE) { + const result = currentData; + if (result == null) { + // Not an error if we re-apply a transaction + existingBuckets = []; + existingLookups = []; + // Log to help with debugging if there was a consistency issue + if (this.options.store_current_data) { + logger.warn( + `Cannot find previous record for update on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}` + ); + } + } else { + existingBuckets = result.buckets; + existingLookups = result.lookups; + if (this.options.store_current_data) { + const data = storage.deserializeBson(result.data) as sync_rules.SqliteRow; + after = storage.mergeToast(after!, data); + } + } + } else if (record.tag == storage.SaveOperationTag.DELETE) { + const result = currentData; + if (result == null) { + // Not an error if we re-apply a transaction + existingBuckets = []; + existingLookups = []; + // Log to help with debugging if there was a consistency issue + if (this.options.store_current_data) { + logger.warn( + `Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}` + ); + } + } else { + existingBuckets = result.buckets; + existingLookups = result.lookups; + } + } + + let afterData: Buffer | undefined; + if (afterId != null && !this.options.store_current_data) { + afterData = storage.serializeBson({}); + } else if (afterId != null) { + try { + afterData = storage.serializeBson(after); + if (afterData!.byteLength > MAX_ROW_SIZE) { + throw new Error(`Row too large: ${afterData?.byteLength}`); + } + } catch (e) { + // Replace with empty values, equivalent to TOAST values + after = Object.fromEntries( + Object.entries(after!).map(([key, value]) => { + return [key, undefined]; + }) + ); + afterData = storage.serializeBson(after); + + container.reporter.captureMessage( + `Data too big on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${e.message}`, + { + level: errors.ErrorSeverity.WARNING, + metadata: { + replication_slot: this.options.slot_name, + table: record.sourceTable.qualifiedName + } + } + ); + } + } + + // 2. Save bucket data + if (beforeId != null && (afterId == null || !storage.replicaIdEquals(beforeId, afterId))) { + // Source ID updated + if (sourceTable.syncData) { + // Delete old record + persistedBatch.saveBucketData({ + source_key: beforeId, + table: sourceTable, + before_buckets: existingBuckets, + evaluated: [] + }); + // Clear this, so we don't also try to REMOVE for the new id + existingBuckets = []; + } + + if (sourceTable.syncParameters) { + // Delete old parameters + persistedBatch.saveParameterData({ + source_key: beforeId, + table: sourceTable, + evaluated: [], + existing_lookups: existingLookups + }); + existingLookups = []; + } + } + + // If we re-apply a transaction, we can end up with a partial row. + // + // We may end up with toasted values, which means the record is not quite valid. + // However, it will be valid by the end of the transaction. + // + // In this case, we don't save the op, but we do save the current data. + if (afterId && after && utils.isCompleteRow(this.options.store_current_data, after)) { + // Insert or update + if (sourceTable.syncData) { + const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({ + record: after, + sourceTable + }); + + for (const error of syncErrors) { + container.reporter.captureMessage( + `Failed to evaluate data query on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.error}`, + { + level: errors.ErrorSeverity.WARNING, + metadata: { + replication_slot: this.options.slot_name, + table: record.sourceTable.qualifiedName + } + } + ); + logger.error( + `Failed to evaluate data query on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.error}` + ); + } + + // Save new one + persistedBatch.saveBucketData({ + source_key: afterId, + evaluated, + table: sourceTable, + before_buckets: existingBuckets + }); + + newBuckets = evaluated.map((e) => { + return { + bucket: e.bucket, + table: e.table, + id: e.id + }; + }); + } + + if (sourceTable.syncParameters) { + // Parameters + const { results: paramEvaluated, errors: paramErrors } = this.sync_rules.evaluateParameterRowWithErrors( + sourceTable, + after + ); + + for (let error of paramErrors) { + container.reporter.captureMessage( + `Failed to evaluate parameter query on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.error}`, + { + level: errors.ErrorSeverity.WARNING, + metadata: { + replication_slot: this.options.slot_name, + table: record.sourceTable.qualifiedName + } + } + ); + logger.error( + `Failed to evaluate parameter query on ${record.sourceTable.qualifiedName}.${after.id}: ${error.error}` + ); + } + + persistedBatch.saveParameterData({ + source_key: afterId, + table: sourceTable, + evaluated: paramEvaluated, + existing_lookups: existingLookups + }); + + newLookups = paramEvaluated.map((p) => { + return storage.serializeLookupBuffer(p.lookup); + }); + } + } + + let result: CurrentDataDecoded | null = null; + + // 5. TOAST: Update current data and bucket list. + if (afterId) { + // Insert or update + result = { + source_key: afterId, + group_id: BigInt(this.group_id), + data: afterData!, + source_table: sourceTable.id, + buckets: newBuckets, + lookups: newLookups + }; + persistedBatch.upsertCurrentData(result); + } + + if (afterId == null || !storage.replicaIdEquals(beforeId, afterId)) { + // Either a delete (afterId == null), or replaced the old replication id + persistedBatch.deleteCurrentData({ + source_table_id: record.sourceTable.id, + source_key: beforeId! + }); + } + + return result; + } + + /** + * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable} + * TODO maybe share this with an abstract class + */ + protected getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] { + return this.sync_rules.event_descriptors.filter((evt) => + [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table)) + ); + } + + protected async withReplicationTransaction(callback: (tx: WrappedConnection) => Promise): Promise { + try { + return await this.db.transaction(async (db) => { + return await callback(db); + }); + } finally { + await this.db.sql` + UPDATE sync_rules + SET + last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }} + WHERE + id = ${{ type: 'int8', value: this.group_id }} + `.execute(); + } + } +} + +/** + * Uses Postgres' NOTIFY functionality to update different processes when the + * active checkpoint has been updated. + */ +export const notifySyncRulesUpdate = async (db: DatabaseClient, update: StatefulCheckpointDecoded) => { + if (update.state != storage.SyncRuleState.ACTIVE) { + return; + } + + await db.query({ + statement: `NOTIFY ${NOTIFICATION_CHANNEL}, '${models.ActiveCheckpointNotification.encode({ active_checkpoint: update })}'` + }); +}; diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts new file mode 100644 index 000000000..1fa23cdcb --- /dev/null +++ b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts @@ -0,0 +1,438 @@ +import { logger } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import * as sync_rules from '@powersync/service-sync-rules'; +import { BatchLimits, models } from '../../types/types.js'; +import { replicaIdToSubkey } from '../../utils/bson.js'; +import { WrappedConnection } from '../../utils/connection/WrappedConnection.js'; + +export type SaveBucketDataOptions = { + /** + * This value will be serialized into a BSON Byte array for storage + */ + source_key: storage.ReplicaId; + table: storage.SourceTable; + before_buckets: models.CurrentBucket[]; + evaluated: sync_rules.EvaluatedRow[]; +}; + +export type SaveParameterDataOptions = { + source_key: storage.ReplicaId; + table: storage.SourceTable; + evaluated: sync_rules.EvaluatedParameters[]; + existing_lookups: Buffer[]; +}; + +export type DeleteCurrentDataOptions = { + source_table_id: bigint; + source_key: storage.ReplicaId; +}; + +export type PostgresPersistedBatchOptions = BatchLimits & { + group_id: number; +}; + +const MAX_TRANSACTION_BATCH_SIZE = 30_000_000; + +const MAX_TRANSACTION_DOC_COUNT = 2_000; + +export class PostgresPersistedBatch { + group_id: number; + + /** + * Very rough estimate of current operations size in bytes + */ + currentSize: number; + + readonly maxTransactionBatchSize: number; + readonly maxTransactionDocCount: number; + + /** + * Ordered set of bucket_data insert operation parameters + */ + protected bucketDataInserts: models.BucketData[]; + protected parameterDataInserts: models.BucketParameters[]; + protected currentDataDeletes: Pick[]; + /** + * This is stored as a map to avoid multiple inserts (or conflicts) for the same key + */ + protected currentDataInserts: Map; + + constructor(options: PostgresPersistedBatchOptions) { + this.group_id = options.group_id; + + this.maxTransactionBatchSize = options.max_estimated_size ?? MAX_TRANSACTION_BATCH_SIZE; + this.maxTransactionDocCount = options.max_record_count ?? MAX_TRANSACTION_DOC_COUNT; + + this.bucketDataInserts = []; + this.parameterDataInserts = []; + this.currentDataDeletes = []; + this.currentDataInserts = new Map(); + this.currentSize = 0; + } + + saveBucketData(options: SaveBucketDataOptions) { + const remaining_buckets = new Map(); + for (const b of options.before_buckets) { + const key = currentBucketKey(b); + remaining_buckets.set(key, b); + } + + const dchecksum = utils.hashDelete(replicaIdToSubkey(options.table.id, options.source_key)); + + const serializedSourceKey = storage.serializeReplicaId(options.source_key); + const hexSourceKey = serializedSourceKey.toString('hex'); + + for (const k of options.evaluated) { + const key = currentBucketKey(k); + remaining_buckets.delete(key); + + const data = JSONBig.stringify(k.data); + const checksum = utils.hashData(k.table, k.id, data); + + this.bucketDataInserts.push({ + group_id: this.group_id, + bucket_name: k.bucket, + op: models.OpType.PUT, + source_table: options.table.id, + source_key: hexSourceKey, + table_name: k.table, + row_id: k.id, + checksum, + data, + op_id: 0, // Will use nextval of sequence + target_op: null + }); + + this.currentSize += k.bucket.length * 2 + data.length * 2 + hexSourceKey.length * 2 + 100; + } + + for (const bd of remaining_buckets.values()) { + // REMOVE operation + this.bucketDataInserts.push({ + group_id: this.group_id, + bucket_name: bd.bucket, + op: models.OpType.REMOVE, + source_table: options.table.id, + source_key: hexSourceKey, + table_name: bd.table, + row_id: bd.id, + checksum: dchecksum, + op_id: 0, // Will use nextval of sequence + target_op: null, + data: null + }); + this.currentSize += bd.bucket.length * 2 + hexSourceKey.length * 2 + 100; + } + } + + saveParameterData(options: SaveParameterDataOptions) { + // This is similar to saving bucket data. + // A key difference is that we don't need to keep the history intact. + // We do need to keep track of recent history though - enough that we can get consistent data for any specific checkpoint. + // Instead of storing per bucket id, we store per "lookup". + // A key difference is that we don't need to store or keep track of anything per-bucket - the entire record is + // either persisted or removed. + // We also don't need to keep history intact. + const { source_key, table, evaluated, existing_lookups } = options; + const serializedSourceKey = storage.serializeReplicaId(source_key); + const hexSourceKey = serializedSourceKey.toString('hex'); + const remaining_lookups = new Map(); + for (const l of existing_lookups) { + remaining_lookups.set(l.toString('base64'), l); + } + + // 1. Insert new entries + for (const result of evaluated) { + const binLookup = storage.serializeLookupBuffer(result.lookup); + const base64 = binLookup.toString('base64'); + remaining_lookups.delete(base64); + const hexLookup = binLookup.toString('hex'); + const serializedBucketParameters = JSONBig.stringify(result.bucket_parameters); + this.parameterDataInserts.push({ + group_id: this.group_id, + source_table: table.id, + source_key: hexSourceKey, + bucket_parameters: serializedBucketParameters, + id: 0, // auto incrementing id + lookup: hexLookup + }); + this.currentSize += hexLookup.length * 2 + serializedBucketParameters.length * 2 + hexSourceKey.length * 2 + 100; + } + + // 2. "REMOVE" entries for any lookup not touched. + for (const lookup of remaining_lookups.values()) { + const hexLookup = lookup.toString('hex'); + this.parameterDataInserts.push({ + group_id: this.group_id, + source_table: table.id, + source_key: hexSourceKey, + bucket_parameters: JSON.stringify([]), + id: 0, // auto incrementing id + lookup: hexLookup + }); + this.currentSize += hexLookup.length * 2 + hexSourceKey.length * 2 + 100; + } + } + + deleteCurrentData(options: DeleteCurrentDataOptions) { + const serializedReplicaId = storage.serializeReplicaId(options.source_key); + this.currentDataDeletes.push({ + group_id: this.group_id, + source_table: options.source_table_id.toString(), + source_key: serializedReplicaId.toString('hex') + }); + this.currentSize += serializedReplicaId.byteLength * 2 + 100; + } + + upsertCurrentData(options: models.CurrentDataDecoded) { + const { source_table, source_key, buckets } = options; + + const serializedReplicaId = storage.serializeReplicaId(source_key); + const hexReplicaId = serializedReplicaId.toString('hex'); + const serializedBuckets = JSONBig.stringify(options.buckets); + + /** + * Only track the last unique ID for this current_data record. + * Applying multiple items in the flush method could cause an + * " + * ON CONFLICT DO UPDATE command cannot affect row a second time + * " + * error. + */ + const key = `${this.group_id}-${source_table}-${hexReplicaId}`; + + this.currentDataInserts.set(key, { + group_id: this.group_id, + source_table: source_table, + source_key: hexReplicaId, + buckets: serializedBuckets, + data: options.data.toString('hex'), + lookups: options.lookups.map((l) => l.toString('hex')) + }); + + this.currentSize += + (options.data?.byteLength ?? 0) + + serializedReplicaId.byteLength + + buckets.length * 2 + + options.lookups.reduce((total, l) => { + return total + l.byteLength; + }, 0) + + 100; + } + + shouldFlushTransaction() { + return ( + this.currentSize >= this.maxTransactionBatchSize || + this.bucketDataInserts.length >= this.maxTransactionDocCount || + this.currentDataInserts.size >= this.maxTransactionDocCount || + this.currentDataDeletes.length >= this.maxTransactionDocCount || + this.parameterDataInserts.length >= this.maxTransactionDocCount + ); + } + + async flush(db: WrappedConnection) { + logger.info( + `powersync_${this.group_id} Flushed ${this.bucketDataInserts.length} + ${this.parameterDataInserts.length} + ${ + this.currentDataInserts.size + this.currentDataDeletes.length + } updates, ${Math.round(this.currentSize / 1024)}kb.` + ); + + await this.flushBucketData(db); + await this.flushParameterData(db); + await this.flushCurrentData(db); + + this.bucketDataInserts = []; + this.parameterDataInserts = []; + this.currentDataDeletes = []; + this.currentDataInserts = new Map(); + this.currentSize = 0; + } + + protected async flushBucketData(db: WrappedConnection) { + if (this.bucketDataInserts.length > 0) { + await db.sql` + WITH + parsed_data AS ( + SELECT + group_id, + bucket_name, + source_table, + decode(source_key, 'hex') AS source_key, -- Decode hex to bytea + table_name, + op, + row_id, + checksum, + data, + target_op + FROM + jsonb_to_recordset(${{ type: 'jsonb', value: this.bucketDataInserts }}::jsonb) AS t ( + group_id integer, + bucket_name text, + source_table text, + source_key text, -- Input as hex string + table_name text, + op text, + row_id text, + checksum bigint, + data text, + target_op bigint + ) + ) + INSERT INTO + bucket_data ( + group_id, + bucket_name, + op_id, + op, + source_table, + source_key, + table_name, + row_id, + checksum, + data, + target_op + ) + SELECT + group_id, + bucket_name, + nextval('op_id_sequence'), + op, + source_table, + source_key, -- Already decoded + table_name, + row_id, + checksum, + data, + target_op + FROM + parsed_data; + `.execute(); + } + } + + protected async flushParameterData(db: WrappedConnection) { + if (this.parameterDataInserts.length > 0) { + await db.sql` + WITH + parsed_data AS ( + SELECT + group_id, + source_table, + decode(source_key, 'hex') AS source_key, -- Decode hex to bytea + decode(lookup, 'hex') AS lookup, -- Decode hex to bytea + bucket_parameters::jsonb AS bucket_parameters + FROM + jsonb_to_recordset(${{ type: 'jsonb', value: this.parameterDataInserts }}::jsonb) AS t ( + group_id integer, + source_table text, + source_key text, -- Input as hex string + lookup text, -- Input as hex string + bucket_parameters text -- Input as stringified jsonb + ) + ) + INSERT INTO + bucket_parameters ( + group_id, + source_table, + source_key, + lookup, + bucket_parameters + ) + SELECT + group_id, + source_table, + source_key, -- Already decoded + lookup, -- Already decoded + bucket_parameters + FROM + parsed_data; + `.execute(); + } + } + + protected async flushCurrentData(db: WrappedConnection) { + if (this.currentDataInserts.size > 0) { + await db.sql` + WITH + parsed_data AS ( + SELECT + group_id, + source_table, + decode(source_key, 'hex') AS source_key, -- Decode hex to bytea + buckets::jsonb AS buckets, + decode(data, 'hex') AS data, -- Decode hex to bytea + ARRAY( + SELECT + decode((value ->> 0)::TEXT, 'hex') + FROM + jsonb_array_elements(lookups::jsonb) AS value + ) AS lookups -- Decode array of hex strings to bytea[] + FROM + jsonb_to_recordset(${{ + type: 'jsonb', + value: Array.from(this.currentDataInserts.values()) + }}::jsonb) AS t ( + group_id integer, + source_table text, + source_key text, -- Input as hex string + buckets text, + data text, -- Input as hex string + lookups text -- Input as stringified JSONB array of hex strings + ) + ) + INSERT INTO + current_data ( + group_id, + source_table, + source_key, + buckets, + data, + lookups + ) + SELECT + group_id, + source_table, + source_key, -- Already decoded + buckets, + data, -- Already decoded + lookups -- Already decoded + FROM + parsed_data + ON CONFLICT (group_id, source_table, source_key) DO + UPDATE + SET + buckets = EXCLUDED.buckets, + data = EXCLUDED.data, + lookups = EXCLUDED.lookups; + `.execute(); + } + + if (this.currentDataDeletes.length > 0) { + await db.sql` + WITH + conditions AS ( + SELECT + group_id, + source_table, + decode(source_key, 'hex') AS source_key -- Decode hex to bytea + FROM + jsonb_to_recordset(${{ type: 'jsonb', value: this.currentDataDeletes }}::jsonb) AS t ( + group_id integer, + source_table text, + source_key text -- Input as hex string + ) + ) + DELETE FROM current_data USING conditions + WHERE + current_data.group_id = conditions.group_id + AND current_data.source_table = conditions.source_table + AND current_data.source_key = conditions.source_key; + `.execute(); + } + } +} + +export function currentBucketKey(b: models.CurrentBucket) { + return `${b.bucket}/${b.table}/${b.id}`; +} diff --git a/modules/module-postgres-storage/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts b/modules/module-postgres-storage/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts new file mode 100644 index 000000000..529e861b0 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts @@ -0,0 +1,179 @@ +import * as framework from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { models } from '../../types/types.js'; +import { DatabaseClient } from '../../utils/connection/DatabaseClient.js'; + +export type PostgresCheckpointAPIOptions = { + db: DatabaseClient; + mode: storage.WriteCheckpointMode; +}; + +export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI { + readonly db: DatabaseClient; + private _mode: storage.WriteCheckpointMode; + + constructor(options: PostgresCheckpointAPIOptions) { + this.db = options.db; + this._mode = options.mode; + } + + get writeCheckpointMode() { + return this._mode; + } + + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { + this._mode = mode; + } + + async batchCreateCustomWriteCheckpoints(checkpoints: storage.CustomWriteCheckpointOptions[]): Promise { + return batchCreateCustomWriteCheckpoints(this.db, checkpoints); + } + + async createCustomWriteCheckpoint(options: storage.CustomWriteCheckpointOptions): Promise { + if (this.writeCheckpointMode !== storage.WriteCheckpointMode.CUSTOM) { + throw new framework.errors.ValidationError( + `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"` + ); + } + + const { checkpoint, user_id, sync_rules_id } = options; + const row = await this.db.sql` + INSERT INTO + custom_write_checkpoints (user_id, write_checkpoint, sync_rules_id) + VALUES + ( + ${{ type: 'varchar', value: user_id }}, + ${{ type: 'int8', value: checkpoint }}, + ${{ type: 'int4', value: sync_rules_id }} + ) + ON CONFLICT DO + UPDATE + SET + write_checkpoint = EXCLUDED.write_checkpoint + RETURNING + *; + ` + .decoded(models.CustomWriteCheckpoint) + .first(); + return row!.write_checkpoint; + } + + async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { + if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) { + throw new framework.errors.ValidationError( + `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"` + ); + } + + const row = await this.db.sql` + INSERT INTO + write_checkpoints (user_id, lsns, write_checkpoint) + VALUES + ( + ${{ type: 'varchar', value: checkpoint.user_id }}, + ${{ type: 'jsonb', value: checkpoint.heads }}, + ${{ type: 'int8', value: 1 }} + ) + ON CONFLICT (user_id) DO + UPDATE + SET + write_checkpoint = write_checkpoints.write_checkpoint + 1, + lsns = EXCLUDED.lsns + RETURNING + *; + ` + .decoded(models.WriteCheckpoint) + .first(); + return row!.write_checkpoint; + } + + async lastWriteCheckpoint(filters: storage.LastWriteCheckpointFilters): Promise { + switch (this.writeCheckpointMode) { + case storage.WriteCheckpointMode.CUSTOM: + if (false == 'sync_rules_id' in filters) { + throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`); + } + return this.lastCustomWriteCheckpoint(filters as storage.CustomWriteCheckpointFilters); + case storage.WriteCheckpointMode.MANAGED: + if (false == 'heads' in filters) { + throw new framework.errors.ValidationError( + `Replication HEAD is required for managed Write Checkpoint filtering` + ); + } + return this.lastManagedWriteCheckpoint(filters as storage.ManagedWriteCheckpointFilters); + } + } + + protected async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters) { + const { user_id, sync_rules_id } = filters; + const row = await this.db.sql` + SELECT + * + FROM + custom_write_checkpoints + WHERE + user_id = ${{ type: 'varchar', value: user_id }} + AND sync_rules_id = ${{ type: 'int4', value: sync_rules_id }} + ` + .decoded(models.CustomWriteCheckpoint) + .first(); + return row?.write_checkpoint ?? null; + } + + protected async lastManagedWriteCheckpoint(filters: storage.ManagedWriteCheckpointFilters) { + const { user_id, heads } = filters; + // TODO: support multiple heads when we need to support multiple connections + const lsn = heads['1']; + if (lsn == null) { + // Can happen if we haven't replicated anything yet. + return null; + } + const row = await this.db.sql` + SELECT + * + FROM + write_checkpoints + WHERE + user_id = ${{ type: 'varchar', value: user_id }} + AND lsns ->> '1' <= ${{ type: 'varchar', value: lsn }}; + ` + .decoded(models.WriteCheckpoint) + .first(); + return row?.write_checkpoint ?? null; + } +} + +export async function batchCreateCustomWriteCheckpoints( + db: DatabaseClient, + checkpoints: storage.CustomWriteCheckpointOptions[] +): Promise { + if (!checkpoints.length) { + return; + } + + await db.sql` + WITH + json_data AS ( + SELECT + jsonb_array_elements(${{ type: 'jsonb', value: JSONBig.stringify(checkpoints) }}) AS + CHECKPOINT + ) + INSERT INTO + custom_write_checkpoints (user_id, write_checkpoint, sync_rules_id) + SELECT + CHECKPOINT ->> 'user_id'::varchar, + ( + CHECKPOINT ->> 'checkpoint' + )::int8, + ( + CHECKPOINT ->> 'sync_rules_id' + )::int4 + FROM + json_data + ON CONFLICT (user_id, sync_rules_id) DO + UPDATE + SET + write_checkpoint = EXCLUDED.write_checkpoint; + `.execute(); +} diff --git a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts new file mode 100644 index 000000000..e472ac681 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts @@ -0,0 +1,67 @@ +import { logger } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { SqlSyncRules } from '@powersync/service-sync-rules'; +import { PostgresLockManager } from '../../locks/PostgresLockManager.js'; +import { models } from '../../types/types.js'; +import { DatabaseClient } from '../../utils/connection/DatabaseClient.js'; + +export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncRulesContent { + public readonly slot_name: string; + + public readonly id: number; + public readonly sync_rules_content: string; + public readonly last_checkpoint_lsn: string | null; + public readonly last_fatal_error: string | null; + public readonly last_keepalive_ts: Date | null; + public readonly last_checkpoint_ts: Date | null; + current_lock: storage.ReplicationLock | null = null; + + constructor( + private db: DatabaseClient, + row: models.SyncRulesDecoded + ) { + this.id = Number(row.id); + this.sync_rules_content = row.content; + this.last_checkpoint_lsn = row.last_checkpoint_lsn; + this.slot_name = row.slot_name; + this.last_fatal_error = row.last_fatal_error; + this.last_checkpoint_ts = row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null; + this.last_keepalive_ts = row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null; + } + + parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules { + return { + id: this.id, + slot_name: this.slot_name, + sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, options) + }; + } + + async lock(): Promise { + const manager = new PostgresLockManager({ + db: this.db, + name: `sync_rules_${this.id}_${this.slot_name}` + }); + const lockHandle = await manager.acquire(); + if (!lockHandle) { + throw new Error(`Sync rules: ${this.id} have been locked by another process for replication.`); + } + + const interval = setInterval(async () => { + try { + await lockHandle.refresh(); + } catch (e) { + logger.error('Failed to refresh lock', e); + clearInterval(interval); + } + }, 30_130); + + return (this.current_lock = { + sync_rules_id: this.id, + release: async () => { + clearInterval(interval); + return lockHandle.release(); + } + }); + } +} diff --git a/modules/module-postgres-storage/src/types/codecs.ts b/modules/module-postgres-storage/src/types/codecs.ts new file mode 100644 index 000000000..c28d45279 --- /dev/null +++ b/modules/module-postgres-storage/src/types/codecs.ts @@ -0,0 +1,32 @@ +import * as t from 'ts-codec'; + +/** + * Wraps a codec which is encoded to a JSON string + */ +export const jsonb = (subCodec: t.Codec) => + t.codec( + 'jsonb', + (decoded: Decoded) => { + return JSON.stringify(subCodec.encode(decoded) as any); + }, + (encoded: string | { data: string }) => { + const s = typeof encoded == 'object' ? encoded.data : encoded; + return subCodec.decode(JSON.parse(s)); + } + ); + +export const bigint = t.codec( + 'bigint', + (decoded: BigInt) => { + return decoded.toString(); + }, + (encoded: string | number) => { + return BigInt(encoded); + } +); + +export const uint8array = t.codec( + 'uint8array', + (d) => d, + (e) => e +); diff --git a/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts b/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts new file mode 100644 index 000000000..adbbc6dd1 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts @@ -0,0 +1,15 @@ +import * as t from 'ts-codec'; +import { bigint } from '../codecs.js'; + +/** + * Notification payload sent via Postgres' NOTIFY API. + * + */ +export const ActiveCheckpoint = t.object({ + id: bigint, + last_checkpoint: t.Null.or(bigint), + last_checkpoint_lsn: t.Null.or(t.string) +}); + +export type ActiveCheckpoint = t.Encoded; +export type ActiveCheckpointDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/ActiveCheckpointNotification.ts b/modules/module-postgres-storage/src/types/models/ActiveCheckpointNotification.ts new file mode 100644 index 000000000..e2d49989b --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/ActiveCheckpointNotification.ts @@ -0,0 +1,14 @@ +import * as t from 'ts-codec'; +import { jsonb } from '../codecs.js'; +import { ActiveCheckpoint } from './ActiveCheckpoint.js'; + +export const ActiveCheckpointPayload = t.object({ + active_checkpoint: ActiveCheckpoint +}); + +export type ActiveCheckpointPayload = t.Encoded; +export type ActiveCheckpointPayloadDecoded = t.Decoded; + +export const ActiveCheckpointNotification = jsonb(ActiveCheckpointPayload); +export type ActiveCheckpointNotification = t.Encoded; +export type ActiveCheckpointNotificationDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/BucketData.ts b/modules/module-postgres-storage/src/types/models/BucketData.ts new file mode 100644 index 000000000..401031190 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/BucketData.ts @@ -0,0 +1,27 @@ +import { framework } from '@powersync/service-core'; +import * as t from 'ts-codec'; +import { bigint } from '../codecs.js'; + +export enum OpType { + PUT = 'PUT', + REMOVE = 'REMOVE', + MOVE = 'MOVE', + CLEAR = 'CLEAR' +} + +export const BucketData = t.object({ + group_id: bigint, + bucket_name: t.string, + op_id: bigint, + op: t.Enum(OpType), + source_table: t.Null.or(t.string), + source_key: t.Null.or(framework.codecs.buffer), + table_name: t.string.or(t.Null), + row_id: t.string.or(t.Null), + checksum: bigint, + data: t.Null.or(t.string), + target_op: t.Null.or(bigint) +}); + +export type BucketData = t.Encoded; +export type BucketDataDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/BucketParameters.ts b/modules/module-postgres-storage/src/types/models/BucketParameters.ts new file mode 100644 index 000000000..5e03bf812 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/BucketParameters.ts @@ -0,0 +1,16 @@ +import { framework } from '@powersync/service-core'; +import * as t from 'ts-codec'; +import { bigint, jsonb } from '../codecs.js'; +import { SQLiteJSONRecord } from './SQLiteJSONValue.js'; + +export const BucketParameters = t.object({ + id: bigint, + group_id: t.number, + source_table: t.string, + source_key: framework.codecs.buffer, + lookup: framework.codecs.buffer, + bucket_parameters: jsonb(t.array(SQLiteJSONRecord)) +}); + +export type BucketParameters = t.Encoded; +export type BucketParametersDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/CurrentData.ts b/modules/module-postgres-storage/src/types/models/CurrentData.ts new file mode 100644 index 000000000..6d4d7e3fe --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/CurrentData.ts @@ -0,0 +1,24 @@ +import { framework } from '@powersync/service-core'; +import * as t from 'ts-codec'; +import { bigint, jsonb } from '../codecs.js'; + +export const CurrentBucket = t.object({ + bucket: t.string, + table: t.string, + id: t.string +}); + +export type CurrentBucket = t.Encoded; +export type CurrentBucketDecoded = t.Decoded; + +export const CurrentData = t.object({ + buckets: jsonb(t.array(CurrentBucket)), + data: framework.codecs.buffer, + group_id: bigint, + lookups: t.array(framework.codecs.buffer), + source_key: framework.codecs.buffer, + source_table: t.string +}); + +export type CurrentData = t.Encoded; +export type CurrentDataDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/Instance.ts b/modules/module-postgres-storage/src/types/models/Instance.ts new file mode 100644 index 000000000..c1e94e2db --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/Instance.ts @@ -0,0 +1,8 @@ +import * as t from 'ts-codec'; + +export const Instance = t.object({ + id: t.string +}); + +export type Instance = t.Encoded; +export type InstanceDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/SQLiteJSONValue.ts b/modules/module-postgres-storage/src/types/models/SQLiteJSONValue.ts new file mode 100644 index 000000000..84b4d789c --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/SQLiteJSONValue.ts @@ -0,0 +1,10 @@ +import * as t from 'ts-codec'; +import { bigint } from '../codecs.js'; + +export const SQLiteJSONValue = t.number.or(t.string).or(bigint).or(t.Null); +export type SQLiteJSONValue = t.Encoded; +export type SQLiteJSONValueDecoded = t.Decoded; + +export const SQLiteJSONRecord = t.record(SQLiteJSONValue); +export type SQLiteJSONRecord = t.Encoded; +export type SQLiteJSONRecordDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/SourceTable.ts b/modules/module-postgres-storage/src/types/models/SourceTable.ts new file mode 100644 index 000000000..4613eb0dd --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/SourceTable.ts @@ -0,0 +1,28 @@ +import * as t from 'ts-codec'; +import { bigint, jsonb } from '../codecs.js'; + +export const ColumnDescriptor = t.object({ + name: t.string, + /** + * The type of the column ie VARCHAR, INT, etc + */ + type: t.string.optional(), + /** + * Some data sources have a type id that can be used to identify the type of the column + */ + typeId: t.number.optional() +}); + +export const SourceTable = t.object({ + id: t.string, + group_id: bigint, + connection_id: bigint, + relation_id: t.Null.or(bigint).or(t.string), + schema_name: t.string, + table_name: t.string, + replica_id_columns: t.Null.or(jsonb(t.array(ColumnDescriptor))), + snapshot_done: t.boolean +}); + +export type SourceTable = t.Encoded; +export type SourceTableDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/SyncRules.ts b/modules/module-postgres-storage/src/types/models/SyncRules.ts new file mode 100644 index 000000000..94b486983 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/SyncRules.ts @@ -0,0 +1,50 @@ +import { framework, storage } from '@powersync/service-core'; +import * as t from 'ts-codec'; +import { bigint } from '../codecs.js'; + +export const SyncRules = t.object({ + id: bigint, + state: t.Enum(storage.SyncRuleState), + /** + * True if initial snapshot has been replicated. + * + * Can only be false if state == PROCESSING. + */ + snapshot_done: t.boolean, + /** + * The last consistent checkpoint. + * + * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction. + */ + last_checkpoint: t.Null.or(bigint), + /** + * The LSN associated with the last consistent checkpoint. + */ + last_checkpoint_lsn: t.Null.or(t.string), + /** + * If set, no new checkpoints may be created < this value. + */ + no_checkpoint_before: t.Null.or(t.string), + slot_name: t.string, + /** + * Last time we persisted a checkpoint. + * + * This may be old if no data is incoming. + */ + last_checkpoint_ts: t.Null.or(framework.codecs.date), + /** + * Last time we persisted a checkpoint or keepalive. + * + * This should stay fairly current while replicating. + */ + last_keepalive_ts: t.Null.or(framework.codecs.date), + /** + * If an error is stopping replication, it will be stored here. + */ + last_fatal_error: t.Null.or(t.string), + keepalive_op: t.Null.or(t.string), + content: t.string +}); + +export type SyncRules = t.Encoded; +export type SyncRulesDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/WriteCheckpoint.ts b/modules/module-postgres-storage/src/types/models/WriteCheckpoint.ts new file mode 100644 index 000000000..1fd74ddc2 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/WriteCheckpoint.ts @@ -0,0 +1,20 @@ +import * as t from 'ts-codec'; +import { bigint, jsonb } from '../codecs.js'; + +export const WriteCheckpoint = t.object({ + user_id: t.string, + lsns: jsonb(t.record(t.string)), + write_checkpoint: bigint +}); + +export type WriteCheckpoint = t.Encoded; +export type WriteCheckpointDecoded = t.Decoded; + +export const CustomWriteCheckpoint = t.object({ + user_id: t.string, + write_checkpoint: bigint, + sync_rules_id: bigint +}); + +export type CustomWriteCheckpoint = t.Encoded; +export type CustomWriteCheckpointDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/models-index.ts b/modules/module-postgres-storage/src/types/models/models-index.ts new file mode 100644 index 000000000..caf39b3b3 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/models-index.ts @@ -0,0 +1,10 @@ +export * from './ActiveCheckpoint.js'; +export * from './ActiveCheckpointNotification.js'; +export * from './BucketData.js'; +export * from './BucketParameters.js'; +export * from './CurrentData.js'; +export * from './Instance.js'; +export * from './SourceTable.js'; +export * from './SQLiteJSONValue.js'; +export * from './SyncRules.js'; +export * from './WriteCheckpoint.js'; diff --git a/modules/module-postgres-storage/src/types/types.ts b/modules/module-postgres-storage/src/types/types.ts new file mode 100644 index 000000000..30fe263f6 --- /dev/null +++ b/modules/module-postgres-storage/src/types/types.ts @@ -0,0 +1,69 @@ +import * as pg_wire from '@powersync/service-jpgwire'; +import * as pg_types from '@powersync/service-module-postgres/types'; +import * as t from 'ts-codec'; +export * as models from './models/models-index.js'; + +export const MAX_BATCH_RECORD_COUNT = 2000; + +export const MAX_BATCH_ESTIMATED_SIZE = 5_000_000; + +export const MAX_BATCH_CURRENT_DATA_SIZE = 50_000_000; + +export const BatchLimits = t.object({ + /** + * Maximum size of operations we write in a single transaction. + */ + max_estimated_size: t.number.optional(), + /** + * Limit number of documents to write in a single transaction. + */ + max_record_count: t.number.optional() +}); + +export type BatchLimits = t.Encoded; + +export const OperationBatchLimits = BatchLimits.and( + t.object({ + /** + * Maximum size of size of current_data documents we lookup at a time. + */ + max_current_data_batch_size: t.number.optional() + }) +); + +export type OperationBatchLimits = t.Encoded; + +export const BaseStorageConfig = t.object({ + /** + * Allow batch operation limits to be configurable. + * Postgres has less batch size restrictions compared to MongoDB. + * Increasing limits can drastically improve replication performance, but + * can come at the cost of higher memory usage or potential issues. + */ + batch_limits: OperationBatchLimits.optional() +}); + +export type BaseStorageConfig = t.Encoded; + +export const PostgresStorageConfig = pg_types.PostgresConnectionConfig.and(BaseStorageConfig); +export type PostgresStorageConfig = t.Encoded; +export type PostgresStorageConfigDecoded = t.Decoded; + +export type RequiredOperationBatchLimits = Required; + +export type NormalizedPostgresStorageConfig = pg_wire.NormalizedConnectionConfig & { + batch_limits: RequiredOperationBatchLimits; +}; + +export const normalizePostgresStorageConfig = ( + baseConfig: PostgresStorageConfigDecoded +): NormalizedPostgresStorageConfig => { + return { + ...pg_types.normalizeConnectionConfig(baseConfig), + batch_limits: { + max_current_data_batch_size: baseConfig.batch_limits?.max_current_data_batch_size ?? MAX_BATCH_CURRENT_DATA_SIZE, + max_estimated_size: baseConfig.batch_limits?.max_estimated_size ?? MAX_BATCH_ESTIMATED_SIZE, + max_record_count: baseConfig.batch_limits?.max_record_count ?? MAX_BATCH_RECORD_COUNT + } + }; +}; diff --git a/modules/module-postgres-storage/src/utils/bson.ts b/modules/module-postgres-storage/src/utils/bson.ts new file mode 100644 index 000000000..c60be1775 --- /dev/null +++ b/modules/module-postgres-storage/src/utils/bson.ts @@ -0,0 +1,17 @@ +import { storage, utils } from '@powersync/service-core'; +import * as uuid from 'uuid'; + +/** + * BSON is used to serialize certain documents for storage in BYTEA columns. + * JSONB columns do not directly support storing binary data which could be required in future. + */ + +export function replicaIdToSubkey(tableId: string, id: storage.ReplicaId): string { + // Hashed UUID from the table and id + if (storage.isUUID(id)) { + // Special case for UUID for backwards-compatiblity + return `${tableId}/${id.toHexString()}`; + } + const repr = storage.serializeBson({ table: tableId, id }); + return uuid.v5(repr, utils.ID_NAMESPACE); +} diff --git a/modules/module-postgres-storage/src/utils/bucket-data.ts b/modules/module-postgres-storage/src/utils/bucket-data.ts new file mode 100644 index 000000000..e4b7f504d --- /dev/null +++ b/modules/module-postgres-storage/src/utils/bucket-data.ts @@ -0,0 +1,25 @@ +import { utils } from '@powersync/service-core'; +import { models } from '../types/types.js'; +import { replicaIdToSubkey } from './bson.js'; + +export const mapOpEntry = (entry: models.BucketDataDecoded) => { + if (entry.op == models.OpType.PUT || entry.op == models.OpType.REMOVE) { + return { + op_id: utils.timestampToOpId(entry.op_id), + op: entry.op, + object_type: entry.table_name ?? undefined, + object_id: entry.row_id ?? undefined, + checksum: Number(entry.checksum), + subkey: replicaIdToSubkey(entry.source_table!, entry.source_key!), + data: entry.data + }; + } else { + // MOVE, CLEAR + + return { + op_id: utils.timestampToOpId(entry.op_id), + op: entry.op, + checksum: Number(entry.checksum) + }; + } +}; diff --git a/modules/module-postgres-storage/src/utils/connection/AbstractPostgresConnection.ts b/modules/module-postgres-storage/src/utils/connection/AbstractPostgresConnection.ts new file mode 100644 index 000000000..1d8dd61e9 --- /dev/null +++ b/modules/module-postgres-storage/src/utils/connection/AbstractPostgresConnection.ts @@ -0,0 +1,108 @@ +import * as framework from '@powersync/lib-services-framework'; +import * as pgwire from '@powersync/service-jpgwire'; +import { pg_utils } from '@powersync/service-module-postgres'; +import * as t from 'ts-codec'; + +export type DecodedSQLQueryExecutor> = { + first: () => Promise | null>; + rows: () => Promise[]>; +}; + +export abstract class AbstractPostgresConnection< + Listener extends framework.DisposableListener = framework.DisposableListener +> extends framework.DisposableObserver { + protected abstract baseConnection: pgwire.PgClient; + + stream(...args: pgwire.Statement[]): AsyncIterableIterator { + return this.baseConnection.stream(...args); + } + + query(...args: pgwire.Statement[]): Promise { + return pg_utils.retriedQuery(this.baseConnection, ...args); + } + + /** + * Template string helper which can be used to execute template SQL strings. + */ + sql(strings: TemplateStringsArray, ...params: pgwire.StatementParam[]) { + const { statement, params: queryParams } = sql(strings, ...params); + + const rows = (): Promise => + this.queryRows({ + statement, + params: queryParams + }); + + const first = async (): Promise => { + const [f] = await rows(); + return f; + }; + + return { + execute: () => + this.query({ + statement, + params + }), + rows, + first, + decoded: >(codec: T): DecodedSQLQueryExecutor => { + return { + first: async () => { + const result = await first(); + return result && codec.decode(result); + }, + rows: async () => { + const results = await rows(); + return results.map((r) => { + return codec.decode(r); + }); + } + }; + } + }; + } + + queryRows(script: string, options?: pgwire.PgSimpleQueryOptions): Promise; + queryRows(...args: pgwire.Statement[] | [...pgwire.Statement[], pgwire.PgExtendedQueryOptions]): Promise; + async queryRows(...args: any[]) { + return pgwire.pgwireRows(await this.query(...args)); + } + + async *streamRows(...args: pgwire.Statement[]): AsyncIterableIterator { + let columns: Array = []; + + for await (const chunk of this.stream(...args)) { + if (chunk.tag == 'RowDescription') { + columns = chunk.payload.map((c, index) => { + return c.name as keyof T; + }); + continue; + } + + if (!chunk.rows.length) { + continue; + } + + yield chunk.rows.map((row) => { + let q: Partial = {}; + for (const [index, c] of columns.entries()) { + q[c] = row[index]; + } + return q as T; + }); + } + } +} + +/** + * Template string helper function which generates PGWire statements. + */ +export const sql = (strings: TemplateStringsArray, ...params: pgwire.StatementParam[]): pgwire.Statement => { + const paramPlaceholders = new Array(params.length).fill('').map((value, index) => `$${index + 1}`); + const joinedQueryStatement = strings.map((query, index) => `${query} ${paramPlaceholders[index] ?? ''}`).join(' '); + return { + statement: joinedQueryStatement, + params + }; +}; diff --git a/modules/module-postgres-storage/src/utils/connection/ConnectionSlot.ts b/modules/module-postgres-storage/src/utils/connection/ConnectionSlot.ts new file mode 100644 index 000000000..c928b088b --- /dev/null +++ b/modules/module-postgres-storage/src/utils/connection/ConnectionSlot.ts @@ -0,0 +1,144 @@ +import { framework } from '@powersync/service-core'; +import * as pgwire from '@powersync/service-jpgwire'; + +export const NOTIFICATION_CHANNEL = 'powersynccheckpoints'; + +export interface NotificationListener extends framework.DisposableListener { + notification?: (payload: pgwire.PgNotification) => void; +} + +export interface ConnectionSlotListener extends NotificationListener { + connectionAvailable?: () => void; + connectionError?: (exception: any) => void; +} + +export type ConnectionLease = { + connection: pgwire.PgConnection; + release: () => void; +}; + +export const MAX_CONNECTION_ATTEMPTS = 5; + +export class ConnectionSlot extends framework.DisposableObserver { + isAvailable: boolean; + isPoking: boolean; + + protected connection: pgwire.PgConnection | null; + + constructor(protected config: pgwire.NormalizedConnectionConfig) { + super(); + this.isAvailable = false; + this.connection = null; + this.isPoking = false; + } + + get isConnected() { + return !!this.connection; + } + + protected async connect() { + const connection = await pgwire.connectPgWire(this.config, { type: 'standard' }); + if (this.hasNotificationListener()) { + await this.configureConnectionNotifications(connection); + } + return connection; + } + + async [Symbol.asyncDispose]() { + await this.connection?.end(); + return super[Symbol.dispose](); + } + + protected async configureConnectionNotifications(connection: pgwire.PgConnection) { + if (connection.onnotification == this.handleNotification) { + return; + } + + connection.onnotification = this.handleNotification; + await connection.query({ + statement: `LISTEN ${NOTIFICATION_CHANNEL}` + }); + } + + registerListener(listener: Partial): () => void { + const dispose = super.registerListener(listener); + if (this.connection && this.hasNotificationListener()) { + this.configureConnectionNotifications(this.connection); + } + return () => { + dispose(); + if (this.connection && !this.hasNotificationListener()) { + this.connection.onnotification = () => {}; + } + }; + } + + protected handleNotification = (payload: pgwire.PgNotification) => { + this.iterateListeners((l) => l.notification?.(payload)); + }; + + protected hasNotificationListener() { + return !!Object.values(this.listeners).find((l) => !!l.notification); + } + + /** + * Test the connection if it can be reached. + */ + async poke() { + if (this.isPoking || (this.isConnected && this.isAvailable == false)) { + return; + } + this.isPoking = true; + for (let retryCounter = 0; retryCounter <= MAX_CONNECTION_ATTEMPTS; retryCounter++) { + try { + const connection = this.connection ?? (await this.connect()); + + await connection.query({ + statement: 'SELECT 1' + }); + + if (!this.connection) { + this.connection = connection; + this.setAvailable(); + } else if (this.isAvailable) { + this.iterateListeners((cb) => cb.connectionAvailable?.()); + } + + // Connection is alive and healthy + break; + } catch (ex) { + // Should be valid for all cases + this.isAvailable = false; + if (this.connection) { + this.connection.onnotification = () => {}; + this.connection.destroy(); + this.connection = null; + } + if (retryCounter >= MAX_CONNECTION_ATTEMPTS) { + this.iterateListeners((cb) => cb.connectionError?.(ex)); + } + } + } + this.isPoking = false; + } + + protected setAvailable() { + this.isAvailable = true; + this.iterateListeners((l) => l.connectionAvailable?.()); + } + + lock(): ConnectionLease | null { + if (!this.isAvailable || !this.connection) { + return null; + } + + this.isAvailable = false; + + return { + connection: this.connection, + release: () => { + this.setAvailable(); + } + }; + } +} diff --git a/modules/module-postgres-storage/src/utils/connection/DatabaseClient.ts b/modules/module-postgres-storage/src/utils/connection/DatabaseClient.ts new file mode 100644 index 000000000..ea1fe6c10 --- /dev/null +++ b/modules/module-postgres-storage/src/utils/connection/DatabaseClient.ts @@ -0,0 +1,193 @@ +import * as pgwire from '@powersync/service-jpgwire'; +import * as pg_types from '@powersync/service-module-postgres/types'; +import pDefer, { DeferredPromise } from 'p-defer'; +import { AbstractPostgresConnection, sql } from './AbstractPostgresConnection.js'; +import { ConnectionLease, ConnectionSlot, NotificationListener } from './ConnectionSlot.js'; +import { WrappedConnection } from './WrappedConnection.js'; + +export const TRANSACTION_CONNECTION_COUNT = 5; + +export const STORAGE_SCHEMA_NAME = 'powersync'; + +const SCHEMA_STATEMENT: pgwire.Statement = { + statement: `SET search_path TO ${STORAGE_SCHEMA_NAME};` +}; + +export class DatabaseClient extends AbstractPostgresConnection { + closed: boolean; + + protected pool: pgwire.PgClient; + protected connections: ConnectionSlot[]; + + protected initialized: Promise; + protected queue: DeferredPromise[]; + + constructor(protected config: pg_types.NormalizedPostgresConnectionConfig) { + super(); + this.closed = false; + this.pool = pgwire.connectPgWirePool(this.config, {}); + this.connections = Array.from({ length: TRANSACTION_CONNECTION_COUNT }, () => { + const slot = new ConnectionSlot(config); + slot.registerListener({ + connectionAvailable: () => this.processConnectionQueue(), + connectionError: (ex) => this.handleConnectionError(ex) + }); + return slot; + }); + this.queue = []; + this.initialized = this.initialize(); + } + + protected get baseConnection() { + return this.pool; + } + + registerListener(listener: Partial): () => void { + let disposeNotification: (() => void) | null = null; + if ('notification' in listener) { + // Pass this on to the first connection slot + // It will only actively listen on the connection once a listener has been registered + disposeNotification = this.connections[0].registerListener({ + notification: listener.notification + }); + delete listener['notification']; + } + + const superDispose = super.registerListener(listener); + return () => { + disposeNotification?.(); + superDispose(); + }; + } + + /** + * There is no direct way to set the default schema with pgwire. + * This hack uses multiple statements in order to always ensure the + * appropriate connection uses the correct schema. + */ + async query(...args: pgwire.Statement[]): Promise { + await this.initialized; + return super.query(...[SCHEMA_STATEMENT, ...args]); + } + + async *stream(...args: pgwire.Statement[]): AsyncIterableIterator { + await this.initialized; + yield* super.stream(...[SCHEMA_STATEMENT, ...args]); + } + + async lockConnection(callback: (db: WrappedConnection) => Promise): Promise { + const { connection, release } = await this.requestConnection(); + + await this.setSchema(connection); + + try { + return await callback(new WrappedConnection(connection)); + } finally { + release(); + } + } + + async transaction(tx: (db: WrappedConnection) => Promise): Promise { + return this.lockConnection(async (db) => { + try { + await db.query(sql`BEGIN`); + const result = await tx(db); + await db.query(sql`COMMIT`); + return result; + } catch (ex) { + await db.query(sql`ROLLBACK`); + throw ex; + } + }); + } + + /** + * Use the `powersync` schema as the default when resolving table names + */ + protected async setSchema(client: pgwire.PgClient) { + await client.query(SCHEMA_STATEMENT); + } + + protected async initialize() { + // Create the schema if it doesn't exist + await this.pool.query({ statement: `CREATE SCHEMA IF NOT EXISTS ${STORAGE_SCHEMA_NAME}` }); + } + + protected async requestConnection(): Promise { + if (this.closed) { + throw new Error('Database client is closed'); + } + + await this.initialized; + + // Queue the operation + const deferred = pDefer(); + this.queue.push(deferred); + + // Poke the slots to check if they are alive + for (const slot of this.connections) { + // No need to await this. Errors are reported asynchronously + slot.poke(); + } + + return deferred.promise; + } + + protected leaseConnectionSlot(): ConnectionLease | null { + const availableSlots = this.connections.filter((s) => s.isAvailable); + for (const slot of availableSlots) { + const lease = slot.lock(); + if (lease) { + return lease; + } + // Possibly some contention detected, keep trying + } + return null; + } + + protected processConnectionQueue() { + if (this.closed && this.queue.length) { + for (const q of this.queue) { + q.reject(new Error('Database has closed while waiting for a connection')); + } + this.queue = []; + } + + if (this.queue.length) { + const lease = this.leaseConnectionSlot(); + if (lease) { + const deferred = this.queue.shift()!; + deferred.resolve(lease); + } + } + } + + /** + * Reports connection errors which might occur from bad configuration or + * a server which is no longer available. + * This fails all pending requests. + */ + protected handleConnectionError(exception: any) { + for (const q of this.queue) { + q.reject(exception); + } + this.queue = []; + } + + async [Symbol.asyncDispose]() { + await this.initialized; + this.closed = true; + + for (const c of this.connections) { + await c[Symbol.asyncDispose](); + } + + await this.pool.end(); + + // Reject all remaining items + for (const q of this.queue) { + q.reject(new Error(`Database is disposed`)); + } + this.queue = []; + } +} diff --git a/modules/module-postgres-storage/src/utils/connection/WrappedConnection.ts b/modules/module-postgres-storage/src/utils/connection/WrappedConnection.ts new file mode 100644 index 000000000..9bd5c6a67 --- /dev/null +++ b/modules/module-postgres-storage/src/utils/connection/WrappedConnection.ts @@ -0,0 +1,11 @@ +import * as pgwire from '@powersync/service-jpgwire'; +import { AbstractPostgresConnection } from './AbstractPostgresConnection.js'; + +/** + * Provides helper functionality to transaction contexts given an existing PGWire connection + */ +export class WrappedConnection extends AbstractPostgresConnection { + constructor(protected baseConnection: pgwire.PgConnection) { + super(); + } +} diff --git a/modules/module-postgres-storage/src/utils/db.ts b/modules/module-postgres-storage/src/utils/db.ts new file mode 100644 index 000000000..6f94fef90 --- /dev/null +++ b/modules/module-postgres-storage/src/utils/db.ts @@ -0,0 +1,18 @@ +import { DatabaseClient } from './connection/DatabaseClient.js'; + +export const dropTables = async (client: DatabaseClient) => { + // Lock a connection for automatic schema search paths + await client.lockConnection(async (db) => { + await db.sql`DROP TABLE IF EXISTS bucket_data`.execute(); + await db.sql`DROP TABLE IF EXISTS bucket_parameters`.execute(); + await db.sql`DROP TABLE IF EXISTS sync_rules`.execute(); + await db.sql`DROP TABLE IF EXISTS instance`.execute(); + await db.sql`DROP TABLE IF EXISTS bucket_data`.execute(); + await db.sql`DROP TABLE IF EXISTS current_data`.execute(); + await db.sql`DROP TABLE IF EXISTS source_tables`.execute(); + await db.sql`DROP TABLE IF EXISTS write_checkpoints`.execute(); + await db.sql`DROP TABLE IF EXISTS custom_write_checkpoints`.execute(); + await db.sql`DROP SEQUENCE IF EXISTS op_id_sequence`.execute(); + await db.sql`DROP SEQUENCE IF EXISTS sync_rules_id_sequence`.execute(); + }); +}; diff --git a/modules/module-postgres-storage/src/utils/ts-codec.ts b/modules/module-postgres-storage/src/utils/ts-codec.ts new file mode 100644 index 000000000..1740b10c6 --- /dev/null +++ b/modules/module-postgres-storage/src/utils/ts-codec.ts @@ -0,0 +1,14 @@ +import * as t from 'ts-codec'; + +/** + * Returns a new codec with a subset of keys. Equivalent to the TypeScript Pick utility. + */ +export const pick = (codec: t.ObjectCodec, keys: Keys[]) => { + // Filter the shape by the specified keys + const newShape = Object.fromEntries( + Object.entries(codec.props.shape).filter(([key]) => keys.includes(key as Keys)) + ) as Pick; + + // Return a new codec with the narrowed shape + return t.object(newShape) as t.ObjectCodec>; +}; diff --git a/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap new file mode 100644 index 000000000..64e792029 --- /dev/null +++ b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -0,0 +1,332 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`sync - postgres > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + }, + ], + "last_op_id": "2", + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - postgres > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": -93886621n, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 499012468, + "count": 4, + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 1859363232n, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153n, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - postgres > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - postgres > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + }, + ], + "last_op_id": "0", + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - postgres > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - postgres > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + }, + ], + "last_op_id": "2", + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466n, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209n, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - postgres > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -852817836, + "count": 1, + }, + ], + "last_op_id": "1", + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - postgres > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + }, + ], + "last_op_id": "0", + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - postgres > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 920318466, + "count": 1, + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466n, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - postgres > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3280762209n, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; diff --git a/modules/module-postgres-storage/test/src/env.ts b/modules/module-postgres-storage/test/src/env.ts new file mode 100644 index 000000000..6047ebcfa --- /dev/null +++ b/modules/module-postgres-storage/test/src/env.ts @@ -0,0 +1,6 @@ +import { utils } from '@powersync/lib-services-framework'; + +export const env = utils.collectEnvironmentVariables({ + PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'), + CI: utils.type.boolean.default('false') +}); diff --git a/modules/module-postgres-storage/test/src/migrations.test.ts b/modules/module-postgres-storage/test/src/migrations.test.ts new file mode 100644 index 000000000..0b3940104 --- /dev/null +++ b/modules/module-postgres-storage/test/src/migrations.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest'; +import { POSTGRES_STORAGE_FACTORY } from './util.js'; + +describe('Migrations', () => { + it('Should have tables declared', async () => { + const { db } = await POSTGRES_STORAGE_FACTORY(); + + const tables = await db.sql` + SELECT + table_schema, + table_name + FROM + information_schema.tables + WHERE + table_type = 'BASE TABLE' + AND table_schema NOT IN ('pg_catalog', 'information_schema') + ORDER BY + table_schema, + table_name; + `.rows<{ table_schema: string; table_name: string }>(); + + expect(tables.find((t) => t.table_name == 'sync_rules')).exist; + }); +}); diff --git a/modules/module-postgres-storage/test/src/setup.ts b/modules/module-postgres-storage/test/src/setup.ts new file mode 100644 index 000000000..802007e89 --- /dev/null +++ b/modules/module-postgres-storage/test/src/setup.ts @@ -0,0 +1,16 @@ +import { container } from '@powersync/lib-services-framework'; +import { Metrics } from '@powersync/service-core'; +import { beforeAll } from 'vitest'; + +beforeAll(async () => { + // Executes for every test file + container.registerDefaults(); + + // The metrics need to be initialized before they can be used + await Metrics.initialise({ + disable_telemetry_sharing: true, + powersync_instance_id: 'test', + internal_metrics_endpoint: 'unused.for.tests.com' + }); + Metrics.getInstance().resetCounters(); +}); diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts new file mode 100644 index 000000000..5977ec220 --- /dev/null +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -0,0 +1,131 @@ +import { storage } from '@powersync/service-core'; +import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests'; +import { describe, expect, test } from 'vitest'; +import { POSTGRES_STORAGE_FACTORY } from './util.js'; + +describe('Sync Bucket Validation', register.registerBucketValidationTests); + +describe('Postgres Sync Bucket Storage', () => { + register.registerDataStorageTests(POSTGRES_STORAGE_FACTORY); + + /** + * The split of returned results can vary depending on storage drivers. + * The large rows here are 2MB large while the default chunk limit is 1mb. + * The Postgres storage driver will detect if the next row will increase the batch + * over the limit and separate that row into a new batch (or single row batch) if applicable. + */ + test('large batch (2)', async () => { + // Test syncing a batch of data that is small in count, + // but large enough in size to be split over multiple returned chunks. + // Similar to the above test, but splits over 1MB chunks. + const sync_rules = test_utils.testRules( + ` + bucket_definitions: + global: + data: + - SELECT id, description FROM "%" + ` + ); + using factory = await POSTGRES_STORAGE_FACTORY(); + const bucketStorage = factory.getInstance(sync_rules); + + const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { + const sourceTable = TEST_TABLE; + + const largeDescription = '0123456789'.repeat(2_000_00); + + await batch.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') + }); + + await batch.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large1', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large1') + }); + + // Large enough to split the returned batch + await batch.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large2', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large2') + }); + + await batch.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test3', + description: 'test3' + }, + afterReplicaId: test_utils.rid('test3') + }); + }); + + const checkpoint = result!.flushed_op; + + const options: storage.BucketDataBatchOptions = {}; + + const batch1 = await test_utils.fromAsync( + bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), options) + ); + expect(test_utils.getBatchData(batch1)).toEqual([ + { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 } + ]); + expect(test_utils.getBatchMeta(batch1)).toEqual({ + after: '0', + has_more: true, + next_after: '1' + }); + + const batch2 = await test_utils.fromAsync( + bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].batch.next_after]]), options) + ); + expect(test_utils.getBatchData(batch2)).toEqual([ + { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 } + ]); + expect(test_utils.getBatchMeta(batch2)).toEqual({ + after: '1', + has_more: true, + next_after: '2' + }); + + const batch3 = await test_utils.fromAsync( + bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].batch.next_after]]), options) + ); + expect(test_utils.getBatchData(batch3)).toEqual([ + { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 } + ]); + expect(test_utils.getBatchMeta(batch3)).toEqual({ + after: '2', + has_more: true, + next_after: '3' + }); + + const batch4 = await test_utils.fromAsync( + bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch3[0].batch.next_after]]), options) + ); + expect(test_utils.getBatchData(batch4)).toEqual([ + { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 } + ]); + expect(test_utils.getBatchMeta(batch4)).toEqual({ + after: '3', + has_more: false, + next_after: '4' + }); + }); +}); diff --git a/modules/module-postgres-storage/test/src/storage_compacting.test.ts b/modules/module-postgres-storage/test/src/storage_compacting.test.ts new file mode 100644 index 000000000..f0b02b696 --- /dev/null +++ b/modules/module-postgres-storage/test/src/storage_compacting.test.ts @@ -0,0 +1,5 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { POSTGRES_STORAGE_FACTORY } from './util.js'; + +describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY, {})); diff --git a/modules/module-postgres-storage/test/src/storage_sync.test.ts b/modules/module-postgres-storage/test/src/storage_sync.test.ts new file mode 100644 index 000000000..d7aae902a --- /dev/null +++ b/modules/module-postgres-storage/test/src/storage_sync.test.ts @@ -0,0 +1,12 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { POSTGRES_STORAGE_FACTORY } from './util.js'; + +/** + * Bucket compacting is not yet implemented. + * This causes the internal compacting test to fail. + * Other tests have been verified manually. + */ +describe('sync - postgres', () => { + register.registerSyncTests(POSTGRES_STORAGE_FACTORY); +}); diff --git a/modules/module-postgres-storage/test/src/util.ts b/modules/module-postgres-storage/test/src/util.ts new file mode 100644 index 000000000..7ea4d5ac4 --- /dev/null +++ b/modules/module-postgres-storage/test/src/util.ts @@ -0,0 +1,63 @@ +import { framework, PowerSyncMigrationManager, ServiceContext, storage } from '@powersync/service-core'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { normalizePostgresStorageConfig } from '../../src//types/types.js'; +import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js'; +import { PostgresBucketStorageFactory } from '../../src/storage/PostgresBucketStorageFactory.js'; +import { env } from './env.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +export const TEST_URI = env.PG_STORAGE_TEST_URL; + +const BASE_CONFIG = { + type: 'postgresql' as const, + uri: TEST_URI, + sslmode: 'disable' as const +}; + +export const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG); + +/** + * Vitest tries to load the migrations via .ts files which fails. + * For tests this links to the relevant .js files correctly + */ +class TestPostgresMigrationAgent extends PostgresMigrationAgent { + getInternalScriptsDir(): string { + return path.resolve(__dirname, '../../dist/migrations/scripts'); + } +} + +export const POSTGRES_STORAGE_FACTORY = async (options?: storage.TestStorageOptions) => { + const migrationManager: PowerSyncMigrationManager = new framework.MigrationManager(); + await using migrationAgent = new TestPostgresMigrationAgent(BASE_CONFIG); + migrationManager.registerMigrationAgent(migrationAgent); + + const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext; + + if (!options?.doNotClear) { + console.log('Running down migrations to clear DB'); + await migrationManager.migrate({ + direction: framework.migrations.Direction.Down, + migrationContext: { + service_context: mockServiceContext + } + }); + } + + // In order to run up migration after + await migrationAgent.resetStore(); + + await migrationManager.migrate({ + direction: framework.migrations.Direction.Up, + migrationContext: { + service_context: mockServiceContext + } + }); + + return new PostgresBucketStorageFactory({ + config: TEST_CONNECTION_OPTIONS, + slot_name_prefix: 'test_' + }); +}; diff --git a/modules/module-postgres-storage/test/tsconfig.json b/modules/module-postgres-storage/test/tsconfig.json new file mode 100644 index 000000000..35074fd52 --- /dev/null +++ b/modules/module-postgres-storage/test/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "baseUrl": "./", + "noEmit": true, + "esModuleInterop": true, + "declarationDir": "dist/@types", + "tsBuildInfoFile": "dist/.tsbuildinfo", + "lib": ["ES2022", "esnext.disposable"], + "skipLibCheck": true, + "sourceMap": true, + "paths": { + "@module/*": ["../src/*"] + } + }, + "include": ["src"], + "references": [ + { + "path": "../" + } + ] +} diff --git a/modules/module-postgres-storage/tsconfig.json b/modules/module-postgres-storage/tsconfig.json new file mode 100644 index 000000000..4cd41de8c --- /dev/null +++ b/modules/module-postgres-storage/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "declarationDir": "dist/@types", + "tsBuildInfoFile": "dist/.tsbuildinfo", + "rootDir": "src", + "target": "ES2022", + "lib": ["ES2022", "esnext.disposable"], + "skipLibCheck": true + }, + "include": ["src"], + "references": [] +} diff --git a/modules/module-postgres-storage/vitest.config.ts b/modules/module-postgres-storage/vitest.config.ts new file mode 100644 index 000000000..885dab34e --- /dev/null +++ b/modules/module-postgres-storage/vitest.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + setupFiles: './test/src/setup.ts', + poolOptions: { + threads: { + singleThread: true + } + }, + pool: 'threads' + } +}); diff --git a/package.json b/package.json index a0ff8792f..b737003f2 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,9 @@ "concurrently": "^8.2.2", "inquirer": "^9.2.7", "npm-check-updates": "^17.1.2", - "prettier": "^3.3.3", + "prettier": "^3.4.1", + "prettier-plugin-embed": "^0.4.15", + "prettier-plugin-sql": "^0.18.1", "rsocket-core": "1.0.0-alpha.3", "rsocket-websocket-client": "1.0.0-alpha.3", "semver": "^7.5.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f62b68b26..689da8912 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,8 +30,14 @@ importers: specifier: ^17.1.2 version: 17.1.3 prettier: - specifier: ^3.3.3 - version: 3.3.3 + specifier: ^3.4.1 + version: 3.4.2 + prettier-plugin-embed: + specifier: ^0.4.15 + version: 0.4.15 + prettier-plugin-sql: + specifier: ^0.18.1 + version: 0.18.1(prettier@3.4.2) rsocket-core: specifier: 1.0.0-alpha.3 version: 1.0.0-alpha.3 @@ -308,6 +314,55 @@ importers: specifier: ^9.0.4 version: 9.0.8 + modules/module-postgres-storage: + dependencies: + '@powersync/lib-services-framework': + specifier: workspace:* + version: link:../../libs/lib-services + '@powersync/service-core': + specifier: workspace:* + version: link:../../packages/service-core + '@powersync/service-core-tests': + specifier: workspace:* + version: link:../../packages/service-core-tests + '@powersync/service-jpgwire': + specifier: workspace:* + version: link:../../packages/jpgwire + '@powersync/service-jsonbig': + specifier: ^0.17.10 + version: 0.17.10 + '@powersync/service-module-postgres': + specifier: workspace:* + version: link:../module-postgres + '@powersync/service-sync-rules': + specifier: workspace:* + version: link:../../packages/sync-rules + '@powersync/service-types': + specifier: workspace:* + version: link:../../packages/types + ix: + specifier: ^5.0.0 + version: 5.0.0 + lru-cache: + specifier: ^10.2.2 + version: 10.4.3 + p-defer: + specifier: ^4.0.1 + version: 4.0.1 + ts-codec: + specifier: ^1.3.0 + version: 1.3.0 + uuid: + specifier: ^9.0.1 + version: 9.0.1 + devDependencies: + '@types/uuid': + specifier: ^9.0.4 + version: 9.0.8 + typescript: + specifier: ^5.2.2 + version: 5.6.2 + packages/jpgwire: dependencies: '@powersync/service-jsonbig': @@ -563,6 +618,9 @@ importers: '@powersync/service-module-postgres': specifier: workspace:* version: link:../modules/module-postgres + '@powersync/service-module-postgres-storage': + specifier: workspace:* + version: link:../modules/module-postgres-storage '@powersync/service-rsocket-router': specifier: workspace:* version: link:../packages/rsocket-router @@ -1256,6 +1314,9 @@ packages: resolution: {integrity: sha512-2GjOxVws+wtbb+xFUJe4Ozzkp/f0Gsna0fje9art76bmz6yfLCW4K3Mf2/M310xMnAIp8eP9hsJ6DYwwZCo1RA==} engines: {node: '>=20.0.0'} + '@powersync/service-jsonbig@0.17.10': + resolution: {integrity: sha512-BgxgUewuw4HFCM9MzuzlIuRKHya6rimNPYqUItt7CO3ySUeUnX8Qn9eZpMxu9AT5Y8zqkSyxvduY36zZueNojg==} + '@prisma/instrumentation@5.16.1': resolution: {integrity: sha512-4m5gRFWnQb8s/yTyGbMZkL7A5uJgqOWcWJxapwcAD0T0kh5sGPEVSQl/zTQvE9aduXhFAxOtC3gO+R8Hb5xO1Q==} @@ -1924,6 +1985,14 @@ packages: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} + dedent@1.5.3: + resolution: {integrity: sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==} + peerDependencies: + babel-plugin-macros: ^3.1.0 + peerDependenciesMeta: + babel-plugin-macros: + optional: true + deep-eql@5.0.2: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} @@ -2100,6 +2169,10 @@ packages: resolution: {integrity: sha512-+SrHpvQ52Q6W9f3wJoJBbAQULJuNEEQwBvlvYwACDhBTLOTMiQ0HYWh4+vC3OivGP2ENcTI1oKlFA2OepJNjhQ==} engines: {node: '>=14'} + find-up-simple@1.0.0: + resolution: {integrity: sha512-q7Us7kcjj2VMePAa02hDAF6d+MzsdsAWEwYyOpwUtlerRBkOEPBCRZrAV4XfcSN8fHAgaD0hP7miwoay6DCprw==} + engines: {node: '>=18'} + find-up@4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} @@ -2497,6 +2570,10 @@ packages: resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} engines: {node: '>=0.10.0'} + jsox@1.2.121: + resolution: {integrity: sha512-9Ag50tKhpTwS6r5wh3MJSAvpSof0UBr39Pto8OnzFT32Z/pAbxAsKHzyvsyMEHVslELvHyO/4/jaQELHk8wDcw==} + hasBin: true + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -2594,6 +2671,9 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} + micro-memoize@4.1.3: + resolution: {integrity: sha512-DzRMi8smUZXT7rCGikRwldEh6eO6qzKiPPopcr1+2EY3AYKpy5fu159PKWwIS9A6IWnrvPKDMcuFtyrroZa8Bw==} + micromatch@4.0.7: resolution: {integrity: sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==} engines: {node: '>=8.6'} @@ -2762,6 +2842,10 @@ packages: engines: {node: ^12.13 || ^14.13 || >=16} hasBin: true + node-sql-parser@4.18.0: + resolution: {integrity: sha512-2YEOR5qlI1zUFbGMLKNfsrR5JUvFg9LxIRVE+xJe962pfVLH0rnItqLzv96XVs1Y1UIR8FxsXAuvX/lYAWZ2BQ==} + engines: {node: '>=8'} + nodemon@3.1.4: resolution: {integrity: sha512-wjPBbFhtpJwmIeY2yP7QF+UKzPfltVGtfce1g/bB15/8vCGZj8uxD62b/b9M9/WVgme0NZudpownKN+c0plXlQ==} engines: {node: '>=10'} @@ -2872,6 +2956,10 @@ packages: resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} engines: {node: '>=12.20'} + p-defer@4.0.1: + resolution: {integrity: sha512-Mr5KC5efvAK5VUptYEIopP1bakB85k2IWXaRC0rsh1uwn1L6M0LVml8OIQ4Gudg4oyZakf7FmeRLkMMtZW1i5A==} + engines: {node: '>=12'} + p-filter@2.1.0: resolution: {integrity: sha512-ZBxxZ5sL2HghephhpGAQdoskxplTwr7ICaehZwLIlfL6acuVgZPm8yBNuRAFBGEqtD/hmUeq9eqLg2ys9Xr/yw==} engines: {node: '>=8'} @@ -2914,6 +3002,10 @@ packages: package-manager-detector@0.2.0: resolution: {integrity: sha512-E385OSk9qDcXhcM9LNSe4sdhx8a9mAPrZ4sMLW+tmxl5ZuGtPUcdFu+MPP2jbgiWAZ6Pfe5soGFMd+0Db5Vrog==} + package-up@5.0.0: + resolution: {integrity: sha512-MQEgDUvXCa3sGvqHg3pzHO8e9gqTCMPVrWUko3vPQGntwegmFo52mZb2abIVTjFnUcW0BcPz0D93jV5Cas1DWA==} + engines: {node: '>=18'} + pacote@15.2.0: resolution: {integrity: sha512-rJVZeIwHTUta23sIZgEIM62WYwbmGbThdbnkt81ravBplQv+HjyroqnLRNH2+sLJHcGZmLRmhPwACqhfTcOmnA==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -3020,13 +3112,22 @@ packages: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} + prettier-plugin-embed@0.4.15: + resolution: {integrity: sha512-9pZVIp3bw2jw+Ge+iAMZ4j+sIVC9cPruZ93H2tj5Wa/3YDFDJ/uYyVWdUGfcFUnv28drhW2Bmome9xSGXsPKOw==} + + prettier-plugin-sql@0.18.1: + resolution: {integrity: sha512-2+Nob2sg7hzLAKJoE6sfgtkhBZCqOzrWHZPvE4Kee/e80oOyI4qwy9vypeltqNBJwTtq3uiKPrCxlT03bBpOaw==} + engines: {node: ^14.18.0 || >=16.0.0} + peerDependencies: + prettier: ^3.0.3 + prettier@2.8.8: resolution: {integrity: sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==} engines: {node: '>=10.13.0'} hasBin: true - prettier@3.3.3: - resolution: {integrity: sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==} + prettier@3.4.2: + resolution: {integrity: sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ==} engines: {node: '>=14'} hasBin: true @@ -3411,6 +3512,10 @@ packages: sprintf-js@1.1.3: resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} + sql-formatter@15.4.9: + resolution: {integrity: sha512-5vmt2HlCAVozxsBZuXWkAki/KGawaK+b5GG5x+BtXOFVpN/8cqppblFUxHl4jxdA0cvo14lABhM+KBnrUapOlw==} + hasBin: true + sqlstring@2.3.3: resolution: {integrity: sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==} engines: {node: '>= 0.6'} @@ -3512,6 +3617,9 @@ packages: through@2.3.8: resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} + tiny-jsonc@1.0.1: + resolution: {integrity: sha512-ik6BCxzva9DoiEfDX/li0L2cWKPPENYvixUprFdl3YPi4bZZUhDnNI9YUkacrv+uIG90dnxR5mNqaoD6UhD6Bw==} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -3625,6 +3733,10 @@ packages: resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} engines: {node: '>=12.20'} + type-fest@4.31.0: + resolution: {integrity: sha512-yCxltHW07Nkhv/1F6wWBr8kz+5BGMfP+RbRSYFnegVb0qV/UMT0G0ElBloPVerqn4M2ZV80Ir1FtCcYv1cT6vQ==} + engines: {node: '>=16'} + typedarray-to-buffer@3.1.5: resolution: {integrity: sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==} @@ -4659,6 +4771,10 @@ snapshots: big-integer: 1.6.51 iconv-lite: 0.6.3 + '@powersync/service-jsonbig@0.17.10': + dependencies: + lossless-json: 2.0.11 + '@prisma/instrumentation@5.16.1': dependencies: '@opentelemetry/api': 1.8.0 @@ -4746,7 +4862,7 @@ snapshots: '@opentelemetry/semantic-conventions': 1.25.1 '@prisma/instrumentation': 5.16.1 '@sentry/core': 8.17.0 - '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.1) + '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1) '@sentry/types': 8.17.0 '@sentry/utils': 8.17.0 optionalDependencies: @@ -4754,7 +4870,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.1)': + '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) @@ -5361,6 +5477,8 @@ snapshots: dependencies: mimic-response: 3.1.0 + dedent@1.5.3: {} + deep-eql@5.0.2: {} deep-extend@0.6.0: {} @@ -5557,6 +5675,8 @@ snapshots: fast-querystring: 1.1.2 safe-regex2: 2.0.0 + find-up-simple@1.0.0: {} + find-up@4.1.0: dependencies: locate-path: 5.0.0 @@ -5940,6 +6060,8 @@ snapshots: jsonpointer@5.0.1: {} + jsox@1.2.121: {} + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -6063,6 +6185,8 @@ snapshots: merge2@1.4.1: {} + micro-memoize@4.1.3: {} + micromatch@4.0.7: dependencies: braces: 3.0.3 @@ -6223,6 +6347,10 @@ snapshots: - bluebird - supports-color + node-sql-parser@4.18.0: + dependencies: + big-integer: 1.6.51 + nodemon@3.1.4: dependencies: chokidar: 3.6.0 @@ -6389,6 +6517,8 @@ snapshots: p-cancelable@3.0.0: {} + p-defer@4.0.1: {} + p-filter@2.1.0: dependencies: p-map: 2.1.0 @@ -6428,6 +6558,10 @@ snapshots: package-manager-detector@0.2.0: {} + package-up@5.0.0: + dependencies: + find-up-simple: 1.0.0 + pacote@15.2.0: dependencies: '@npmcli/git': 4.1.0 @@ -6541,9 +6675,28 @@ snapshots: dependencies: xtend: 4.0.2 + prettier-plugin-embed@0.4.15: + dependencies: + '@types/estree': 1.0.5 + dedent: 1.5.3 + micro-memoize: 4.1.3 + package-up: 5.0.0 + tiny-jsonc: 1.0.1 + type-fest: 4.31.0 + transitivePeerDependencies: + - babel-plugin-macros + + prettier-plugin-sql@0.18.1(prettier@3.4.2): + dependencies: + jsox: 1.2.121 + node-sql-parser: 4.18.0 + prettier: 3.4.2 + sql-formatter: 15.4.9 + tslib: 2.6.3 + prettier@2.8.8: {} - prettier@3.3.3: {} + prettier@3.4.2: {} proc-log@3.0.0: {} @@ -6927,6 +7080,12 @@ snapshots: sprintf-js@1.1.3: {} + sql-formatter@15.4.9: + dependencies: + argparse: 2.0.1 + get-stdin: 8.0.0 + nearley: 2.20.1 + sqlstring@2.3.3: {} ssri@10.0.6: @@ -7023,6 +7182,8 @@ snapshots: through@2.3.8: {} + tiny-jsonc@1.0.1: {} + tinybench@2.9.0: {} tinyexec@0.3.0: {} @@ -7126,6 +7287,8 @@ snapshots: type-fest@2.19.0: {} + type-fest@4.31.0: {} + typedarray-to-buffer@3.1.5: dependencies: is-typedarray: 1.0.0 diff --git a/service/Dockerfile b/service/Dockerfile index 4acd8d355..e857cc85d 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -17,6 +17,7 @@ COPY libs/lib-services/package.json libs/lib-services/tsconfig.json libs/lib-ser COPY libs/lib-mongodb/package.json libs/lib-mongodb/tsconfig.json libs/lib-mongodb/ COPY modules/module-postgres/package.json modules/module-postgres/tsconfig.json modules/module-postgres/ +COPY modules/module-postgres-storage/package.json modules/module-postgres-storage/tsconfig.json modules/module-postgres-storage/ COPY modules/module-mongodb/package.json modules/module-mongodb/tsconfig.json modules/module-mongodb/ COPY modules/module-mongodb-storage/package.json modules/module-mongodb-storage/tsconfig.json modules/module-mongodb-storage/ COPY modules/module-mysql/package.json modules/module-mysql/tsconfig.json modules/module-mysql/ @@ -38,6 +39,7 @@ COPY libs/lib-services/src libs/lib-services/src/ COPY libs/lib-mongodb/src libs/lib-mongodb/src/ COPY modules/module-postgres/src modules/module-postgres/src/ +COPY modules/module-postgres-storage/src modules/module-postgres-storage/src/ COPY modules/module-mongodb/src modules/module-mongodb/src/ COPY modules/module-mongodb-storage/src modules/module-mongodb-storage/src/ COPY modules/module-mysql/src modules/module-mysql/src/ diff --git a/service/package.json b/service/package.json index 780150a1d..1f0ffa523 100644 --- a/service/package.json +++ b/service/package.json @@ -17,6 +17,7 @@ "@powersync/service-core": "workspace:*", "@powersync/lib-services-framework": "workspace:*", "@powersync/service-module-postgres": "workspace:*", + "@powersync/service-module-postgres-storage": "workspace:*", "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-mysql": "workspace:*", diff --git a/service/src/entry.ts b/service/src/entry.ts index e133c04e4..06a062a5b 100644 --- a/service/src/entry.ts +++ b/service/src/entry.ts @@ -5,6 +5,7 @@ import { MongoModule } from '@powersync/service-module-mongodb'; import { MongoStorageModule } from '@powersync/service-module-mongodb-storage'; import { MySQLModule } from '@powersync/service-module-mysql'; import { PostgresModule } from '@powersync/service-module-postgres'; +import { PostgresStorageModule } from '@powersync/service-module-postgres-storage'; import { startServer } from './runners/server.js'; import { startStreamRunner } from './runners/stream-worker.js'; import { startUnifiedRunner } from './runners/unified-runner.js'; @@ -15,7 +16,13 @@ container.registerDefaults(); container.register(ContainerImplementation.REPORTER, createSentryReporter()); const moduleManager = new core.modules.ModuleManager(); -moduleManager.register([new PostgresModule(), new MySQLModule(), new MongoModule(), new MongoStorageModule()]); +moduleManager.register([ + new PostgresModule(), + new MySQLModule(), + new MongoModule(), + new MongoStorageModule(), + new PostgresStorageModule() +]); // This is a bit of a hack. Commands such as the teardown command or even migrations might // want access to the ModuleManager in order to use modules container.register(core.ModuleManager, moduleManager); diff --git a/tsconfig.json b/tsconfig.json index 4eaac0158..338c428af 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,6 +19,9 @@ { "path": "./modules/module-postgres" }, + { + "path": "./modules/module-postgres-storage" + }, { "path": "./modules/module-mysql" }, From a745f3386f9d078687b9c52232c795152d11c645 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 8 Jan 2025 15:29:54 +0200 Subject: [PATCH 02/50] wip test factory --- modules/module-postgres-storage/src/index.ts | 1 + .../PostgresTestStorageFactoryGenerator.ts | 51 +++++++++++++++++++ .../src/storage/storage-index.ts | 1 + .../module-postgres-storage/test/src/util.ts | 1 - 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts create mode 100644 modules/module-postgres-storage/src/storage/storage-index.ts diff --git a/modules/module-postgres-storage/src/index.ts b/modules/module-postgres-storage/src/index.ts index c89905886..b5876e818 100644 --- a/modules/module-postgres-storage/src/index.ts +++ b/modules/module-postgres-storage/src/index.ts @@ -3,4 +3,5 @@ export * from './storage/PostgresBucketStorageFactory.js'; export * from './migrations/PostgresMigrationAgent.js'; +export * from './storage/storage-index.js'; export * from './types/types.js'; diff --git a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts new file mode 100644 index 000000000..3e1d192e6 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts @@ -0,0 +1,51 @@ +import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOptions } from '@powersync/service-core'; +import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'; +import { normalizePostgresStorageConfig } from '../types/types.js'; +import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; + +export type PostgresTestStorageOptions = { + url: string; +}; + +export const PostgresTestStorageFactoryGenerator = (factoryOptions: PostgresTestStorageOptions) => { + return async (options?: TestStorageOptions) => { + const migrationManager: PowerSyncMigrationManager = new framework.MigrationManager(); + + const BASE_CONFIG = { + type: 'postgresql' as const, + uri: factoryOptions.url, + sslmode: 'disable' as const + }; + + const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG); + + await using migrationAgent = new PostgresMigrationAgent(BASE_CONFIG); + migrationManager.registerMigrationAgent(migrationAgent); + + const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext; + + if (!options?.doNotClear) { + await migrationManager.migrate({ + direction: framework.migrations.Direction.Down, + migrationContext: { + service_context: mockServiceContext + } + }); + } + + // In order to run up migration after + await migrationAgent.resetStore(); + + await migrationManager.migrate({ + direction: framework.migrations.Direction.Up, + migrationContext: { + service_context: mockServiceContext + } + }); + + return new PostgresBucketStorageFactory({ + config: TEST_CONNECTION_OPTIONS, + slot_name_prefix: 'test_' + }); + }; +}; diff --git a/modules/module-postgres-storage/src/storage/storage-index.ts b/modules/module-postgres-storage/src/storage/storage-index.ts new file mode 100644 index 000000000..b95f1eaa8 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/storage-index.ts @@ -0,0 +1 @@ +export * from './PostgresTestStorageFactoryGenerator.js'; diff --git a/modules/module-postgres-storage/test/src/util.ts b/modules/module-postgres-storage/test/src/util.ts index 7ea4d5ac4..95c376103 100644 --- a/modules/module-postgres-storage/test/src/util.ts +++ b/modules/module-postgres-storage/test/src/util.ts @@ -37,7 +37,6 @@ export const POSTGRES_STORAGE_FACTORY = async (options?: storage.TestStorageOpti const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext; if (!options?.doNotClear) { - console.log('Running down migrations to clear DB'); await migrationManager.migrate({ direction: framework.migrations.Direction.Down, migrationContext: { From 997ed4e6c84b2c67fbf9135adb0de3384b553bf4 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jan 2025 09:07:15 +0200 Subject: [PATCH 03/50] Add postgres storage to MySQL and Postgres replicator tests --- modules/module-mongodb/package.json | 3 +- modules/module-mysql/package.json | 3 +- .../test/src/BinLogStream.test.ts | 17 +++++---- modules/module-mysql/test/src/env.ts | 5 ++- modules/module-mysql/test/src/util.ts | 5 +++ .../PostgresTestStorageFactoryGenerator.ts | 16 ++++++-- .../connection/AbstractPostgresConnection.ts | 3 +- .../src/utils/connection/DatabaseClient.ts | 5 ++- .../module-postgres-storage/test/src/util.ts | 38 +++---------------- modules/module-postgres/package.json | 3 +- modules/module-postgres/test/src/env.ts | 5 ++- .../test/src/large_batch.test.ts | 19 +++++++++- .../test/src/schema_changes.test.ts | 9 ++++- .../test/src/slow_tests.test.ts | 14 ++++++- modules/module-postgres/test/src/util.ts | 5 +++ .../test/src/wal_stream.test.ts | 9 ++++- .../test/src/wal_stream_utils.ts | 1 + pnpm-lock.yaml | 13 ++++++- 18 files changed, 111 insertions(+), 62 deletions(-) diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index 640dea3b9..b93e76845 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -42,6 +42,7 @@ "devDependencies": { "@types/uuid": "^9.0.4", "@powersync/service-core-tests": "workspace:*", - "@powersync/service-module-mongodb-storage": "workspace:*" + "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-postgres-storage": "workspace:*" } } diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 33b4b1bdf..d86bd005d 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -46,6 +46,7 @@ "@types/async": "^3.2.24", "@types/uuid": "^9.0.4", "@powersync/service-core-tests": "workspace:*", - "@powersync/service-module-mongodb-storage": "workspace:*" + "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-postgres-storage": "workspace:*" } } diff --git a/modules/module-mysql/test/src/BinLogStream.test.ts b/modules/module-mysql/test/src/BinLogStream.test.ts index 7295935ad..2ed560980 100644 --- a/modules/module-mysql/test/src/BinLogStream.test.ts +++ b/modules/module-mysql/test/src/BinLogStream.test.ts @@ -3,7 +3,8 @@ import { putOp, removeOp } from '@powersync/service-core-tests'; import { v4 as uuid } from 'uuid'; import { describe, expect, test } from 'vitest'; import { BinlogStreamTestContext } from './BinlogStreamUtils.js'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { env } from './env.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; const BASIC_SYNC_RULES = ` bucket_definitions: @@ -12,13 +13,13 @@ bucket_definitions: - SELECT id, description FROM "test_data" `; -describe( - ' Binlog stream - mongodb', - function () { - defineBinlogStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY); - }, - { timeout: 20_000 } -); +describe.skipIf(!env.TEST_MONGO_STORAGE)(' Binlog stream - mongodb', { timeout: 20_000 }, function () { + defineBinlogStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY); +}); + +describe.skipIf(!env.TEST_POSTGRES_STORAGE)(' Binlog stream - postgres', { timeout: 20_000 }, function () { + defineBinlogStreamTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); +}); function defineBinlogStreamTests(factory: storage.TestStorageFactory) { test('Replicate basic values', async () => { diff --git a/modules/module-mysql/test/src/env.ts b/modules/module-mysql/test/src/env.ts index 53ecef648..063745e4a 100644 --- a/modules/module-mysql/test/src/env.ts +++ b/modules/module-mysql/test/src/env.ts @@ -3,6 +3,9 @@ import { utils } from '@powersync/lib-services-framework'; export const env = utils.collectEnvironmentVariables({ MYSQL_TEST_URI: utils.type.string.default('mysql://root:mypassword@localhost:3306/mydatabase'), MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'), + PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'), CI: utils.type.boolean.default('false'), - SLOW_TESTS: utils.type.boolean.default('false') + SLOW_TESTS: utils.type.boolean.default('false'), + TEST_MONGO_STORAGE: utils.type.boolean.default('true'), + TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') }); diff --git a/modules/module-mysql/test/src/util.ts b/modules/module-mysql/test/src/util.ts index 597f03b17..ffb797032 100644 --- a/modules/module-mysql/test/src/util.ts +++ b/modules/module-mysql/test/src/util.ts @@ -1,6 +1,7 @@ import * as types from '@module/types/types.js'; import { getMySQLVersion, isVersionAtLeast } from '@module/utils/mysql-utils.js'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; +import * as postgres_storage from '@powersync/service-module-postgres-storage'; import mysqlPromise from 'mysql2/promise'; import { env } from './env.js'; @@ -16,6 +17,10 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageF isCI: env.CI }); +export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTestStorageFactoryGenerator({ + url: env.PG_STORAGE_TEST_URL +}); + export async function clearTestDb(connection: mysqlPromise.Connection) { const version = await getMySQLVersion(connection); if (isVersionAtLeast(version, '8.4.0')) { diff --git a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts index 3e1d192e6..e27fc9204 100644 --- a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts +++ b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts @@ -1,10 +1,16 @@ import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOptions } from '@powersync/service-core'; +import { PostgresConnectionConfig } from '@powersync/service-module-postgres/types'; import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'; import { normalizePostgresStorageConfig } from '../types/types.js'; import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; export type PostgresTestStorageOptions = { url: string; + /** + * Vitest can cause issues when loading .ts files for migrations. + * This allows for providing a custom PostgresMigrationAgent. + */ + migrationAgent?: (config: PostgresConnectionConfig) => PostgresMigrationAgent; }; export const PostgresTestStorageFactoryGenerator = (factoryOptions: PostgresTestStorageOptions) => { @@ -19,7 +25,9 @@ export const PostgresTestStorageFactoryGenerator = (factoryOptions: PostgresTest const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG); - await using migrationAgent = new PostgresMigrationAgent(BASE_CONFIG); + await using migrationAgent = factoryOptions.migrationAgent + ? factoryOptions.migrationAgent(BASE_CONFIG) + : new PostgresMigrationAgent(BASE_CONFIG); migrationManager.registerMigrationAgent(migrationAgent); const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext; @@ -31,10 +39,10 @@ export const PostgresTestStorageFactoryGenerator = (factoryOptions: PostgresTest service_context: mockServiceContext } }); - } - // In order to run up migration after - await migrationAgent.resetStore(); + // In order to run up migration after + await migrationAgent.resetStore(); + } await migrationManager.migrate({ direction: framework.migrations.Direction.Up, diff --git a/modules/module-postgres-storage/src/utils/connection/AbstractPostgresConnection.ts b/modules/module-postgres-storage/src/utils/connection/AbstractPostgresConnection.ts index 1d8dd61e9..a5c773be9 100644 --- a/modules/module-postgres-storage/src/utils/connection/AbstractPostgresConnection.ts +++ b/modules/module-postgres-storage/src/utils/connection/AbstractPostgresConnection.ts @@ -1,6 +1,5 @@ import * as framework from '@powersync/lib-services-framework'; import * as pgwire from '@powersync/service-jpgwire'; -import { pg_utils } from '@powersync/service-module-postgres'; import * as t from 'ts-codec'; export type DecodedSQLQueryExecutor> = { @@ -18,7 +17,7 @@ export abstract class AbstractPostgresConnection< } query(...args: pgwire.Statement[]): Promise { - return pg_utils.retriedQuery(this.baseConnection, ...args); + return this.baseConnection.query(...args); } /** diff --git a/modules/module-postgres-storage/src/utils/connection/DatabaseClient.ts b/modules/module-postgres-storage/src/utils/connection/DatabaseClient.ts index ea1fe6c10..7265f73af 100644 --- a/modules/module-postgres-storage/src/utils/connection/DatabaseClient.ts +++ b/modules/module-postgres-storage/src/utils/connection/DatabaseClient.ts @@ -1,4 +1,5 @@ import * as pgwire from '@powersync/service-jpgwire'; +import { pg_utils } from '@powersync/service-module-postgres'; import * as pg_types from '@powersync/service-module-postgres/types'; import pDefer, { DeferredPromise } from 'p-defer'; import { AbstractPostgresConnection, sql } from './AbstractPostgresConnection.js'; @@ -67,7 +68,9 @@ export class DatabaseClient extends AbstractPostgresConnection { await this.initialized; - return super.query(...[SCHEMA_STATEMENT, ...args]); + // Retry pool queries. Note that we can't retry queries in a transaction + // since a failed query will end the transaction. + return pg_utils.retriedQuery(this.pool, ...[SCHEMA_STATEMENT, ...args]); } async *stream(...args: pgwire.Statement[]): AsyncIterableIterator { diff --git a/modules/module-postgres-storage/test/src/util.ts b/modules/module-postgres-storage/test/src/util.ts index 95c376103..3f0cd4428 100644 --- a/modules/module-postgres-storage/test/src/util.ts +++ b/modules/module-postgres-storage/test/src/util.ts @@ -1,9 +1,8 @@ -import { framework, PowerSyncMigrationManager, ServiceContext, storage } from '@powersync/service-core'; import path from 'path'; import { fileURLToPath } from 'url'; import { normalizePostgresStorageConfig } from '../../src//types/types.js'; import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js'; -import { PostgresBucketStorageFactory } from '../../src/storage/PostgresBucketStorageFactory.js'; +import { PostgresTestStorageFactoryGenerator } from '../../src/storage/PostgresTestStorageFactoryGenerator.js'; import { env } from './env.js'; const __filename = fileURLToPath(import.meta.url); @@ -29,34 +28,7 @@ class TestPostgresMigrationAgent extends PostgresMigrationAgent { } } -export const POSTGRES_STORAGE_FACTORY = async (options?: storage.TestStorageOptions) => { - const migrationManager: PowerSyncMigrationManager = new framework.MigrationManager(); - await using migrationAgent = new TestPostgresMigrationAgent(BASE_CONFIG); - migrationManager.registerMigrationAgent(migrationAgent); - - const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext; - - if (!options?.doNotClear) { - await migrationManager.migrate({ - direction: framework.migrations.Direction.Down, - migrationContext: { - service_context: mockServiceContext - } - }); - } - - // In order to run up migration after - await migrationAgent.resetStore(); - - await migrationManager.migrate({ - direction: framework.migrations.Direction.Up, - migrationContext: { - service_context: mockServiceContext - } - }); - - return new PostgresBucketStorageFactory({ - config: TEST_CONNECTION_OPTIONS, - slot_name_prefix: 'test_' - }); -}; +export const POSTGRES_STORAGE_FACTORY = PostgresTestStorageFactoryGenerator({ + url: env.PG_STORAGE_TEST_URL, + migrationAgent: (config) => new TestPostgresMigrationAgent(config) +}); diff --git a/modules/module-postgres/package.json b/modules/module-postgres/package.json index 282cece2f..759696535 100644 --- a/modules/module-postgres/package.json +++ b/modules/module-postgres/package.json @@ -43,6 +43,7 @@ "devDependencies": { "@types/uuid": "^9.0.4", "@powersync/service-core-tests": "workspace:*", - "@powersync/service-module-mongodb-storage": "workspace:*" + "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-postgres-storage": "workspace:*" } } diff --git a/modules/module-postgres/test/src/env.ts b/modules/module-postgres/test/src/env.ts index 214b75ca6..58b69e235 100644 --- a/modules/module-postgres/test/src/env.ts +++ b/modules/module-postgres/test/src/env.ts @@ -2,7 +2,10 @@ import { utils } from '@powersync/lib-services-framework'; export const env = utils.collectEnvironmentVariables({ PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'), + PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'), MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'), CI: utils.type.boolean.default('false'), - SLOW_TESTS: utils.type.boolean.default('false') + SLOW_TESTS: utils.type.boolean.default('false'), + TEST_MONGO_STORAGE: utils.type.boolean.default('true'), + TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') }); diff --git a/modules/module-postgres/test/src/large_batch.test.ts b/modules/module-postgres/test/src/large_batch.test.ts index 4d49a259f..453ec6153 100644 --- a/modules/module-postgres/test/src/large_batch.test.ts +++ b/modules/module-postgres/test/src/large_batch.test.ts @@ -3,10 +3,14 @@ import * as timers from 'timers/promises'; import { describe, expect, test } from 'vitest'; import { populateData } from '../../dist/utils/populate_test_data.js'; import { env } from './env.js'; -import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_CONNECTION_OPTIONS } from './util.js'; +import { + INITIALIZED_MONGO_STORAGE_FACTORY, + INITIALIZED_POSTGRES_STORAGE_FACTORY, + TEST_CONNECTION_OPTIONS +} from './util.js'; import { WalStreamTestContext } from './wal_stream_utils.js'; -describe('batch replication tests - mongodb', { timeout: 120_000 }, function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('batch replication tests - mongodb', { timeout: 120_000 }, function () { // These are slow but consistent tests. // Not run on every test run, but we do run on CI, or when manually debugging issues. if (env.CI || env.SLOW_TESTS) { @@ -17,6 +21,17 @@ describe('batch replication tests - mongodb', { timeout: 120_000 }, function () } }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('batch replication tests - postgres', { timeout: 120_000 }, function () { + // These are slow but consistent tests. + // Not run on every test run, but we do run on CI, or when manually debugging issues. + if (env.CI || env.SLOW_TESTS) { + defineBatchTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); + } else { + // Need something in this file. + test('no-op', () => {}); + } +}); + const BASIC_SYNC_RULES = `bucket_definitions: global: data: diff --git a/modules/module-postgres/test/src/schema_changes.test.ts b/modules/module-postgres/test/src/schema_changes.test.ts index c6bb3de23..a890c3fcf 100644 --- a/modules/module-postgres/test/src/schema_changes.test.ts +++ b/modules/module-postgres/test/src/schema_changes.test.ts @@ -3,13 +3,18 @@ import * as timers from 'timers/promises'; import { describe, expect, test } from 'vitest'; import { storage } from '@powersync/service-core'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { env } from './env.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; import { WalStreamTestContext } from './wal_stream_utils.js'; -describe('schema changes', { timeout: 20_000 }, function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('schema changes - mongodb', { timeout: 20_000 }, function () { defineTests(INITIALIZED_MONGO_STORAGE_FACTORY); }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('schema changes - postgres', { timeout: 20_000 }, function () { + defineTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); +}); + const BASIC_SYNC_RULES = ` bucket_definitions: global: diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index fd02ed90b..d6634219a 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -7,6 +7,7 @@ import { connectPgPool, getClientCheckpoint, INITIALIZED_MONGO_STORAGE_FACTORY, + INITIALIZED_POSTGRES_STORAGE_FACTORY, TEST_CONNECTION_OPTIONS } from './util.js'; @@ -19,7 +20,7 @@ import { test_utils } from '@powersync/service-core-tests'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; import * as timers from 'node:timers/promises'; -describe('slow tests - mongodb', function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('slow tests - mongodb', function () { // These are slow, inconsistent tests. // Not run on every test run, but we do run on CI, or when manually debugging issues. if (env.CI || env.SLOW_TESTS) { @@ -30,6 +31,17 @@ describe('slow tests - mongodb', function () { } }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('slow tests - postgres', function () { + // These are slow, inconsistent tests. + // Not run on every test run, but we do run on CI, or when manually debugging issues. + if (env.CI || env.SLOW_TESTS) { + defineSlowTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); + } else { + // Need something in this file. + test('no-op', () => {}); + } +}); + function defineSlowTests(factory: storage.TestStorageFactory) { let walStream: WalStream | undefined; let connections: PgManager | undefined; diff --git a/modules/module-postgres/test/src/util.ts b/modules/module-postgres/test/src/util.ts index 7499dfd1b..8e5d3a528 100644 --- a/modules/module-postgres/test/src/util.ts +++ b/modules/module-postgres/test/src/util.ts @@ -5,6 +5,7 @@ import { logger } from '@powersync/lib-services-framework'; import { BucketStorageFactory, OpId } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; +import * as postgres_storage from '@powersync/service-module-postgres-storage'; import { env } from './env.js'; export const TEST_URI = env.PG_TEST_URL; @@ -14,6 +15,10 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageF isCI: env.CI }); +export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTestStorageFactoryGenerator({ + url: env.PG_STORAGE_TEST_URL +}); + export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({ type: 'postgresql', uri: TEST_URI, diff --git a/modules/module-postgres/test/src/wal_stream.test.ts b/modules/module-postgres/test/src/wal_stream.test.ts index f444696b4..fea606643 100644 --- a/modules/module-postgres/test/src/wal_stream.test.ts +++ b/modules/module-postgres/test/src/wal_stream.test.ts @@ -4,7 +4,8 @@ import { putOp, removeOp } from '@powersync/service-core-tests'; import { pgwireRows } from '@powersync/service-jpgwire'; import * as crypto from 'crypto'; import { describe, expect, test } from 'vitest'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { env } from './env.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; import { WalStreamTestContext } from './wal_stream_utils.js'; const BASIC_SYNC_RULES = ` @@ -14,10 +15,14 @@ bucket_definitions: - SELECT id, description FROM "test_data" `; -describe('wal stream - mongodb', { timeout: 20_000 }, function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('wal stream - mongodb', { timeout: 20_000 }, function () { defineWalStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY); }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('wal stream - postgres', { timeout: 20_000 }, function () { + defineWalStreamTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); +}); + function defineWalStreamTests(factory: storage.TestStorageFactory) { test('replicating basic values', async () => { await using context = await WalStreamTestContext.open(factory); diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index ae549b70e..2bd8d64d6 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -46,6 +46,7 @@ export class WalStreamTestContext implements AsyncDisposable { await this.streamPromise; await this.connectionManager.destroy(); this.storage?.[Symbol.dispose](); + this.factory?.[Symbol.dispose](); } get pool() { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 689da8912..a5a560e17 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -163,6 +163,9 @@ importers: '@powersync/service-module-mongodb-storage': specifier: workspace:* version: link:../module-mongodb-storage + '@powersync/service-module-postgres-storage': + specifier: workspace:* + version: link:../module-postgres-storage '@types/uuid': specifier: ^9.0.4 version: 9.0.8 @@ -258,6 +261,9 @@ importers: '@powersync/service-module-mongodb-storage': specifier: workspace:* version: link:../module-mongodb-storage + '@powersync/service-module-postgres-storage': + specifier: workspace:* + version: link:../module-postgres-storage '@types/async': specifier: ^3.2.24 version: 3.2.24 @@ -310,6 +316,9 @@ importers: '@powersync/service-module-mongodb-storage': specifier: workspace:* version: link:../module-mongodb-storage + '@powersync/service-module-postgres-storage': + specifier: workspace:* + version: link:../module-postgres-storage '@types/uuid': specifier: ^9.0.4 version: 9.0.8 @@ -4862,7 +4871,7 @@ snapshots: '@opentelemetry/semantic-conventions': 1.25.1 '@prisma/instrumentation': 5.16.1 '@sentry/core': 8.17.0 - '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1) + '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.1) '@sentry/types': 8.17.0 '@sentry/utils': 8.17.0 optionalDependencies: @@ -4870,7 +4879,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1)': + '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.1)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) From 3846a9938cd3c37559d157f58caee5a3c430109c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jan 2025 09:38:00 +0200 Subject: [PATCH 04/50] Fixes for MySQL tests: Store relation_id as text. Update BIGINT max value. --- .../src/replication/BinLogStream.ts | 19 +++++++++++++++---- .../migrations/scripts/1684951997326-init.ts | 2 +- .../src/storage/PostgresCompactor.ts | 3 ++- .../src/storage/PostgresSyncRulesStorage.ts | 13 +++++++------ .../storage/batch/PostgresPersistedBatch.ts | 15 +++++---------- .../src/types/codecs.ts | 2 ++ .../src/types/models/SourceTable.ts | 2 +- 7 files changed, 33 insertions(+), 23 deletions(-) diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 483f292f9..c4a06988b 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -6,12 +6,12 @@ import { ColumnDescriptor, framework, getUuidReplicaIdentityBson, Metrics, stora import mysql, { FieldPacket } from 'mysql2'; import { BinLogEvent, StartOptions, TableMapEntry } from '@powersync/mysql-zongji'; +import mysqlPromise from 'mysql2/promise'; import * as common from '../common/common-index.js'; -import * as zongji_utils from './zongji/zongji-utils.js'; -import { MySQLConnectionManager } from './MySQLConnectionManager.js'; import { isBinlogStillAvailable, ReplicatedGTID, toColumnDescriptors } from '../common/common-index.js'; -import mysqlPromise from 'mysql2/promise'; import { createRandomServerId, escapeMysqlTableName } from '../utils/mysql-utils.js'; +import { MySQLConnectionManager } from './MySQLConnectionManager.js'; +import * as zongji_utils from './zongji/zongji-utils.js'; export interface BinLogStreamOptions { connections: MySQLConnectionManager; @@ -75,8 +75,19 @@ export class BinLogStream { } get connectionId() { + const { connectionId } = this.connections; // Default to 1 if not set - return this.connections.connectionId ? Number.parseInt(this.connections.connectionId) : 1; + if (!connectionId) { + return 1; + } + /** + * This is often `"default"` (string) which will parse to `NaN` + */ + const parsed = Number.parseInt(connectionId); + if (isNaN(parsed)) { + return 1; + } + return parsed; } get stopped() { diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index 3e10324d9..fe4b5b329 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -102,7 +102,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { id TEXT PRIMARY KEY, group_id integer NOT NULL, connection_id integer NOT NULL, - relation_id integer, + relation_id text, schema_name text NOT NULL, table_name text NOT NULL, replica_id_columns jsonb, diff --git a/modules/module-postgres-storage/src/storage/PostgresCompactor.ts b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts index 103a8f165..9423b940c 100644 --- a/modules/module-postgres-storage/src/storage/PostgresCompactor.ts +++ b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts @@ -2,6 +2,7 @@ import { logger } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import * as t from 'ts-codec'; +import { BIGINT_MAX } from '../types/codecs.js'; import { models } from '../types/types.js'; import { sql } from '../utils/connection/AbstractPostgresConnection.js'; import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; @@ -109,7 +110,7 @@ export class PostgresCompactor { bucketUpper = `${bucket}[\uFFFF`; } - let upperOpIdLimit = BigInt('9223372036854775807'); // 2^63 - 1 + let upperOpIdLimit = BIGINT_MAX; while (true) { const batch = await this.db.sql` diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index e75a55e3e..883b984ee 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -3,7 +3,7 @@ import { storage, utils } from '@powersync/service-core'; import * as sync_rules from '@powersync/service-sync-rules'; import * as t from 'ts-codec'; import * as uuid from 'uuid'; -import { bigint } from '../types/codecs.js'; +import { bigint, BIGINT_MAX } from '../types/codecs.js'; import { models, RequiredOperationBatchLimits } from '../types/types.js'; import { replicaIdToSubkey } from '../utils/bson.js'; import { mapOpEntry } from '../utils/bucket-data.js'; @@ -156,7 +156,7 @@ export class PostgresSyncRulesStorage WHERE group_id = ${{ type: 'int4', value: group_id }} AND connection_id = ${{ type: 'int4', value: connection_id }} - AND relation_id = ${{ type: 'int4', value: objectId }} + AND relation_id = ${{ type: 'varchar', value: objectId.toString() }} AND schema_name = ${{ type: 'varchar', value: schema }} AND table_name = ${{ type: 'varchar', value: table }} AND replica_id_columns = ${{ type: 'jsonb', value: columns }} @@ -181,7 +181,8 @@ export class PostgresSyncRulesStorage ${{ type: 'varchar', value: uuid.v4() }}, ${{ type: 'int4', value: group_id }}, ${{ type: 'int4', value: connection_id }}, - ${{ type: 'int4', value: objectId }}, + --- The objectId can be string | number, we store it as a string and decode when querying + ${{ type: 'varchar', value: objectId.toString() }}, ${{ type: 'varchar', value: schema }}, ${{ type: 'varchar', value: table }}, ${{ type: 'jsonb', value: columns }} @@ -217,7 +218,7 @@ export class PostgresSyncRulesStorage AND connection_id = ${{ type: 'int4', value: connection_id }} AND id != ${{ type: 'varchar', value: sourceTableRow!.id }} AND ( - relation_id = ${{ type: 'int4', value: objectId }} + relation_id = ${{ type: 'varchar', value: objectId.toString() }} OR ( schema_name = ${{ type: 'varchar', value: schema }} AND table_name = ${{ type: 'varchar', value: table }} @@ -234,7 +235,7 @@ export class PostgresSyncRulesStorage new storage.SourceTable( doc.id, connection_tag, - Number(doc.relation_id ?? 0), + doc.relation_id ?? 0, doc.schema_name, doc.table_name, doc.replica_id_columns?.map((c) => ({ @@ -340,7 +341,7 @@ export class PostgresSyncRulesStorage return; } - const end = checkpoint ? BigInt(checkpoint) : BigInt(2) ** BigInt(64) - BigInt(1); + const end = checkpoint ?? BIGINT_MAX; const filters = Array.from(dataBuckets.entries()).map(([name, start]) => ({ bucket_name: name, start: start diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts index 1fa23cdcb..740231a7e 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts @@ -2,7 +2,7 @@ import { logger } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; import * as sync_rules from '@powersync/service-sync-rules'; -import { BatchLimits, models } from '../../types/types.js'; +import { models, RequiredOperationBatchLimits } from '../../types/types.js'; import { replicaIdToSubkey } from '../../utils/bson.js'; import { WrappedConnection } from '../../utils/connection/WrappedConnection.js'; @@ -28,14 +28,10 @@ export type DeleteCurrentDataOptions = { source_key: storage.ReplicaId; }; -export type PostgresPersistedBatchOptions = BatchLimits & { +export type PostgresPersistedBatchOptions = RequiredOperationBatchLimits & { group_id: number; }; -const MAX_TRANSACTION_BATCH_SIZE = 30_000_000; - -const MAX_TRANSACTION_DOC_COUNT = 2_000; - export class PostgresPersistedBatch { group_id: number; @@ -61,8 +57,8 @@ export class PostgresPersistedBatch { constructor(options: PostgresPersistedBatchOptions) { this.group_id = options.group_id; - this.maxTransactionBatchSize = options.max_estimated_size ?? MAX_TRANSACTION_BATCH_SIZE; - this.maxTransactionDocCount = options.max_record_count ?? MAX_TRANSACTION_DOC_COUNT; + this.maxTransactionBatchSize = options.max_estimated_size; + this.maxTransactionDocCount = options.max_record_count; this.bucketDataInserts = []; this.parameterDataInserts = []; @@ -399,8 +395,7 @@ export class PostgresPersistedBatch { lookups -- Already decoded FROM parsed_data - ON CONFLICT (group_id, source_table, source_key) DO - UPDATE + ON CONFLICT (group_id, source_table, source_key) DO UPDATE SET buckets = EXCLUDED.buckets, data = EXCLUDED.data, diff --git a/modules/module-postgres-storage/src/types/codecs.ts b/modules/module-postgres-storage/src/types/codecs.ts index c28d45279..99fa0b344 100644 --- a/modules/module-postgres-storage/src/types/codecs.ts +++ b/modules/module-postgres-storage/src/types/codecs.ts @@ -1,5 +1,7 @@ import * as t from 'ts-codec'; +export const BIGINT_MAX = BigInt('9223372036854775807'); + /** * Wraps a codec which is encoded to a JSON string */ diff --git a/modules/module-postgres-storage/src/types/models/SourceTable.ts b/modules/module-postgres-storage/src/types/models/SourceTable.ts index 4613eb0dd..04095467a 100644 --- a/modules/module-postgres-storage/src/types/models/SourceTable.ts +++ b/modules/module-postgres-storage/src/types/models/SourceTable.ts @@ -17,7 +17,7 @@ export const SourceTable = t.object({ id: t.string, group_id: bigint, connection_id: bigint, - relation_id: t.Null.or(bigint).or(t.string), + relation_id: t.Null.or(t.number).or(t.string), schema_name: t.string, table_name: t.string, replica_id_columns: t.Null.or(jsonb(t.array(ColumnDescriptor))), From ee891a84fb6be45f51d43d8c84e54eaab5f26ec2 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jan 2025 12:07:39 +0200 Subject: [PATCH 05/50] Add Postgres storage to MongoDB replicator tests. Fix bug: last_checkpoint should be coalesced when committing updates. --- .../module-mongodb/test/src/change_stream.test.ts | 9 +++++++-- .../module-mongodb/test/src/change_stream_utils.ts | 4 ++++ modules/module-mongodb/test/src/env.ts | 5 ++++- modules/module-mongodb/test/src/slow_tests.test.ts | 13 +++++++++++-- modules/module-mongodb/test/src/util.ts | 5 +++++ .../src/storage/batch/PostgresBucketBatch.ts | 5 ++++- 6 files changed, 35 insertions(+), 6 deletions(-) diff --git a/modules/module-mongodb/test/src/change_stream.test.ts b/modules/module-mongodb/test/src/change_stream.test.ts index a35fcd67e..a5d429349 100644 --- a/modules/module-mongodb/test/src/change_stream.test.ts +++ b/modules/module-mongodb/test/src/change_stream.test.ts @@ -7,7 +7,8 @@ import * as mongo from 'mongodb'; import { setTimeout } from 'node:timers/promises'; import { describe, expect, test, vi } from 'vitest'; import { ChangeStreamTestContext } from './change_stream_utils.js'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { env } from './env.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; const BASIC_SYNC_RULES = ` bucket_definitions: @@ -16,10 +17,14 @@ bucket_definitions: - SELECT _id as id, description FROM "test_data" `; -describe('change stream - mongodb', { timeout: 20_000 }, function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('change stream - mongodb', { timeout: 20_000 }, function () { defineChangeStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY); }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('change stream - postgres', { timeout: 20_000 }, function () { + defineChangeStreamTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); +}); + function defineChangeStreamTests(factory: storage.TestStorageFactory) { test('replicating basic values', async () => { await using context = await ChangeStreamTestContext.open(factory); diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index 2209f4112..994f8cb9c 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -35,6 +35,7 @@ export class ChangeStreamTestContext { async dispose() { this.abortController.abort(); + this.factory[Symbol.dispose](); await this.streamPromise?.catch((e) => e); await this.connectionManager.destroy(); } @@ -141,6 +142,7 @@ export async function getClientCheckpoint( ): Promise { const start = Date.now(); const lsn = await createCheckpoint(client, db); + console.log('created checkpoint pushing lsn to', lsn); // This old API needs a persisted checkpoint id. // Since we don't use LSNs anymore, the only way to get that is to wait. @@ -154,9 +156,11 @@ export async function getClientCheckpoint( throw new Error('No sync rules available'); } if (cp.lsn && cp.lsn >= lsn) { + console.log('active checkpoint has replicated created checkpoint', cp?.lsn); return cp.checkpoint; } + console.log('active checkpoint is still behind created checkpoint', cp?.lsn); await new Promise((resolve) => setTimeout(resolve, 30)); } diff --git a/modules/module-mongodb/test/src/env.ts b/modules/module-mongodb/test/src/env.ts index 7bfe03857..ad0f171ec 100644 --- a/modules/module-mongodb/test/src/env.ts +++ b/modules/module-mongodb/test/src/env.ts @@ -3,6 +3,9 @@ import { utils } from '@powersync/lib-services-framework'; export const env = utils.collectEnvironmentVariables({ MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'), MONGO_TEST_DATA_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test_data'), + PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'), CI: utils.type.boolean.default('false'), - SLOW_TESTS: utils.type.boolean.default('false') + SLOW_TESTS: utils.type.boolean.default('false'), + TEST_MONGO_STORAGE: utils.type.boolean.default('true'), + TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') }); diff --git a/modules/module-mongodb/test/src/slow_tests.test.ts b/modules/module-mongodb/test/src/slow_tests.test.ts index 4225cfbba..ef7c24630 100644 --- a/modules/module-mongodb/test/src/slow_tests.test.ts +++ b/modules/module-mongodb/test/src/slow_tests.test.ts @@ -4,9 +4,9 @@ import { setTimeout } from 'node:timers/promises'; import { describe, expect, test } from 'vitest'; import { ChangeStreamTestContext, setSnapshotHistorySeconds } from './change_stream_utils.js'; import { env } from './env.js'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; -describe('change stream slow tests - mongodb', { timeout: 60_000 }, function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('change stream slow tests - mongodb', { timeout: 60_000 }, function () { if (env.CI || env.SLOW_TESTS) { defineSlowTests(INITIALIZED_MONGO_STORAGE_FACTORY); } else { @@ -15,6 +15,15 @@ describe('change stream slow tests - mongodb', { timeout: 60_000 }, function () } }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('change stream slow tests - postgres', { timeout: 60_000 }, function () { + if (env.CI || env.SLOW_TESTS) { + defineSlowTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); + } else { + // Need something in this file. + test('no-op', () => {}); + } +}); + function defineSlowTests(factory: storage.TestStorageFactory) { test('replicating snapshot with lots of data', async () => { await using context = await ChangeStreamTestContext.open(factory); diff --git a/modules/module-mongodb/test/src/util.ts b/modules/module-mongodb/test/src/util.ts index 01312adb5..7a11bd4d7 100644 --- a/modules/module-mongodb/test/src/util.ts +++ b/modules/module-mongodb/test/src/util.ts @@ -1,6 +1,7 @@ import * as types from '@module/types/types.js'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; +import * as postgres_storage from '@powersync/service-module-postgres-storage'; import * as mongo from 'mongodb'; import { env } from './env.js'; @@ -16,6 +17,10 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageF isCI: env.CI }); +export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTestStorageFactoryGenerator({ + url: env.PG_STORAGE_TEST_URL +}); + export async function clearTestDb(db: mongo.Db) { await db.dropDatabase(); } diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index 3ac629319..5ba29e7cc 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -324,7 +324,10 @@ export class PostgresBucketBatch last_fatal_error = ${{ type: 'varchar', value: update.last_fatal_error }}, snapshot_done = ${{ type: 'bool', value: update.snapshot_done }}, last_keepalive_ts = ${{ type: 1184, value: update.last_keepalive_ts }}, - last_checkpoint = ${{ type: 'int8', value: update.last_checkpoint }}, + last_checkpoint = COALESCE( + ${{ type: 'int8', value: update.last_checkpoint }}, + last_checkpoint + ), last_checkpoint_ts = ${{ type: 1184, value: update.last_checkpoint_ts }}, last_checkpoint_lsn = ${{ type: 'varchar', value: update.last_checkpoint_lsn }} WHERE From 501df227be9f404ed07db18f035ea62efd7842ed Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jan 2025 15:38:12 +0200 Subject: [PATCH 06/50] git move --- libs/lib-postgres/CHANGELOG.md | 1 + libs/lib-postgres/LICENSE | 67 +++++++++++++++++++ libs/lib-postgres/README.md | 3 + libs/lib-postgres/package.json | 39 +++++++++++ .../src/locks}/locks/PostgresLockManager.ts | 0 libs/lib-postgres/tsconfig.json | 12 ++++ libs/lib-postgres/vitest.config.ts | 3 + tsconfig.json | 3 + 8 files changed, 128 insertions(+) create mode 100644 libs/lib-postgres/CHANGELOG.md create mode 100644 libs/lib-postgres/LICENSE create mode 100644 libs/lib-postgres/README.md create mode 100644 libs/lib-postgres/package.json rename {modules/module-postgres-storage/src => libs/lib-postgres/src/locks}/locks/PostgresLockManager.ts (100%) create mode 100644 libs/lib-postgres/tsconfig.json create mode 100644 libs/lib-postgres/vitest.config.ts diff --git a/libs/lib-postgres/CHANGELOG.md b/libs/lib-postgres/CHANGELOG.md new file mode 100644 index 000000000..3a699be57 --- /dev/null +++ b/libs/lib-postgres/CHANGELOG.md @@ -0,0 +1 @@ +# @powersync/lib-service-postgres diff --git a/libs/lib-postgres/LICENSE b/libs/lib-postgres/LICENSE new file mode 100644 index 000000000..c8efd46cc --- /dev/null +++ b/libs/lib-postgres/LICENSE @@ -0,0 +1,67 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2023-2024 Journey Mobile, Inc. + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under these Terms and Conditions, as indicated by our inclusion of these Terms and Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, Redistribution and Trademark clauses below, we hereby grant you the right to use, copy, modify, create derivative works, publicly perform, publicly display and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use means making the Software available to others in a commercial product or service that: + +1. substitutes for the Software; +2. substitutes for any other product or service we offer using the Software that exists as of the date we make the Software available; or +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; +2. for non-commercial education; +3. for non-commercial research; and +4. in connection with professional services that you provide to a licensee using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our patents, the license grant above includes a license under our patents. If you make a claim against any party that the Software infringes or contributes to the infringement of any patent, then your patent license to the Software ends immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of the Software. +If you redistribute any copies, modifications or derivatives of the Software, you must include a copy of or a link to these Terms and Conditions and not remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of the Software, you have no right under these Terms and Conditions to use our trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under the Apache License, Version 2.0 that is effective on the second anniversary of the date we make the Software available. On or after that date, you may use the Software under the Apache License, Version 2.0, in which case the following will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/libs/lib-postgres/README.md b/libs/lib-postgres/README.md new file mode 100644 index 000000000..1cf2cf1de --- /dev/null +++ b/libs/lib-postgres/README.md @@ -0,0 +1,3 @@ +# PowerSync Service Postgres + +Library for common Postgres logic used in the PowerSync service. diff --git a/libs/lib-postgres/package.json b/libs/lib-postgres/package.json new file mode 100644 index 000000000..d3958eb57 --- /dev/null +++ b/libs/lib-postgres/package.json @@ -0,0 +1,39 @@ +{ + "name": "@powersync/lib-service-postgres", + "repository": "https://github.com/powersync-ja/powersync-service", + "types": "dist/index.d.ts", + "version": "0.0.0", + "main": "dist/index.js", + "license": "FSL-1.1-Apache-2.0", + "type": "module", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsc -b", + "build:tests": "tsc -b test/tsconfig.json", + "clean": "rm -rf ./dist && tsc -b --clean", + "test": "vitest" + }, + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "default": "./dist/index.js" + }, + "./types": { + "import": "./dist/types/types.js", + "require": "./dist/types/types.js", + "default": "./dist/types/types.js" + } + }, + "dependencies": { + "@powersync/lib-services-framework": "workspace:*", + "@powersync/service-types": "workspace:*", + "@powersync/service-jpgwire": "workspace:*", + "@powersync/service-sync-rules": "workspace:*", + "ts-codec": "^1.3.0", + "uri-js": "^4.4.1" + }, + "devDependencies": {} +} diff --git a/modules/module-postgres-storage/src/locks/PostgresLockManager.ts b/libs/lib-postgres/src/locks/locks/PostgresLockManager.ts similarity index 100% rename from modules/module-postgres-storage/src/locks/PostgresLockManager.ts rename to libs/lib-postgres/src/locks/locks/PostgresLockManager.ts diff --git a/libs/lib-postgres/tsconfig.json b/libs/lib-postgres/tsconfig.json new file mode 100644 index 000000000..a0ae425c6 --- /dev/null +++ b/libs/lib-postgres/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "references": [] +} diff --git a/libs/lib-postgres/vitest.config.ts b/libs/lib-postgres/vitest.config.ts new file mode 100644 index 000000000..94ede10e2 --- /dev/null +++ b/libs/lib-postgres/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({}); diff --git a/tsconfig.json b/tsconfig.json index 338c428af..fc9861cc0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -37,6 +37,9 @@ { "path": "./libs/lib-mongodb" }, + { + "path": "./libs/lib-postgres" + }, { "path": "./packages/types" }, From a86472a2af2eb5e1568b47c520ab82819feefa0f Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jan 2025 16:27:24 +0200 Subject: [PATCH 07/50] share logic in lib postgres --- libs/lib-postgres/package.json | 10 +- .../connection/AbstractPostgresConnection.ts | 0 .../src/db}/connection/ConnectionSlot.ts | 23 ++- .../src/db}/connection/DatabaseClient.ts | 61 +++++-- .../src/db}/connection/WrappedConnection.ts | 0 libs/lib-postgres/src/db/db-index.ts | 4 + libs/lib-postgres/src/index.ts | 11 ++ .../locks/{locks => }/PostgresLockManager.ts | 5 +- libs/lib-postgres/src/locks/locks-index.ts | 1 + libs/lib-postgres/src/types/types.ts | 149 +++++++++++++++++ libs/lib-postgres/src/utils/pgwire_utils.ts | 49 ++++++ libs/lib-postgres/src/utils/utils-index.ts | 1 + libs/lib-postgres/test/src/config.test.ts | 12 ++ libs/lib-postgres/test/tsconfig.json | 18 +++ modules/module-postgres-storage/package.json | 2 +- .../src/migrations/PostgresMigrationAgent.ts | 19 ++- .../src/migrations/PostgresMigrationStore.ts | 9 +- .../src/migrations/migration-utils.ts | 14 ++ .../migrations/scripts/1684951997326-init.ts | 10 +- .../src/module/PostgresStorageModule.ts | 9 +- .../storage/PostgresBucketStorageFactory.ts | 16 +- .../src/storage/PostgresCompactor.ts | 6 +- .../src/storage/PostgresStorageProvider.ts | 14 +- .../src/storage/PostgresSyncRulesStorage.ts | 7 +- .../PostgresTestStorageFactoryGenerator.ts | 5 +- .../src/storage/batch/PostgresBucketBatch.ts | 23 ++- .../storage/batch/PostgresPersistedBatch.ts | 10 +- .../checkpoints/PostgresWriteCheckpointAPI.ts | 17 +- .../PostgresPersistedSyncRulesContent.ts | 8 +- .../src/types/types.ts | 32 ++-- .../module-postgres-storage/src/utils/db.ts | 13 +- modules/module-postgres-storage/tsconfig.json | 24 ++- modules/module-postgres/package.json | 1 + .../src/api/PostgresRouteAPIAdapter.ts | 19 ++- .../src/auth/SupabaseKeyCollector.ts | 4 +- .../src/replication/WalStream.ts | 4 +- .../src/replication/replication-utils.ts | 26 ++- modules/module-postgres/src/types/types.ts | 152 ++---------------- .../module-postgres/src/utils/pgwire_utils.ts | 47 +----- modules/module-postgres/test/src/util.ts | 4 +- modules/module-postgres/tsconfig.json | 3 + pnpm-lock.yaml | 44 ++++- service/Dockerfile | 2 + 43 files changed, 552 insertions(+), 336 deletions(-) rename {modules/module-postgres-storage/src/utils => libs/lib-postgres/src/db}/connection/AbstractPostgresConnection.ts (100%) rename {modules/module-postgres-storage/src/utils => libs/lib-postgres/src/db}/connection/ConnectionSlot.ts (87%) rename {modules/module-postgres-storage/src/utils => libs/lib-postgres/src/db}/connection/DatabaseClient.ts (78%) rename {modules/module-postgres-storage/src/utils => libs/lib-postgres/src/db}/connection/WrappedConnection.ts (100%) create mode 100644 libs/lib-postgres/src/db/db-index.ts create mode 100644 libs/lib-postgres/src/index.ts rename libs/lib-postgres/src/locks/{locks => }/PostgresLockManager.ts (93%) create mode 100644 libs/lib-postgres/src/locks/locks-index.ts create mode 100644 libs/lib-postgres/src/types/types.ts create mode 100644 libs/lib-postgres/src/utils/pgwire_utils.ts create mode 100644 libs/lib-postgres/src/utils/utils-index.ts create mode 100644 libs/lib-postgres/test/src/config.test.ts create mode 100644 libs/lib-postgres/test/tsconfig.json create mode 100644 modules/module-postgres-storage/src/migrations/migration-utils.ts diff --git a/libs/lib-postgres/package.json b/libs/lib-postgres/package.json index d3958eb57..a62cee473 100644 --- a/libs/lib-postgres/package.json +++ b/libs/lib-postgres/package.json @@ -29,11 +29,15 @@ }, "dependencies": { "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-types": "workspace:*", "@powersync/service-jpgwire": "workspace:*", "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", + "p-defer": "^4.0.1", "ts-codec": "^1.3.0", - "uri-js": "^4.4.1" + "uri-js": "^4.4.1", + "uuid": "^9.0.1" }, - "devDependencies": {} + "devDependencies": { + "@types/uuid": "^9.0.4" + } } diff --git a/modules/module-postgres-storage/src/utils/connection/AbstractPostgresConnection.ts b/libs/lib-postgres/src/db/connection/AbstractPostgresConnection.ts similarity index 100% rename from modules/module-postgres-storage/src/utils/connection/AbstractPostgresConnection.ts rename to libs/lib-postgres/src/db/connection/AbstractPostgresConnection.ts diff --git a/modules/module-postgres-storage/src/utils/connection/ConnectionSlot.ts b/libs/lib-postgres/src/db/connection/ConnectionSlot.ts similarity index 87% rename from modules/module-postgres-storage/src/utils/connection/ConnectionSlot.ts rename to libs/lib-postgres/src/db/connection/ConnectionSlot.ts index c928b088b..76dd18c1a 100644 --- a/modules/module-postgres-storage/src/utils/connection/ConnectionSlot.ts +++ b/libs/lib-postgres/src/db/connection/ConnectionSlot.ts @@ -1,8 +1,6 @@ -import { framework } from '@powersync/service-core'; +import * as framework from '@powersync/lib-services-framework'; import * as pgwire from '@powersync/service-jpgwire'; -export const NOTIFICATION_CHANNEL = 'powersynccheckpoints'; - export interface NotificationListener extends framework.DisposableListener { notification?: (payload: pgwire.PgNotification) => void; } @@ -17,6 +15,11 @@ export type ConnectionLease = { release: () => void; }; +export type ConnectionSlotOptions = { + config: pgwire.NormalizedConnectionConfig; + notificationChannels?: string[]; +}; + export const MAX_CONNECTION_ATTEMPTS = 5; export class ConnectionSlot extends framework.DisposableObserver { @@ -25,7 +28,7 @@ export class ConnectionSlot extends framework.DisposableObserver): () => void { diff --git a/modules/module-postgres-storage/src/utils/connection/DatabaseClient.ts b/libs/lib-postgres/src/db/connection/DatabaseClient.ts similarity index 78% rename from modules/module-postgres-storage/src/utils/connection/DatabaseClient.ts rename to libs/lib-postgres/src/db/connection/DatabaseClient.ts index 7265f73af..8e29d136c 100644 --- a/modules/module-postgres-storage/src/utils/connection/DatabaseClient.ts +++ b/libs/lib-postgres/src/db/connection/DatabaseClient.ts @@ -1,19 +1,24 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import * as pgwire from '@powersync/service-jpgwire'; -import { pg_utils } from '@powersync/service-module-postgres'; -import * as pg_types from '@powersync/service-module-postgres/types'; import pDefer, { DeferredPromise } from 'p-defer'; import { AbstractPostgresConnection, sql } from './AbstractPostgresConnection.js'; import { ConnectionLease, ConnectionSlot, NotificationListener } from './ConnectionSlot.js'; import { WrappedConnection } from './WrappedConnection.js'; -export const TRANSACTION_CONNECTION_COUNT = 5; - -export const STORAGE_SCHEMA_NAME = 'powersync'; - -const SCHEMA_STATEMENT: pgwire.Statement = { - statement: `SET search_path TO ${STORAGE_SCHEMA_NAME};` +export type DatabaseClientOptions = { + config: lib_postgres.NormalizedBasePostgresConnectionConfig; + /** + * Optional schema which will be used as the default search path + */ + schema?: string; + /** + * Notification channels to listen to. + */ + notificationChannels?: string[]; }; +export const TRANSACTION_CONNECTION_COUNT = 5; + export class DatabaseClient extends AbstractPostgresConnection { closed: boolean; @@ -23,12 +28,12 @@ export class DatabaseClient extends AbstractPostgresConnection; protected queue: DeferredPromise[]; - constructor(protected config: pg_types.NormalizedPostgresConnectionConfig) { + constructor(protected options: DatabaseClientOptions) { super(); this.closed = false; - this.pool = pgwire.connectPgWirePool(this.config, {}); + this.pool = pgwire.connectPgWirePool(options.config); this.connections = Array.from({ length: TRANSACTION_CONNECTION_COUNT }, () => { - const slot = new ConnectionSlot(config); + const slot = new ConnectionSlot({ config: options.config, notificationChannels: options.notificationChannels }); slot.registerListener({ connectionAvailable: () => this.processConnectionQueue(), connectionError: (ex) => this.handleConnectionError(ex) @@ -43,6 +48,16 @@ export class DatabaseClient extends AbstractPostgresConnection): () => void { let disposeNotification: (() => void) | null = null; if ('notification' in listener) { @@ -70,12 +85,20 @@ export class DatabaseClient extends AbstractPostgresConnection { await this.initialized; - yield* super.stream(...[SCHEMA_STATEMENT, ...args]); + const { schemaStatement } = this; + if (schemaStatement) { + args.unshift(schemaStatement); + } + yield* super.stream(...args); } async lockConnection(callback: (db: WrappedConnection) => Promise): Promise { @@ -108,12 +131,18 @@ export class DatabaseClient extends AbstractPostgresConnection { diff --git a/modules/module-postgres-storage/src/utils/connection/WrappedConnection.ts b/libs/lib-postgres/src/db/connection/WrappedConnection.ts similarity index 100% rename from modules/module-postgres-storage/src/utils/connection/WrappedConnection.ts rename to libs/lib-postgres/src/db/connection/WrappedConnection.ts diff --git a/libs/lib-postgres/src/db/db-index.ts b/libs/lib-postgres/src/db/db-index.ts new file mode 100644 index 000000000..8b6017e74 --- /dev/null +++ b/libs/lib-postgres/src/db/db-index.ts @@ -0,0 +1,4 @@ +export * from './connection/AbstractPostgresConnection.js'; +export * from './connection/ConnectionSlot.js'; +export * from './connection/DatabaseClient.js'; +export * from './connection/WrappedConnection.js'; diff --git a/libs/lib-postgres/src/index.ts b/libs/lib-postgres/src/index.ts new file mode 100644 index 000000000..0b47d2dd8 --- /dev/null +++ b/libs/lib-postgres/src/index.ts @@ -0,0 +1,11 @@ +export * from './db/db-index.js'; +export * as db from './db/db-index.js'; + +export * from './locks/locks-index.js'; +export * as locks from './locks/locks-index.js'; + +export * from './types/types.js'; +export * as types from './types/types.js'; + +export * from './utils/utils-index.js'; +export * as utils from './utils/utils-index.js'; diff --git a/libs/lib-postgres/src/locks/locks/PostgresLockManager.ts b/libs/lib-postgres/src/locks/PostgresLockManager.ts similarity index 93% rename from libs/lib-postgres/src/locks/locks/PostgresLockManager.ts rename to libs/lib-postgres/src/locks/PostgresLockManager.ts index b8aa23d51..fc5d49cfe 100644 --- a/libs/lib-postgres/src/locks/locks/PostgresLockManager.ts +++ b/libs/lib-postgres/src/locks/PostgresLockManager.ts @@ -1,7 +1,6 @@ -import { framework } from '@powersync/service-core'; +import * as framework from '@powersync/lib-services-framework'; import { v4 as uuidv4 } from 'uuid'; -import { sql } from '../utils/connection/AbstractPostgresConnection.js'; -import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; +import { DatabaseClient, sql } from '../db/db-index.js'; const DEFAULT_LOCK_TIMEOUT = 60_000; // 1 minute diff --git a/libs/lib-postgres/src/locks/locks-index.ts b/libs/lib-postgres/src/locks/locks-index.ts new file mode 100644 index 000000000..06f0e730e --- /dev/null +++ b/libs/lib-postgres/src/locks/locks-index.ts @@ -0,0 +1 @@ +export * from './PostgresLockManager.js'; diff --git a/libs/lib-postgres/src/types/types.ts b/libs/lib-postgres/src/types/types.ts new file mode 100644 index 000000000..3d147e10b --- /dev/null +++ b/libs/lib-postgres/src/types/types.ts @@ -0,0 +1,149 @@ +import * as service_types from '@powersync/service-types'; +import * as t from 'ts-codec'; +import * as urijs from 'uri-js'; + +export interface NormalizedBasePostgresConnectionConfig { + id: string; + tag: string; + + hostname: string; + port: number; + database: string; + + username: string; + password: string; + + sslmode: 'verify-full' | 'verify-ca' | 'disable'; + cacert: string | undefined; + + client_certificate: string | undefined; + client_private_key: string | undefined; +} + +export const POSTGRES_CONNECTION_TYPE = 'postgresql' as const; + +export const BasePostgresConnectionConfig = t.object({ + /** Unique identifier for the connection - optional when a single connection is present. */ + id: t.string.optional(), + /** Additional meta tag for connection */ + tag: t.string.optional(), + type: t.literal(POSTGRES_CONNECTION_TYPE), + uri: t.string.optional(), + hostname: t.string.optional(), + port: service_types.configFile.portCodec.optional(), + username: t.string.optional(), + password: t.string.optional(), + database: t.string.optional(), + + /** Defaults to verify-full */ + sslmode: t.literal('verify-full').or(t.literal('verify-ca')).or(t.literal('disable')).optional(), + /** Required for verify-ca, optional for verify-full */ + cacert: t.string.optional(), + + client_certificate: t.string.optional(), + client_private_key: t.string.optional(), + + /** Expose database credentials */ + demo_database: t.boolean.optional(), + + /** + * Prefix for the slot name. Defaults to "powersync_" + */ + slot_name_prefix: t.string.optional() +}); + +export type BasePostgresConnectionConfig = t.Encoded; +export type BasePostgresConnectionConfigDecoded = t.Decoded; + +/** + * Validate and normalize connection options. + * + * Returns destructured options. + */ +export function normalizeConnectionConfig(options: BasePostgresConnectionConfigDecoded) { + let uri: urijs.URIComponents; + if (options.uri) { + uri = urijs.parse(options.uri); + if (uri.scheme != 'postgresql' && uri.scheme != 'postgres') { + `Invalid URI - protocol must be postgresql, got ${uri.scheme}`; + } else if (uri.scheme != 'postgresql') { + uri.scheme = 'postgresql'; + } + } else { + uri = urijs.parse('postgresql:///'); + } + + const hostname = options.hostname ?? uri.host ?? ''; + const port = validatePort(options.port ?? uri.port ?? 5432); + + const database = options.database ?? uri.path?.substring(1) ?? ''; + + const [uri_username, uri_password] = (uri.userinfo ?? '').split(':'); + + const username = options.username ?? uri_username ?? ''; + const password = options.password ?? uri_password ?? ''; + + const sslmode = options.sslmode ?? 'verify-full'; // Configuration not supported via URI + const cacert = options.cacert; + + if (sslmode == 'verify-ca' && cacert == null) { + throw new Error('Explicit cacert is required for sslmode=verify-ca'); + } + + if (hostname == '') { + throw new Error(`hostname required`); + } + + if (username == '') { + throw new Error(`username required`); + } + + if (password == '') { + throw new Error(`password required`); + } + + if (database == '') { + throw new Error(`database required`); + } + + return { + id: options.id ?? 'default', + tag: options.tag ?? 'default', + + hostname, + port, + database, + + username, + password, + sslmode, + cacert, + + client_certificate: options.client_certificate ?? undefined, + client_private_key: options.client_private_key ?? undefined + } satisfies NormalizedBasePostgresConnectionConfig; +} + +/** + * Check whether the port is in a "safe" range. + * + * We do not support connecting to "privileged" ports. + */ +export function validatePort(port: string | number): number { + if (typeof port == 'string') { + port = parseInt(port); + } + if (port < 1024) { + throw new Error(`Port ${port} not supported`); + } + return port; +} + +/** + * Construct a postgres URI, without username, password or ssl options. + * + * Only contains hostname, port, database. + */ +export function baseUri(options: NormalizedBasePostgresConnectionConfig) { + return `postgresql://${options.hostname}:${options.port}/${options.database}`; +} diff --git a/libs/lib-postgres/src/utils/pgwire_utils.ts b/libs/lib-postgres/src/utils/pgwire_utils.ts new file mode 100644 index 000000000..230fef7b8 --- /dev/null +++ b/libs/lib-postgres/src/utils/pgwire_utils.ts @@ -0,0 +1,49 @@ +// Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218 + +import * as pgwire from '@powersync/service-jpgwire'; +import { SqliteJsonValue } from '@powersync/service-sync-rules'; + +import { logger } from '@powersync/lib-services-framework'; + +export function escapeIdentifier(identifier: string) { + return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`; +} + +export function autoParameter(arg: SqliteJsonValue | boolean): pgwire.StatementParam { + if (arg == null) { + return { type: 'varchar', value: null }; + } else if (typeof arg == 'string') { + return { type: 'varchar', value: arg }; + } else if (typeof arg == 'number') { + if (Number.isInteger(arg)) { + return { type: 'int8', value: arg }; + } else { + return { type: 'float8', value: arg }; + } + } else if (typeof arg == 'boolean') { + return { type: 'bool', value: arg }; + } else if (typeof arg == 'bigint') { + return { type: 'int8', value: arg }; + } else { + throw new Error(`Unsupported query parameter: ${typeof arg}`); + } +} + +export async function retriedQuery(db: pgwire.PgClient, ...statements: pgwire.Statement[]): Promise; +export async function retriedQuery(db: pgwire.PgClient, query: string): Promise; + +/** + * Retry a simple query - up to 2 attempts total. + */ +export async function retriedQuery(db: pgwire.PgClient, ...args: any[]) { + for (let tries = 2; ; tries--) { + try { + return await db.query(...args); + } catch (e) { + if (tries == 1) { + throw e; + } + logger.warn('Query error, retrying', e); + } + } +} diff --git a/libs/lib-postgres/src/utils/utils-index.ts b/libs/lib-postgres/src/utils/utils-index.ts new file mode 100644 index 000000000..5bfe85120 --- /dev/null +++ b/libs/lib-postgres/src/utils/utils-index.ts @@ -0,0 +1 @@ +export * from './pgwire_utils.js'; diff --git a/libs/lib-postgres/test/src/config.test.ts b/libs/lib-postgres/test/src/config.test.ts new file mode 100644 index 000000000..7b2dc52cc --- /dev/null +++ b/libs/lib-postgres/test/src/config.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, test } from 'vitest'; +import { normalizeConnectionConfig } from '../../src/types/types.js'; + +describe('config', () => { + test('Should resolve database', () => { + const normalized = normalizeConnectionConfig({ + type: 'postgresql', + uri: 'postgresql://localhost:4321/powersync_test' + }); + expect(normalized.database).equals('powersync_test'); + }); +}); diff --git a/libs/lib-postgres/test/tsconfig.json b/libs/lib-postgres/test/tsconfig.json new file mode 100644 index 000000000..4ce408172 --- /dev/null +++ b/libs/lib-postgres/test/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "baseUrl": "./", + "noEmit": true, + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true, + "paths": {} + }, + "include": ["src"], + "references": [ + { + "path": "../" + } + ] +} diff --git a/modules/module-postgres-storage/package.json b/modules/module-postgres-storage/package.json index 134b4627d..46c17ad5b 100644 --- a/modules/module-postgres-storage/package.json +++ b/modules/module-postgres-storage/package.json @@ -27,11 +27,11 @@ }, "dependencies": { "@powersync/lib-services-framework": "workspace:*", + "@powersync/lib-service-postgres": "workspace:*", "@powersync/service-core": "workspace:*", "@powersync/service-core-tests": "workspace:*", "@powersync/service-jpgwire": "workspace:*", "@powersync/service-jsonbig": "^0.17.10", - "@powersync/service-module-postgres": "workspace:*", "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "ix": "^5.0.0", diff --git a/modules/module-postgres-storage/src/migrations/PostgresMigrationAgent.ts b/modules/module-postgres-storage/src/migrations/PostgresMigrationAgent.ts index 07ed3a94a..ba6831d06 100644 --- a/modules/module-postgres-storage/src/migrations/PostgresMigrationAgent.ts +++ b/modules/module-postgres-storage/src/migrations/PostgresMigrationAgent.ts @@ -1,10 +1,12 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import * as framework from '@powersync/lib-services-framework'; import { migrations } from '@powersync/service-core'; -import * as pg_types from '@powersync/service-module-postgres/types'; import * as path from 'path'; import { fileURLToPath } from 'url'; -import { PostgresLockManager } from '../locks/PostgresLockManager.js'; -import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; + +import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js'; + +import { STORAGE_SCHEMA_NAME } from '../utils/db.js'; import { PostgresMigrationStore } from './PostgresMigrationStore.js'; const __filename = fileURLToPath(import.meta.url); @@ -16,16 +18,19 @@ export class PostgresMigrationAgent extends migrations.AbstractPowerSyncMigratio store: framework.MigrationStore; locks: framework.LockManager; - protected db: DatabaseClient; + protected db: lib_postgres.DatabaseClient; - constructor(config: pg_types.PostgresConnectionConfig) { + constructor(config: PostgresStorageConfigDecoded) { super(); - this.db = new DatabaseClient(pg_types.normalizeConnectionConfig(config)); + this.db = new lib_postgres.DatabaseClient({ + config: normalizePostgresStorageConfig(config), + schema: STORAGE_SCHEMA_NAME + }); this.store = new PostgresMigrationStore({ db: this.db }); - this.locks = new PostgresLockManager({ + this.locks = new lib_postgres.PostgresLockManager({ name: 'migrations', db: this.db }); diff --git a/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts b/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts index 6923d6a5d..02c45f01e 100644 --- a/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts +++ b/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts @@ -1,9 +1,9 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { migrations } from '@powersync/lib-services-framework'; -import { sql } from '../utils/connection/AbstractPostgresConnection.js'; -import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; +import { sql } from '../utils/db.js'; export type PostgresMigrationStoreOptions = { - db: DatabaseClient; + db: lib_postgres.DatabaseClient; }; export class PostgresMigrationStore implements migrations.MigrationStore { @@ -59,8 +59,7 @@ export class PostgresMigrationStore implements migrations.MigrationStore { ${{ type: 'varchar', value: state.last_run }}, ${{ type: 'jsonb', value: state.log }} ) - ON CONFLICT (id) DO - UPDATE + ON CONFLICT (id) DO UPDATE SET last_run = EXCLUDED.last_run, LOG = EXCLUDED.log; diff --git a/modules/module-postgres-storage/src/migrations/migration-utils.ts b/modules/module-postgres-storage/src/migrations/migration-utils.ts new file mode 100644 index 000000000..be50c98cc --- /dev/null +++ b/modules/module-postgres-storage/src/migrations/migration-utils.ts @@ -0,0 +1,14 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { configFile } from '@powersync/service-types'; +import { isPostgresStorageConfig, normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js'; +import { STORAGE_SCHEMA_NAME } from '../utils/db.js'; + +export const openMigrationDB = (config: configFile.BaseStorageConfig) => { + if (!isPostgresStorageConfig(config)) { + throw new Error(`Input storage configuration is not for Postgres`); + } + return new lib_postgres.DatabaseClient({ + config: normalizePostgresStorageConfig(PostgresStorageConfig.decode(config)), + schema: STORAGE_SCHEMA_NAME + }); +}; diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index fe4b5b329..a8c48a568 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -1,13 +1,13 @@ import { migrations } from '@powersync/service-core'; -import { PostgresConnectionConfig, normalizeConnectionConfig } from '@powersync/service-module-postgres/types'; -import { DatabaseClient } from '../../utils/connection/DatabaseClient.js'; + import { dropTables } from '../../utils/db.js'; +import { openMigrationDB } from '../migration-utils.js'; export const up: migrations.PowerSyncMigrationFunction = async (context) => { const { service_context: { configuration } } = context; - await using client = new DatabaseClient(normalizeConnectionConfig(configuration.storage as PostgresConnectionConfig)); + await using client = openMigrationDB(configuration.storage); /** * Request an explicit connection which will automatically set the search @@ -137,7 +137,7 @@ export const down: migrations.PowerSyncMigrationFunction = async (context) => { const { service_context: { configuration } } = context; - await using db = new DatabaseClient(normalizeConnectionConfig(configuration.storage as PostgresConnectionConfig)); + using client = openMigrationDB(configuration.storage); - await dropTables(db); + await dropTables(client); }; diff --git a/modules/module-postgres-storage/src/module/PostgresStorageModule.ts b/modules/module-postgres-storage/src/module/PostgresStorageModule.ts index 2dc4b18db..90cbbb430 100644 --- a/modules/module-postgres-storage/src/module/PostgresStorageModule.ts +++ b/modules/module-postgres-storage/src/module/PostgresStorageModule.ts @@ -1,7 +1,8 @@ import { modules, system } from '@powersync/service-core'; -import * as pg_types from '@powersync/service-module-postgres/types'; + import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'; import { PostgresStorageProvider } from '../storage/PostgresStorageProvider.js'; +import { isPostgresStorageConfig, PostgresStorageConfig } from '../types/types.js'; export class PostgresStorageModule extends modules.AbstractModule { constructor() { @@ -16,8 +17,10 @@ export class PostgresStorageModule extends modules.AbstractModule { // Register the ability to use Postgres as a BucketStorage storageEngine.registerProvider(new PostgresStorageProvider()); - if (pg_types.isPostgresConfig(context.configuration.storage)) { - context.migrations.registerMigrationAgent(new PostgresMigrationAgent(context.configuration.storage)); + if (isPostgresStorageConfig(context.configuration.storage)) { + context.migrations.registerMigrationAgent( + new PostgresMigrationAgent(PostgresStorageConfig.decode(context.configuration.storage)) + ); } } diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index d34cff7ff..709cd2b6c 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -7,9 +7,11 @@ import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js'; import { LRUCache } from 'lru-cache/min'; import * as timers from 'timers/promises'; import * as uuid from 'uuid'; -import { PostgresLockManager } from '../locks/PostgresLockManager.js'; + +import * as lib_postgres from '@powersync/lib-service-postgres'; import { models, NormalizedPostgresStorageConfig } from '../types/types.js'; -import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; + +import { NOTIFICATION_CHANNEL, STORAGE_SCHEMA_NAME } from '../utils/db.js'; import { notifySyncRulesUpdate } from './batch/PostgresBucketBatch.js'; import { PostgresSyncRulesStorage } from './PostgresSyncRulesStorage.js'; import { PostgresPersistedSyncRulesContent } from './sync-rules/PostgresPersistedSyncRulesContent.js'; @@ -23,7 +25,7 @@ export class PostgresBucketStorageFactory extends framework.DisposableObserver implements storage.BucketStorageFactory { - readonly db: DatabaseClient; + readonly db: lib_postgres.DatabaseClient; public readonly slot_name_prefix: string; protected notificationConnection: pg_wire.PgConnection | null; @@ -57,7 +59,11 @@ export class PostgresBucketStorageFactory constructor(protected options: PostgresBucketStorageOptions) { super(); - this.db = new DatabaseClient(options.config); + this.db = new lib_postgres.DatabaseClient({ + config: options.config, + schema: STORAGE_SCHEMA_NAME, + notificationChannels: [NOTIFICATION_CHANNEL] + }); this.slot_name_prefix = options.slot_name_prefix; this.notificationConnection = null; @@ -150,7 +156,7 @@ export class PostgresBucketStorageFactory if (instanceRow) { return instanceRow.id; } - const lockManager = new PostgresLockManager({ + const lockManager = new lib_postgres.PostgresLockManager({ db: this.db, name: `instance-id-insertion-lock` }); diff --git a/modules/module-postgres-storage/src/storage/PostgresCompactor.ts b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts index 9423b940c..94d647881 100644 --- a/modules/module-postgres-storage/src/storage/PostgresCompactor.ts +++ b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts @@ -1,11 +1,11 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { logger } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import * as t from 'ts-codec'; import { BIGINT_MAX } from '../types/codecs.js'; import { models } from '../types/types.js'; -import { sql } from '../utils/connection/AbstractPostgresConnection.js'; -import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; +import { sql } from '../utils/db.js'; import { pick } from '../utils/ts-codec.js'; import { encodedCacheKey } from './batch/OperationBatch.js'; @@ -62,7 +62,7 @@ export class PostgresCompactor { private buckets: string[] | undefined; constructor( - private db: DatabaseClient, + private db: lib_postgres.DatabaseClient, private group_id: number, options?: PostgresCompactOptions ) { diff --git a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts index 276ba1064..7125ec6a8 100644 --- a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts +++ b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts @@ -1,28 +1,28 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; -import { POSTGRES_CONNECTION_TYPE } from '@powersync/service-module-postgres/types'; -import { normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js'; +import { isPostgresStorageConfig, normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js'; import { dropTables } from '../utils/db.js'; import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; export class PostgresStorageProvider implements storage.BucketStorageProvider { get type() { - return POSTGRES_CONNECTION_TYPE; + return lib_postgres.POSTGRES_CONNECTION_TYPE; } async getStorage(options: storage.GetStorageOptions): Promise { const { resolvedConfig } = options; const { storage } = resolvedConfig; - if (storage.type != POSTGRES_CONNECTION_TYPE) { + if (!isPostgresStorageConfig(storage)) { // This should not be reached since the generation should be managed externally. throw new Error( - `Cannot create Postgres bucket storage with provided config ${storage.type} !== ${POSTGRES_CONNECTION_TYPE}` + `Cannot create Postgres bucket storage with provided config ${storage.type} !== ${lib_postgres.POSTGRES_CONNECTION_TYPE}` ); } - const decodedConfig = PostgresStorageConfig.decode(storage as any); + const decodedConfig = PostgresStorageConfig.decode(storage); const normalizedConfig = normalizePostgresStorageConfig(decodedConfig); const storageFactory = new PostgresBucketStorageFactory({ config: normalizedConfig, @@ -33,7 +33,7 @@ export class PostgresStorageProvider implements storage.BucketStorageProvider { shutDown: async () => storageFactory.db[Symbol.asyncDispose](), tearDown: async () => { logger.info(`Tearing down Postgres storage: ${normalizedConfig.database}...`); - await dropTables(storage.db); + await dropTables(storageFactory.db); await storageFactory.db[Symbol.asyncDispose](); return true; } diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 883b984ee..b179f2152 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -1,3 +1,4 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { DisposableObserver } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import * as sync_rules from '@powersync/service-sync-rules'; @@ -7,7 +8,7 @@ import { bigint, BIGINT_MAX } from '../types/codecs.js'; import { models, RequiredOperationBatchLimits } from '../types/types.js'; import { replicaIdToSubkey } from '../utils/bson.js'; import { mapOpEntry } from '../utils/bucket-data.js'; -import { DatabaseClient } from '../utils/connection/DatabaseClient.js'; + import { pick } from '../utils/ts-codec.js'; import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js'; import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js'; @@ -16,7 +17,7 @@ import { PostgresCompactor } from './PostgresCompactor.js'; export type PostgresSyncRulesStorageOptions = { factory: PostgresBucketStorageFactory; - db: DatabaseClient; + db: lib_postgres.DatabaseClient; sync_rules: storage.PersistedSyncRulesContent; write_checkpoint_mode?: storage.WriteCheckpointMode; batchLimits: RequiredOperationBatchLimits; @@ -31,7 +32,7 @@ export class PostgresSyncRulesStorage public readonly slot_name: string; public readonly factory: PostgresBucketStorageFactory; - protected db: DatabaseClient; + protected db: lib_postgres.DatabaseClient; protected writeCheckpointAPI: PostgresWriteCheckpointAPI; // TODO we might be able to share this in an abstract class diff --git a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts index e27fc9204..183a2fd81 100644 --- a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts +++ b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts @@ -1,7 +1,6 @@ import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOptions } from '@powersync/service-core'; -import { PostgresConnectionConfig } from '@powersync/service-module-postgres/types'; import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'; -import { normalizePostgresStorageConfig } from '../types/types.js'; +import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js'; import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; export type PostgresTestStorageOptions = { @@ -10,7 +9,7 @@ export type PostgresTestStorageOptions = { * Vitest can cause issues when loading .ts files for migrations. * This allows for providing a custom PostgresMigrationAgent. */ - migrationAgent?: (config: PostgresConnectionConfig) => PostgresMigrationAgent; + migrationAgent?: (config: PostgresStorageConfigDecoded) => PostgresMigrationAgent; }; export const PostgresTestStorageFactoryGenerator = (factoryOptions: PostgresTestStorageOptions) => { diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index 5ba29e7cc..608536094 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -1,3 +1,4 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { container, DisposableObserver, errors, logger } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import * as sync_rules from '@powersync/service-sync-rules'; @@ -5,17 +6,13 @@ import * as timers from 'timers/promises'; import * as t from 'ts-codec'; import { CurrentBucket, CurrentData, CurrentDataDecoded } from '../../types/models/CurrentData.js'; import { models, RequiredOperationBatchLimits } from '../../types/types.js'; -import { sql } from '../../utils/connection/AbstractPostgresConnection.js'; -import { NOTIFICATION_CHANNEL } from '../../utils/connection/ConnectionSlot.js'; -import { DatabaseClient } from '../../utils/connection/DatabaseClient.js'; -import { WrappedConnection } from '../../utils/connection/WrappedConnection.js'; +import { NOTIFICATION_CHANNEL } from '../../utils/db.js'; import { pick } from '../../utils/ts-codec.js'; import { batchCreateCustomWriteCheckpoints } from '../checkpoints/PostgresWriteCheckpointAPI.js'; import { cacheKey, encodedCacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PostgresPersistedBatch } from './PostgresPersistedBatch.js'; - export interface PostgresBucketBatchOptions { - db: DatabaseClient; + db: lib_postgres.DatabaseClient; sync_rules: sync_rules.SqlSyncRules; group_id: number; slot_name: string; @@ -46,7 +43,7 @@ export class PostgresBucketBatch { public last_flushed_op: bigint | null = null; - protected db: DatabaseClient; + protected db: lib_postgres.DatabaseClient; protected group_id: number; protected last_checkpoint_lsn: string | null; protected no_checkpoint_before_lsn: string; @@ -148,7 +145,7 @@ export class PostgresBucketBatch while (lastBatchCount == BATCH_LIMIT) { lastBatchCount = 0; - for await (const rows of this.db.streamRows>(sql` + for await (const rows of this.db.streamRows>(lib_postgres.sql` SELECT buckets, lookups, @@ -445,7 +442,7 @@ export class PostgresBucketBatch }); } - protected async replicateBatch(db: WrappedConnection, batch: OperationBatch) { + protected async replicateBatch(db: lib_postgres.WrappedConnection, batch: OperationBatch) { let sizes: Map | undefined = undefined; if (this.options.store_current_data && !this.options.skip_existing_rows) { // We skip this step if we don't store current_data, since the sizes will @@ -472,7 +469,7 @@ export class PostgresBucketBatch source_table: string; source_key: storage.ReplicaId; data_size: number; - }>(sql` + }>(lib_postgres.sql` WITH filter_data AS ( SELECT @@ -854,7 +851,9 @@ export class PostgresBucketBatch ); } - protected async withReplicationTransaction(callback: (tx: WrappedConnection) => Promise): Promise { + protected async withReplicationTransaction( + callback: (tx: lib_postgres.WrappedConnection) => Promise + ): Promise { try { return await this.db.transaction(async (db) => { return await callback(db); @@ -875,7 +874,7 @@ export class PostgresBucketBatch * Uses Postgres' NOTIFY functionality to update different processes when the * active checkpoint has been updated. */ -export const notifySyncRulesUpdate = async (db: DatabaseClient, update: StatefulCheckpointDecoded) => { +export const notifySyncRulesUpdate = async (db: lib_postgres.DatabaseClient, update: StatefulCheckpointDecoded) => { if (update.state != storage.SyncRuleState.ACTIVE) { return; } diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts index 740231a7e..2c803dd9c 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts @@ -1,10 +1,10 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { logger } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; import * as sync_rules from '@powersync/service-sync-rules'; import { models, RequiredOperationBatchLimits } from '../../types/types.js'; import { replicaIdToSubkey } from '../../utils/bson.js'; -import { WrappedConnection } from '../../utils/connection/WrappedConnection.js'; export type SaveBucketDataOptions = { /** @@ -227,7 +227,7 @@ export class PostgresPersistedBatch { ); } - async flush(db: WrappedConnection) { + async flush(db: lib_postgres.WrappedConnection) { logger.info( `powersync_${this.group_id} Flushed ${this.bucketDataInserts.length} + ${this.parameterDataInserts.length} + ${ this.currentDataInserts.size + this.currentDataDeletes.length @@ -245,7 +245,7 @@ export class PostgresPersistedBatch { this.currentSize = 0; } - protected async flushBucketData(db: WrappedConnection) { + protected async flushBucketData(db: lib_postgres.WrappedConnection) { if (this.bucketDataInserts.length > 0) { await db.sql` WITH @@ -307,7 +307,7 @@ export class PostgresPersistedBatch { } } - protected async flushParameterData(db: WrappedConnection) { + protected async flushParameterData(db: lib_postgres.WrappedConnection) { if (this.parameterDataInserts.length > 0) { await db.sql` WITH @@ -347,7 +347,7 @@ export class PostgresPersistedBatch { } } - protected async flushCurrentData(db: WrappedConnection) { + protected async flushCurrentData(db: lib_postgres.WrappedConnection) { if (this.currentDataInserts.size > 0) { await db.sql` WITH diff --git a/modules/module-postgres-storage/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts b/modules/module-postgres-storage/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts index 529e861b0..a12c405bf 100644 --- a/modules/module-postgres-storage/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +++ b/modules/module-postgres-storage/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts @@ -1,16 +1,16 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import * as framework from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; import { models } from '../../types/types.js'; -import { DatabaseClient } from '../../utils/connection/DatabaseClient.js'; export type PostgresCheckpointAPIOptions = { - db: DatabaseClient; + db: lib_postgres.DatabaseClient; mode: storage.WriteCheckpointMode; }; export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI { - readonly db: DatabaseClient; + readonly db: lib_postgres.DatabaseClient; private _mode: storage.WriteCheckpointMode; constructor(options: PostgresCheckpointAPIOptions) { @@ -47,8 +47,7 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI { ${{ type: 'int8', value: checkpoint }}, ${{ type: 'int4', value: sync_rules_id }} ) - ON CONFLICT DO - UPDATE + ON CONFLICT DO UPDATE SET write_checkpoint = EXCLUDED.write_checkpoint RETURNING @@ -75,8 +74,7 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI { ${{ type: 'jsonb', value: checkpoint.heads }}, ${{ type: 'int8', value: 1 }} ) - ON CONFLICT (user_id) DO - UPDATE + ON CONFLICT (user_id) DO UPDATE SET write_checkpoint = write_checkpoints.write_checkpoint + 1, lsns = EXCLUDED.lsns @@ -145,7 +143,7 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI { } export async function batchCreateCustomWriteCheckpoints( - db: DatabaseClient, + db: lib_postgres.DatabaseClient, checkpoints: storage.CustomWriteCheckpointOptions[] ): Promise { if (!checkpoints.length) { @@ -171,8 +169,7 @@ export async function batchCreateCustomWriteCheckpoints( )::int4 FROM json_data - ON CONFLICT (user_id, sync_rules_id) DO - UPDATE + ON CONFLICT (user_id, sync_rules_id) DO UPDATE SET write_checkpoint = EXCLUDED.write_checkpoint; `.execute(); diff --git a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts index e472ac681..c3cf5da6e 100644 --- a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +++ b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts @@ -1,9 +1,9 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import { SqlSyncRules } from '@powersync/service-sync-rules'; -import { PostgresLockManager } from '../../locks/PostgresLockManager.js'; + import { models } from '../../types/types.js'; -import { DatabaseClient } from '../../utils/connection/DatabaseClient.js'; export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncRulesContent { public readonly slot_name: string; @@ -17,7 +17,7 @@ export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncR current_lock: storage.ReplicationLock | null = null; constructor( - private db: DatabaseClient, + private db: lib_postgres.DatabaseClient, row: models.SyncRulesDecoded ) { this.id = Number(row.id); @@ -38,7 +38,7 @@ export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncR } async lock(): Promise { - const manager = new PostgresLockManager({ + const manager = new lib_postgres.PostgresLockManager({ db: this.db, name: `sync_rules_${this.id}_${this.slot_name}` }); diff --git a/modules/module-postgres-storage/src/types/types.ts b/modules/module-postgres-storage/src/types/types.ts index 30fe263f6..80ff067b4 100644 --- a/modules/module-postgres-storage/src/types/types.ts +++ b/modules/module-postgres-storage/src/types/types.ts @@ -1,5 +1,6 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import * as pg_wire from '@powersync/service-jpgwire'; -import * as pg_types from '@powersync/service-module-postgres/types'; +import { configFile } from '@powersync/service-types'; import * as t from 'ts-codec'; export * as models from './models/models-index.js'; @@ -33,19 +34,18 @@ export const OperationBatchLimits = BatchLimits.and( export type OperationBatchLimits = t.Encoded; -export const BaseStorageConfig = t.object({ - /** - * Allow batch operation limits to be configurable. - * Postgres has less batch size restrictions compared to MongoDB. - * Increasing limits can drastically improve replication performance, but - * can come at the cost of higher memory usage or potential issues. - */ - batch_limits: OperationBatchLimits.optional() -}); - -export type BaseStorageConfig = t.Encoded; +export const PostgresStorageConfig = configFile.BaseStorageConfig.and(lib_postgres.BasePostgresConnectionConfig).and( + t.object({ + /** + * Allow batch operation limits to be configurable. + * Postgres has less batch size restrictions compared to MongoDB. + * Increasing limits can drastically improve replication performance, but + * can come at the cost of higher memory usage or potential issues. + */ + batch_limits: OperationBatchLimits.optional() + }) +); -export const PostgresStorageConfig = pg_types.PostgresConnectionConfig.and(BaseStorageConfig); export type PostgresStorageConfig = t.Encoded; export type PostgresStorageConfigDecoded = t.Decoded; @@ -59,7 +59,7 @@ export const normalizePostgresStorageConfig = ( baseConfig: PostgresStorageConfigDecoded ): NormalizedPostgresStorageConfig => { return { - ...pg_types.normalizeConnectionConfig(baseConfig), + ...lib_postgres.normalizeConnectionConfig(baseConfig), batch_limits: { max_current_data_batch_size: baseConfig.batch_limits?.max_current_data_batch_size ?? MAX_BATCH_CURRENT_DATA_SIZE, max_estimated_size: baseConfig.batch_limits?.max_estimated_size ?? MAX_BATCH_ESTIMATED_SIZE, @@ -67,3 +67,7 @@ export const normalizePostgresStorageConfig = ( } }; }; + +export const isPostgresStorageConfig = (config: configFile.BaseStorageConfig): config is PostgresStorageConfig => { + return config.type == lib_postgres.POSTGRES_CONNECTION_TYPE; +}; diff --git a/modules/module-postgres-storage/src/utils/db.ts b/modules/module-postgres-storage/src/utils/db.ts index 6f94fef90..500cb3aa8 100644 --- a/modules/module-postgres-storage/src/utils/db.ts +++ b/modules/module-postgres-storage/src/utils/db.ts @@ -1,6 +1,15 @@ -import { DatabaseClient } from './connection/DatabaseClient.js'; +import * as lib_postgres from '@powersync/lib-service-postgres'; -export const dropTables = async (client: DatabaseClient) => { +export const STORAGE_SCHEMA_NAME = 'powersync'; + +export const NOTIFICATION_CHANNEL = 'powersynccheckpoints'; + +/** + * Re export for prettier to detect the tag better + */ +export const sql = lib_postgres.sql; + +export const dropTables = async (client: lib_postgres.DatabaseClient) => { // Lock a connection for automatic schema search paths await client.lockConnection(async (db) => { await db.sql`DROP TABLE IF EXISTS bucket_data`.execute(); diff --git a/modules/module-postgres-storage/tsconfig.json b/modules/module-postgres-storage/tsconfig.json index 4cd41de8c..edfd2bf74 100644 --- a/modules/module-postgres-storage/tsconfig.json +++ b/modules/module-postgres-storage/tsconfig.json @@ -10,5 +10,27 @@ "skipLibCheck": true }, "include": ["src"], - "references": [] + "references": [ + { + "path": "../../packages/types" + }, + { + "path": "../../packages/jsonbig" + }, + { + "path": "../../packages/jpgwire" + }, + { + "path": "../../packages/sync-rules" + }, + { + "path": "../../packages/service-core" + }, + { + "path": "../../libs/lib-services" + }, + { + "path": "../../libs/lib-postgres" + } + ] } diff --git a/modules/module-postgres/package.json b/modules/module-postgres/package.json index 759696535..c86566a32 100644 --- a/modules/module-postgres/package.json +++ b/modules/module-postgres/package.json @@ -29,6 +29,7 @@ }, "dependencies": { "@powersync/lib-services-framework": "workspace:*", + "@powersync/lib-service-postgres": "workspace:*", "@powersync/service-core": "workspace:*", "@powersync/service-jpgwire": "workspace:*", "@powersync/service-jsonbig": "workspace:*", diff --git a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts index 76edcea68..2600657e1 100644 --- a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts +++ b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts @@ -1,13 +1,12 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { api, ParseSyncRulesOptions } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; - import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import * as replication_utils from '../replication/replication-utils.js'; -import * as types from '../types/types.js'; -import * as pg_utils from '../utils/pgwire_utils.js'; import { getDebugTableInfo } from '../replication/replication-utils.js'; import { PUBLICATION_NAME } from '../replication/WalStream.js'; +import * as types from '../types/types.js'; export class PostgresRouteAPIAdapter implements api.RouteAPI { connectionTag: string; @@ -53,7 +52,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { }; try { - await pg_utils.retriedQuery(this.pool, `SELECT 'PowerSync connection test'`); + await lib_postgres.retriedQuery(this.pool, `SELECT 'PowerSync connection test'`); } catch (e) { return { ...base, @@ -94,7 +93,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { try { const result = await this.pool.query({ statement: query, - params: params.map(pg_utils.autoParameter) + params: params.map(lib_postgres.autoParameter) }); return service_types.internal_routes.ExecuteSqlResponse.encode({ @@ -146,7 +145,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { if (tablePattern.isWildcard) { patternResult.tables = []; const prefix = tablePattern.tablePrefix; - const results = await pg_utils.retriedQuery(this.pool, { + const results = await lib_postgres.retriedQuery(this.pool, { statement: `SELECT c.oid AS relid, c.relname AS table_name FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace @@ -169,7 +168,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { patternResult.tables.push(details); } } else { - const results = await pg_utils.retriedQuery(this.pool, { + const results = await lib_postgres.retriedQuery(this.pool, { statement: `SELECT c.oid AS relid, c.relname AS table_name FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace @@ -215,7 +214,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { async getReplicationLag(options: api.ReplicationLagOptions): Promise { const { bucketStorage } = options; const slotName = bucketStorage.slot_name; - const results = await pg_utils.retriedQuery(this.pool, { + const results = await lib_postgres.retriedQuery(this.pool, { statement: `SELECT slot_name, confirmed_flush_lsn, @@ -237,7 +236,7 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`, // However, on Aurora (Postgres compatible), it can return an entirely different LSN, // causing the write checkpoints to never be replicated back to the client. // For those, we need to use pg_current_wal_lsn() instead. - const { results } = await pg_utils.retriedQuery( + const { results } = await lib_postgres.retriedQuery( this.pool, { statement: `SELECT pg_current_wal_lsn() as lsn` }, { statement: `SELECT pg_logical_emit_message(false, 'powersync', 'ping')` } @@ -250,7 +249,7 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`, async getConnectionSchema(): Promise { // https://github.com/Borvik/vscode-postgres/blob/88ec5ed061a0c9bced6c5d4ec122d0759c3f3247/src/language/server.ts - const results = await pg_utils.retriedQuery( + const results = await lib_postgres.retriedQuery( this.pool, `SELECT tbl.schemaname, diff --git a/modules/module-postgres/src/auth/SupabaseKeyCollector.ts b/modules/module-postgres/src/auth/SupabaseKeyCollector.ts index af300f251..187d68a80 100644 --- a/modules/module-postgres/src/auth/SupabaseKeyCollector.ts +++ b/modules/module-postgres/src/auth/SupabaseKeyCollector.ts @@ -1,9 +1,9 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { auth } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import * as jose from 'jose'; import * as types from '../types/types.js'; -import * as pgwire_utils from '../utils/pgwire_utils.js'; /** * Fetches key from the Supabase database. @@ -39,7 +39,7 @@ export class SupabaseKeyCollector implements auth.KeyCollector { let row: { jwt_secret: string }; try { const rows = pgwire.pgwireRows( - await pgwire_utils.retriedQuery(this.pool, `SELECT current_setting('app.settings.jwt_secret') as jwt_secret`) + await lib_postgres.retriedQuery(this.pool, `SELECT current_setting('app.settings.jwt_secret') as jwt_secret`) ); row = rows[0] as any; } catch (e) { diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index 34c168f90..92b8a2031 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -1,8 +1,10 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { container, errors, logger } from '@powersync/lib-services-framework'; import { getUuidReplicaIdentityBson, Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules'; import * as pg_utils from '../utils/pgwire_utils.js'; + import { PgManager } from './PgManager.js'; import { getPgOutputRelation, getRelId } from './PgRelation.js'; import { checkSourceConfiguration, getReplicationIdentityColumns } from './replication-utils.js'; @@ -63,7 +65,7 @@ export class WalStream { // Ping to speed up cancellation of streaming replication // We're not using pg_snapshot here, since it could be in the middle of // an initial replication transaction. - const promise = pg_utils.retriedQuery( + const promise = lib_postgres.retriedQuery( this.connections.pool, `SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')` ); diff --git a/modules/module-postgres/src/replication/replication-utils.ts b/modules/module-postgres/src/replication/replication-utils.ts index c6b1e3fe1..6dde04035 100644 --- a/modules/module-postgres/src/replication/replication-utils.ts +++ b/modules/module-postgres/src/replication/replication-utils.ts @@ -1,13 +1,11 @@ import * as pgwire from '@powersync/service-jpgwire'; +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { logger } from '@powersync/lib-services-framework'; import { PatternResult, storage } from '@powersync/service-core'; -import * as pgwire_utils from '../utils/pgwire_utils.js'; -import { ReplicationIdentity } from './PgRelation.js'; import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; -import * as pg_utils from '../utils/pgwire_utils.js'; -import * as util from '../utils/pgwire_utils.js'; -import { logger } from '@powersync/lib-services-framework'; +import { ReplicationIdentity } from './PgRelation.js'; export interface ReplicaIdentityResult { replicationColumns: storage.ColumnDescriptor[]; @@ -20,7 +18,7 @@ export async function getPrimaryKeyColumns( mode: 'primary' | 'replident' ): Promise { const indexFlag = mode == 'primary' ? `i.indisprimary` : `i.indisreplident`; - const attrRows = await pgwire_utils.retriedQuery(db, { + const attrRows = await lib_postgres.retriedQuery(db, { statement: `SELECT a.attname as name, a.atttypid as typeid, t.typname as type, a.attnum as attnum FROM pg_index i JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY (i.indkey) @@ -41,7 +39,7 @@ export async function getPrimaryKeyColumns( } export async function getAllColumns(db: pgwire.PgClient, relationId: number): Promise { - const attrRows = await pgwire_utils.retriedQuery(db, { + const attrRows = await lib_postgres.retriedQuery(db, { statement: `SELECT a.attname as name, a.atttypid as typeid, t.typname as type, a.attnum as attnum FROM pg_attribute a JOIN pg_type t ON a.atttypid = t.oid @@ -62,7 +60,7 @@ export async function getReplicationIdentityColumns( db: pgwire.PgClient, relationId: number ): Promise { - const rows = await pgwire_utils.retriedQuery(db, { + const rows = await lib_postgres.retriedQuery(db, { statement: `SELECT CASE relreplident WHEN 'd' THEN 'default' WHEN 'n' THEN 'nothing' @@ -95,7 +93,7 @@ WHERE oid = $1::oid LIMIT 1`, export async function checkSourceConfiguration(db: pgwire.PgClient, publicationName: string): Promise { // Check basic config - await pgwire_utils.retriedQuery( + await lib_postgres.retriedQuery( db, `DO $$ BEGIN @@ -113,7 +111,7 @@ $$ LANGUAGE plpgsql;` ); // Check that publication exists - const rs = await pgwire_utils.retriedQuery(db, { + const rs = await lib_postgres.retriedQuery(db, { statement: `SELECT * FROM pg_publication WHERE pubname = $1`, params: [{ type: 'varchar', value: publicationName }] }); @@ -158,7 +156,7 @@ export async function getDebugTablesInfo(options: GetDebugTablesInfoOptions): Pr if (tablePattern.isWildcard) { patternResult.tables = []; const prefix = tablePattern.tablePrefix; - const results = await util.retriedQuery(db, { + const results = await lib_postgres.retriedQuery(db, { statement: `SELECT c.oid AS relid, c.relname AS table_name FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace @@ -189,7 +187,7 @@ export async function getDebugTablesInfo(options: GetDebugTablesInfoOptions): Pr patternResult.tables.push(details); } } else { - const results = await util.retriedQuery(db, { + const results = await lib_postgres.retriedQuery(db, { statement: `SELECT c.oid AS relid, c.relname AS table_name FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace @@ -284,14 +282,14 @@ export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Prom let selectError = null; try { - await pg_utils.retriedQuery(db, `SELECT * FROM ${sourceTable.escapedIdentifier} LIMIT 1`); + await lib_postgres.retriedQuery(db, `SELECT * FROM ${sourceTable.escapedIdentifier} LIMIT 1`); } catch (e) { selectError = { level: 'fatal', message: e.message }; } let replicateError = null; - const publications = await pg_utils.retriedQuery(db, { + const publications = await lib_postgres.retriedQuery(db, { statement: `SELECT tablename FROM pg_publication_tables WHERE pubname = $1 AND schemaname = $2 AND tablename = $3`, params: [ { type: 'varchar', value: publicationName }, diff --git a/modules/module-postgres/src/types/types.ts b/modules/module-postgres/src/types/types.ts index 3629a8cb7..4de2ac0ed 100644 --- a/modules/module-postgres/src/types/types.ts +++ b/modules/module-postgres/src/types/types.ts @@ -1,56 +1,18 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import * as service_types from '@powersync/service-types'; import * as t from 'ts-codec'; -import * as urijs from 'uri-js'; -export const POSTGRES_CONNECTION_TYPE = 'postgresql' as const; - -export interface NormalizedPostgresConnectionConfig { - id: string; - tag: string; - - hostname: string; - port: number; - database: string; - - username: string; - password: string; - - sslmode: 'verify-full' | 'verify-ca' | 'disable'; - cacert: string | undefined; - - client_certificate: string | undefined; - client_private_key: string | undefined; -} +// Maintain backwards compatibility by exporting these +export const validatePort = lib_postgres.validatePort; +export const baseUri = lib_postgres.baseUri; +export type NormalizedPostgresConnectionConfig = lib_postgres.NormalizedBasePostgresConnectionConfig; +export const POSTGRES_CONNECTION_TYPE = lib_postgres.POSTGRES_CONNECTION_TYPE; export const PostgresConnectionConfig = service_types.configFile.DataSourceConfig.and( + lib_postgres.BasePostgresConnectionConfig +).and( t.object({ - type: t.literal(POSTGRES_CONNECTION_TYPE), - /** Unique identifier for the connection - optional when a single connection is present. */ - id: t.string.optional(), - /** Tag used as reference in sync rules. Defaults to "default". Does not have to be unique. */ - tag: t.string.optional(), - uri: t.string.optional(), - hostname: t.string.optional(), - port: service_types.configFile.portCodec.optional(), - username: t.string.optional(), - password: t.string.optional(), - database: t.string.optional(), - - /** Defaults to verify-full */ - sslmode: t.literal('verify-full').or(t.literal('verify-ca')).or(t.literal('disable')).optional(), - /** Required for verify-ca, optional for verify-full */ - cacert: t.string.optional(), - - client_certificate: t.string.optional(), - client_private_key: t.string.optional(), - - /** Expose database credentials */ - demo_database: t.boolean.optional(), - - /** - * Prefix for the slot name. Defaults to "powersync_" - */ - slot_name_prefix: t.string.optional() + // Add any replication connection specific config here in future }) ); @@ -64,101 +26,19 @@ export type PostgresConnectionConfig = t.Decoded SQLite row. @@ -28,46 +26,3 @@ export function constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.Pg const record = pgwire.decodeTuple(message.relation, rawData); return toSyncRulesRow(record); } - -export function escapeIdentifier(identifier: string) { - return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`; -} - -export function autoParameter(arg: SqliteJsonValue | boolean): pgwire.StatementParam { - if (arg == null) { - return { type: 'varchar', value: null }; - } else if (typeof arg == 'string') { - return { type: 'varchar', value: arg }; - } else if (typeof arg == 'number') { - if (Number.isInteger(arg)) { - return { type: 'int8', value: arg }; - } else { - return { type: 'float8', value: arg }; - } - } else if (typeof arg == 'boolean') { - return { type: 'bool', value: arg }; - } else if (typeof arg == 'bigint') { - return { type: 'int8', value: arg }; - } else { - throw new Error(`Unsupported query parameter: ${typeof arg}`); - } -} - -export async function retriedQuery(db: pgwire.PgClient, ...statements: pgwire.Statement[]): Promise; -export async function retriedQuery(db: pgwire.PgClient, query: string): Promise; - -/** - * Retry a simple query - up to 2 attempts total. - */ -export async function retriedQuery(db: pgwire.PgClient, ...args: any[]) { - for (let tries = 2; ; tries--) { - try { - return await db.query(...args); - } catch (e) { - if (tries == 1) { - throw e; - } - logger.warn('Query error, retrying', e); - } - } -} diff --git a/modules/module-postgres/test/src/util.ts b/modules/module-postgres/test/src/util.ts index 8e5d3a528..dca5521fa 100644 --- a/modules/module-postgres/test/src/util.ts +++ b/modules/module-postgres/test/src/util.ts @@ -1,6 +1,6 @@ import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js'; import * as types from '@module/types/types.js'; -import * as pg_utils from '@module/utils/pgwire_utils.js'; +import * as lib_postgres from '@powersync/lib-service-postgres'; import { logger } from '@powersync/lib-services-framework'; import { BucketStorageFactory, OpId } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; @@ -45,7 +45,7 @@ export async function clearTestDb(db: pgwire.PgClient) { for (let row of tableRows) { const name = row.table_name; if (name.startsWith('test_')) { - await db.query(`DROP TABLE public.${pg_utils.escapeIdentifier(name)}`); + await db.query(`DROP TABLE public.${lib_postgres.escapeIdentifier(name)}`); } } } diff --git a/modules/module-postgres/tsconfig.json b/modules/module-postgres/tsconfig.json index 9ceadec40..77a56fc5a 100644 --- a/modules/module-postgres/tsconfig.json +++ b/modules/module-postgres/tsconfig.json @@ -26,6 +26,9 @@ }, { "path": "../../libs/lib-services" + }, + { + "path": "../../libs/lib-postgres" } ] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a5a560e17..cf7938377 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -84,6 +84,37 @@ importers: specifier: ^4.4.1 version: 4.4.1 + libs/lib-postgres: + dependencies: + '@powersync/lib-services-framework': + specifier: workspace:* + version: link:../lib-services + '@powersync/service-jpgwire': + specifier: workspace:* + version: link:../../packages/jpgwire + '@powersync/service-sync-rules': + specifier: workspace:* + version: link:../../packages/sync-rules + '@powersync/service-types': + specifier: workspace:* + version: link:../../packages/types + p-defer: + specifier: ^4.0.1 + version: 4.0.1 + ts-codec: + specifier: ^1.3.0 + version: 1.3.0 + uri-js: + specifier: ^4.4.1 + version: 4.4.1 + uuid: + specifier: ^9.0.1 + version: 9.0.1 + devDependencies: + '@types/uuid': + specifier: ^9.0.4 + version: 9.0.8 + libs/lib-services: dependencies: ajv: @@ -276,6 +307,9 @@ importers: modules/module-postgres: dependencies: + '@powersync/lib-service-postgres': + specifier: workspace:* + version: link:../../libs/lib-postgres '@powersync/lib-services-framework': specifier: workspace:* version: link:../../libs/lib-services @@ -325,6 +359,9 @@ importers: modules/module-postgres-storage: dependencies: + '@powersync/lib-service-postgres': + specifier: workspace:* + version: link:../../libs/lib-postgres '@powersync/lib-services-framework': specifier: workspace:* version: link:../../libs/lib-services @@ -340,9 +377,6 @@ importers: '@powersync/service-jsonbig': specifier: ^0.17.10 version: 0.17.10 - '@powersync/service-module-postgres': - specifier: workspace:* - version: link:../module-postgres '@powersync/service-sync-rules': specifier: workspace:* version: link:../../packages/sync-rules @@ -4871,7 +4905,7 @@ snapshots: '@opentelemetry/semantic-conventions': 1.25.1 '@prisma/instrumentation': 5.16.1 '@sentry/core': 8.17.0 - '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.1) + '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1) '@sentry/types': 8.17.0 '@sentry/utils': 8.17.0 optionalDependencies: @@ -4879,7 +4913,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.1)': + '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) diff --git a/service/Dockerfile b/service/Dockerfile index e857cc85d..ea01a10fe 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -15,6 +15,7 @@ COPY packages/types/package.json packages/types/tsconfig.json packages/types/ COPY libs/lib-services/package.json libs/lib-services/tsconfig.json libs/lib-services/ COPY libs/lib-mongodb/package.json libs/lib-mongodb/tsconfig.json libs/lib-mongodb/ +COPY libs/lib-postgres/package.json libs/lib-postgres/tsconfig.json libs/lib-postgres/ COPY modules/module-postgres/package.json modules/module-postgres/tsconfig.json modules/module-postgres/ COPY modules/module-postgres-storage/package.json modules/module-postgres-storage/tsconfig.json modules/module-postgres-storage/ @@ -37,6 +38,7 @@ COPY packages/types/src packages/types/src/ COPY libs/lib-services/src libs/lib-services/src/ COPY libs/lib-mongodb/src libs/lib-mongodb/src/ +COPY libs/lib-postgres/src libs/lib-postgres/src/ COPY modules/module-postgres/src modules/module-postgres/src/ COPY modules/module-postgres-storage/src modules/module-postgres-storage/src/ From 556b8d1c3cff54b7a9431bd5128480f67175c730 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jan 2025 16:44:04 +0200 Subject: [PATCH 08/50] Add postgres storage to CI --- .github/workflows/test.yml | 36 +++++++++++++++++++++++ modules/module-postgres-storage/README.md | 3 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f5fbd055..e2d3a3358 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,6 +104,18 @@ jobs: -d postgres:${{ matrix.postgres-version }} \ -c wal_level=logical + - name: Start PostgreSQL (Storage) + run: | + docker run \ + --health-cmd pg_isready \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 5 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=powersync_storage_test \ + -p 5431:5432 \ + -d postgres:${{ matrix.postgres-version }} + - name: Start MongoDB uses: supercharge/mongodb-github-action@1.8.0 with: @@ -176,6 +188,18 @@ jobs: mongodb-version: '6.0' mongodb-replica-set: test-rs + - name: Start PostgreSQL (Storage) + run: | + docker run \ + --health-cmd pg_isready \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 5 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=powersync_storage_test \ + -p 5431:5432 \ + -d postgres:16 + - name: Setup NodeJS uses: actions/setup-node@v4 with: @@ -229,6 +253,18 @@ jobs: mongodb-version: ${{ matrix.mongodb-version }} mongodb-replica-set: test-rs + - name: Start PostgreSQL (Storage) + run: | + docker run \ + --health-cmd pg_isready \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 5 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=powersync_storage_test \ + -p 5431:5432 \ + -d postgres:16 + - name: Setup Node.js uses: actions/setup-node@v4 with: diff --git a/modules/module-postgres-storage/README.md b/modules/module-postgres-storage/README.md index 566a55500..3c96bbeab 100644 --- a/modules/module-postgres-storage/README.md +++ b/modules/module-postgres-storage/README.md @@ -26,11 +26,12 @@ storage: uri: !env PS_STORAGE_SOURCE_URI ``` +**IMPORTANT**: A separate Postgres server is currently required for replication connections (if using Postgres for replication) and storage. Using the same server might cause unexpected results. ### Connection credentials -The Postgres bucket storage implementation requires write access to the provided Postgres database. The module will create a `powersync` schema in the provided database which will contain all the tables and data used for bucket storage. Ensure that the provided credentials specified in the `uri` or `username`, `password` configuration fields has the appropriate write access. +The Postgres bucket storage implementation requires write access to the provided Postgres database. The module will create a `powersync` schema in the provided database which will contain all the tables and data used for bucket storage. Ensure that the provided credentials specified in the `uri` or `username`, `password` configuration fields have the appropriate write access. A sample user could be created with: From 63d8e3f96fcf0710bba02de8656534210de5dab3 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jan 2025 16:51:22 +0200 Subject: [PATCH 09/50] fix core test --- libs/lib-postgres/test/src/config.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/lib-postgres/test/src/config.test.ts b/libs/lib-postgres/test/src/config.test.ts index 7b2dc52cc..35f84d220 100644 --- a/libs/lib-postgres/test/src/config.test.ts +++ b/libs/lib-postgres/test/src/config.test.ts @@ -5,7 +5,7 @@ describe('config', () => { test('Should resolve database', () => { const normalized = normalizeConnectionConfig({ type: 'postgresql', - uri: 'postgresql://localhost:4321/powersync_test' + uri: 'postgresql://postgres:postgres@localhost:4321/powersync_test' }); expect(normalized.database).equals('powersync_test'); }); From a255cb7b26fecd433d93196b01972974e0c0ffe1 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jan 2025 19:06:53 +0200 Subject: [PATCH 10/50] dispose clients --- modules/module-mysql/test/src/BinlogStreamUtils.ts | 1 + .../src/migrations/scripts/1684951997326-init.ts | 2 +- modules/module-postgres/test/src/slow_tests.test.ts | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/module-mysql/test/src/BinlogStreamUtils.ts b/modules/module-mysql/test/src/BinlogStreamUtils.ts index be6c3064a..866e111c8 100644 --- a/modules/module-mysql/test/src/BinlogStreamUtils.ts +++ b/modules/module-mysql/test/src/BinlogStreamUtils.ts @@ -49,6 +49,7 @@ export class BinlogStreamTestContext { this.abortController.abort(); await this.streamPromise; await this.connectionManager.end(); + this.factory[Symbol.dispose](); } [Symbol.asyncDispose]() { diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index a8c48a568..3b7417035 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -137,7 +137,7 @@ export const down: migrations.PowerSyncMigrationFunction = async (context) => { const { service_context: { configuration } } = context; - using client = openMigrationDB(configuration.storage); + await using client = openMigrationDB(configuration.storage); await dropTables(client); }; diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index d6634219a..b1b94e5c7 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -91,7 +91,7 @@ function defineSlowTests(factory: storage.TestStorageFactory) { const replicationConnection = await connections.replicationConnection(); const pool = connections.pool; await clearTestDb(pool); - const f = (await factory()) as mongo_storage.storage.MongoBucketStorage; + using f = (await factory()) as mongo_storage.storage.MongoBucketStorage; const syncRuleContent = ` bucket_definitions: @@ -243,7 +243,7 @@ bucket_definitions: async () => { const pool = await connectPgPool(); await clearTestDb(pool); - const f = await factory(); + using f = await factory(); const syncRuleContent = ` bucket_definitions: From 7121d068e8fb79d754066d19866462b74a446669 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 10 Jan 2025 08:46:39 +0200 Subject: [PATCH 11/50] fix postgres slow tests --- modules/module-postgres-storage/src/index.ts | 5 +- .../src/storage/PostgresCompactor.ts | 1 + .../src/storage/storage-index.ts | 4 + .../src/utils/utils-index.ts | 4 + .../test/src/slow_tests.test.ts | 136 ++++++++++++++---- 5 files changed, 119 insertions(+), 31 deletions(-) create mode 100644 modules/module-postgres-storage/src/utils/utils-index.ts diff --git a/modules/module-postgres-storage/src/index.ts b/modules/module-postgres-storage/src/index.ts index b5876e818..3665b4e09 100644 --- a/modules/module-postgres-storage/src/index.ts +++ b/modules/module-postgres-storage/src/index.ts @@ -1,7 +1,10 @@ export * from './module/PostgresStorageModule.js'; -export * from './storage/PostgresBucketStorageFactory.js'; export * from './migrations/PostgresMigrationAgent.js'; +export * from './utils/utils-index.js'; +export * as utils from './utils/utils-index.js'; + export * from './storage/storage-index.js'; +export * as storage from './storage/storage-index.js'; export * from './types/types.js'; diff --git a/modules/module-postgres-storage/src/storage/PostgresCompactor.ts b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts index 94d647881..4fcbdccc1 100644 --- a/modules/module-postgres-storage/src/storage/PostgresCompactor.ts +++ b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts @@ -320,6 +320,7 @@ export class PostgresCompactor { } if (!gotAnOp) { + await db.sql`COMMIT`.execute(); done = true; return; } diff --git a/modules/module-postgres-storage/src/storage/storage-index.ts b/modules/module-postgres-storage/src/storage/storage-index.ts index b95f1eaa8..b97b6a966 100644 --- a/modules/module-postgres-storage/src/storage/storage-index.ts +++ b/modules/module-postgres-storage/src/storage/storage-index.ts @@ -1 +1,5 @@ +export * from './PostgresBucketStorageFactory.js'; +export * from './PostgresCompactor.js'; +export * from './PostgresStorageProvider.js'; +export * from './PostgresSyncRulesStorage.js'; export * from './PostgresTestStorageFactoryGenerator.js'; diff --git a/modules/module-postgres-storage/src/utils/utils-index.ts b/modules/module-postgres-storage/src/utils/utils-index.ts new file mode 100644 index 000000000..65f808ff7 --- /dev/null +++ b/modules/module-postgres-storage/src/utils/utils-index.ts @@ -0,0 +1,4 @@ +export * from './bson.js'; +export * from './bucket-data.js'; +export * from './db.js'; +export * from './ts-codec.js'; diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index b1b94e5c7..f6a85071d 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -18,6 +18,7 @@ import { PgManager } from '@module/replication/PgManager.js'; import { storage } from '@powersync/service-core'; import { test_utils } from '@powersync/service-core-tests'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; +import * as postgres_storage from '@powersync/service-module-postgres-storage'; import * as timers from 'node:timers/promises'; describe.skipIf(!env.TEST_MONGO_STORAGE)('slow tests - mongodb', function () { @@ -91,7 +92,7 @@ function defineSlowTests(factory: storage.TestStorageFactory) { const replicationConnection = await connections.replicationConnection(); const pool = connections.pool; await clearTestDb(pool); - using f = (await factory()) as mongo_storage.storage.MongoBucketStorage; + using f = await factory(); const syncRuleContent = ` bucket_definitions: @@ -186,15 +187,50 @@ bucket_definitions: } const checkpoint = BigInt((await storage.getCheckpoint()).checkpoint); - const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) - .filter((row) => row._id.o <= checkpoint) - .map(mongo_storage.storage.mapOpEntry); - await storage.compact({ maxOpId: checkpoint }); - const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) - .filter((row) => row._id.o <= checkpoint) - .map(mongo_storage.storage.mapOpEntry); - - test_utils.validateCompactedBucket(opsBefore, opsAfter); + if (f instanceof mongo_storage.storage.MongoBucketStorage) { + const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) + .filter((row) => row._id.o <= checkpoint) + .map(mongo_storage.storage.mapOpEntry); + await storage.compact({ maxOpId: checkpoint }); + const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) + .filter((row) => row._id.o <= checkpoint) + .map(mongo_storage.storage.mapOpEntry); + + test_utils.validateCompactedBucket(opsBefore, opsAfter); + } else if (f instanceof postgres_storage.PostgresBucketStorageFactory) { + const { db } = f; + const opsBefore = ( + await db.sql` + SELECT + * + FROM + bucket_data + WHERE + op_id <= ${{ type: 'int8', value: checkpoint }} + ORDER BY + op_id ASC + ` + .decoded(postgres_storage.models.BucketData) + .rows() + ).map(postgres_storage.utils.mapOpEntry); + await storage.compact({ maxOpId: checkpoint }); + const opsAfter = ( + await db.sql` + SELECT + * + FROM + bucket_data + WHERE + op_id <= ${{ type: 'int8', value: checkpoint }} + ORDER BY + op_id ASC + ` + .decoded(postgres_storage.models.BucketData) + .rows() + ).map(postgres_storage.utils.mapOpEntry); + + test_utils.validateCompactedBucket(opsBefore, opsAfter); + } } }; @@ -208,26 +244,66 @@ bucket_definitions: // Wait for replication to finish let checkpoint = await getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS }); - // Check that all inserts have been deleted again - const docs = await f.db.current_data.find().toArray(); - const transformed = docs.map((doc) => { - return bson.deserialize(doc.data.buffer) as SqliteRow; - }); - expect(transformed).toEqual([]); - - // Check that each PUT has a REMOVE - const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray(); - - // All a single bucket in this test - const bucket = ops.map((op) => mongo_storage.storage.mapOpEntry(op)); - const reduced = test_utils.reduceBucket(bucket); - expect(reduced).toMatchObject([ - { - op_id: '0', - op: 'CLEAR' - } - // Should contain no additional data - ]); + if (f instanceof mongo_storage.storage.MongoBucketStorage) { + // Check that all inserts have been deleted again + const docs = await f.db.current_data.find().toArray(); + const transformed = docs.map((doc) => { + return bson.deserialize(doc.data.buffer) as SqliteRow; + }); + expect(transformed).toEqual([]); + + // Check that each PUT has a REMOVE + const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray(); + + // All a single bucket in this test + const bucket = ops.map((op) => mongo_storage.storage.mapOpEntry(op)); + const reduced = test_utils.reduceBucket(bucket); + expect(reduced).toMatchObject([ + { + op_id: '0', + op: 'CLEAR' + } + // Should contain no additional data + ]); + } else if (f instanceof postgres_storage.storage.PostgresBucketStorageFactory) { + const { db } = f; + // Check that all inserts have been deleted again + const docs = await db.sql` + SELECT + * + FROM + current_data + ` + .decoded(postgres_storage.models.CurrentData) + .rows(); + const transformed = docs.map((doc) => { + return bson.deserialize(doc.data) as SqliteRow; + }); + expect(transformed).toEqual([]); + + // Check that each PUT has a REMOVE + const ops = await db.sql` + SELECT + * + FROM + bucket_data + ORDER BY + op_id ASC + ` + .decoded(postgres_storage.models.BucketData) + .rows(); + + // All a single bucket in this test + const bucket = ops.map((op) => postgres_storage.utils.mapOpEntry(op)); + const reduced = test_utils.reduceBucket(bucket); + expect(reduced).toMatchObject([ + { + op_id: '0', + op: 'CLEAR' + } + // Should contain no additional data + ]); + } } abortController.abort(); From 98f46a1b4fd0b81532e833531808dd26f4e9bd00 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 10 Jan 2025 09:02:54 +0200 Subject: [PATCH 12/50] bump timeout --- modules/module-postgres/test/src/large_batch.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/module-postgres/test/src/large_batch.test.ts b/modules/module-postgres/test/src/large_batch.test.ts index 453ec6153..e5671f6e6 100644 --- a/modules/module-postgres/test/src/large_batch.test.ts +++ b/modules/module-postgres/test/src/large_batch.test.ts @@ -21,7 +21,8 @@ describe.skipIf(!env.TEST_MONGO_STORAGE)('batch replication tests - mongodb', { } }); -describe.skipIf(!env.TEST_POSTGRES_STORAGE)('batch replication tests - postgres', { timeout: 120_000 }, function () { +// TODO verify Postgres performance +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('batch replication tests - postgres', { timeout: 240_000 }, function () { // These are slow but consistent tests. // Not run on every test run, but we do run on CI, or when manually debugging issues. if (env.CI || env.SLOW_TESTS) { From 73de76eb0876c855972cc4210a8b4ae29e2ec93b Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 10 Jan 2025 10:42:15 +0200 Subject: [PATCH 13/50] fix truncation --- .../src/storage/batch/PostgresBucketBatch.ts | 52 ++++++++++--------- .../storage/batch/PostgresPersistedBatch.ts | 12 ++++- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index 608536094..5816aff98 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -6,7 +6,7 @@ import * as timers from 'timers/promises'; import * as t from 'ts-codec'; import { CurrentBucket, CurrentData, CurrentDataDecoded } from '../../types/models/CurrentData.js'; import { models, RequiredOperationBatchLimits } from '../../types/types.js'; -import { NOTIFICATION_CHANNEL } from '../../utils/db.js'; +import { NOTIFICATION_CHANNEL, sql } from '../../utils/db.js'; import { pick } from '../../utils/ts-codec.js'; import { batchCreateCustomWriteCheckpoints } from '../checkpoints/PostgresWriteCheckpointAPI.js'; import { cacheKey, encodedCacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; @@ -145,26 +145,27 @@ export class PostgresBucketBatch while (lastBatchCount == BATCH_LIMIT) { lastBatchCount = 0; - for await (const rows of this.db.streamRows>(lib_postgres.sql` - SELECT - buckets, - lookups, - source_key - FROM - current_data - WHERE - group_id = ${{ type: 'int8', value: this.group_id }} - AND source_table = ${{ type: 'varchar', value: sourceTable.id }} - LIMIT - ${{ type: 'int4', value: BATCH_LIMIT }} - `)) { - lastBatchCount += rows.length; - processedCount += rows.length; - await this.withReplicationTransaction(async (db) => { - const persistedBatch = new PostgresPersistedBatch({ - group_id: this.group_id, - ...this.options.batch_limits - }); + await this.withReplicationTransaction(async (db) => { + const persistedBatch = new PostgresPersistedBatch({ + group_id: this.group_id, + ...this.options.batch_limits + }); + + for await (const rows of db.streamRows>(sql` + SELECT + buckets, + lookups, + source_key + FROM + current_data + WHERE + group_id = ${{ type: 'int8', value: this.group_id }} + AND source_table = ${{ type: 'varchar', value: sourceTable.id }} + LIMIT + ${{ type: 'int4', value: BATCH_LIMIT }} + `)) { + lastBatchCount += rows.length; + processedCount += rows.length; const decodedRows = rows.map((row) => codec.decode(row)); for (const value of decodedRows) { @@ -181,13 +182,14 @@ export class PostgresBucketBatch source_key: value.source_key }); persistedBatch.deleteCurrentData({ - source_key: value.source_key, + // This is serialized since we got it from a DB query + serialized_source_key: value.source_key, source_table_id: sourceTable.id }); } - await persistedBatch.flush(db); - }); - } + } + await persistedBatch.flush(db); + }); } if (processedCount == 0) { // The op sequence should not have progressed diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts index 2c803dd9c..b41a8b723 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts @@ -25,7 +25,15 @@ export type SaveParameterDataOptions = { export type DeleteCurrentDataOptions = { source_table_id: bigint; - source_key: storage.ReplicaId; + /** + * ReplicaID which needs to be serialized in order to be queried + * or inserted into the DB + */ + source_key?: storage.ReplicaId; + /** + * Optionally provide the serialized source key directly + */ + serialized_source_key?: Buffer; }; export type PostgresPersistedBatchOptions = RequiredOperationBatchLimits & { @@ -172,7 +180,7 @@ export class PostgresPersistedBatch { } deleteCurrentData(options: DeleteCurrentDataOptions) { - const serializedReplicaId = storage.serializeReplicaId(options.source_key); + const serializedReplicaId = options.serialized_source_key ?? storage.serializeReplicaId(options.source_key); this.currentDataDeletes.push({ group_id: this.group_id, source_table: options.source_table_id.toString(), From 0c682bbf3aa61e704b0cd86b41c026540333ea04 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 10 Jan 2025 11:14:38 +0200 Subject: [PATCH 14/50] attempt to fix racey mysql test --- modules/module-mysql/src/replication/BinLogStream.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index c4a06988b..cddb22cab 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -372,6 +372,7 @@ AND table_type = 'BASE TABLE';`, } ); } + await this.storage.autoActivate(); } private getTable(tableId: string): storage.SourceTable { From 653626aca154c767de035f653c38640e31ae0e4c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 10 Jan 2025 13:57:32 +0200 Subject: [PATCH 15/50] fix sync rules termination bug --- .../src/storage/PostgresSyncRulesStorage.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index b179f2152..51956f2df 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -487,6 +487,8 @@ export class PostgresSyncRulesStorage SET state = ${{ type: 'varchar', value: storage.SyncRuleState.TERMINATED }}, snapshot_done = ${{ type: 'bool', value: false }} + WHERE + id = ${{ type: 'int8', value: this.group_id }} `.execute(); } From 9fd0481fc1c82fed87a78a2c995944cb3f03b654 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 10 Jan 2025 15:45:45 +0200 Subject: [PATCH 16/50] cleanup --- .../src/db/connection/DatabaseClient.ts | 16 +++++++++- .../test/src/change_stream_utils.ts | 4 --- modules/module-postgres-storage/README.md | 29 ++++++++++--------- .../src/storage/batch/PostgresBucketBatch.ts | 1 + 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/libs/lib-postgres/src/db/connection/DatabaseClient.ts b/libs/lib-postgres/src/db/connection/DatabaseClient.ts index 8e29d136c..b69ca930d 100644 --- a/libs/lib-postgres/src/db/connection/DatabaseClient.ts +++ b/libs/lib-postgres/src/db/connection/DatabaseClient.ts @@ -139,7 +139,21 @@ export class DatabaseClient extends AbstractPostgresConnection { const start = Date.now(); const lsn = await createCheckpoint(client, db); - console.log('created checkpoint pushing lsn to', lsn); // This old API needs a persisted checkpoint id. // Since we don't use LSNs anymore, the only way to get that is to wait. @@ -156,11 +155,8 @@ export async function getClientCheckpoint( throw new Error('No sync rules available'); } if (cp.lsn && cp.lsn >= lsn) { - console.log('active checkpoint has replicated created checkpoint', cp?.lsn); return cp.checkpoint; } - - console.log('active checkpoint is still behind created checkpoint', cp?.lsn); await new Promise((resolve) => setTimeout(resolve, 30)); } diff --git a/modules/module-postgres-storage/README.md b/modules/module-postgres-storage/README.md index 3c96bbeab..fd90da2f5 100644 --- a/modules/module-postgres-storage/README.md +++ b/modules/module-postgres-storage/README.md @@ -33,35 +33,38 @@ A separate Postgres server is currently required for replication connections (if The Postgres bucket storage implementation requires write access to the provided Postgres database. The module will create a `powersync` schema in the provided database which will contain all the tables and data used for bucket storage. Ensure that the provided credentials specified in the `uri` or `username`, `password` configuration fields have the appropriate write access. -A sample user could be created with: +A sample user could be created with the following + +If a `powersync` schema should be created manually ```sql --- Create the user with a password CREATE USER powersync_storage_user WITH PASSWORD 'secure_password'; --- Optionally create a PowerSync schema and make the user its owner CREATE SCHEMA IF NOT EXISTS powersync AUTHORIZATION powersync_storage_user; --- OR: Allow PowerSync to create schemas in the database -GRANT CREATE ON DATABASE example_database TO powersync_storage_user; - --- Set default privileges for objects created by powersync_storage_user in the database --- (Ensures the user gets full access to tables they create in any schema) -ALTER DEFAULT PRIVILEGES FOR ROLE powersync_storage_user -GRANT ALL PRIVILEGES ON TABLES TO powersync_storage_user; +GRANT CONNECT ON DATABASE postgres TO powersync_storage_user; --- [if the schema was pre-created] Grant usage and privileges on the powersync schema GRANT USAGE ON SCHEMA powersync TO powersync_storage_user; --- [if the schema was pre-created] GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA powersync TO powersync_storage_user; ``` +If the PowerSync service should create a `powersync` schema + +```sql +CREATE USER powersync_storage_user +WITH + PASSWORD 'secure_password'; + +-- The user should only have access to the schema it created +GRANT CREATE ON DATABASE postgres TO powersync_storage_user; +``` + ### Batching -Replication data is persisted via batch operations. Batching ensures performant, memory optimized writes. Batches are limited in size. Increasing batch size limits can reduce the amount of server round-trips which increases performance, but will result in higher memory usage and potential server issues. +Replication data is persisted via batch operations. Batching ensures performant, memory optimized writes. Batches are limited in size. Increasing batch size limits could reduce the amount of server round-trips which could increase performance (in some circumstances), but will result in higher memory usage and potential server issues. Batch size limits are defaulted and can optionally be configured in the configuration. diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index 5816aff98..e6f7b3df5 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -11,6 +11,7 @@ import { pick } from '../../utils/ts-codec.js'; import { batchCreateCustomWriteCheckpoints } from '../checkpoints/PostgresWriteCheckpointAPI.js'; import { cacheKey, encodedCacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PostgresPersistedBatch } from './PostgresPersistedBatch.js'; + export interface PostgresBucketBatchOptions { db: lib_postgres.DatabaseClient; sync_rules: sync_rules.SqlSyncRules; From af47872ff5ddfbabff4b6d5fccd510601cc2238e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 10 Jan 2025 15:57:24 +0200 Subject: [PATCH 17/50] added changeset --- .changeset/green-trainers-yell.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/green-trainers-yell.md diff --git a/.changeset/green-trainers-yell.md b/.changeset/green-trainers-yell.md new file mode 100644 index 000000000..46ca799a2 --- /dev/null +++ b/.changeset/green-trainers-yell.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-module-postgres-storage': minor +'@powersync/service-module-postgres': minor +'@powersync/lib-service-postgres': minor +--- + +Initial release of Postgres bucket storage. From 198b6aaa13bc3f9e76729c057a0d7c7ed8856118 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Sat, 11 Jan 2025 10:32:57 +0200 Subject: [PATCH 18/50] improve sync performance --- .../connection/AbstractPostgresConnection.ts | 4 +- .../src/db/connection/ConnectionSlot.ts | 2 + .../src/db/connection/DatabaseClient.ts | 20 +++- .../migrations/scripts/1684951997326-init.ts | 13 +++ .../storage/PostgresBucketStorageFactory.ts | 8 ++ .../src/storage/PostgresSyncRulesStorage.ts | 108 ++++++++++-------- .../PostgresTestStorageFactoryGenerator.ts | 62 +++++----- .../storage/batch/PostgresPersistedBatch.ts | 12 +- 8 files changed, 144 insertions(+), 85 deletions(-) diff --git a/libs/lib-postgres/src/db/connection/AbstractPostgresConnection.ts b/libs/lib-postgres/src/db/connection/AbstractPostgresConnection.ts index a5c773be9..3484ac81c 100644 --- a/libs/lib-postgres/src/db/connection/AbstractPostgresConnection.ts +++ b/libs/lib-postgres/src/db/connection/AbstractPostgresConnection.ts @@ -16,7 +16,9 @@ export abstract class AbstractPostgresConnection< return this.baseConnection.stream(...args); } - query(...args: pgwire.Statement[]): Promise { + query(script: string, options?: pgwire.PgSimpleQueryOptions): Promise; + query(...args: pgwire.Statement[]): Promise; + query(...args: any[]): Promise { return this.baseConnection.query(...args); } diff --git a/libs/lib-postgres/src/db/connection/ConnectionSlot.ts b/libs/lib-postgres/src/db/connection/ConnectionSlot.ts index 76dd18c1a..7a6f54bc1 100644 --- a/libs/lib-postgres/src/db/connection/ConnectionSlot.ts +++ b/libs/lib-postgres/src/db/connection/ConnectionSlot.ts @@ -8,6 +8,7 @@ export interface NotificationListener extends framework.DisposableListener { export interface ConnectionSlotListener extends NotificationListener { connectionAvailable?: () => void; connectionError?: (exception: any) => void; + connectionCreated?: (connection: pgwire.PgConnection) => Promise; } export type ConnectionLease = { @@ -41,6 +42,7 @@ export class ConnectionSlot extends framework.DisposableObserver l.connectionCreated?.(connection)); if (this.hasNotificationListener()) { await this.configureConnectionNotifications(connection); } diff --git a/libs/lib-postgres/src/db/connection/DatabaseClient.ts b/libs/lib-postgres/src/db/connection/DatabaseClient.ts index b69ca930d..4e9520865 100644 --- a/libs/lib-postgres/src/db/connection/DatabaseClient.ts +++ b/libs/lib-postgres/src/db/connection/DatabaseClient.ts @@ -17,9 +17,13 @@ export type DatabaseClientOptions = { notificationChannels?: string[]; }; +export type DatabaseClientListener = NotificationListener & { + connectionCreated?: (connection: pgwire.PgConnection) => Promise; +}; + export const TRANSACTION_CONNECTION_COUNT = 5; -export class DatabaseClient extends AbstractPostgresConnection { +export class DatabaseClient extends AbstractPostgresConnection { closed: boolean; protected pool: pgwire.PgClient; @@ -36,7 +40,8 @@ export class DatabaseClient extends AbstractPostgresConnection this.processConnectionQueue(), - connectionError: (ex) => this.handleConnectionError(ex) + connectionError: (ex) => this.handleConnectionError(ex), + connectionCreated: (connection) => this.iterateAsyncListeners(async (l) => l.connectionCreated?.(connection)) }); return slot; }); @@ -58,7 +63,7 @@ export class DatabaseClient extends AbstractPostgresConnection): () => void { + registerListener(listener: Partial): () => void { let disposeNotification: (() => void) | null = null; if ('notification' in listener) { // Pass this on to the first connection slot @@ -81,13 +86,18 @@ export class DatabaseClient extends AbstractPostgresConnection { + query(script: string, options?: pgwire.PgSimpleQueryOptions): Promise; + query(...args: pgwire.Statement[]): Promise; + async query(...args: any[]): Promise { await this.initialized; // Retry pool queries. Note that we can't retry queries in a transaction // since a failed query will end the transaction. + const { schemaStatement } = this; - if (schemaStatement) { + if (typeof args[0] == 'object' && schemaStatement) { args.unshift(schemaStatement); + } else if (typeof args[0] == 'string' && schemaStatement) { + args[0] = `${schemaStatement.statement}; ${args[0]}`; } return lib_postgres.retriedQuery(this.pool, ...args); } diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index 3b7417035..fe3a9d63c 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -43,6 +43,19 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { ) `.execute(); + /** + * Rough comparison: + * Creating these indexes causes an initial replication of 2.5million rows + * to take about 10minutes to complete, compared to about 7.5 minutes without indexes. + * + * The time to fetch operations for the 2.5mil rows is 2min, 35 seconds with indexes versus + * 2min 21 seconds without indexes (probably in the margin of error). + * + * Not including them since they impact initial replication more than providing any noticeable benefit. + */ + // await db.sql`CREATE INDEX idx_bucket_data_composite ON bucket_data (group_id, bucket_name, op_id); `.execute(); + // await db.sql`CREATE INDEX idx_bucket_data_group_id ON bucket_data (group_id);`.execute(); + await db.sql`CREATE TABLE instance (id TEXT PRIMARY KEY) `.execute(); await db.sql` diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index 709cd2b6c..2be0c39aa 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -66,6 +66,9 @@ export class PostgresBucketStorageFactory }); this.slot_name_prefix = options.slot_name_prefix; + this.db.registerListener({ + connectionCreated: async (connection) => this.prepareStatements(connection) + }); this.notificationConnection = null; } @@ -74,6 +77,11 @@ export class PostgresBucketStorageFactory await this.db[Symbol.asyncDispose](); } + async prepareStatements(connection: pg_wire.PgConnection) { + // It should be possible to prepare statements for some common operations here. + // This has not been implemented yet. + } + getInstance(syncRules: storage.PersistedSyncRulesContent): storage.SyncRulesBucketStorage { const storage = new PostgresSyncRulesStorage({ factory: this, diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 51956f2df..38773b415 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -2,13 +2,13 @@ import * as lib_postgres from '@powersync/lib-service-postgres'; import { DisposableObserver } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import * as sync_rules from '@powersync/service-sync-rules'; -import * as t from 'ts-codec'; import * as uuid from 'uuid'; -import { bigint, BIGINT_MAX } from '../types/codecs.js'; +import { BIGINT_MAX } from '../types/codecs.js'; import { models, RequiredOperationBatchLimits } from '../types/types.js'; import { replicaIdToSubkey } from '../utils/bson.js'; import { mapOpEntry } from '../utils/bucket-data.js'; +import { StatementParam } from '@powersync/service-jpgwire'; import { pick } from '../utils/ts-codec.js'; import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js'; import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js'; @@ -356,50 +356,70 @@ export class PostgresSyncRulesStorage let targetOp: bigint | null = null; let rowCount = 0; - for await (const rawRows of this.db.streamRows({ - statement: /* sql */ ` - WITH - filter_data AS ( - SELECT - FILTER ->> 'bucket_name' AS bucket_name, - (FILTER ->> 'start')::BIGINT AS start_op_id - FROM - jsonb_array_elements($1::jsonb) AS FILTER - ) - SELECT - b.*, - octet_length(b.data) AS data_size - FROM - bucket_data b - JOIN filter_data f ON b.bucket_name = f.bucket_name - AND b.op_id > f.start_op_id - AND b.op_id <= $2 - WHERE - b.group_id = $3 - ORDER BY - b.bucket_name ASC, - b.op_id ASC - LIMIT - $4 - `, + /** + * It is possible to perform this query with JSONB join. e.g. + * ```sql + * WITH + * filter_data AS ( + * SELECT + * FILTER ->> 'bucket_name' AS bucket_name, + * (FILTER ->> 'start')::BIGINT AS start_op_id + * FROM + * jsonb_array_elements($1::jsonb) AS FILTER + * ) + * SELECT + * b.*, + * octet_length(b.data) AS data_size + * FROM + * bucket_data b + * JOIN filter_data f ON b.bucket_name = f.bucket_name + * AND b.op_id > f.start_op_id + * AND b.op_id <= $2 + * WHERE + * b.group_id = $3 + * ORDER BY + * b.bucket_name ASC, + * b.op_id ASC + * LIMIT + * $4; + * ``` + * Which might be better for large volumes of buckets, but in testing the JSON method + * was significantly slower than the method below. Syncing 2.5 million rows in a single + * bucket takes 2 minutes and 11 seconds with the method below. With the JSON method + * 1 million rows were only synced before a 5 minute timeout. + */ + for await (const rows of this.db.streamRows({ + statement: ` + SELECT + * + FROM + bucket_data + WHERE + group_id = $1 + and op_id <= $2 + and ( + ${filters.map((f, index) => `(bucket_name = $${index + 4} and op_id > $${index + 5})`).join(' OR ')} + ) + ORDER BY + bucket_name ASC, + op_id ASC + LIMIT + $3;`, params: [ - { type: 'jsonb', value: filters }, - { type: 'int8', value: end }, { type: 'int8', value: this.group_id }, - { type: 'int4', value: rowLimit + 1 } // Increase the row limit by 1 in order to detect hasMore + { type: 'int8', value: end }, + { type: 'int4', value: rowLimit + 1 }, + ...filters.flatMap((f) => [ + { type: 'varchar' as const, value: f.bucket_name }, + { type: 'int8' as const, value: f.start } satisfies StatementParam + ]) ] })) { - const rows = rawRows.map((q) => { - return models.BucketData.and( - t.object({ - data_size: t.Null.or(bigint) - }) - ).decode(q as any); - }); - - for (const row of rows) { + const decodedRows = rows.map((r) => models.BucketData.decode(r as any)); + + for (const row of decodedRows) { const { bucket_name } = row; - const rowSize = row.data_size ? Number(row.data_size) : 0; + const rowSize = row.data ? row.data.length : 0; if ( currentBatch == null || @@ -457,15 +477,13 @@ export class PostgresSyncRulesStorage currentBatch.data.push(entry); currentBatch.next_after = entry.op_id; - // Obtained from pg_column_size(data) AS data_size - // We could optimize this and persist the column instead of calculating - // on query. - batchSize += row.data_size ? Number(row.data_size) : 0; + batchSize += rowSize; // Manually track the total rows yielded rowCount++; } } + if (currentBatch != null) { const yieldBatch = currentBatch; currentBatch = null; diff --git a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts index 183a2fd81..ec30b9118 100644 --- a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts +++ b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts @@ -14,45 +14,51 @@ export type PostgresTestStorageOptions = { export const PostgresTestStorageFactoryGenerator = (factoryOptions: PostgresTestStorageOptions) => { return async (options?: TestStorageOptions) => { - const migrationManager: PowerSyncMigrationManager = new framework.MigrationManager(); + try { + const migrationManager: PowerSyncMigrationManager = new framework.MigrationManager(); - const BASE_CONFIG = { - type: 'postgresql' as const, - uri: factoryOptions.url, - sslmode: 'disable' as const - }; + const BASE_CONFIG = { + type: 'postgresql' as const, + uri: factoryOptions.url, + sslmode: 'disable' as const + }; - const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG); + const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG); - await using migrationAgent = factoryOptions.migrationAgent - ? factoryOptions.migrationAgent(BASE_CONFIG) - : new PostgresMigrationAgent(BASE_CONFIG); - migrationManager.registerMigrationAgent(migrationAgent); + await using migrationAgent = factoryOptions.migrationAgent + ? factoryOptions.migrationAgent(BASE_CONFIG) + : new PostgresMigrationAgent(BASE_CONFIG); + migrationManager.registerMigrationAgent(migrationAgent); - const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext; + const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext; + + if (!options?.doNotClear) { + await migrationManager.migrate({ + direction: framework.migrations.Direction.Down, + migrationContext: { + service_context: mockServiceContext + } + }); + + // In order to run up migration after + await migrationAgent.resetStore(); + } - if (!options?.doNotClear) { await migrationManager.migrate({ - direction: framework.migrations.Direction.Down, + direction: framework.migrations.Direction.Up, migrationContext: { service_context: mockServiceContext } }); - // In order to run up migration after - await migrationAgent.resetStore(); + return new PostgresBucketStorageFactory({ + config: TEST_CONNECTION_OPTIONS, + slot_name_prefix: 'test_' + }); + } catch (ex) { + // Vitest does not display these errors nicely when using the `await using` syntx + console.error(ex, ex.cause); + throw ex; } - - await migrationManager.migrate({ - direction: framework.migrations.Direction.Up, - migrationContext: { - service_context: mockServiceContext - } - }); - - return new PostgresBucketStorageFactory({ - config: TEST_CONNECTION_OPTIONS, - slot_name_prefix: 'test_' - }); }; }; diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts index b41a8b723..0573764ae 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts @@ -108,7 +108,7 @@ export class PostgresPersistedBatch { target_op: null }); - this.currentSize += k.bucket.length * 2 + data.length * 2 + hexSourceKey.length * 2 + 100; + this.currentSize += k.bucket.length + data.length + hexSourceKey.length + 100; } for (const bd of remaining_buckets.values()) { @@ -126,7 +126,7 @@ export class PostgresPersistedBatch { target_op: null, data: null }); - this.currentSize += bd.bucket.length * 2 + hexSourceKey.length * 2 + 100; + this.currentSize += bd.bucket.length + hexSourceKey.length + 100; } } @@ -161,7 +161,7 @@ export class PostgresPersistedBatch { id: 0, // auto incrementing id lookup: hexLookup }); - this.currentSize += hexLookup.length * 2 + serializedBucketParameters.length * 2 + hexSourceKey.length * 2 + 100; + this.currentSize += hexLookup.length + serializedBucketParameters.length + hexSourceKey.length + 100; } // 2. "REMOVE" entries for any lookup not touched. @@ -175,7 +175,7 @@ export class PostgresPersistedBatch { id: 0, // auto incrementing id lookup: hexLookup }); - this.currentSize += hexLookup.length * 2 + hexSourceKey.length * 2 + 100; + this.currentSize += hexLookup.length + hexSourceKey.length + 100; } } @@ -186,7 +186,7 @@ export class PostgresPersistedBatch { source_table: options.source_table_id.toString(), source_key: serializedReplicaId.toString('hex') }); - this.currentSize += serializedReplicaId.byteLength * 2 + 100; + this.currentSize += serializedReplicaId.byteLength + 100; } upsertCurrentData(options: models.CurrentDataDecoded) { @@ -218,7 +218,7 @@ export class PostgresPersistedBatch { this.currentSize += (options.data?.byteLength ?? 0) + serializedReplicaId.byteLength + - buckets.length * 2 + + buckets.length + options.lookups.reduce((total, l) => { return total + l.byteLength; }, 0) + From 705b4723c9c3d3025f9b26682cc59fbd2c390e1a Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Sat, 11 Jan 2025 10:48:16 +0200 Subject: [PATCH 19/50] fix bug --- .../src/migrations/scripts/1684951997326-init.ts | 2 ++ .../src/storage/PostgresSyncRulesStorage.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index fe3a9d63c..9c6756d72 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -47,9 +47,11 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { * Rough comparison: * Creating these indexes causes an initial replication of 2.5million rows * to take about 10minutes to complete, compared to about 7.5 minutes without indexes. + * [For comparison MongoDB took 5min 22 seconds to replicate the same data]. * * The time to fetch operations for the 2.5mil rows is 2min, 35 seconds with indexes versus * 2min 21 seconds without indexes (probably in the margin of error). + * [For comparison MongoDB took 1 minute for the same data]. * * Not including them since they impact initial replication more than providing any noticeable benefit. */ diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 38773b415..0d3e55f21 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -398,7 +398,7 @@ export class PostgresSyncRulesStorage group_id = $1 and op_id <= $2 and ( - ${filters.map((f, index) => `(bucket_name = $${index + 4} and op_id > $${index + 5})`).join(' OR ')} + ${filters.map((f, index) => `(bucket_name = $${index * 2 + 4} and op_id > $${index * 2 + 5})`).join(' OR ')} ) ORDER BY bucket_name ASC, From dc76df15f1c938c8250c7fca796a26bd4acdcf31 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 08:17:08 +0200 Subject: [PATCH 20/50] cleanup --- .../storage/PostgresBucketStorageFactory.ts | 1 - .../src/storage/PostgresCompactor.ts | 1 - .../src/storage/batch/OperationBatch.ts | 32 ++----------------- packages/service-core/src/util/utils.ts | 26 +++++++++++++++ 4 files changed, 28 insertions(+), 32 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index 2be0c39aa..4dfa99b3b 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -31,7 +31,6 @@ export class PostgresBucketStorageFactory protected notificationConnection: pg_wire.PgConnection | null; private sharedIterator = new sync.BroadcastIterable((signal) => this.watchActiveCheckpoint(signal)); - // TODO we might be able to share this private readonly storageCache = new LRUCache({ max: 3, fetchMethod: async (id) => { diff --git a/modules/module-postgres-storage/src/storage/PostgresCompactor.ts b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts index 4fcbdccc1..b81ba5f78 100644 --- a/modules/module-postgres-storage/src/storage/PostgresCompactor.ts +++ b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts @@ -135,7 +135,6 @@ export class PostgresCompactor { ${{ type: 'int4', value: this.moveBatchQueryLimit }} ` .decoded( - // TODO maybe a subtype pick(models.BucketData, ['op', 'source_table', 'table_name', 'source_key', 'row_id', 'op_id', 'bucket_name']) ) .rows(); diff --git a/modules/module-postgres-storage/src/storage/batch/OperationBatch.ts b/modules/module-postgres-storage/src/storage/batch/OperationBatch.ts index f00139031..2b91fab68 100644 --- a/modules/module-postgres-storage/src/storage/batch/OperationBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/OperationBatch.ts @@ -3,9 +3,7 @@ * There are some subtle differences in this implementation. */ -import { ToastableSqliteRow } from '@powersync/service-sync-rules'; - -import { storage } from '@powersync/service-core'; +import { storage, utils } from '@powersync/service-core'; import { RequiredOperationBatchLimits } from '../../types/types.js'; /** @@ -84,7 +82,7 @@ export class RecordOperation { this.beforeId = beforeId; this.internalBeforeKey = cacheKey(record.sourceTable.id, beforeId); this.internalAfterKey = afterId ? cacheKey(record.sourceTable.id, afterId) : null; - this.estimatedSize = estimateRowSize(record.before) + estimateRowSize(record.after); + this.estimatedSize = utils.estimateRowSize(record.before) + utils.estimateRowSize(record.after); } } @@ -101,29 +99,3 @@ export function cacheKey(sourceTableId: string, id: storage.ReplicaId) { export function encodedCacheKey(sourceTableId: string, storedKey: Buffer) { return `${sourceTableId}.${storedKey.toString('base64')}`; } - -/** - * Estimate in-memory size of row. - */ -function estimateRowSize(record: ToastableSqliteRow | undefined) { - if (record == null) { - return 12; - } - let size = 0; - for (let [key, value] of Object.entries(record)) { - size += 12 + key.length; - // number | string | null | bigint | Uint8Array - if (value == null) { - size += 4; - } else if (typeof value == 'number') { - size += 8; - } else if (typeof value == 'bigint') { - size += 8; - } else if (typeof value == 'string') { - size += value.length; - } else if (value instanceof Uint8Array) { - size += value.byteLength; - } - } - return size; -} diff --git a/packages/service-core/src/util/utils.ts b/packages/service-core/src/util/utils.ts index b34cf7491..673ad2091 100644 --- a/packages/service-core/src/util/utils.ts +++ b/packages/service-core/src/util/utils.ts @@ -222,3 +222,29 @@ export function flatstr(s: string) { function rowKey(entry: OplogEntry) { return `${entry.object_type}/${entry.object_id}/${entry.subkey}`; } + +/** + * Estimate in-memory size of row. + */ +export function estimateRowSize(record: sync_rules.ToastableSqliteRow | undefined) { + if (record == null) { + return 12; + } + let size = 0; + for (let [key, value] of Object.entries(record)) { + size += 12 + key.length; + // number | string | null | bigint | Uint8Array + if (value == null) { + size += 4; + } else if (typeof value == 'number') { + size += 8; + } else if (typeof value == 'bigint') { + size += 8; + } else if (typeof value == 'string') { + size += value.length; + } else if (value instanceof Uint8Array) { + size += value.byteLength; + } + } + return size; +} From 7524d3b15e08efd1f9869a9aabfb00b36cdf2fd2 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 09:58:59 +0200 Subject: [PATCH 21/50] cleanup --- .changeset/green-trainers-yell.md | 1 + .../src/db/connection/ConnectionSlot.ts | 12 +++++++++-- .../src/db/connection/DatabaseClient.ts | 5 +++-- .../src/replication/BinLogStream.ts | 1 - .../test/src/BinlogStreamUtils.ts | 1 + .../src/storage/PostgresSyncRulesStorage.ts | 20 +++++++++---------- .../test/src/large_batch.test.ts | 1 - 7 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.changeset/green-trainers-yell.md b/.changeset/green-trainers-yell.md index 46ca799a2..a6b35c77a 100644 --- a/.changeset/green-trainers-yell.md +++ b/.changeset/green-trainers-yell.md @@ -2,6 +2,7 @@ '@powersync/service-module-postgres-storage': minor '@powersync/service-module-postgres': minor '@powersync/lib-service-postgres': minor +'@powersync/service-core': minor --- Initial release of Postgres bucket storage. diff --git a/libs/lib-postgres/src/db/connection/ConnectionSlot.ts b/libs/lib-postgres/src/db/connection/ConnectionSlot.ts index 7a6f54bc1..4775470a0 100644 --- a/libs/lib-postgres/src/db/connection/ConnectionSlot.ts +++ b/libs/lib-postgres/src/db/connection/ConnectionSlot.ts @@ -28,12 +28,14 @@ export class ConnectionSlot extends framework.DisposableObserver | null; constructor(protected options: ConnectionSlotOptions) { super(); this.isAvailable = false; this.connection = null; this.isPoking = false; + this.connectingPromise = null; } get isConnected() { @@ -41,7 +43,9 @@ export class ConnectionSlot extends framework.DisposableObserver l.connectionCreated?.(connection)); if (this.hasNotificationListener()) { await this.configureConnectionNotifications(connection); @@ -50,7 +54,8 @@ export class ConnectionSlot extends framework.DisposableObserver { + if (!this.options.notificationChannels?.includes(payload.channel)) { + return; + } this.iterateListeners((l) => l.notification?.(payload)); }; diff --git a/libs/lib-postgres/src/db/connection/DatabaseClient.ts b/libs/lib-postgres/src/db/connection/DatabaseClient.ts index 4e9520865..702e9566e 100644 --- a/libs/lib-postgres/src/db/connection/DatabaseClient.ts +++ b/libs/lib-postgres/src/db/connection/DatabaseClient.ts @@ -90,8 +90,6 @@ export class DatabaseClient extends AbstractPostgresConnection; async query(...args: any[]): Promise { await this.initialized; - // Retry pool queries. Note that we can't retry queries in a transaction - // since a failed query will end the transaction. const { schemaStatement } = this; if (typeof args[0] == 'object' && schemaStatement) { @@ -99,6 +97,9 @@ export class DatabaseClient extends AbstractPostgresConnection> 0)::text, 'hex') -- Decode the hex string to bytea @@ -519,7 +519,7 @@ export class PostgresSyncRulesStorage FROM sync_rules WHERE - id = ${{ type: 'int4', value: this.group_id }} + id = ${{ type: 'int8', value: this.group_id }} ` .decoded(pick(models.SyncRules, ['snapshot_done', 'last_checkpoint_lsn', 'state'])) .first(); @@ -580,7 +580,7 @@ export class PostgresSyncRulesStorage FROM sync_rules WHERE - id = ${{ type: 'int4', value: this.group_id }} + id = ${{ type: 'int8', value: this.group_id }} ` .decoded(pick(models.SyncRules, ['state'])) .first(); @@ -591,7 +591,7 @@ export class PostgresSyncRulesStorage SET state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }} WHERE - id = ${{ type: 'int4', value: this.group_id }} + id = ${{ type: 'int8', value: this.group_id }} `.execute(); } @@ -601,7 +601,7 @@ export class PostgresSyncRulesStorage state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }} WHERE state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }} - AND id != ${{ type: 'int4', value: this.group_id }} + AND id != ${{ type: 'int8', value: this.group_id }} `.execute(); }); } diff --git a/modules/module-postgres/test/src/large_batch.test.ts b/modules/module-postgres/test/src/large_batch.test.ts index e5671f6e6..2806bad2f 100644 --- a/modules/module-postgres/test/src/large_batch.test.ts +++ b/modules/module-postgres/test/src/large_batch.test.ts @@ -21,7 +21,6 @@ describe.skipIf(!env.TEST_MONGO_STORAGE)('batch replication tests - mongodb', { } }); -// TODO verify Postgres performance describe.skipIf(!env.TEST_POSTGRES_STORAGE)('batch replication tests - postgres', { timeout: 240_000 }, function () { // These are slow but consistent tests. // Not run on every test run, but we do run on CI, or when manually debugging issues. From b67044154edb07b83d3ebd5322bc35bc2185539a Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 09:59:46 +0200 Subject: [PATCH 22/50] add env note --- .env.template | 1 + 1 file changed, 1 insertion(+) diff --git a/.env.template b/.env.template index 3082d8ec7..4b9d70da3 100644 --- a/.env.template +++ b/.env.template @@ -1,4 +1,5 @@ # Connections for tests MONGO_TEST_URL="mongodb://localhost:27017/powersync_test" PG_TEST_URL="postgres://postgres:postgres@localhost:5432/powersync_test" +# Note that this uses a separate server on a different port PG_STORAGE_TEST_URL="postgres://postgres:postgres@localhost:5431/powersync_storage_test" \ No newline at end of file From b9009d84e209c865a337aba5172d5cfe56170cb3 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 10:39:47 +0200 Subject: [PATCH 23/50] add primary keys to tables --- .../migrations/scripts/1684951997326-init.ts | 27 +++---------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index 9c6756d72..e935f108a 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -13,7 +13,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { * Request an explicit connection which will automatically set the search * path to the powersync schema */ - await client.lockConnection(async (db) => { + await client.transaction(async (db) => { await db.sql` CREATE SEQUENCE op_id_sequence AS int8 START WITH @@ -31,7 +31,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { group_id integer NOT NULL, bucket_name TEXT NOT NULL, op_id bigint NOT NULL, - CONSTRAINT unique_id UNIQUE (group_id, bucket_name, op_id), + CONSTRAINT unique_id PRIMARY KEY (group_id, bucket_name, op_id), op text NOT NULL, source_table TEXT, source_key bytea, @@ -43,21 +43,6 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { ) `.execute(); - /** - * Rough comparison: - * Creating these indexes causes an initial replication of 2.5million rows - * to take about 10minutes to complete, compared to about 7.5 minutes without indexes. - * [For comparison MongoDB took 5min 22 seconds to replicate the same data]. - * - * The time to fetch operations for the 2.5mil rows is 2min, 35 seconds with indexes versus - * 2min 21 seconds without indexes (probably in the margin of error). - * [For comparison MongoDB took 1 minute for the same data]. - * - * Not including them since they impact initial replication more than providing any noticeable benefit. - */ - // await db.sql`CREATE INDEX idx_bucket_data_composite ON bucket_data (group_id, bucket_name, op_id); `.execute(); - // await db.sql`CREATE INDEX idx_bucket_data_group_id ON bucket_data (group_id);`.execute(); - await db.sql`CREATE TABLE instance (id TEXT PRIMARY KEY) `.execute(); await db.sql` @@ -101,15 +86,13 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { group_id integer NOT NULL, source_table TEXT NOT NULL, source_key bytea NOT NULL, - CONSTRAINT unique_current_data_id UNIQUE (group_id, source_table, source_key), + CONSTRAINT unique_current_data_id PRIMARY KEY (group_id, source_table, source_key), buckets jsonb NOT NULL, data bytea NOT NULL, lookups bytea[] NOT NULL ); `.execute(); - await db.sql`CREATE INDEX current_data_lookup ON current_data (group_id, source_table, source_key)`.execute(); - await db.sql` CREATE TABLE source_tables ( --- This is currently a TEXT column to make the (shared) tests easier to integrate @@ -135,14 +118,12 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { ) `.execute(); - await db.sql`CREATE INDEX write_checkpoint_by_user ON write_checkpoints (user_id)`.execute(); - await db.sql` CREATE TABLE custom_write_checkpoints ( user_id text NOT NULL, write_checkpoint BIGINT NOT NULL, sync_rules_id integer NOT NULL, - CONSTRAINT unique_user_sync UNIQUE (user_id, sync_rules_id) + CONSTRAINT unique_user_sync PRIMARY KEY (user_id, sync_rules_id) ); `.execute(); }); From 07bbf1925c3fe6783a1e20967c3cc182ac8c805a Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 10:48:07 +0200 Subject: [PATCH 24/50] update max row size to match mongo --- .../src/storage/batch/PostgresBucketBatch.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index e6f7b3df5..92a339b1b 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -35,8 +35,11 @@ export interface PostgresBucketBatchOptions { const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(storage.SyncRuleState) })); type StatefulCheckpointDecoded = t.Decoded; -// The limits here are not as strict as MongoDB -const MAX_ROW_SIZE = 40_000_000; +/** + * 15MB. Currently matches MongoDB. + * This could be increased in future. + */ +const MAX_ROW_SIZE = 15 * 1024 * 1024; export class PostgresBucketBatch extends DisposableObserver From 6fbd9f89aa9d33f9a8c5e47db3484eb5467d48d9 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 14:50:40 +0200 Subject: [PATCH 25/50] set keepalive_op to bigint --- .../src/migrations/scripts/1684951997326-init.ts | 2 +- .../src/storage/batch/PostgresBucketBatch.ts | 8 ++++---- .../module-postgres-storage/src/types/models/SyncRules.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index e935f108a..ea928fd68 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -56,7 +56,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { slot_name TEXT, last_checkpoint_ts TIMESTAMP WITH TIME ZONE, last_keepalive_ts TIMESTAMP WITH TIME ZONE, - keepalive_op TEXT, + keepalive_op BIGINT, last_fatal_error TEXT, content TEXT NOT NULL ); diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index 92a339b1b..ae390816e 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -20,7 +20,7 @@ export interface PostgresBucketBatchOptions { last_checkpoint_lsn: string | null; no_checkpoint_before_lsn: string; store_current_data: boolean; - keep_alive_op?: string | null; + keep_alive_op?: bigint | null; /** * Set to true for initial replication. */ @@ -70,7 +70,7 @@ export class PostgresBucketBatch this.batch = null; this.persisted_op = null; if (options.keep_alive_op) { - this.persisted_op = BigInt(options.keep_alive_op); + this.persisted_op = options.keep_alive_op; } } @@ -299,7 +299,7 @@ export class PostgresBucketBatch await this.db.sql` UPDATE sync_rules SET - keepalive_op = ${{ type: 'varchar', value: this.persisted_op == null ? null : String(this.persisted_op) }} + keepalive_op = ${{ type: 'int8', value: this.persisted_op }} WHERE id = ${{ type: 'int4', value: this.group_id }} `.execute(); @@ -323,7 +323,7 @@ export class PostgresBucketBatch const doc = await this.db.sql` UPDATE sync_rules SET - keepalive_op = ${{ type: 'varchar', value: update.keepalive_op }}, + keepalive_op = ${{ type: 'int8', value: update.keepalive_op }}, last_fatal_error = ${{ type: 'varchar', value: update.last_fatal_error }}, snapshot_done = ${{ type: 'bool', value: update.snapshot_done }}, last_keepalive_ts = ${{ type: 1184, value: update.last_keepalive_ts }}, diff --git a/modules/module-postgres-storage/src/types/models/SyncRules.ts b/modules/module-postgres-storage/src/types/models/SyncRules.ts index 94b486983..4e4589f92 100644 --- a/modules/module-postgres-storage/src/types/models/SyncRules.ts +++ b/modules/module-postgres-storage/src/types/models/SyncRules.ts @@ -42,7 +42,7 @@ export const SyncRules = t.object({ * If an error is stopping replication, it will be stored here. */ last_fatal_error: t.Null.or(t.string), - keepalive_op: t.Null.or(t.string), + keepalive_op: t.Null.or(bigint), content: t.string }); From 40d434fdaafaa74b5f47b3443f2072db68166aa5 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 15:19:57 +0200 Subject: [PATCH 26/50] Cleanup sync_rules id sequence usage. Use integer/number type for group_id throughout. --- .../migrations/scripts/1684951997326-init.ts | 2 +- .../storage/PostgresBucketStorageFactory.ts | 6 +-- .../src/storage/PostgresSyncRulesStorage.ts | 38 +++++++++---------- .../src/storage/batch/PostgresBucketBatch.ts | 10 ++--- .../src/types/models/ActiveCheckpoint.ts | 3 +- .../src/types/models/BucketData.ts | 3 +- .../src/types/models/BucketParameters.ts | 3 +- .../src/types/models/CurrentData.ts | 5 ++- .../src/types/models/SourceTable.ts | 5 ++- .../src/types/models/SyncRules.ts | 3 +- .../src/utils/ts-codec.ts | 21 ++++++++++ 11 files changed, 63 insertions(+), 36 deletions(-) diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index ea928fd68..229ee2b36 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -47,7 +47,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { await db.sql` CREATE TABLE sync_rules ( - id BIGSERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY, state TEXT NOT NULL, snapshot_done BOOLEAN NOT NULL DEFAULT FALSE, last_checkpoint BIGINT, diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index 4dfa99b3b..f4150b64c 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -117,7 +117,7 @@ export class PostgresBucketStorageFactory FROM bucket_data WHERE - group_id = ${{ type: 'int8', value: active_sync_rules.id }} + group_id = ${{ type: 'int4', value: active_sync_rules.id }} `.first<{ operations_size_bytes: bigint }>(); const parameterData = await this.db.sql` @@ -129,7 +129,7 @@ export class PostgresBucketStorageFactory FROM bucket_parameters WHERE - group_id = ${{ type: 'int8', value: active_sync_rules.id }} + group_id = ${{ type: 'int4', value: active_sync_rules.id }} `.first<{ parameter_size_bytes: bigint }>(); const currentData = await this.db.sql` @@ -141,7 +141,7 @@ export class PostgresBucketStorageFactory FROM current_data WHERE - group_id = ${{ type: 'int8', value: active_sync_rules.id }} + group_id = ${{ type: 'int4', value: active_sync_rules.id }} `.first<{ current_size_bytes: bigint }>(); return { diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 60000971f..ab5701979 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -82,7 +82,7 @@ export class PostgresSyncRulesStorage SET last_fatal_error = ${{ type: 'varchar', value: message }} WHERE - id = ${{ type: 'int8', value: this.group_id }}; + id = ${{ type: 'int4', value: this.group_id }}; `.execute(); } @@ -126,7 +126,7 @@ export class PostgresSyncRulesStorage FROM sync_rules WHERE - id = ${{ type: 'int8', value: this.group_id }} + id = ${{ type: 'int4', value: this.group_id }} ` .decoded(pick(models.SyncRules, ['last_checkpoint', 'last_checkpoint_lsn'])) .first(); @@ -155,7 +155,7 @@ export class PostgresSyncRulesStorage FROM source_tables WHERE - group_id = ${{ type: 'int8', value: group_id }} + group_id = ${{ type: 'int4', value: group_id }} AND connection_id = ${{ type: 'int4', value: connection_id }} AND relation_id = ${{ type: 'varchar', value: objectId.toString() }} AND schema_name = ${{ type: 'varchar', value: schema }} @@ -180,7 +180,7 @@ export class PostgresSyncRulesStorage VALUES ( ${{ type: 'varchar', value: uuid.v4() }}, - ${{ type: 'int8', value: group_id }}, + ${{ type: 'int4', value: group_id }}, ${{ type: 'int4', value: connection_id }}, --- The objectId can be string | number, we store it as a string and decode when querying ${{ type: 'varchar', value: objectId.toString() }}, @@ -215,7 +215,7 @@ export class PostgresSyncRulesStorage FROM source_tables WHERE - group_id = ${{ type: 'int8', value: group_id }} + group_id = ${{ type: 'int4', value: group_id }} AND connection_id = ${{ type: 'int4', value: connection_id }} AND id != ${{ type: 'varchar', value: sourceTableRow!.id }} AND ( @@ -263,7 +263,7 @@ export class PostgresSyncRulesStorage FROM sync_rules WHERE - id = ${{ type: 'int8', value: this.group_id }} + id = ${{ type: 'int4', value: this.group_id }} ` .decoded(pick(models.SyncRules, ['last_checkpoint_lsn', 'no_checkpoint_before', 'keepalive_op'])) .first(); @@ -307,7 +307,7 @@ export class PostgresSyncRulesStorage FROM bucket_parameters WHERE - group_id = ${{ type: 'int8', value: this.group_id }} + group_id = ${{ type: 'int4', value: this.group_id }} AND lookup = ANY ( SELECT decode((FILTER ->> 0)::text, 'hex') -- Decode the hex string to bytea @@ -406,7 +406,7 @@ export class PostgresSyncRulesStorage LIMIT $3;`, params: [ - { type: 'int8', value: this.group_id }, + { type: 'int4', value: this.group_id }, { type: 'int8', value: end }, { type: 'int4', value: rowLimit + 1 }, ...filters.flatMap((f) => [ @@ -506,7 +506,7 @@ export class PostgresSyncRulesStorage state = ${{ type: 'varchar', value: storage.SyncRuleState.TERMINATED }}, snapshot_done = ${{ type: 'bool', value: false }} WHERE - id = ${{ type: 'int8', value: this.group_id }} + id = ${{ type: 'int4', value: this.group_id }} `.execute(); } @@ -519,7 +519,7 @@ export class PostgresSyncRulesStorage FROM sync_rules WHERE - id = ${{ type: 'int8', value: this.group_id }} + id = ${{ type: 'int4', value: this.group_id }} ` .decoded(pick(models.SyncRules, ['snapshot_done', 'last_checkpoint_lsn', 'state'])) .first(); @@ -544,31 +544,31 @@ export class PostgresSyncRulesStorage last_checkpoint = NULL, no_checkpoint_before = NULL WHERE - id = ${{ type: 'int8', value: this.group_id }} + id = ${{ type: 'int4', value: this.group_id }} `.execute(); await this.db.sql` DELETE FROM bucket_data WHERE - group_id = ${{ type: 'int8', value: this.group_id }} + group_id = ${{ type: 'int4', value: this.group_id }} `.execute(); await this.db.sql` DELETE FROM bucket_parameters WHERE - group_id = ${{ type: 'int8', value: this.group_id }} + group_id = ${{ type: 'int4', value: this.group_id }} `.execute(); await this.db.sql` DELETE FROM current_data WHERE - group_id = ${{ type: 'int8', value: this.group_id }} + group_id = ${{ type: 'int4', value: this.group_id }} `.execute(); await this.db.sql` DELETE FROM source_tables WHERE - group_id = ${{ type: 'int8', value: this.group_id }} + group_id = ${{ type: 'int4', value: this.group_id }} `.execute(); } @@ -580,7 +580,7 @@ export class PostgresSyncRulesStorage FROM sync_rules WHERE - id = ${{ type: 'int8', value: this.group_id }} + id = ${{ type: 'int4', value: this.group_id }} ` .decoded(pick(models.SyncRules, ['state'])) .first(); @@ -591,7 +591,7 @@ export class PostgresSyncRulesStorage SET state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }} WHERE - id = ${{ type: 'int8', value: this.group_id }} + id = ${{ type: 'int4', value: this.group_id }} `.execute(); } @@ -601,7 +601,7 @@ export class PostgresSyncRulesStorage state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }} WHERE state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }} - AND id != ${{ type: 'int8', value: this.group_id }} + AND id != ${{ type: 'int4', value: this.group_id }} `.execute(); }); } @@ -642,7 +642,7 @@ export class PostgresSyncRulesStorage AND b.op_id > f.start_op_id AND b.op_id <= f.end_op_id WHERE - b.group_id = ${{ type: 'int8', value: this.group_id }} + b.group_id = ${{ type: 'int4', value: this.group_id }} GROUP BY b.bucket_name; `.rows<{ bucket: string; checksum_total: bigint; total: bigint; has_clear_op: number }>(); diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index ae390816e..c26840d14 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -163,7 +163,7 @@ export class PostgresBucketBatch FROM current_data WHERE - group_id = ${{ type: 'int8', value: this.group_id }} + group_id = ${{ type: 'int4', value: this.group_id }} AND source_table = ${{ type: 'varchar', value: sourceTable.id }} LIMIT ${{ type: 'int4', value: BATCH_LIMIT }} @@ -376,7 +376,7 @@ export class PostgresBucketBatch last_fatal_error = ${{ type: 'varchar', value: null }}, last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }} WHERE - id = ${{ type: 'int8', value: this.group_id }} + id = ${{ type: 'int4', value: this.group_id }} RETURNING id, state, @@ -421,7 +421,7 @@ export class PostgresBucketBatch no_checkpoint_before = ${{ type: 'varchar', value: no_checkpoint_before_lsn }}, last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }} WHERE - id = ${{ type: 'int8', value: this.group_id }} + id = ${{ type: 'int4', value: this.group_id }} `.execute(); } }); @@ -827,7 +827,7 @@ export class PostgresBucketBatch // Insert or update result = { source_key: afterId, - group_id: BigInt(this.group_id), + group_id: this.group_id, data: afterData!, source_table: sourceTable.id, buckets: newBuckets, @@ -870,7 +870,7 @@ export class PostgresBucketBatch SET last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }} WHERE - id = ${{ type: 'int8', value: this.group_id }} + id = ${{ type: 'int4', value: this.group_id }} `.execute(); } } diff --git a/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts b/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts index adbbc6dd1..e1640dea5 100644 --- a/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts +++ b/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts @@ -1,4 +1,5 @@ import * as t from 'ts-codec'; +import { pgwire_number } from '../../utils/ts-codec.js'; import { bigint } from '../codecs.js'; /** @@ -6,7 +7,7 @@ import { bigint } from '../codecs.js'; * */ export const ActiveCheckpoint = t.object({ - id: bigint, + id: pgwire_number, last_checkpoint: t.Null.or(bigint), last_checkpoint_lsn: t.Null.or(t.string) }); diff --git a/modules/module-postgres-storage/src/types/models/BucketData.ts b/modules/module-postgres-storage/src/types/models/BucketData.ts index 401031190..95947d725 100644 --- a/modules/module-postgres-storage/src/types/models/BucketData.ts +++ b/modules/module-postgres-storage/src/types/models/BucketData.ts @@ -1,5 +1,6 @@ import { framework } from '@powersync/service-core'; import * as t from 'ts-codec'; +import { pgwire_number } from '../../utils/ts-codec.js'; import { bigint } from '../codecs.js'; export enum OpType { @@ -10,7 +11,7 @@ export enum OpType { } export const BucketData = t.object({ - group_id: bigint, + group_id: pgwire_number, bucket_name: t.string, op_id: bigint, op: t.Enum(OpType), diff --git a/modules/module-postgres-storage/src/types/models/BucketParameters.ts b/modules/module-postgres-storage/src/types/models/BucketParameters.ts index 5e03bf812..27c11a177 100644 --- a/modules/module-postgres-storage/src/types/models/BucketParameters.ts +++ b/modules/module-postgres-storage/src/types/models/BucketParameters.ts @@ -1,11 +1,12 @@ import { framework } from '@powersync/service-core'; import * as t from 'ts-codec'; +import { pgwire_number } from '../../utils/ts-codec.js'; import { bigint, jsonb } from '../codecs.js'; import { SQLiteJSONRecord } from './SQLiteJSONValue.js'; export const BucketParameters = t.object({ id: bigint, - group_id: t.number, + group_id: pgwire_number, source_table: t.string, source_key: framework.codecs.buffer, lookup: framework.codecs.buffer, diff --git a/modules/module-postgres-storage/src/types/models/CurrentData.ts b/modules/module-postgres-storage/src/types/models/CurrentData.ts index 6d4d7e3fe..4659b4988 100644 --- a/modules/module-postgres-storage/src/types/models/CurrentData.ts +++ b/modules/module-postgres-storage/src/types/models/CurrentData.ts @@ -1,6 +1,7 @@ import { framework } from '@powersync/service-core'; import * as t from 'ts-codec'; -import { bigint, jsonb } from '../codecs.js'; +import { pgwire_number } from '../../utils/ts-codec.js'; +import { jsonb } from '../codecs.js'; export const CurrentBucket = t.object({ bucket: t.string, @@ -14,7 +15,7 @@ export type CurrentBucketDecoded = t.Decoded; export const CurrentData = t.object({ buckets: jsonb(t.array(CurrentBucket)), data: framework.codecs.buffer, - group_id: bigint, + group_id: pgwire_number, lookups: t.array(framework.codecs.buffer), source_key: framework.codecs.buffer, source_table: t.string diff --git a/modules/module-postgres-storage/src/types/models/SourceTable.ts b/modules/module-postgres-storage/src/types/models/SourceTable.ts index 04095467a..00714843d 100644 --- a/modules/module-postgres-storage/src/types/models/SourceTable.ts +++ b/modules/module-postgres-storage/src/types/models/SourceTable.ts @@ -1,4 +1,5 @@ import * as t from 'ts-codec'; +import { pgwire_number } from '../../utils/ts-codec.js'; import { bigint, jsonb } from '../codecs.js'; export const ColumnDescriptor = t.object({ @@ -15,9 +16,9 @@ export const ColumnDescriptor = t.object({ export const SourceTable = t.object({ id: t.string, - group_id: bigint, + group_id: pgwire_number, connection_id: bigint, - relation_id: t.Null.or(t.number).or(t.string), + relation_id: t.Null.or(pgwire_number).or(t.string), schema_name: t.string, table_name: t.string, replica_id_columns: t.Null.or(jsonb(t.array(ColumnDescriptor))), diff --git a/modules/module-postgres-storage/src/types/models/SyncRules.ts b/modules/module-postgres-storage/src/types/models/SyncRules.ts index 4e4589f92..c26a5f58a 100644 --- a/modules/module-postgres-storage/src/types/models/SyncRules.ts +++ b/modules/module-postgres-storage/src/types/models/SyncRules.ts @@ -1,9 +1,10 @@ import { framework, storage } from '@powersync/service-core'; import * as t from 'ts-codec'; +import { pgwire_number } from '../../utils/ts-codec.js'; import { bigint } from '../codecs.js'; export const SyncRules = t.object({ - id: bigint, + id: pgwire_number, state: t.Enum(storage.SyncRuleState), /** * True if initial snapshot has been replicated. diff --git a/modules/module-postgres-storage/src/utils/ts-codec.ts b/modules/module-postgres-storage/src/utils/ts-codec.ts index 1740b10c6..c7f828a37 100644 --- a/modules/module-postgres-storage/src/utils/ts-codec.ts +++ b/modules/module-postgres-storage/src/utils/ts-codec.ts @@ -12,3 +12,24 @@ export const pick = (code // Return a new codec with the narrowed shape return t.object(newShape) as t.ObjectCodec>; }; + +/** + * PGWire returns INTEGER columns as a `bigint`. + * This does a decode operation to `number`. + */ +export const pgwire_number = t.codec( + 'pg_number', + (decoded: number) => decoded, + (encoded: bigint | number) => { + if (typeof encoded == 'number') { + return encoded; + } + if (typeof encoded !== 'bigint') { + throw new Error(`Expected either number or bigint for value`); + } + if (encoded > BigInt(Number.MAX_SAFE_INTEGER) || encoded < BigInt(Number.MIN_SAFE_INTEGER)) { + throw new RangeError('BigInt value is out of safe integer range for conversion to Number.'); + } + return Number(encoded); + } +); From 405521e3ea96ee7a0cf9ca67228453e8bb23f87d Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 15:36:40 +0200 Subject: [PATCH 27/50] fix lock conflict. --- .../src/locks/PostgresLockManager.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/libs/lib-postgres/src/locks/PostgresLockManager.ts b/libs/lib-postgres/src/locks/PostgresLockManager.ts index fc5d49cfe..4f01f5156 100644 --- a/libs/lib-postgres/src/locks/PostgresLockManager.ts +++ b/libs/lib-postgres/src/locks/PostgresLockManager.ts @@ -52,37 +52,39 @@ export class PostgresLockManager extends framework.locks.AbstractLockManager { protected async _acquireId(): Promise { const now = new Date(); - const expiredTs = new Date(now.getTime() - this.timeout); + const nowISO = now.toISOString(); + const expiredTs = new Date(now.getTime() - this.timeout).toISOString(); const lockId = uuidv4(); try { // Attempt to acquire or refresh the lock - const res = await this.db.query(sql` + const res = await this.db.queryRows<{ lock_id: string }>(sql` INSERT INTO locks (name, lock_id, ts) VALUES ( ${{ type: 'varchar', value: this.name }}, ${{ type: 'uuid', value: lockId }}, - ${{ type: 1184, value: now.toISOString() }} + ${{ type: 1184, value: nowISO }} ) ON CONFLICT (name) DO UPDATE SET lock_id = CASE - WHEN locks.ts <= $4 THEN $2 + WHEN locks.ts <= ${{ type: 1184, value: expiredTs }} THEN ${{ type: 'uuid', value: lockId }} ELSE locks.lock_id END, ts = CASE - WHEN locks.ts <= $4 THEN $3 + WHEN locks.ts <= ${{ type: 1184, value: expiredTs }} THEN ${{ + type: 1184, + value: nowISO + }} ELSE locks.ts END - WHERE - locks.ts <= ${{ type: 1184, value: expiredTs.toISOString() }} RETURNING lock_id; `); - if (res.rows.length === 0) { + if (res.length == 0 || res[0].lock_id !== lockId) { // Lock is active and could not be acquired return null; } From 07ca4b71a75ee4f78fad9ec91db944a2d0335f41 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 15:47:13 +0200 Subject: [PATCH 28/50] move comment. --- .../src/db/connection/DatabaseClient.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/libs/lib-postgres/src/db/connection/DatabaseClient.ts b/libs/lib-postgres/src/db/connection/DatabaseClient.ts index 702e9566e..89d7ca815 100644 --- a/libs/lib-postgres/src/db/connection/DatabaseClient.ts +++ b/libs/lib-postgres/src/db/connection/DatabaseClient.ts @@ -23,6 +23,12 @@ export type DatabaseClientListener = NotificationListener & { export const TRANSACTION_CONNECTION_COUNT = 5; +/** + * This provides access to Postgres via the PGWire library. + * A connection pool is used for individual query executions while + * a custom pool of connections is available for transactions or other operations + * which require being executed on the same connection. + */ export class DatabaseClient extends AbstractPostgresConnection { closed: boolean; @@ -81,16 +87,15 @@ export class DatabaseClient extends AbstractPostgresConnection; query(...args: pgwire.Statement[]): Promise; async query(...args: any[]): Promise { await this.initialized; - + /** + * There is no direct way to set the default schema with pgwire. + * This hack uses multiple statements in order to always ensure the + * appropriate connection (in the pool) uses the correct schema. + */ const { schemaStatement } = this; if (typeof args[0] == 'object' && schemaStatement) { args.unshift(schemaStatement); From a06c715f9a2bd89016f029df83543f54f48af2bf Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 15:54:32 +0200 Subject: [PATCH 29/50] update readme configuration. Remove batch settings from documentation. --- modules/module-postgres-storage/README.md | 27 +---------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/modules/module-postgres-storage/README.md b/modules/module-postgres-storage/README.md index fd90da2f5..47ed5194b 100644 --- a/modules/module-postgres-storage/README.md +++ b/modules/module-postgres-storage/README.md @@ -6,7 +6,7 @@ This module provides a `BucketStorageProvider` which uses a Postgres database fo Postgres storage can be enabled by selecting the appropriate storage `type` and providing connection details for a Postgres server. -The storage connection configuration extends the configuration for a Postgres replication source, thus it accepts and supports the same configurations fields. +The storage connection configuration supports the same fields as the Postgres replication connection configuration. A sample YAML configuration could look like @@ -61,28 +61,3 @@ WITH -- The user should only have access to the schema it created GRANT CREATE ON DATABASE postgres TO powersync_storage_user; ``` - -### Batching - -Replication data is persisted via batch operations. Batching ensures performant, memory optimized writes. Batches are limited in size. Increasing batch size limits could reduce the amount of server round-trips which could increase performance (in some circumstances), but will result in higher memory usage and potential server issues. - -Batch size limits are defaulted and can optionally be configured in the configuration. - -```yaml -# Connection settings for sync bucket storage -storage: - type: postgresql - # This accepts the same parameters as a Postgres replication source connection - uri: !env PS_STORAGE_SOURCE_URI - batch_limits: - # Maximum estimated byte size of operations in a single batch. - # Defaults to 5 megabytes. - max_estimated_size: 5000000 - # Maximum number of records present in a single batch. - # Defaults to 2000 records. - # Increasing this limit can improve replication times for large volumes of data. - max_record_count: 2000 - # Maximum byte size of size of current_data documents we lookup at a time. - # Defaults to 50 megabytes. - max_current_data_batch_size: 50000000 -``` From c03e56e8a7659c0c54a86c1fcf9185fcbc132d8a Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 17:38:34 +0200 Subject: [PATCH 30/50] better size estimation --- .../storage/PostgresBucketStorageFactory.ts | 43 ++++--------------- 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index f4150b64c..2253b6782 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -110,44 +110,17 @@ export class PostgresBucketStorageFactory }; } - const bucketData = await this.db.sql` + const sizes = await this.db.sql` SELECT - --- This can differ from the octet_length - sum(pg_column_size(data)) AS operations_size_bytes - FROM - bucket_data - WHERE - group_id = ${{ type: 'int4', value: active_sync_rules.id }} - `.first<{ operations_size_bytes: bigint }>(); - - const parameterData = await this.db.sql` - SELECT - --- This can differ from the octet_length - ( - sum(pg_column_size(bucket_parameters)) + sum(pg_column_size(lookup)) + sum(pg_column_size(source_key)) - ) AS parameter_size_bytes - FROM - bucket_parameters - WHERE - group_id = ${{ type: 'int4', value: active_sync_rules.id }} - `.first<{ parameter_size_bytes: bigint }>(); - - const currentData = await this.db.sql` - SELECT - --- This can differ from the octet_length - ( - sum(pg_column_size(data)) + sum(pg_column_size(lookups)) + sum(pg_column_size(source_key)) + sum(pg_column_size(buckets)) - ) AS current_size_bytes - FROM - current_data - WHERE - group_id = ${{ type: 'int4', value: active_sync_rules.id }} - `.first<{ current_size_bytes: bigint }>(); + pg_total_relation_size('current_data') AS current_size_bytes, + pg_total_relation_size('bucket_parameters') AS parameter_size_bytes, + pg_total_relation_size('bucket_data') AS operations_size_bytes; + `.first<{ current_size_bytes: bigint; parameter_size_bytes: bigint; operations_size_bytes: bigint }>(); return { - operations_size_bytes: Number(bucketData!.operations_size_bytes), - parameters_size_bytes: Number(parameterData!.parameter_size_bytes), - replication_size_bytes: Number(currentData!.current_size_bytes) + operations_size_bytes: Number(sizes!.operations_size_bytes), + parameters_size_bytes: Number(sizes!.parameter_size_bytes), + replication_size_bytes: Number(sizes!.current_size_bytes) }; } From fc3963868d2096eda9dcd4273ca4b42f86a28176 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 17:52:51 +0200 Subject: [PATCH 31/50] update to pg_relation_size to not include index size. --- .../src/storage/PostgresBucketStorageFactory.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index 2253b6782..ab0d10623 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -112,9 +112,9 @@ export class PostgresBucketStorageFactory const sizes = await this.db.sql` SELECT - pg_total_relation_size('current_data') AS current_size_bytes, - pg_total_relation_size('bucket_parameters') AS parameter_size_bytes, - pg_total_relation_size('bucket_data') AS operations_size_bytes; + pg_relation_size('current_data') AS current_size_bytes, + pg_relation_size('bucket_parameters') AS parameter_size_bytes, + pg_relation_size('bucket_data') AS operations_size_bytes; `.first<{ current_size_bytes: bigint; parameter_size_bytes: bigint; operations_size_bytes: bigint }>(); return { From 8c4a7e987176b7bf513ede20649f4589d9794f4a Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 18:13:07 +0200 Subject: [PATCH 32/50] test pg_total_relation_size with vitest snapshot --- .../test/src/__snapshots__/storage.test.ts.snap | 9 +++++++++ .../src/storage/PostgresBucketStorageFactory.ts | 6 +++--- .../test/src/__snapshots__/storage.test.ts.snap | 9 +++++++++ .../src/tests/register-data-storage-tests.ts | 6 +----- 4 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 modules/module-mongodb-storage/test/src/__snapshots__/storage.test.ts.snap create mode 100644 modules/module-postgres-storage/test/src/__snapshots__/storage.test.ts.snap diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/storage.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/storage.test.ts.snap new file mode 100644 index 000000000..7c072c0a1 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/__snapshots__/storage.test.ts.snap @@ -0,0 +1,9 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`Mongo Sync Bucket Storage > empty storage metrics 1`] = ` +{ + "operations_size_bytes": 0, + "parameters_size_bytes": 0, + "replication_size_bytes": 0, +} +`; diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index ab0d10623..2253b6782 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -112,9 +112,9 @@ export class PostgresBucketStorageFactory const sizes = await this.db.sql` SELECT - pg_relation_size('current_data') AS current_size_bytes, - pg_relation_size('bucket_parameters') AS parameter_size_bytes, - pg_relation_size('bucket_data') AS operations_size_bytes; + pg_total_relation_size('current_data') AS current_size_bytes, + pg_total_relation_size('bucket_parameters') AS parameter_size_bytes, + pg_total_relation_size('bucket_data') AS operations_size_bytes; `.first<{ current_size_bytes: bigint; parameter_size_bytes: bigint; operations_size_bytes: bigint }>(); return { diff --git a/modules/module-postgres-storage/test/src/__snapshots__/storage.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/storage.test.ts.snap new file mode 100644 index 000000000..10eb83682 --- /dev/null +++ b/modules/module-postgres-storage/test/src/__snapshots__/storage.test.ts.snap @@ -0,0 +1,9 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`Postgres Sync Bucket Storage > empty storage metrics 1`] = ` +{ + "operations_size_bytes": 16384, + "parameters_size_bytes": 32768, + "replication_size_bytes": 16384, +} +`; diff --git a/packages/service-core-tests/src/tests/register-data-storage-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-tests.ts index f0d4d781f..1c4c89343 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-tests.ts @@ -1507,11 +1507,7 @@ bucket_definitions: await storage.autoActivate(); const metrics2 = await f.getStorageMetrics(); - expect(metrics2).toEqual({ - operations_size_bytes: 0, - parameters_size_bytes: 0, - replication_size_bytes: 0 - }); + expect(metrics2).toMatchSnapshot(); }); test('invalidate cached parsed sync rules', async () => { From efed5342526ce94432536f35e5d5bf11de5bc8b4 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 18:16:05 +0200 Subject: [PATCH 33/50] add beta readme note --- modules/module-postgres-storage/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/module-postgres-storage/README.md b/modules/module-postgres-storage/README.md index 47ed5194b..f53bc947e 100644 --- a/modules/module-postgres-storage/README.md +++ b/modules/module-postgres-storage/README.md @@ -2,6 +2,10 @@ This module provides a `BucketStorageProvider` which uses a Postgres database for persistence. +## Beta + +This feature is currently in a beta release. See [here](https://docs.powersync.com/resources/feature-status#feature-status) for more details. + ## Configuration Postgres storage can be enabled by selecting the appropriate storage `type` and providing connection details for a Postgres server. From d17d896552b0f7d3f527c0bf685976b972c558d5 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 13 Jan 2025 18:26:26 +0200 Subject: [PATCH 34/50] update postgres replication storage snapshot --- .../test/src/__snapshots__/schema_changes.test.ts.snap | 5 +++++ modules/module-postgres/test/src/schema_changes.test.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 modules/module-postgres/test/src/__snapshots__/schema_changes.test.ts.snap diff --git a/modules/module-postgres/test/src/__snapshots__/schema_changes.test.ts.snap b/modules/module-postgres/test/src/__snapshots__/schema_changes.test.ts.snap new file mode 100644 index 000000000..2ba06c6fe --- /dev/null +++ b/modules/module-postgres/test/src/__snapshots__/schema_changes.test.ts.snap @@ -0,0 +1,5 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`schema changes - mongodb > add to publication (not in sync rules) 1`] = `0`; + +exports[`schema changes - postgres > add to publication (not in sync rules) 1`] = `16384`; diff --git a/modules/module-postgres/test/src/schema_changes.test.ts b/modules/module-postgres/test/src/schema_changes.test.ts index a890c3fcf..3fffb3f6d 100644 --- a/modules/module-postgres/test/src/schema_changes.test.ts +++ b/modules/module-postgres/test/src/schema_changes.test.ts @@ -437,7 +437,7 @@ function defineTests(factory: storage.TestStorageFactory) { expect(data).toMatchObject([]); const metrics = await storage.factory.getStorageMetrics(); - expect(metrics.replication_size_bytes).toEqual(0); + expect(metrics.replication_size_bytes).toMatchSnapshot(); }); test('replica identity nothing', async () => { From af9a6a18b1d73ecae7ccdc5f1da22407b0ca8247 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 08:53:38 +0200 Subject: [PATCH 35/50] cleanup slot poking --- libs/lib-postgres/src/db/connection/DatabaseClient.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libs/lib-postgres/src/db/connection/DatabaseClient.ts b/libs/lib-postgres/src/db/connection/DatabaseClient.ts index 89d7ca815..1fbd048c9 100644 --- a/libs/lib-postgres/src/db/connection/DatabaseClient.ts +++ b/libs/lib-postgres/src/db/connection/DatabaseClient.ts @@ -77,6 +77,8 @@ export class DatabaseClient extends AbstractPostgresConnection(); this.queue.push(deferred); + this.pokeSlots(); + + return deferred.promise; + } + + protected pokeSlots() { // Poke the slots to check if they are alive for (const slot of this.connections) { // No need to await this. Errors are reported asynchronously slot.poke(); } - - return deferred.promise; } protected leaseConnectionSlot(): ConnectionLease | null { From 7cb338d9f45fedc343c98d15960839bfd18fd70c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 09:53:55 +0200 Subject: [PATCH 36/50] add note about ts-codec. Make buffer codex a bit clearer. --- .../src/types/codecs.ts | 56 +++++++++++++++++++ .../src/types/models/BucketData.ts | 5 +- .../src/types/models/BucketParameters.ts | 7 +-- .../src/types/models/CurrentData.ts | 9 ++- 4 files changed, 65 insertions(+), 12 deletions(-) diff --git a/modules/module-postgres-storage/src/types/codecs.ts b/modules/module-postgres-storage/src/types/codecs.ts index 99fa0b344..561757c6a 100644 --- a/modules/module-postgres-storage/src/types/codecs.ts +++ b/modules/module-postgres-storage/src/types/codecs.ts @@ -2,6 +2,42 @@ import * as t from 'ts-codec'; export const BIGINT_MAX = BigInt('9223372036854775807'); +/** + * The use of ts-codec: + * We currently use pgwire for Postgres queries. This library provides fine-grained control + * over parameter typings and efficient streaming of query responses. Additionally, configuring + * pgwire with default certificates allows us to use the same connection configuration process + * for both replication and storage libraries. + * + * Unfortunately, ORM driver support for pgwire is limited, so we rely on pure SQL queries in the + * absence of writing an ORM driver from scratch. + * + * [Opinion]: Writing pure SQL queries throughout a codebase can be daunting from a maintenance + * and debugging perspective. For example, row response types are often declared when performing a query: + * + * ```typescript + * const rows = await db.queryRows(`SELECT one, two FROM my_table`); + * ``` + * This type declaration suggests `rows` is an array of `MyRowType` objects, even though no validation + * is enforced. Adding a field to the `MyRowType` interface without updating the query could easily + * introduce subtle bugs. Similarly, type mismatches between SQL results and TypeScript interfaces, such as + * a `Date` field returned as a `string`, require manual conversion. + * + * `ts-codec` is not an ORM, but it simplifies working with pure SQL query responses in several ways: + * + * - **Validations**: The `decode` operation ensures that the returned row matches the expected object + * structure, throwing an error if it doesn't. + * - **Decoding Columns**: pgwire already decodes common SQLite types, but `ts-codec` adds an extra layer + * for JS-native values. For instance, `jsonb` columns are returned as `JsonContainer`/`string` and can + * be automatically parsed into objects. Similarly, fields like `group_id` are converted from `Bigint` + * to `Number` for easier use. + * - **Encoded Forms**: A single `ts-codec` type definition can infer both encoded and decoded forms. This + * is especially useful for persisted batch operations that rely on JSON query parameters for bulk inserts. + * Collections like `bucket_data`, `current_data`, and `bucket_parameters` use encoded/decoded types, making + * changes easier to manage and validate. While some manual encoding is done for intermediate values (e.g., + * size estimation), these types are validated with `ts-codec` to ensure consistency. + */ + /** * Wraps a codec which is encoded to a JSON string */ @@ -32,3 +68,23 @@ export const uint8array = t.codec( (d) => d, (e) => e ); + +/** + * PGWire returns BYTEA values as Uint8Array instances. + * We also serialize to a hex string for bulk inserts. + */ +export const hexBuffer = t.codec( + 'hexBuffer', + (decoded: Buffer) => { + return decoded.toString('hex'); + }, + (encoded: string | Uint8Array) => { + if (encoded instanceof Uint8Array) { + return Buffer.from(encoded); + } + if (typeof encoded !== 'string') { + throw new Error(`Expected either a Buffer instance or hex encoded buffer string`); + } + return Buffer.from(encoded, 'hex'); + } +); diff --git a/modules/module-postgres-storage/src/types/models/BucketData.ts b/modules/module-postgres-storage/src/types/models/BucketData.ts index 95947d725..3e3463638 100644 --- a/modules/module-postgres-storage/src/types/models/BucketData.ts +++ b/modules/module-postgres-storage/src/types/models/BucketData.ts @@ -1,7 +1,6 @@ -import { framework } from '@powersync/service-core'; import * as t from 'ts-codec'; import { pgwire_number } from '../../utils/ts-codec.js'; -import { bigint } from '../codecs.js'; +import { bigint, hexBuffer } from '../codecs.js'; export enum OpType { PUT = 'PUT', @@ -16,7 +15,7 @@ export const BucketData = t.object({ op_id: bigint, op: t.Enum(OpType), source_table: t.Null.or(t.string), - source_key: t.Null.or(framework.codecs.buffer), + source_key: t.Null.or(hexBuffer), table_name: t.string.or(t.Null), row_id: t.string.or(t.Null), checksum: bigint, diff --git a/modules/module-postgres-storage/src/types/models/BucketParameters.ts b/modules/module-postgres-storage/src/types/models/BucketParameters.ts index 27c11a177..9d67a2b59 100644 --- a/modules/module-postgres-storage/src/types/models/BucketParameters.ts +++ b/modules/module-postgres-storage/src/types/models/BucketParameters.ts @@ -1,15 +1,14 @@ -import { framework } from '@powersync/service-core'; import * as t from 'ts-codec'; import { pgwire_number } from '../../utils/ts-codec.js'; -import { bigint, jsonb } from '../codecs.js'; +import { bigint, hexBuffer, jsonb } from '../codecs.js'; import { SQLiteJSONRecord } from './SQLiteJSONValue.js'; export const BucketParameters = t.object({ id: bigint, group_id: pgwire_number, source_table: t.string, - source_key: framework.codecs.buffer, - lookup: framework.codecs.buffer, + source_key: hexBuffer, + lookup: hexBuffer, bucket_parameters: jsonb(t.array(SQLiteJSONRecord)) }); diff --git a/modules/module-postgres-storage/src/types/models/CurrentData.ts b/modules/module-postgres-storage/src/types/models/CurrentData.ts index 4659b4988..74fec3639 100644 --- a/modules/module-postgres-storage/src/types/models/CurrentData.ts +++ b/modules/module-postgres-storage/src/types/models/CurrentData.ts @@ -1,7 +1,6 @@ -import { framework } from '@powersync/service-core'; import * as t from 'ts-codec'; import { pgwire_number } from '../../utils/ts-codec.js'; -import { jsonb } from '../codecs.js'; +import { hexBuffer, jsonb } from '../codecs.js'; export const CurrentBucket = t.object({ bucket: t.string, @@ -14,10 +13,10 @@ export type CurrentBucketDecoded = t.Decoded; export const CurrentData = t.object({ buckets: jsonb(t.array(CurrentBucket)), - data: framework.codecs.buffer, + data: hexBuffer, group_id: pgwire_number, - lookups: t.array(framework.codecs.buffer), - source_key: framework.codecs.buffer, + lookups: t.array(hexBuffer), + source_key: hexBuffer, source_table: t.string }); From c5179760a2f06ed6f54fe481dee9cdacb2521c2b Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 10:45:08 +0200 Subject: [PATCH 37/50] move pgwire_number to codecs file --- .../src/types/codecs.ts | 21 +++++++++++++++++++ .../src/types/models/ActiveCheckpoint.ts | 3 +-- .../src/types/models/BucketData.ts | 3 +-- .../src/types/models/BucketParameters.ts | 3 +-- .../src/types/models/CurrentData.ts | 3 +-- .../src/types/models/SourceTable.ts | 3 +-- .../src/types/models/SyncRules.ts | 3 +-- 7 files changed, 27 insertions(+), 12 deletions(-) diff --git a/modules/module-postgres-storage/src/types/codecs.ts b/modules/module-postgres-storage/src/types/codecs.ts index 561757c6a..643a47640 100644 --- a/modules/module-postgres-storage/src/types/codecs.ts +++ b/modules/module-postgres-storage/src/types/codecs.ts @@ -88,3 +88,24 @@ export const hexBuffer = t.codec( return Buffer.from(encoded, 'hex'); } ); + +/** + * PGWire returns INTEGER columns as a `bigint`. + * This does a decode operation to `number`. + */ +export const pgwire_number = t.codec( + 'pg_number', + (decoded: number) => decoded, + (encoded: bigint | number) => { + if (typeof encoded == 'number') { + return encoded; + } + if (typeof encoded !== 'bigint') { + throw new Error(`Expected either number or bigint for value`); + } + if (encoded > BigInt(Number.MAX_SAFE_INTEGER) || encoded < BigInt(Number.MIN_SAFE_INTEGER)) { + throw new RangeError('BigInt value is out of safe integer range for conversion to Number.'); + } + return Number(encoded); + } +); diff --git a/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts b/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts index e1640dea5..f55ba29ed 100644 --- a/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts +++ b/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts @@ -1,6 +1,5 @@ import * as t from 'ts-codec'; -import { pgwire_number } from '../../utils/ts-codec.js'; -import { bigint } from '../codecs.js'; +import { bigint, pgwire_number } from '../codecs.js'; /** * Notification payload sent via Postgres' NOTIFY API. diff --git a/modules/module-postgres-storage/src/types/models/BucketData.ts b/modules/module-postgres-storage/src/types/models/BucketData.ts index 3e3463638..757e58165 100644 --- a/modules/module-postgres-storage/src/types/models/BucketData.ts +++ b/modules/module-postgres-storage/src/types/models/BucketData.ts @@ -1,6 +1,5 @@ import * as t from 'ts-codec'; -import { pgwire_number } from '../../utils/ts-codec.js'; -import { bigint, hexBuffer } from '../codecs.js'; +import { bigint, hexBuffer, pgwire_number } from '../codecs.js'; export enum OpType { PUT = 'PUT', diff --git a/modules/module-postgres-storage/src/types/models/BucketParameters.ts b/modules/module-postgres-storage/src/types/models/BucketParameters.ts index 9d67a2b59..8c36d9a86 100644 --- a/modules/module-postgres-storage/src/types/models/BucketParameters.ts +++ b/modules/module-postgres-storage/src/types/models/BucketParameters.ts @@ -1,6 +1,5 @@ import * as t from 'ts-codec'; -import { pgwire_number } from '../../utils/ts-codec.js'; -import { bigint, hexBuffer, jsonb } from '../codecs.js'; +import { bigint, hexBuffer, jsonb, pgwire_number } from '../codecs.js'; import { SQLiteJSONRecord } from './SQLiteJSONValue.js'; export const BucketParameters = t.object({ diff --git a/modules/module-postgres-storage/src/types/models/CurrentData.ts b/modules/module-postgres-storage/src/types/models/CurrentData.ts index 74fec3639..828d9a8c0 100644 --- a/modules/module-postgres-storage/src/types/models/CurrentData.ts +++ b/modules/module-postgres-storage/src/types/models/CurrentData.ts @@ -1,6 +1,5 @@ import * as t from 'ts-codec'; -import { pgwire_number } from '../../utils/ts-codec.js'; -import { hexBuffer, jsonb } from '../codecs.js'; +import { hexBuffer, jsonb, pgwire_number } from '../codecs.js'; export const CurrentBucket = t.object({ bucket: t.string, diff --git a/modules/module-postgres-storage/src/types/models/SourceTable.ts b/modules/module-postgres-storage/src/types/models/SourceTable.ts index 00714843d..1bf5e2dd3 100644 --- a/modules/module-postgres-storage/src/types/models/SourceTable.ts +++ b/modules/module-postgres-storage/src/types/models/SourceTable.ts @@ -1,6 +1,5 @@ import * as t from 'ts-codec'; -import { pgwire_number } from '../../utils/ts-codec.js'; -import { bigint, jsonb } from '../codecs.js'; +import { bigint, jsonb, pgwire_number } from '../codecs.js'; export const ColumnDescriptor = t.object({ name: t.string, diff --git a/modules/module-postgres-storage/src/types/models/SyncRules.ts b/modules/module-postgres-storage/src/types/models/SyncRules.ts index c26a5f58a..8edc5eea4 100644 --- a/modules/module-postgres-storage/src/types/models/SyncRules.ts +++ b/modules/module-postgres-storage/src/types/models/SyncRules.ts @@ -1,7 +1,6 @@ import { framework, storage } from '@powersync/service-core'; import * as t from 'ts-codec'; -import { pgwire_number } from '../../utils/ts-codec.js'; -import { bigint } from '../codecs.js'; +import { bigint, pgwire_number } from '../codecs.js'; export const SyncRules = t.object({ id: pgwire_number, From 79da9db294fcb0b96d5052f78caaac0f6d3b4090 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 10:45:36 +0200 Subject: [PATCH 38/50] cleanup --- .../src/utils/ts-codec.ts | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/modules/module-postgres-storage/src/utils/ts-codec.ts b/modules/module-postgres-storage/src/utils/ts-codec.ts index c7f828a37..1740b10c6 100644 --- a/modules/module-postgres-storage/src/utils/ts-codec.ts +++ b/modules/module-postgres-storage/src/utils/ts-codec.ts @@ -12,24 +12,3 @@ export const pick = (code // Return a new codec with the narrowed shape return t.object(newShape) as t.ObjectCodec>; }; - -/** - * PGWire returns INTEGER columns as a `bigint`. - * This does a decode operation to `number`. - */ -export const pgwire_number = t.codec( - 'pg_number', - (decoded: number) => decoded, - (encoded: bigint | number) => { - if (typeof encoded == 'number') { - return encoded; - } - if (typeof encoded !== 'bigint') { - throw new Error(`Expected either number or bigint for value`); - } - if (encoded > BigInt(Number.MAX_SAFE_INTEGER) || encoded < BigInt(Number.MIN_SAFE_INTEGER)) { - throw new RangeError('BigInt value is out of safe integer range for conversion to Number.'); - } - return Number(encoded); - } -); From d006e90fd91ce61e6a6bc89a4a89536344008512 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 12:17:42 +0200 Subject: [PATCH 39/50] update bucket_parameters types --- .../src/migrations/scripts/1684951997326-init.ts | 4 +++- .../src/storage/PostgresSyncRulesStorage.ts | 3 ++- .../src/storage/batch/PostgresPersistedBatch.ts | 4 ++-- .../src/types/models/BucketParameters.ts | 6 +++--- .../src/types/models/SQLiteJSONValue.ts | 10 ---------- .../src/types/models/models-index.ts | 1 - 6 files changed, 10 insertions(+), 18 deletions(-) delete mode 100644 modules/module-postgres-storage/src/types/models/SQLiteJSONValue.ts diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index 229ee2b36..a387f0f65 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -69,7 +69,9 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { source_table TEXT NOT NULL, source_key bytea NOT NULL, lookup bytea NOT NULL, - bucket_parameters jsonb NOT NULL + --- Stored as text which is stringified with JSONBig + --- BigInts are not standard JSON, storing as JSONB seems risky + bucket_parameters text NOT NULL ); `.execute(); diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index ab5701979..b0361a783 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -1,6 +1,7 @@ import * as lib_postgres from '@powersync/lib-service-postgres'; import { DisposableObserver } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; import * as sync_rules from '@powersync/service-sync-rules'; import * as uuid from 'uuid'; import { BIGINT_MAX } from '../types/codecs.js'; @@ -328,7 +329,7 @@ export class PostgresSyncRulesStorage .rows(); const groupedParameters = rows.map((row) => { - return row.bucket_parameters; + return JSONBig.parse(row.bucket_parameters) as sync_rules.SqliteJsonRow; }); return groupedParameters.flat(); } diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts index 0573764ae..8dcb7eb0d 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts @@ -325,14 +325,14 @@ export class PostgresPersistedBatch { source_table, decode(source_key, 'hex') AS source_key, -- Decode hex to bytea decode(lookup, 'hex') AS lookup, -- Decode hex to bytea - bucket_parameters::jsonb AS bucket_parameters + bucket_parameters FROM jsonb_to_recordset(${{ type: 'jsonb', value: this.parameterDataInserts }}::jsonb) AS t ( group_id integer, source_table text, source_key text, -- Input as hex string lookup text, -- Input as hex string - bucket_parameters text -- Input as stringified jsonb + bucket_parameters text -- Input as stringified JSON ) ) INSERT INTO diff --git a/modules/module-postgres-storage/src/types/models/BucketParameters.ts b/modules/module-postgres-storage/src/types/models/BucketParameters.ts index 8c36d9a86..b0829fd3b 100644 --- a/modules/module-postgres-storage/src/types/models/BucketParameters.ts +++ b/modules/module-postgres-storage/src/types/models/BucketParameters.ts @@ -1,6 +1,6 @@ import * as t from 'ts-codec'; -import { bigint, hexBuffer, jsonb, pgwire_number } from '../codecs.js'; -import { SQLiteJSONRecord } from './SQLiteJSONValue.js'; +// import { JsonContainer } from '@powersync/service-jsonbig'; +import { bigint, hexBuffer, pgwire_number } from '../codecs.js'; export const BucketParameters = t.object({ id: bigint, @@ -8,7 +8,7 @@ export const BucketParameters = t.object({ source_table: t.string, source_key: hexBuffer, lookup: hexBuffer, - bucket_parameters: jsonb(t.array(SQLiteJSONRecord)) + bucket_parameters: t.string }); export type BucketParameters = t.Encoded; diff --git a/modules/module-postgres-storage/src/types/models/SQLiteJSONValue.ts b/modules/module-postgres-storage/src/types/models/SQLiteJSONValue.ts deleted file mode 100644 index 84b4d789c..000000000 --- a/modules/module-postgres-storage/src/types/models/SQLiteJSONValue.ts +++ /dev/null @@ -1,10 +0,0 @@ -import * as t from 'ts-codec'; -import { bigint } from '../codecs.js'; - -export const SQLiteJSONValue = t.number.or(t.string).or(bigint).or(t.Null); -export type SQLiteJSONValue = t.Encoded; -export type SQLiteJSONValueDecoded = t.Decoded; - -export const SQLiteJSONRecord = t.record(SQLiteJSONValue); -export type SQLiteJSONRecord = t.Encoded; -export type SQLiteJSONRecordDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/models-index.ts b/modules/module-postgres-storage/src/types/models/models-index.ts index caf39b3b3..b346ffd67 100644 --- a/modules/module-postgres-storage/src/types/models/models-index.ts +++ b/modules/module-postgres-storage/src/types/models/models-index.ts @@ -5,6 +5,5 @@ export * from './BucketParameters.js'; export * from './CurrentData.js'; export * from './Instance.js'; export * from './SourceTable.js'; -export * from './SQLiteJSONValue.js'; export * from './SyncRules.js'; export * from './WriteCheckpoint.js'; From 198fe30f82dff2e8cd838c86d7e942ce1684ed13 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 13:50:29 +0200 Subject: [PATCH 40/50] store relation_id as json in order to store number or string type --- .../migrations/scripts/1684951997326-init.ts | 2 +- .../src/storage/PostgresSyncRulesStorage.ts | 11 ++++---- .../src/types/codecs.ts | 25 +++++++++++++++++++ .../src/types/models/BucketParameters.ts | 1 - .../src/types/models/SourceTable.ts | 8 ++++-- 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index a387f0f65..0cd26784e 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -102,7 +102,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { id TEXT PRIMARY KEY, group_id integer NOT NULL, connection_id integer NOT NULL, - relation_id text, + relation_id jsonb, schema_name text NOT NULL, table_name text NOT NULL, replica_id_columns jsonb, diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index b0361a783..6dc929ef9 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -10,6 +10,7 @@ import { replicaIdToSubkey } from '../utils/bson.js'; import { mapOpEntry } from '../utils/bucket-data.js'; import { StatementParam } from '@powersync/service-jpgwire'; +import { StoredRelationId } from '../types/models/SourceTable.js'; import { pick } from '../utils/ts-codec.js'; import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js'; import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js'; @@ -158,7 +159,7 @@ export class PostgresSyncRulesStorage WHERE group_id = ${{ type: 'int4', value: group_id }} AND connection_id = ${{ type: 'int4', value: connection_id }} - AND relation_id = ${{ type: 'varchar', value: objectId.toString() }} + AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }} AND schema_name = ${{ type: 'varchar', value: schema }} AND table_name = ${{ type: 'varchar', value: table }} AND replica_id_columns = ${{ type: 'jsonb', value: columns }} @@ -183,8 +184,8 @@ export class PostgresSyncRulesStorage ${{ type: 'varchar', value: uuid.v4() }}, ${{ type: 'int4', value: group_id }}, ${{ type: 'int4', value: connection_id }}, - --- The objectId can be string | number, we store it as a string and decode when querying - ${{ type: 'varchar', value: objectId.toString() }}, + --- The objectId can be string | number, we store it as jsonb value + ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}, ${{ type: 'varchar', value: schema }}, ${{ type: 'varchar', value: table }}, ${{ type: 'jsonb', value: columns }} @@ -220,7 +221,7 @@ export class PostgresSyncRulesStorage AND connection_id = ${{ type: 'int4', value: connection_id }} AND id != ${{ type: 'varchar', value: sourceTableRow!.id }} AND ( - relation_id = ${{ type: 'varchar', value: objectId.toString() }} + relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }} OR ( schema_name = ${{ type: 'varchar', value: schema }} AND table_name = ${{ type: 'varchar', value: table }} @@ -237,7 +238,7 @@ export class PostgresSyncRulesStorage new storage.SourceTable( doc.id, connection_tag, - doc.relation_id ?? 0, + doc.relation_id?.object_id ?? 0, doc.schema_name, doc.table_name, doc.replica_id_columns?.map((c) => ({ diff --git a/modules/module-postgres-storage/src/types/codecs.ts b/modules/module-postgres-storage/src/types/codecs.ts index 643a47640..9f0824ee1 100644 --- a/modules/module-postgres-storage/src/types/codecs.ts +++ b/modules/module-postgres-storage/src/types/codecs.ts @@ -53,6 +53,21 @@ export const jsonb = (subCodec: t.Codec) => } ); +/** + * Just performs a pure JSON.parse for the decoding step + */ +export const jsonb_raw = () => + t.codec( + 'jsonb_raw', + (decoded: Decoded) => { + return JSON.stringify(decoded); + }, + (encoded: string | { data: string }) => { + const s = typeof encoded == 'object' ? encoded.data : encoded; + return JSON.parse(s); + } + ); + export const bigint = t.codec( 'bigint', (decoded: BigInt) => { @@ -109,3 +124,13 @@ export const pgwire_number = t.codec( return Number(encoded); } ); + +/** + * A codec which contains the same type on the input and output. + */ +export const IdentityCodec = () => + t.codec( + 'identity', + (encoded) => encoded, + (decoded) => decoded + ); diff --git a/modules/module-postgres-storage/src/types/models/BucketParameters.ts b/modules/module-postgres-storage/src/types/models/BucketParameters.ts index b0829fd3b..e4744c895 100644 --- a/modules/module-postgres-storage/src/types/models/BucketParameters.ts +++ b/modules/module-postgres-storage/src/types/models/BucketParameters.ts @@ -1,5 +1,4 @@ import * as t from 'ts-codec'; -// import { JsonContainer } from '@powersync/service-jsonbig'; import { bigint, hexBuffer, pgwire_number } from '../codecs.js'; export const BucketParameters = t.object({ diff --git a/modules/module-postgres-storage/src/types/models/SourceTable.ts b/modules/module-postgres-storage/src/types/models/SourceTable.ts index 1bf5e2dd3..1673bc959 100644 --- a/modules/module-postgres-storage/src/types/models/SourceTable.ts +++ b/modules/module-postgres-storage/src/types/models/SourceTable.ts @@ -1,5 +1,9 @@ import * as t from 'ts-codec'; -import { bigint, jsonb, pgwire_number } from '../codecs.js'; +import { bigint, jsonb, jsonb_raw, pgwire_number } from '../codecs.js'; + +export type StoredRelationId = { + object_id: string | number; +}; export const ColumnDescriptor = t.object({ name: t.string, @@ -17,7 +21,7 @@ export const SourceTable = t.object({ id: t.string, group_id: pgwire_number, connection_id: bigint, - relation_id: t.Null.or(pgwire_number).or(t.string), + relation_id: t.Null.or(jsonb_raw()), schema_name: t.string, table_name: t.string, replica_id_columns: t.Null.or(jsonb(t.array(ColumnDescriptor))), From 91da74dd7461e87801fffba8f3457ed2ad21096d Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 13:59:07 +0200 Subject: [PATCH 41/50] remove duplicated lines --- .../src/storage/batch/PostgresBucketBatch.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index c26840d14..d7596b2bc 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -104,12 +104,6 @@ export class PostgresBucketBatch logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`); - if (!sourceTable.syncData && !sourceTable.syncParameters) { - return null; - } - - logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`); - this.batch ??= new OperationBatch(this.options.batch_limits); this.batch.push(new RecordOperation(record)); From 7f3ea55f1c59dbb667481a54a0d1d6aa5ef19a54 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 14:44:55 +0200 Subject: [PATCH 42/50] Updated BucketStorageFactory to use AsyncDisposable --- .../src/db/connection/ConnectionSlot.ts | 10 +++-- .../src/utils/DisposableObserver.ts | 7 +++- .../src/storage/MongoBucketStorage.ts | 4 ++ .../implementation/MongoStorageProvider.ts | 15 ++++--- .../test/src/change_stream_utils.ts | 2 +- .../test/src/BinlogStreamUtils.ts | 2 +- .../storage/PostgresBucketStorageFactory.ts | 6 +-- .../test/src/slow_tests.test.ts | 4 +- .../test/src/wal_stream_utils.ts | 2 +- .../src/tests/register-compacting-tests.ts | 6 +-- .../src/tests/register-data-storage-tests.ts | 42 +++++++++---------- .../src/tests/register-sync-tests.ts | 12 +++--- .../service-core/src/storage/BucketStorage.ts | 8 +++- 13 files changed, 69 insertions(+), 51 deletions(-) diff --git a/libs/lib-postgres/src/db/connection/ConnectionSlot.ts b/libs/lib-postgres/src/db/connection/ConnectionSlot.ts index 4775470a0..b2b543b6c 100644 --- a/libs/lib-postgres/src/db/connection/ConnectionSlot.ts +++ b/libs/lib-postgres/src/db/connection/ConnectionSlot.ts @@ -27,6 +27,8 @@ export class ConnectionSlot extends framework.DisposableObserver | null; @@ -36,6 +38,7 @@ export class ConnectionSlot extends framework.DisposableObserver void; } -export interface DisposableObserverClient extends ObserverClient, Disposable { +export interface ManagedObserverClient extends ObserverClient { /** * Registers a listener that is automatically disposed when the parent is disposed. * This is useful for disposing nested listeners. @@ -15,6 +15,11 @@ export interface DisposableObserverClient extends registerManagedListener: (parent: DisposableObserverClient, cb: Partial) => () => void; } +export interface DisposableObserverClient extends ManagedObserverClient, Disposable {} +export interface AsyncDisposableObserverClient + extends ManagedObserverClient, + AsyncDisposable {} + export class DisposableObserver extends BaseObserver implements DisposableObserverClient diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 9cbf67738..0edbb7f9f 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -62,6 +62,10 @@ export class MongoBucketStorage this.slot_name_prefix = options.slot_name_prefix; } + async [Symbol.asyncDispose]() { + super[Symbol.dispose](); + } + getInstance(options: storage.PersistedSyncRulesContent): MongoSyncBucketStorage { let { id, slot_name } = options; if ((typeof id as any) == 'bigint') { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts index a147bfe08..cfa010b26 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts @@ -23,13 +23,16 @@ export class MongoStorageProvider implements storage.BucketStorageProvider { const client = lib_mongo.db.createMongoClient(decodedConfig); const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database }); - + const factory = new MongoBucketStorage(database, { + // TODO currently need the entire resolved config due to this + slot_name_prefix: resolvedConfig.slot_name_prefix + }); return { - storage: new MongoBucketStorage(database, { - // TODO currently need the entire resolved config due to this - slot_name_prefix: resolvedConfig.slot_name_prefix - }), - shutDown: () => client.close(), + storage: factory, + shutDown: async () => { + await factory[Symbol.asyncDispose](); + await client.close(); + }, tearDown: () => { logger.info(`Tearing down storage: ${database.db.namespace}...`); return database.db.dropDatabase(); diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index 41347beac..b386eafbc 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -36,7 +36,7 @@ export class ChangeStreamTestContext { async dispose() { this.abortController.abort(); - this.factory[Symbol.dispose](); + await this.factory[Symbol.asyncDispose](); await this.streamPromise?.catch((e) => e); await this.connectionManager.destroy(); } diff --git a/modules/module-mysql/test/src/BinlogStreamUtils.ts b/modules/module-mysql/test/src/BinlogStreamUtils.ts index 62729d48b..5f7062960 100644 --- a/modules/module-mysql/test/src/BinlogStreamUtils.ts +++ b/modules/module-mysql/test/src/BinlogStreamUtils.ts @@ -49,7 +49,7 @@ export class BinlogStreamTestContext { this.abortController.abort(); await this.streamPromise; await this.connectionManager.end(); - this.factory[Symbol.dispose](); + await this.factory[Symbol.asyncDispose](); } [Symbol.asyncDispose]() { diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index 2253b6782..6710a1e26 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -28,7 +28,6 @@ export class PostgresBucketStorageFactory readonly db: lib_postgres.DatabaseClient; public readonly slot_name_prefix: string; - protected notificationConnection: pg_wire.PgConnection | null; private sharedIterator = new sync.BroadcastIterable((signal) => this.watchActiveCheckpoint(signal)); private readonly storageCache = new LRUCache({ @@ -68,11 +67,10 @@ export class PostgresBucketStorageFactory this.db.registerListener({ connectionCreated: async (connection) => this.prepareStatements(connection) }); - this.notificationConnection = null; } - async [Symbol.dispose]() { - await this.notificationConnection?.end(); + async [Symbol.asyncDispose]() { + super[Symbol.dispose](); await this.db[Symbol.asyncDispose](); } diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index f6a85071d..438d3bb98 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -92,7 +92,7 @@ function defineSlowTests(factory: storage.TestStorageFactory) { const replicationConnection = await connections.replicationConnection(); const pool = connections.pool; await clearTestDb(pool); - using f = await factory(); + await using f = await factory(); const syncRuleContent = ` bucket_definitions: @@ -319,7 +319,7 @@ bucket_definitions: async () => { const pool = await connectPgPool(); await clearTestDb(pool); - using f = await factory(); + await using f = await factory(); const syncRuleContent = ` bucket_definitions: diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index 2bd8d64d6..f25d6d083 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -46,7 +46,7 @@ export class WalStreamTestContext implements AsyncDisposable { await this.streamPromise; await this.connectionManager.destroy(); this.storage?.[Symbol.dispose](); - this.factory?.[Symbol.dispose](); + await this.factory?.[Symbol.asyncDispose](); } get pool() { diff --git a/packages/service-core-tests/src/tests/register-compacting-tests.ts b/packages/service-core-tests/src/tests/register-compacting-tests.ts index e7ee55ab7..97c5ed1b1 100644 --- a/packages/service-core-tests/src/tests/register-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-compacting-tests.ts @@ -26,7 +26,7 @@ bucket_definitions: data: [select * from test] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -128,7 +128,7 @@ bucket_definitions: data: [select * from test] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -238,7 +238,7 @@ bucket_definitions: data: [select * from test] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { diff --git a/packages/service-core-tests/src/tests/register-data-storage-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-tests.ts index 1c4c89343..01e337be7 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-tests.ts @@ -36,7 +36,7 @@ bucket_definitions: data: [] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -84,7 +84,7 @@ bucket_definitions: ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -208,7 +208,7 @@ bucket_definitions: ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -253,7 +253,7 @@ bucket_definitions: ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -299,7 +299,7 @@ bucket_definitions: - SELECT id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -367,7 +367,7 @@ bucket_definitions: ); const sync_rules = sync_rules_content.parsed(test_utils.PARSE_OPTIONS).sync_rules; - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules_content); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -418,7 +418,7 @@ bucket_definitions: ); const sync_rules = sync_rules_content.parsed(test_utils.PARSE_OPTIONS).sync_rules; - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules_content); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -493,7 +493,7 @@ bucket_definitions: ); const sync_rules = sync_rules_content.parsed(test_utils.PARSE_OPTIONS).sync_rules; - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules_content); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -581,7 +581,7 @@ bucket_definitions: - SELECT client_id as id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); @@ -647,7 +647,7 @@ bucket_definitions: - SELECT id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -724,7 +724,7 @@ bucket_definitions: - SELECT id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -846,7 +846,7 @@ bucket_definitions: ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -888,7 +888,7 @@ bucket_definitions: - SELECT id, description FROM "test" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); // Pre-setup @@ -1046,7 +1046,7 @@ bucket_definitions: { name: 'description', type: 'VARCHAR', typeId: 25 } ]); } - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const sourceTable = test_utils.makeTestTable('test', ['id', 'description']); @@ -1154,7 +1154,7 @@ bucket_definitions: ]); } - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const sourceTable = test_utils.makeTestTable('test', ['id', 'description']); @@ -1252,7 +1252,7 @@ bucket_definitions: - SELECT id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -1351,7 +1351,7 @@ bucket_definitions: - SELECT id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -1422,7 +1422,7 @@ bucket_definitions: data: [] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); let isDisposed = false; @@ -1461,7 +1461,7 @@ bucket_definitions: data: [] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); let isDisposed = false; @@ -1494,7 +1494,7 @@ bucket_definitions: }); test('empty storage metrics', async () => { - using f = await generateStorageFactory({ dropAll: true }); + await using f = await generateStorageFactory({ dropAll: true }); const metrics = await f.getStorageMetrics(); expect(metrics).toEqual({ operations_size_bytes: 0, @@ -1522,7 +1522,7 @@ bucket_definitions: ` ); - using bucketStorageFactory = await generateStorageFactory(); + await using bucketStorageFactory = await generateStorageFactory(); const syncBucketStorage = bucketStorageFactory.getInstance(sync_rules_content); const parsedSchema1 = syncBucketStorage.getParsedSyncRules({ diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index 9736f8cdb..2cf14cb54 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -33,7 +33,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { const tracker = new sync.RequestTracker(); test('sync global data', async () => { - using f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES @@ -128,7 +128,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { }); test('expired token', async () => { - const f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES @@ -155,7 +155,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { }); test('sync updates to global data', async () => { - using f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES @@ -216,7 +216,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { }); test('expiring token', async () => { - using f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES @@ -254,7 +254,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { // This is expected to be rare in practice, but it is important to handle // this case correctly to maintain consistency on the client. - using f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES @@ -391,7 +391,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { }); test('write checkpoint', async () => { - using f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index eaa212355..ff7b653f5 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -1,4 +1,8 @@ -import { DisposableListener, DisposableObserverClient } from '@powersync/lib-services-framework'; +import { + AsyncDisposableObserverClient, + DisposableListener, + DisposableObserverClient +} from '@powersync/lib-services-framework'; import { EvaluatedParameters, EvaluatedRow, @@ -56,7 +60,7 @@ export interface BucketStorageFactoryListener extends DisposableListener { replicationEvent: (event: ReplicationEventPayload) => void; } -export interface BucketStorageFactory extends DisposableObserverClient { +export interface BucketStorageFactory extends AsyncDisposableObserverClient { /** * Update sync rules from configuration, if changed. */ From c39ea45cb71ad6e7c2ed29e178e56856d63b7f71 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 14:45:01 +0200 Subject: [PATCH 43/50] added changeset --- .changeset/big-books-remember.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/big-books-remember.md diff --git a/.changeset/big-books-remember.md b/.changeset/big-books-remember.md new file mode 100644 index 000000000..97f75dc27 --- /dev/null +++ b/.changeset/big-books-remember.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-core-tests': minor +'@powersync/service-core': minor +--- + +Updated BucketStorageFactory to use AsyncDisposable From 2318c6de272f23d7658a483682f0962cefc0db93 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 14:56:41 +0200 Subject: [PATCH 44/50] fix mongo tests --- modules/module-mongodb/test/src/change_stream_utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index b386eafbc..6d21ee817 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -36,9 +36,9 @@ export class ChangeStreamTestContext { async dispose() { this.abortController.abort(); - await this.factory[Symbol.asyncDispose](); await this.streamPromise?.catch((e) => e); await this.connectionManager.destroy(); + await this.factory[Symbol.asyncDispose](); } async [Symbol.asyncDispose]() { From 2d0f8fddaa1853c1251de8bfd097c04038892828 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 15:06:14 +0200 Subject: [PATCH 45/50] remove mystery todo update comment --- modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts | 1 - .../src/storage/PostgresBucketStorageFactory.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 0edbb7f9f..87f577eca 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -110,7 +110,6 @@ export class MongoBucketStorage // In both the below cases, we create a new sync rules instance. // The current one will continue erroring until the next one has finished processing. - // TODO: Update if (next != null && next.slot_name == slot_name) { // We need to redo the "next" sync rules await this.updateSyncRules({ diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index 6710a1e26..e2e6c4540 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -252,7 +252,6 @@ export class PostgresBucketStorageFactory // In both the below cases, we create a new sync rules instance. // The current one will continue erroring until the next one has finished processing. - // TODO: Update if (next != null && next.slot_name == slot_name) { // We need to redo the "next" sync rules await this.updateSyncRules({ From 444bc954f67a8520bcfbd235e2747b50414247c0 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jan 2025 16:35:47 +0200 Subject: [PATCH 46/50] reset metrics between tests --- modules/module-mongodb-storage/test/src/setup.ts | 6 +++++- modules/module-mongodb/test/src/setup.ts | 5 +++++ modules/module-mysql/test/src/setup.ts | 6 +++++- modules/module-postgres/test/src/setup.ts | 6 +++++- packages/service-core-tests/src/test-utils/metrics-utils.ts | 4 ++++ 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/setup.ts b/modules/module-mongodb-storage/test/src/setup.ts index debe66011..43946a1aa 100644 --- a/modules/module-mongodb-storage/test/src/setup.ts +++ b/modules/module-mongodb-storage/test/src/setup.ts @@ -1,9 +1,13 @@ import { container } from '@powersync/lib-services-framework'; import { test_utils } from '@powersync/service-core-tests'; -import { beforeAll } from 'vitest'; +import { beforeAll, beforeEach } from 'vitest'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); await test_utils.initMetrics(); }); + +beforeEach(async () => { + await test_utils.resetMetrics(); +}); diff --git a/modules/module-mongodb/test/src/setup.ts b/modules/module-mongodb/test/src/setup.ts index fe127d8a9..c89537d5d 100644 --- a/modules/module-mongodb/test/src/setup.ts +++ b/modules/module-mongodb/test/src/setup.ts @@ -1,5 +1,6 @@ import { container } from '@powersync/lib-services-framework'; import { test_utils } from '@powersync/service-core-tests'; +import { beforeEach } from 'node:test'; import { beforeAll } from 'vitest'; beforeAll(async () => { @@ -8,3 +9,7 @@ beforeAll(async () => { await test_utils.initMetrics(); }); + +beforeEach(async () => { + await test_utils.resetMetrics(); +}); diff --git a/modules/module-mysql/test/src/setup.ts b/modules/module-mysql/test/src/setup.ts index debe66011..43946a1aa 100644 --- a/modules/module-mysql/test/src/setup.ts +++ b/modules/module-mysql/test/src/setup.ts @@ -1,9 +1,13 @@ import { container } from '@powersync/lib-services-framework'; import { test_utils } from '@powersync/service-core-tests'; -import { beforeAll } from 'vitest'; +import { beforeAll, beforeEach } from 'vitest'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); await test_utils.initMetrics(); }); + +beforeEach(async () => { + await test_utils.resetMetrics(); +}); diff --git a/modules/module-postgres/test/src/setup.ts b/modules/module-postgres/test/src/setup.ts index debe66011..43946a1aa 100644 --- a/modules/module-postgres/test/src/setup.ts +++ b/modules/module-postgres/test/src/setup.ts @@ -1,9 +1,13 @@ import { container } from '@powersync/lib-services-framework'; import { test_utils } from '@powersync/service-core-tests'; -import { beforeAll } from 'vitest'; +import { beforeAll, beforeEach } from 'vitest'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); await test_utils.initMetrics(); }); + +beforeEach(async () => { + await test_utils.resetMetrics(); +}); diff --git a/packages/service-core-tests/src/test-utils/metrics-utils.ts b/packages/service-core-tests/src/test-utils/metrics-utils.ts index 9fe704d78..8f61d250b 100644 --- a/packages/service-core-tests/src/test-utils/metrics-utils.ts +++ b/packages/service-core-tests/src/test-utils/metrics-utils.ts @@ -6,5 +6,9 @@ export const initMetrics = async () => { powersync_instance_id: 'test', internal_metrics_endpoint: 'unused.for.tests.com' }); + await resetMetrics(); +}; + +export const resetMetrics = async () => { Metrics.getInstance().resetCounters(); }; From 483b94ee35b82a5c981dff952bc26a7537c644da Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 15 Jan 2025 09:33:28 +0200 Subject: [PATCH 47/50] Improve migration logic. --- .changeset/heavy-llamas-move.md | 6 + .../src/migrations/AbstractMigrationAgent.ts | 25 +++- .../test/src/migrations.test.ts | 10 ++ .../src/migrations/PostgresMigrationStore.ts | 18 +-- .../PostgresTestStorageFactoryGenerator.ts | 3 - .../src/types/models/Migration.ts | 19 +++ .../src/types/models/models-index.ts | 1 + .../test/src/migrations.test.ts | 10 ++ .../test/tsconfig.json | 5 +- .../src/tests/register-migration-tests.ts | 130 ++++++++++++++++++ .../src/tests/tests-index.ts | 1 + 11 files changed, 211 insertions(+), 17 deletions(-) create mode 100644 .changeset/heavy-llamas-move.md create mode 100644 modules/module-mongodb-storage/test/src/migrations.test.ts create mode 100644 modules/module-postgres-storage/src/types/models/Migration.ts create mode 100644 packages/service-core-tests/src/tests/register-migration-tests.ts diff --git a/.changeset/heavy-llamas-move.md b/.changeset/heavy-llamas-move.md new file mode 100644 index 000000000..26ff6abfd --- /dev/null +++ b/.changeset/heavy-llamas-move.md @@ -0,0 +1,6 @@ +--- +'@powersync/service-core-tests': minor +'@powersync/lib-services-framework': minor +--- + +Improved migrations logic. Up migrations can be executed correctly after down migrations. diff --git a/libs/lib-services/src/migrations/AbstractMigrationAgent.ts b/libs/lib-services/src/migrations/AbstractMigrationAgent.ts index bfb465501..dca69023b 100644 --- a/libs/lib-services/src/migrations/AbstractMigrationAgent.ts +++ b/libs/lib-services/src/migrations/AbstractMigrationAgent.ts @@ -102,6 +102,7 @@ export abstract class AbstractMigrationAgent { return migration.name === params.state!.last_run; }); @@ -112,8 +113,28 @@ export abstract class AbstractMigrationAgent b.timestamp.getTime() - a.timestamp.getTime()) + .find((log) => log.name == last_run); + + // If we are migrating up: + // If the last run was an up migration: + // Then we want to start at the next migration index + // If after a previous Down migration + // Then we need to start at the current migration index + + // If we are migrating down: + // If the previous migration was a down migration + // Then we need to start at the next index + // If the previous migration was an up migration + // Then we want to include the last run (up) migration + if ( + (params.direction === defs.Direction.Up && lastLogEntry?.direction != defs.Direction.Down) || + (params.direction == defs.Direction.Down && lastLogEntry?.direction == defs.Direction.Down) + ) { index += 1; } } diff --git a/modules/module-mongodb-storage/test/src/migrations.test.ts b/modules/module-mongodb-storage/test/src/migrations.test.ts new file mode 100644 index 000000000..e4728fa04 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/migrations.test.ts @@ -0,0 +1,10 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { MongoMigrationAgent } from '../../src/migrations/MongoMigrationAgent.js'; +import { env } from './env.js'; + +const MIGRATION_AGENT_FACTORY = () => { + return new MongoMigrationAgent({ type: 'mongodb', uri: env.MONGO_TEST_URL }); +}; + +describe('Mongo Migrations Store', () => register.registerMigrationTests(MIGRATION_AGENT_FACTORY)); diff --git a/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts b/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts index 02c45f01e..11b1d68c7 100644 --- a/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts +++ b/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts @@ -1,5 +1,6 @@ import * as lib_postgres from '@powersync/lib-service-postgres'; import { migrations } from '@powersync/lib-services-framework'; +import { models } from '../types/types.js'; import { sql } from '../utils/db.js'; export type PostgresMigrationStoreOptions = { @@ -28,7 +29,7 @@ export class PostgresMigrationStore implements migrations.MigrationStore { } async load(): Promise { - const res = await this.db.queryRows<{ last_run: string; log: string }>(sql` + const res = await this.db.sql` SELECT last_run, LOG @@ -36,26 +37,27 @@ export class PostgresMigrationStore implements migrations.MigrationStore { migrations LIMIT 1 - `); + ` + .decoded(models.Migration) + .first(); - if (res.length === 0) { + if (!res) { return; } - const { last_run, log } = res[0]; - return { - last_run: last_run, - log: log ? JSON.parse(log) : [] + last_run: res.last_run, + log: res.log }; } async save(state: migrations.MigrationState): Promise { await this.db.query(sql` INSERT INTO - migrations (last_run, LOG) + migrations (id, last_run, LOG) VALUES ( + 1, ${{ type: 'varchar', value: state.last_run }}, ${{ type: 'jsonb', value: state.log }} ) diff --git a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts index ec30b9118..1739beeba 100644 --- a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts +++ b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts @@ -39,9 +39,6 @@ export const PostgresTestStorageFactoryGenerator = (factoryOptions: PostgresTest service_context: mockServiceContext } }); - - // In order to run up migration after - await migrationAgent.resetStore(); } await migrationManager.migrate({ diff --git a/modules/module-postgres-storage/src/types/models/Migration.ts b/modules/module-postgres-storage/src/types/models/Migration.ts new file mode 100644 index 000000000..3cf283c58 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/Migration.ts @@ -0,0 +1,19 @@ +import { framework } from '@powersync/service-core'; +import * as t from 'ts-codec'; +import { jsonb } from '../codecs.js'; + +export const Migration = t.object({ + last_run: t.string, + log: jsonb( + t.array( + t.object({ + name: t.string, + direction: t.Enum(framework.migrations.Direction), + timestamp: framework.codecs.date + }) + ) + ) +}); + +export type Migration = t.Encoded; +export type MigrationDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/models-index.ts b/modules/module-postgres-storage/src/types/models/models-index.ts index b346ffd67..fb5574608 100644 --- a/modules/module-postgres-storage/src/types/models/models-index.ts +++ b/modules/module-postgres-storage/src/types/models/models-index.ts @@ -4,6 +4,7 @@ export * from './BucketData.js'; export * from './BucketParameters.js'; export * from './CurrentData.js'; export * from './Instance.js'; +export * from './Migration.js'; export * from './SourceTable.js'; export * from './SyncRules.js'; export * from './WriteCheckpoint.js'; diff --git a/modules/module-postgres-storage/test/src/migrations.test.ts b/modules/module-postgres-storage/test/src/migrations.test.ts index 0b3940104..ee3acd955 100644 --- a/modules/module-postgres-storage/test/src/migrations.test.ts +++ b/modules/module-postgres-storage/test/src/migrations.test.ts @@ -1,7 +1,17 @@ import { describe, expect, it } from 'vitest'; + +import { register } from '@powersync/service-core-tests'; +import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js'; +import { env } from './env.js'; import { POSTGRES_STORAGE_FACTORY } from './util.js'; +const MIGRATION_AGENT_FACTORY = () => { + return new PostgresMigrationAgent({ type: 'postgresql', uri: env.PG_STORAGE_TEST_URL, sslmode: 'disable' }); +}; + describe('Migrations', () => { + register.registerMigrationTests(MIGRATION_AGENT_FACTORY); + it('Should have tables declared', async () => { const { db } = await POSTGRES_STORAGE_FACTORY(); diff --git a/modules/module-postgres-storage/test/tsconfig.json b/modules/module-postgres-storage/test/tsconfig.json index 35074fd52..486ab8eb7 100644 --- a/modules/module-postgres-storage/test/tsconfig.json +++ b/modules/module-postgres-storage/test/tsconfig.json @@ -9,10 +9,7 @@ "tsBuildInfoFile": "dist/.tsbuildinfo", "lib": ["ES2022", "esnext.disposable"], "skipLibCheck": true, - "sourceMap": true, - "paths": { - "@module/*": ["../src/*"] - } + "sourceMap": true }, "include": ["src"], "references": [ diff --git a/packages/service-core-tests/src/tests/register-migration-tests.ts b/packages/service-core-tests/src/tests/register-migration-tests.ts new file mode 100644 index 000000000..e3ccd5e51 --- /dev/null +++ b/packages/service-core-tests/src/tests/register-migration-tests.ts @@ -0,0 +1,130 @@ +import { AbstractPowerSyncMigrationAgent, framework, PowerSyncMigrationManager } from '@powersync/service-core'; +import { expect, test, vi } from 'vitest'; + +const generateTestMigrations = (length: number, start: number = 0) => { + const results: string[] = []; + return { + results, + tests: Array.from({ length }).map((v, index) => { + const i = index + start; + return { + down: vi.fn(async () => { + results.push(`down - ${i}`); + }), + up: vi.fn(async () => { + results.push(`up - ${i}`); + }), + name: i.toString() + }; + }) + }; +}; + +/** + * Reset the factory as part of disposal. This helps cleanup after tests. + */ +const managedResetAgent = (factory: () => AbstractPowerSyncMigrationAgent) => { + const agent = factory(); + return { + agent, + // Reset the store for the next tests + [Symbol.asyncDispose]: () => agent.resetStore() + }; +}; + +export const registerMigrationTests = (migrationAgentFactory: () => AbstractPowerSyncMigrationAgent) => { + test('Should run migrations correctly', async () => { + await using manager = new framework.migrations.MigrationManager() as PowerSyncMigrationManager; + // Disposal is executed in reverse order. The store will be reset before the manage disposes it. + await using managedAgent = managedResetAgent(migrationAgentFactory); + + await managedAgent.agent.resetStore(); + + manager.registerMigrationAgent(managedAgent.agent); + + const length = 10; + const { tests, results } = generateTestMigrations(length); + manager.registerMigrations(tests); + + await manager.migrate({ + direction: framework.migrations.Direction.Up + }); + + const upLogs = Array.from({ length }).map((v, index) => `up - ${index}`); + + expect(results).deep.equals(upLogs); + + // Running up again should not run any migrations + await manager.migrate({ + direction: framework.migrations.Direction.Up + }); + + expect(results.length).equals(length); + + // Clear the results + results.splice(0, length); + + await manager.migrate({ + direction: framework.migrations.Direction.Down + }); + + const downLogs = Array.from({ length }) + .map((v, index) => `down - ${index}`) + .reverse(); + expect(results).deep.equals(downLogs); + + // Running down again should not run any additional migrations + await manager.migrate({ + direction: framework.migrations.Direction.Down + }); + + expect(results.length).equals(length); + + // Clear the results + results.splice(0, length); + + // Running up should run the up migrations again + await manager.migrate({ + direction: framework.migrations.Direction.Up + }); + + expect(results).deep.equals(upLogs); + }); + + test('Should run migrations with additions', async () => { + await using manager = new framework.migrations.MigrationManager() as PowerSyncMigrationManager; + // Disposal is executed in reverse order. The store will be reset before the manage disposes it. + await using managedAgent = managedResetAgent(migrationAgentFactory); + + await managedAgent.agent.resetStore(); + + manager.registerMigrationAgent(managedAgent.agent); + + const length = 10; + const { tests, results } = generateTestMigrations(length); + manager.registerMigrations(tests); + + await manager.migrate({ + direction: framework.migrations.Direction.Up + }); + + const upLogs = Array.from({ length }).map((v, index) => `up - ${index}`); + + expect(results).deep.equals(upLogs); + + // Add a new migration + const { results: newResults, tests: newTests } = generateTestMigrations(1, 10); + manager.registerMigrations(newTests); + + // Running up again should not run any migrations + await manager.migrate({ + direction: framework.migrations.Direction.Up + }); + + // The original tests should not have been executed again + expect(results.length).equals(length); + + // The new migration should have been executed + expect(newResults).deep.equals([`up - ${10}`]); + }); +}; diff --git a/packages/service-core-tests/src/tests/tests-index.ts b/packages/service-core-tests/src/tests/tests-index.ts index 4f0e017fc..558f8c5a1 100644 --- a/packages/service-core-tests/src/tests/tests-index.ts +++ b/packages/service-core-tests/src/tests/tests-index.ts @@ -1,4 +1,5 @@ export * from './register-bucket-validation-tests.js'; export * from './register-compacting-tests.js'; export * from './register-data-storage-tests.js'; +export * from './register-migration-tests.js'; export * from './register-sync-tests.js'; From 5a723fda4827cfad69207670758cb80563a0a065 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 15 Jan 2025 09:42:22 +0200 Subject: [PATCH 48/50] fix typo --- .../src/tests/register-migration-tests.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-migration-tests.ts b/packages/service-core-tests/src/tests/register-migration-tests.ts index e3ccd5e51..e6ac8faa2 100644 --- a/packages/service-core-tests/src/tests/register-migration-tests.ts +++ b/packages/service-core-tests/src/tests/register-migration-tests.ts @@ -35,7 +35,7 @@ const managedResetAgent = (factory: () => AbstractPowerSyncMigrationAgent) => { export const registerMigrationTests = (migrationAgentFactory: () => AbstractPowerSyncMigrationAgent) => { test('Should run migrations correctly', async () => { await using manager = new framework.migrations.MigrationManager() as PowerSyncMigrationManager; - // Disposal is executed in reverse order. The store will be reset before the manage disposes it. + // Disposal is executed in reverse order. The store will be reset before the manager disposes it. await using managedAgent = managedResetAgent(migrationAgentFactory); await managedAgent.agent.resetStore(); @@ -93,7 +93,7 @@ export const registerMigrationTests = (migrationAgentFactory: () => AbstractPowe test('Should run migrations with additions', async () => { await using manager = new framework.migrations.MigrationManager() as PowerSyncMigrationManager; - // Disposal is executed in reverse order. The store will be reset before the manage disposes it. + // Disposal is executed in reverse order. The store will be reset before the manager disposes it. await using managedAgent = managedResetAgent(migrationAgentFactory); await managedAgent.agent.resetStore(); @@ -125,6 +125,6 @@ export const registerMigrationTests = (migrationAgentFactory: () => AbstractPowe expect(results.length).equals(length); // The new migration should have been executed - expect(newResults).deep.equals([`up - ${10}`]); + expect(newResults).deep.equals([`up - 10`]); }); }; From 9556a4762c9c9f770de8070a9a5297419036a0c7 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 15 Jan 2025 10:10:58 +0200 Subject: [PATCH 49/50] cleanup --- .../src/migrations/AbstractMigrationAgent.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libs/lib-services/src/migrations/AbstractMigrationAgent.ts b/libs/lib-services/src/migrations/AbstractMigrationAgent.ts index dca69023b..3be96acf3 100644 --- a/libs/lib-services/src/migrations/AbstractMigrationAgent.ts +++ b/libs/lib-services/src/migrations/AbstractMigrationAgent.ts @@ -120,6 +120,11 @@ export abstract class AbstractMigrationAgent b.timestamp.getTime() - a.timestamp.getTime()) .find((log) => log.name == last_run); + // There should be a log entry for this + if (!lastLogEntry) { + throw new Error(`Could not find last migration log entry for ${last_run}`); + } + // If we are migrating up: // If the last run was an up migration: // Then we want to start at the next migration index @@ -132,8 +137,8 @@ export abstract class AbstractMigrationAgent Date: Wed, 15 Jan 2025 15:47:41 +0200 Subject: [PATCH 50/50] add public config in order for publish task to succeed --- modules/module-postgres-storage/package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/module-postgres-storage/package.json b/modules/module-postgres-storage/package.json index 46c17ad5b..441c2f22a 100644 --- a/modules/module-postgres-storage/package.json +++ b/modules/module-postgres-storage/package.json @@ -5,6 +5,9 @@ "version": "0.0.1", "main": "dist/index.js", "type": "module", + "publishConfig": { + "access": "public" + }, "scripts": { "build": "tsc -b", "build:tests": "tsc -b test/tsconfig.json",