diff --git a/.github/workflows/db-migration-backwards-compatibility.yaml b/.github/workflows/db-migration-backwards-compatibility.yaml index adcc34c963..c09ce95553 100644 --- a/.github/workflows/db-migration-backwards-compatibility.yaml +++ b/.github/workflows/db-migration-backwards-compatibility.yaml @@ -193,6 +193,17 @@ jobs: wait-for: 30s log-output-if: true + - name: Start run-cron-jobs in background + uses: JarvusInnovations/background-action@v1.0.7 + if: ${{ hashFiles('apps/backend/scripts/run-cron-jobs.ts') != '' }} + with: + run: pnpm -C apps/backend run with-env:dev tsx scripts/run-cron-jobs.ts --log-order=stream & + wait-on: | + http://localhost:8102 + tail: true + wait-for: 30s + log-output-if: true + - name: Wait 10 seconds run: sleep 10 @@ -230,4 +241,3 @@ jobs: steps: - name: No migration changes detected run: echo "No changes to migrations folder detected. Skipping backwards compatibility test." - diff --git a/.github/workflows/e2e-api-tests.yaml b/.github/workflows/e2e-api-tests.yaml index c59c38879d..6aa1e55e2d 100644 --- a/.github/workflows/e2e-api-tests.yaml +++ b/.github/workflows/e2e-api-tests.yaml @@ -19,6 +19,9 @@ jobs: NODE_ENV: test STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe" + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" strategy: matrix: @@ -100,6 +103,9 @@ jobs: - name: Wait on Svix run: pnpx wait-on tcp:localhost:8113 + + - name: Wait on QStash + run: pnpx wait-on tcp:localhost:8125 - name: Initialize database run: pnpm run db:init @@ -140,20 +146,29 @@ jobs: tail: true wait-for: 30s log-output-if: true + - name: Start run-cron-jobs in background + uses: JarvusInnovations/background-action@v1.0.7 + with: + run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream & + wait-on: | + http://localhost:8102 + tail: true + wait-for: 30s + log-output-if: true - name: Wait 10 seconds run: sleep 10 - name: Run tests - run: pnpm test ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} + run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} - - name: Run tests again, to make sure they are stable (attempt 1) + - name: Run tests again (attempt 1) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} + run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} - - name: Run tests again, to make sure they are stable (attempt 2) + - name: Run tests again (attempt 2) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} + run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} - name: Verify data integrity run: pnpm run verify-data-integrity --no-bail diff --git a/.github/workflows/e2e-custom-base-port-api-tests.yaml b/.github/workflows/e2e-custom-base-port-api-tests.yaml index 14802828ff..1a1cf52efe 100644 --- a/.github/workflows/e2e-custom-base-port-api-tests.yaml +++ b/.github/workflows/e2e-custom-base-port-api-tests.yaml @@ -19,6 +19,9 @@ jobs: STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:6728/stackframe" NEXT_PUBLIC_STACK_PORT_PREFIX: "67" + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" strategy: matrix: @@ -94,6 +97,9 @@ jobs: - name: Wait on Svix run: pnpx wait-on tcp:localhost:6713 + + - name: Wait on QStash + run: pnpx wait-on tcp:localhost:6725 - name: Initialize database run: pnpm run db:init @@ -134,20 +140,29 @@ jobs: tail: true wait-for: 30s log-output-if: true + - name: Start run-cron-jobs in background + uses: JarvusInnovations/background-action@v1.0.7 + with: + run: pnpm -C apps/backend run run-cron-jobs --log-order=stream & + wait-on: | + http://localhost:6702 + tail: true + wait-for: 30s + log-output-if: true - name: Wait 10 seconds run: sleep 10 - name: Run tests - run: pnpm test + run: pnpm test run - - name: Run tests again, to make sure they are stable (attempt 1) + - name: Run tests again (attempt 1) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test + run: pnpm test run - - name: Run tests again, to make sure they are stable (attempt 2) + - name: Run tests again (attempt 2) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test + run: pnpm test run - name: Verify data integrity run: pnpm run verify-data-integrity --no-bail diff --git a/.github/workflows/e2e-source-of-truth-api-tests.yaml b/.github/workflows/e2e-source-of-truth-api-tests.yaml index cb036f26cc..99b66e51b6 100644 --- a/.github/workflows/e2e-source-of-truth-api-tests.yaml +++ b/.github/workflows/e2e-source-of-truth-api-tests.yaml @@ -17,9 +17,13 @@ jobs: env: NODE_ENV: test STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes + STACK_ACCESS_TOKEN_EXPIRATION_TIME: 30m STACK_OVERRIDE_SOURCE_OF_TRUTH: '{"type": "postgres", "connectionString": "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/source-of-truth-db?schema=sot-schema"}' STACK_TEST_SOURCE_OF_TRUTH: true STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe" + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" strategy: matrix: @@ -95,6 +99,9 @@ jobs: - name: Wait on Svix run: pnpx wait-on tcp:localhost:8113 + + - name: Wait on QStash + run: pnpx wait-on tcp:localhost:8125 - name: Create source-of-truth database and schema run: | @@ -140,20 +147,29 @@ jobs: tail: true wait-for: 30s log-output-if: true + - name: Start run-cron-jobs in background + uses: JarvusInnovations/background-action@v1.0.7 + with: + run: pnpm -C apps/backend run run-cron-jobs --log-order=stream & + wait-on: | + http://localhost:8102 + tail: true + wait-for: 30s + log-output-if: true - name: Wait 10 seconds run: sleep 10 - name: Run tests - run: pnpm test + run: pnpm test run --exclude "**/external-db-sync*.test.ts" # external-db-sync does not support external sot - - name: Run tests again, to make sure they are stable (attempt 1) + - name: Run tests again (attempt 1) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test + run: pnpm test run --exclude "**/external-db-sync*.test.ts" - - name: Run tests again, to make sure they are stable (attempt 2) + - name: Run tests again (attempt 2) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test + run: pnpm test run --exclude "**/external-db-sync*.test.ts" - name: Verify data integrity run: pnpm run verify-data-integrity --no-bail diff --git a/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml b/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml index 6c1f64f45e..65179046fb 100644 --- a/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml +++ b/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml @@ -19,6 +19,9 @@ jobs: runs-on: ubicloud-standard-16 env: NEXT_PUBLIC_STACK_PORT_PREFIX: "69" + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" steps: - uses: actions/checkout@v6 @@ -38,7 +41,7 @@ jobs: run: pnpm run restart-dev-environment - name: Run tests - run: pnpm run test --reporter=verbose + run: pnpm run test run --reporter=verbose - name: Print dev server logs run: cat dev-server.log.untracked.txt diff --git a/.github/workflows/restart-dev-and-test.yaml b/.github/workflows/restart-dev-and-test.yaml index 831148e4e7..6d091edb3c 100644 --- a/.github/workflows/restart-dev-and-test.yaml +++ b/.github/workflows/restart-dev-and-test.yaml @@ -17,6 +17,10 @@ env: jobs: restart-dev-and-test: runs-on: ubicloud-standard-16 + env: + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" steps: - uses: actions/checkout@v6 @@ -36,7 +40,7 @@ jobs: run: pnpm run restart-dev-environment - name: Run tests - run: pnpm run test --reporter=verbose + run: pnpm run test run --reporter=verbose - name: Print dev server logs run: cat dev-server.log.untracked.txt diff --git a/.github/workflows/setup-tests-with-custom-base-port.yaml b/.github/workflows/setup-tests-with-custom-base-port.yaml index b3d15e503f..0c9ac4a6ca 100644 --- a/.github/workflows/setup-tests-with-custom-base-port.yaml +++ b/.github/workflows/setup-tests-with-custom-base-port.yaml @@ -19,6 +19,9 @@ jobs: runs-on: ubicloud-standard-16 env: NEXT_PUBLIC_STACK_PORT_PREFIX: "69" + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" steps: - uses: actions/checkout@v6 @@ -46,4 +49,5 @@ jobs: tail: true wait-for: 120s log-output-if: true - - run: pnpm run test --reporter=verbose + - name: Run tests + run: pnpm run test run --reporter=verbose diff --git a/.github/workflows/setup-tests.yaml b/.github/workflows/setup-tests.yaml index d20748c93e..66374120c5 100644 --- a/.github/workflows/setup-tests.yaml +++ b/.github/workflows/setup-tests.yaml @@ -17,6 +17,10 @@ env: jobs: setup-tests: runs-on: ubicloud-standard-16 + env: + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" steps: - uses: actions/checkout@v6 @@ -43,4 +47,5 @@ jobs: tail: true wait-for: 120s log-output-if: true - - run: pnpm run test --reporter=verbose + - name: Run tests + run: pnpm run test run --reporter=verbose diff --git a/apps/backend/.env.development b/apps/backend/.env.development index a11b38c8e4..55d58dda44 100644 --- a/apps/backend/.env.development +++ b/apps/backend/.env.development @@ -35,8 +35,8 @@ STACK_DATABASE_REPLICATION_WAIT_STRATEGY=pg-stat-replication STACK_EMAIL_HOST=127.0.0.1 STACK_EMAIL_PORT=${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}29 STACK_EMAIL_SECURE=false -STACK_EMAIL_USERNAME=does not matter, ignored by Inbucket -STACK_EMAIL_PASSWORD=does not matter, ignored by Inbucket +STACK_EMAIL_USERNAME="does not matter, ignored by Inbucket" +STACK_EMAIL_PASSWORD="does not matter, ignored by Inbucket" STACK_EMAIL_SENDER=noreply@example.com STACK_ACCESS_TOKEN_EXPIRATION_TIME=60s @@ -50,7 +50,7 @@ STACK_ARTIFICIAL_DEVELOPMENT_DELAY_MS=500 STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING=yes -STACK_INTEGRATION_CLIENTS_CONFIG=[{"client_id": "neon-local", "client_secret": "neon-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}, {"client_id": "custom-local", "client_secret": "custom-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}] +STACK_INTEGRATION_CLIENTS_CONFIG='[{"client_id": "neon-local", "client_secret": "neon-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}, {"client_id": "custom-local", "client_secret": "custom-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}]' CRON_SECRET=mock_cron_secret STACK_FREESTYLE_API_KEY=mock_stack_freestyle_key STACK_VERCEL_SANDBOX_TOKEN=vercel_sandbox_disabled_for_local_development diff --git a/apps/backend/package.json b/apps/backend/package.json index 8feb477d8f..e5257b80d1 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -10,7 +10,8 @@ "with-env": "dotenv -c --", "with-env:dev": "dotenv -c development --", "with-env:prod": "dotenv -c production --", - "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\"", + "with-env:test": "dotenv -c test --", + "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue,cron-jobs\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\" \"pnpm run run-cron-jobs\"", "dev:inspect": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--inspect\" pnpm run dev", "dev:profile": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--experimental-cpu-prof\" pnpm run dev", "build": "pnpm run codegen && next build", @@ -42,6 +43,8 @@ "codegen-docs:watch": "pnpm run with-env tsx watch --exclude '**/node_modules/**' --clear-screen=false scripts/generate-openapi-fumadocs.ts", "generate-keys": "pnpm run with-env tsx scripts/generate-keys.ts", "db-seed-script": "pnpm run db:seed", + "run-cron-jobs": "pnpm run with-env:dev tsx scripts/run-cron-jobs.ts", + "run-cron-jobs:test": "pnpm run with-env:test tsx scripts/run-cron-jobs.ts", "verify-data-integrity": "pnpm run with-env:dev tsx scripts/verify-data-integrity/index.ts", "run-email-queue": "pnpm run with-env:dev tsx scripts/run-email-queue.ts" }, diff --git a/apps/backend/prisma/migrations/20251125030551_external_db_sync/migration.sql b/apps/backend/prisma/migrations/20251125030551_external_db_sync/migration.sql new file mode 100644 index 0000000000..e90bcf2451 --- /dev/null +++ b/apps/backend/prisma/migrations/20251125030551_external_db_sync/migration.sql @@ -0,0 +1,105 @@ +-- Creates a global sequence starting at 1 with increment of 11 for tracking row changes. +-- This sequence is used to order data changes across all tables in the database. +CREATE SEQUENCE global_seq_id + AS BIGINT + START 1 + INCREMENT BY 11 + NO MINVALUE + NO MAXVALUE; + +-- SPLIT_STATEMENT_SENTINEL +-- Adds sequenceId column to ContactChannel and ProjectUser tables. +-- This column stores the sequence number from global_seq_id to track when each row was last modified. +ALTER TABLE "ContactChannel" ADD COLUMN "sequenceId" BIGINT; + +-- SPLIT_STATEMENT_SENTINEL +ALTER TABLE "ProjectUser" ADD COLUMN "sequenceId" BIGINT; + +-- SPLIT_STATEMENT_SENTINEL +-- Creates unique indexes on sequenceId columns to ensure no duplicate sequence IDs exist. +-- This guarantees each row has a unique position in the change sequence. +CREATE UNIQUE INDEX "ContactChannel_sequenceId_key" ON "ContactChannel"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE UNIQUE INDEX "ProjectUser_sequenceId_key" ON "ProjectUser"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates composite indexes on (tenancyId, sequenceId) for efficient sync-engine queries. +-- These allow fast lookups of rows by tenant ordered by sequence number. +CREATE INDEX "ProjectUser_tenancyId_sequenceId_idx" ON "ProjectUser"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "ContactChannel_tenancyId_sequenceId_idx" ON "ContactChannel"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates OutgoingRequest table to queue sync requests to external databases. +-- Each request stores the QStash options for making HTTP requests and tracks when fulfillment started. +CREATE TABLE "OutgoingRequest" ( + "id" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "deduplicationKey" TEXT, + "qstashOptions" JSONB NOT NULL, + "startedFulfillingAt" TIMESTAMP(3), + + CONSTRAINT "OutgoingRequest_pkey" PRIMARY KEY ("id"), + CONSTRAINT "OutgoingRequest_deduplicationKey_key" UNIQUE ("deduplicationKey") +); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "OutgoingRequest_startedFulfillingAt_deduplicationKey_idx" ON "OutgoingRequest"("startedFulfillingAt", "deduplicationKey"); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates composite index on startedFulfillingAt and createdAt for efficient querying of pending requests in order. +-- This allows fast lookups of pending requests (WHERE startedFulfillingAt IS NULL) ordered by createdAt. +CREATE INDEX "OutgoingRequest_startedFulfillingAt_createdAt_idx" ON "OutgoingRequest"("startedFulfillingAt", "createdAt"); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates DeletedRow table to log information about deleted rows from other tables. +-- Stores the primary key and full data of deleted rows so external databases can be notified of deletions. +CREATE TABLE "DeletedRow" ( + "id" UUID NOT NULL, + "tenancyId" UUID NOT NULL, + "tableName" TEXT NOT NULL, + "sequenceId" BIGINT, + "primaryKey" JSONB NOT NULL, + "data" JSONB, + "deletedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "startedFulfillingAt" TIMESTAMP(3), + + CONSTRAINT "DeletedRow_pkey" PRIMARY KEY ("id") +); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates indexes on DeletedRow table for efficient querying by sequence, table name, and tenant. +CREATE UNIQUE INDEX "DeletedRow_sequenceId_key" ON "DeletedRow"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "DeletedRow_tableName_idx" ON "DeletedRow"("tableName"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "DeletedRow_tenancyId_idx" ON "DeletedRow"("tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates composite index for efficient querying of deleted rows by tenant and table, ordered by sequence. +CREATE INDEX "DeletedRow_tenancyId_tableName_sequenceId_idx" ON "DeletedRow"("tenancyId", "tableName", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- Adds shouldUpdateSequenceId flag to track which rows need their sequenceId updated. +ALTER TABLE "ProjectUser" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE; + +-- SPLIT_STATEMENT_SENTINEL +ALTER TABLE "ContactChannel" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE; + +-- SPLIT_STATEMENT_SENTINEL +ALTER TABLE "DeletedRow" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE; + +-- SPLIT_STATEMENT_SENTINEL +-- Creates indexes on (shouldUpdateSequenceId, tenancyId) to quickly find rows that need updates +-- and support ORDER BY tenancyId for less fragmented updates. +CREATE INDEX "ProjectUser_shouldUpdateSequenceId_idx" ON "ProjectUser"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "ContactChannel_shouldUpdateSequenceId_idx" ON "ContactChannel"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "DeletedRow_shouldUpdateSequenceId_idx" ON "DeletedRow"("shouldUpdateSequenceId", "tenancyId"); diff --git a/apps/backend/prisma/migrations/20260204014127_external_db_metadata/migration.sql b/apps/backend/prisma/migrations/20260204014127_external_db_metadata/migration.sql new file mode 100644 index 0000000000..aa0a767c0b --- /dev/null +++ b/apps/backend/prisma/migrations/20260204014127_external_db_metadata/migration.sql @@ -0,0 +1,24 @@ +-- DropIndex +DROP INDEX "ContactChannel_shouldUpdateSequenceId_idx"; + +-- DropIndex +DROP INDEX "DeletedRow_shouldUpdateSequenceId_idx"; + +-- DropIndex +DROP INDEX "ProjectUser_shouldUpdateSequenceId_idx"; + +-- CreateTable +CREATE TABLE "ExternalDbSyncMetadata" ( + "id" TEXT NOT NULL DEFAULT gen_random_uuid(), + "singleton" "BooleanTrue" NOT NULL DEFAULT 'TRUE', + "sequencerEnabled" BOOLEAN NOT NULL DEFAULT true, + "pollerEnabled" BOOLEAN NOT NULL DEFAULT true, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "ExternalDbSyncMetadata_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "ExternalDbSyncMetadata_singleton_key" ON "ExternalDbSyncMetadata"("singleton"); + diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index ed724cc0e3..e0cf9744d7 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -94,6 +94,18 @@ model EnvironmentConfigOverride { @@id([projectId, branchId]) } +model ExternalDbSyncMetadata { + id String @id @default(dbgenerated("gen_random_uuid()")) + + singleton BooleanTrue @unique @default(TRUE) + + sequencerEnabled Boolean @default(true) + pollerEnabled Boolean @default(true) + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + model Team { tenancyId String @db.Uuid teamId String @default(uuid()) @db.Uuid @@ -190,6 +202,9 @@ model ProjectUser { updatedAt DateTime @updatedAt lastActiveAt DateTime @default(now()) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + displayName String? serverMetadata Json? clientReadOnlyMetadata Json? @@ -227,6 +242,8 @@ model ProjectUser { @@index([tenancyId, displayName(sort: Desc)], name: "ProjectUser_displayName_desc") @@index([tenancyId, createdAt(sort: Asc)], name: "ProjectUser_createdAt_asc") @@index([tenancyId, createdAt(sort: Desc)], name: "ProjectUser_createdAt_desc") + @@index([tenancyId, sequenceId], name: "ProjectUser_tenancyId_sequenceId_idx") + // Partial index for external db sync backfill lives in migration SQL. } // This should be renamed to "OAuthAccount" as it is not always bound to a user @@ -273,6 +290,9 @@ model ContactChannel { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + type ContactChannelType isPrimary BooleanTrue? usedForAuth BooleanTrue? @@ -288,6 +308,8 @@ model ContactChannel { @@unique([tenancyId, projectUserId, type, value]) // only one contact channel per project with the same value and type can be used for auth @@unique([tenancyId, type, value, usedForAuth]) + @@index([tenancyId, sequenceId], name: "ContactChannel_tenancyId_sequenceId_idx") + // Partial index for external db sync backfill lives in migration SQL (WHERE shouldUpdateSequenceId = TRUE). } model AuthMethod { @@ -1058,3 +1080,38 @@ model SubscriptionInvoice { @@id([tenancyId, id]) @@unique([tenancyId, stripeInvoiceId]) } + +model OutgoingRequest { + id String @id @default(uuid()) @db.Uuid + + createdAt DateTime @default(now()) + + qstashOptions Json + startedFulfillingAt DateTime? + deduplicationKey String? + + @@unique([deduplicationKey]) + @@index([startedFulfillingAt, createdAt]) + @@index([startedFulfillingAt, deduplicationKey]) +} + +model DeletedRow { + id String @id @default(uuid()) @db.Uuid + tenancyId String @db.Uuid + tableName String + + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + + primaryKey Json + data Json? + + deletedAt DateTime @default(now()) + startedFulfillingAt DateTime? + + @@index([tableName]) + @@index([tenancyId]) + // composite index for efficient querying of deleted rows by tenant and table, ordered by sequence + @@index([tenancyId, tableName, sequenceId]) + // Partial index for external db sync backfill lives in migration SQL (WHERE shouldUpdateSequenceId = TRUE). +} diff --git a/apps/backend/scripts/db-migrations.ts b/apps/backend/scripts/db-migrations.ts index 14733a6873..4f997ec7c8 100644 --- a/apps/backend/scripts/db-migrations.ts +++ b/apps/backend/scripts/db-migrations.ts @@ -91,9 +91,9 @@ const generateMigrationFile = async () => { 'prisma', 'migrate', 'diff', - '--from-url', + '--from-config-datasource', diffUrl, - '--to-schema-datamodel', + '--to-schema', 'prisma/schema.prisma', '--script', ], diff --git a/apps/backend/scripts/db-migrations.tsup.config.ts b/apps/backend/scripts/db-migrations.tsup.config.ts index 72bb27ce99..5dda9ba65b 100644 --- a/apps/backend/scripts/db-migrations.tsup.config.ts +++ b/apps/backend/scripts/db-migrations.tsup.config.ts @@ -12,6 +12,9 @@ const nodeBuiltins = builtinModules.flatMap((m) => [m, `node:${m}`]); // tsup config to build the self-hosting migration script so it can be // run in the Docker container with no extra dependencies. +type EsbuildPlugin = NonNullable[number]; +const basePlugin = createBasePlugin({}) as unknown as EsbuildPlugin; + export default defineConfig({ entry: ['scripts/db-migrations.ts'], format: ['esm'], @@ -32,7 +35,6 @@ const __filename = __fileURLToPath(import.meta.url); const __dirname = __dirname_fn(__filename); const require = __createRequire(import.meta.url);`, }, - esbuildPlugins: [ - createBasePlugin({}), - ], + // Cast to tsup's esbuild plugin type to avoid esbuild version mismatch in typecheck. + esbuildPlugins: [basePlugin], } satisfies Options); diff --git a/apps/backend/scripts/run-cron-jobs.ts b/apps/backend/scripts/run-cron-jobs.ts new file mode 100644 index 0000000000..98b9680ce5 --- /dev/null +++ b/apps/backend/scripts/run-cron-jobs.ts @@ -0,0 +1,44 @@ +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError, StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; +import { runAsynchronously, wait } from "@stackframe/stack-shared/dist/utils/promises"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; + +const endpoints = [ + "/api/latest/internal/external-db-sync/sequencer", + "/api/latest/internal/external-db-sync/poller", +]; + +async function main() { + console.log("Starting cron jobs..."); + const cronSecret = getEnvVariable('CRON_SECRET'); + + const baseUrl = `http://localhost:${getEnvVariable('NEXT_PUBLIC_STACK_PORT_PREFIX', '81')}02`; + + const run = async (endpoint: string) => { + console.log(`Running ${endpoint}...`); + const res = await fetch(`${baseUrl}${endpoint}`, { + headers: { 'Authorization': `Bearer ${cronSecret}` }, + }); + if (!res.ok) throw new StackAssertionError(`Failed to call ${endpoint}: ${res.status} ${res.statusText}\n${await res.text()}`, { res }); + console.log(`${endpoint} completed.`); + }; + + for (const endpoint of endpoints) { + runAsynchronously(async () => { + while (true) { + const runResult = await Result.fromPromise(run(endpoint)); + if (runResult.status === "error") { + captureError("run-cron-jobs", runResult.error); + } + // Vercel only guarantees minute-granularity for cron jobs, so we randomize the interval + await wait(Math.random() * 120_000); + } + }); + } +} + +// eslint-disable-next-line no-restricted-syntax +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/apps/backend/src/app/api/latest/contact-channels/crud.tsx b/apps/backend/src/app/api/latest/contact-channels/crud.tsx index abc0edaa14..d5f3081f96 100644 --- a/apps/backend/src/app/api/latest/contact-channels/crud.tsx +++ b/apps/backend/src/app/api/latest/contact-channels/crud.tsx @@ -1,5 +1,6 @@ import { demoteAllContactChannelsToNonPrimary, setContactChannelAsPrimaryById } from "@/lib/contact-channel"; import { normalizeEmail } from "@/lib/emails"; +import { markProjectUserForExternalDbSync, recordExternalDbSyncDeletion, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { ensureContactChannelDoesNotExists, ensureContactChannelExists } from "@/lib/request-checks"; import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client"; import { createCrudHandlers } from "@/route-handlers/crud-handler"; @@ -121,6 +122,11 @@ export const contactChannelsCrudHandlers = createLazyProxy(() => createCrudHandl }); } + await markProjectUserForExternalDbSync(tx, { + tenancyId: auth.tenancy.id, + projectUserId: data.user_id, + }); + return await tx.contactChannel.findUnique({ where: { tenancyId_projectUserId_id: { @@ -188,7 +194,7 @@ export const contactChannelsCrudHandlers = createLazyProxy(() => createCrudHandl }); } - return await tx.contactChannel.update({ + const updated = await tx.contactChannel.update({ where: { tenancyId_projectUserId_id: { tenancyId: auth.tenancy.id, @@ -196,13 +202,20 @@ export const contactChannelsCrudHandlers = createLazyProxy(() => createCrudHandl id: params.contact_channel_id || throwErr("Missing contact channel id"), }, }, - data: { + data: withExternalDbSyncUpdate({ value: value, isVerified: data.is_verified ?? (value ? false : undefined), // if value is updated and is_verified is not provided, set to false usedForAuth: data.used_for_auth !== undefined ? (data.used_for_auth ? 'TRUE' : null) : undefined, isPrimary: data.is_primary !== undefined ? (data.is_primary ? 'TRUE' : null) : undefined, - }, + }), + }); + + await markProjectUserForExternalDbSync(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, }); + + return updated; }); return contactChannelToCrud(updatedContactChannel); @@ -224,6 +237,13 @@ export const contactChannelsCrudHandlers = createLazyProxy(() => createCrudHandl contactChannelId: params.contact_channel_id || throwErr("Missing contact channel id"), }); + await recordExternalDbSyncDeletion(tx, { + tableName: "ContactChannel", + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + contactChannelId: params.contact_channel_id || throwErr("Missing contact channel id"), + }); + await tx.contactChannel.delete({ where: { tenancyId_projectUserId_id: { @@ -233,6 +253,11 @@ export const contactChannelsCrudHandlers = createLazyProxy(() => createCrudHandl }, }, }); + + await markProjectUserForExternalDbSync(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); }); }, onList: async ({ query, auth }) => { diff --git a/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx b/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx index 53d981fcf6..fa2dfef55b 100644 --- a/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx +++ b/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx @@ -1,4 +1,5 @@ import { sendEmailFromDefaultTemplate } from "@/lib/emails"; +import { markProjectUserForExternalDbSync, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { getSoleTenancyFromProjectBranch } from "@/lib/tenancies"; import { getPrismaClientForTenancy } from "@/prisma-client"; import { createVerificationCodeHandler } from "@/route-handlers/verification-code-handler"; @@ -68,9 +69,14 @@ export const contactChannelVerificationCodeHandler = createVerificationCodeHandl await prisma.contactChannel.update({ where: uniqueKeys, - data: { + data: withExternalDbSyncUpdate({ isVerified: true, - } + }), + }); + + await markProjectUserForExternalDbSync(prisma, { + tenancyId: tenancy.id, + projectUserId: data.user_id, }); return { diff --git a/apps/backend/src/app/api/latest/internal/config/override/[level]/route.tsx b/apps/backend/src/app/api/latest/internal/config/override/[level]/route.tsx index 019cce5bdb..b9e0af9ef9 100644 --- a/apps/backend/src/app/api/latest/internal/config/override/[level]/route.tsx +++ b/apps/backend/src/app/api/latest/internal/config/override/[level]/route.tsx @@ -1,4 +1,5 @@ import { getBranchConfigOverrideQuery, getEnvironmentConfigOverrideQuery, overrideBranchConfigOverride, overrideEnvironmentConfigOverride, setBranchConfigOverride, setBranchConfigOverrideSource, setEnvironmentConfigOverride } from "@/lib/config"; +import { enqueueExternalDbSync } from "@/lib/external-db-sync-queue"; import { globalPrismaClient, rawQuery } from "@/prisma-client"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { branchConfigSchema, environmentConfigSchema, getConfigOverrideErrors, migrateConfigOverride } from "@stackframe/stack-shared/dist/config/schema"; @@ -10,6 +11,19 @@ type BranchConfigSourceApi = yup.InferType; const levelSchema = yupString().oneOf(["branch", "environment"]).defined(); +function shouldEnqueueExternalDbSync(config: unknown): boolean { + if (!config || typeof config !== "object") return false; + const configRecord = config as Record; + if (Object.prototype.hasOwnProperty.call(configRecord, "dbSync.externalDatabases")) { + return true; + } + const dbSync = configRecord.dbSync; + if (dbSync && typeof dbSync === "object") { + return Object.prototype.hasOwnProperty.call(dbSync as Record, "externalDatabases"); + } + return false; +} + const levelConfigs = { branch: { schema: branchConfigSchema, @@ -165,6 +179,10 @@ export const PUT = createSmartRouteHandler({ source: req.body.source as BranchConfigSourceApi, }); + if (req.params.level === "environment" && shouldEnqueueExternalDbSync(parsedConfig)) { + await enqueueExternalDbSync(req.auth.tenancy.id); + } + return { statusCode: 200 as const, bodyType: "success" as const, @@ -202,10 +220,13 @@ export const PATCH = createSmartRouteHandler({ config: parsedConfig, }); + if (req.params.level === "environment" && shouldEnqueueExternalDbSync(parsedConfig)) { + await enqueueExternalDbSync(req.auth.tenancy.id); + } + return { statusCode: 200 as const, bodyType: "success" as const, }; }, }); - diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/fusebox/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/fusebox/route.ts new file mode 100644 index 0000000000..1039e7fa67 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/fusebox/route.ts @@ -0,0 +1,98 @@ +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { + adaptSchema, + adminAuthTypeSchema, + yupBoolean, + yupNumber, + yupObject, + yupString, +} from "@stackframe/stack-shared/dist/schema-fields"; +import { KnownErrors } from "@stackframe/stack-shared"; +import { getExternalDbSyncFusebox, updateExternalDbSyncFusebox } from "@/lib/external-db-sync-metadata"; + +const fuseboxResponseSchema = yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + ok: yupBoolean().defined(), + sequencer_enabled: yupBoolean().defined(), + poller_enabled: yupBoolean().defined(), + }).defined(), +}); + +const fuseboxRequestSchema = yupObject({ + auth: yupObject({ + type: adminAuthTypeSchema, + tenancy: adaptSchema, + }).defined(), + body: yupObject({ + sequencer_enabled: yupBoolean().defined(), + poller_enabled: yupBoolean().defined(), + }).defined(), + method: yupString().oneOf(["POST"]).defined(), +}); + +const fuseboxGetRequestSchema = yupObject({ + auth: yupObject({ + type: adminAuthTypeSchema, + tenancy: adaptSchema, + }).defined(), + method: yupString().oneOf(["GET"]).defined(), +}); + +function ensureInternalProject(projectId: string) { + if (projectId !== "internal") { + throw new KnownErrors.ExpectedInternalProject(); + } +} + +export const GET = createSmartRouteHandler({ + metadata: { + summary: "Get external DB sync fusebox settings", + description: "Returns enablement flags for the external DB sync pipeline.", + tags: ["External DB Sync"], + hidden: true, + }, + request: fuseboxGetRequestSchema, + response: fuseboxResponseSchema, + handler: async ({ auth }) => { + ensureInternalProject(auth.tenancy.project.id); + const fusebox = await getExternalDbSyncFusebox(); + return { + statusCode: 200, + bodyType: "json" as const, + body: { + ok: true, + sequencer_enabled: fusebox.sequencerEnabled, + poller_enabled: fusebox.pollerEnabled, + }, + }; + }, +}); + +export const POST = createSmartRouteHandler({ + metadata: { + summary: "Update external DB sync fusebox settings", + description: "Updates enablement flags for the external DB sync pipeline.", + tags: ["External DB Sync"], + hidden: true, + }, + request: fuseboxRequestSchema, + response: fuseboxResponseSchema, + handler: async ({ auth, body }) => { + ensureInternalProject(auth.tenancy.project.id); + const fusebox = await updateExternalDbSyncFusebox({ + sequencerEnabled: body.sequencer_enabled, + pollerEnabled: body.poller_enabled, + }); + return { + statusCode: 200, + bodyType: "json" as const, + body: { + ok: true, + sequencer_enabled: fusebox.sequencerEnabled, + poller_enabled: fusebox.pollerEnabled, + }, + }; + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts new file mode 100644 index 0000000000..6e82f8bb48 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts @@ -0,0 +1,289 @@ +import type { OutgoingRequest } from "@/generated/prisma/client"; +import { getExternalDbSyncFusebox } from "@/lib/external-db-sync-metadata"; +import { upstash } from "@/lib/upstash"; +import { globalPrismaClient, retryTransaction } from "@/prisma-client"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { traceSpan } from "@/utils/telemetry"; +import { + yupBoolean, + yupNumber, + yupObject, + yupString, + yupTuple, +} from "@stackframe/stack-shared/dist/schema-fields"; +import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError, StackAssertionError, StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { wait } from "@stackframe/stack-shared/dist/utils/promises"; +import type { PublishBatchRequest } from "@upstash/qstash"; + +const DEFAULT_MAX_DURATION_MS = 3 * 60 * 1000; +const DIRECT_SYNC_ENV = "STACK_EXTERNAL_DB_SYNC_DIRECT"; +const POLLER_CLAIM_LIMIT_ENV = "STACK_EXTERNAL_DB_SYNC_POLL_CLAIM_LIMIT"; +const DEFAULT_POLL_CLAIM_LIMIT = 1000; + +function parseMaxDurationMs(value: string | undefined): number { + if (!value) return DEFAULT_MAX_DURATION_MS; + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new StatusError(400, "maxDurationMs must be a positive integer"); + } + return parsed; +} + +function parseStopWhenIdle(value: string | undefined): boolean { + if (!value) return false; + if (value === "true") return true; + if (value === "false") return false; + throw new StatusError(400, "stopWhenIdle must be 'true' or 'false'"); +} + +function directSyncEnabled(): boolean { + return getEnvVariable(DIRECT_SYNC_ENV, "") === "true"; +} + +function getPollerClaimLimit(): number { + const rawValue = getEnvVariable(POLLER_CLAIM_LIMIT_ENV, ""); + if (!rawValue) return DEFAULT_POLL_CLAIM_LIMIT; + const parsed = Number.parseInt(rawValue, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new StackAssertionError( + `${POLLER_CLAIM_LIMIT_ENV} must be a positive integer. Received: ${JSON.stringify(rawValue)}` + ); + } + return parsed; +} + +function getLocalApiBaseUrl(): string { + const prefix = getEnvVariable("NEXT_PUBLIC_STACK_PORT_PREFIX", "81"); + return `http://localhost:${prefix}02`; +} + +export const GET = createSmartRouteHandler({ + metadata: { + summary: "Poll outgoing requests and push to QStash", + description: + "Internal endpoint invoked by Vercel Cron to process pending outgoing requests.", + tags: ["External DB Sync"], + hidden: true, + }, + request: yupObject({ + auth: yupObject({}).nullable().optional(), + method: yupString().oneOf(["GET"]).defined(), + headers: yupObject({ + authorization: yupTuple([yupString().defined()]).defined(), + }).defined(), + query: yupObject({ + maxDurationMs: yupString().optional(), + stopWhenIdle: yupString().optional(), + }).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + ok: yupBoolean().defined(), + requests_processed: yupNumber().defined(), + }).defined(), + }), + handler: async ({ headers, query }) => { + const authHeader = headers.authorization[0]; + if (authHeader !== `Bearer ${getEnvVariable("CRON_SECRET")}`) { + throw new StatusError(401, "Unauthorized"); + } + + return await traceSpan("external-db-sync.poller", async (span) => { + const startTime = performance.now(); + const maxDurationMs = parseMaxDurationMs(query.maxDurationMs); + const stopWhenIdle = parseStopWhenIdle(query.stopWhenIdle); + const pollIntervalMs = 50; + const staleClaimIntervalMinutes = 5; + const pollerClaimLimit = getPollerClaimLimit(); + + span.setAttribute("stack.external-db-sync.max-duration-ms", maxDurationMs); + span.setAttribute("stack.external-db-sync.stop-when-idle", stopWhenIdle); + span.setAttribute("stack.external-db-sync.poll-interval-ms", pollIntervalMs); + span.setAttribute("stack.external-db-sync.poller-claim-limit", pollerClaimLimit); + span.setAttribute("stack.external-db-sync.direct-sync", directSyncEnabled()); + span.setAttribute("stack.external-db-sync.stale-claim-minutes", staleClaimIntervalMinutes); + + let totalRequestsProcessed = 0; + let iterationCount = 0; + + async function claimPendingRequests(): Promise { + return await traceSpan("external-db-sync.poller.claimPendingRequests", async (claimSpan) => { + const requests = await globalPrismaClient.$queryRaw` + UPDATE "OutgoingRequest" + SET "startedFulfillingAt" = NOW() + WHERE "id" IN ( + SELECT id + FROM "OutgoingRequest" + WHERE "startedFulfillingAt" IS NULL + LIMIT ${pollerClaimLimit} + FOR UPDATE SKIP LOCKED + ) + RETURNING *; + `; + claimSpan.setAttribute("stack.external-db-sync.claimed-count", requests.length); + return requests; + }); + } + + async function deleteOutgoingRequest(id: string): Promise { + await retryTransaction(globalPrismaClient, async (tx) => { + await tx.outgoingRequest.delete({ where: { id } }); + }); + } + + async function deleteOutgoingRequests(ids: string[]): Promise { + if (ids.length === 0) return; + await retryTransaction(globalPrismaClient, async (tx) => { + await tx.outgoingRequest.deleteMany({ where: { id: { in: ids } } }); + }); + } + async function processRequest(request: OutgoingRequest): Promise { + // Prisma JsonValue doesn't carry a precise shape for this JSON blob. + const options = request.qstashOptions as any; + const baseUrl = getEnvVariable("NEXT_PUBLIC_STACK_API_URL"); + + let fullUrl = new URL(options.url, baseUrl).toString(); + + // In dev/test, QStash runs in Docker so "localhost" won't work. + // Replace with "host.docker.internal" to reach the host machine. + if (getNodeEnvironment().includes("development") || getNodeEnvironment().includes("test")) { + const url = new URL(fullUrl); + if (url.hostname === "localhost" || url.hostname === "127.0.0.1") { + url.hostname = "host.docker.internal"; + fullUrl = url.toString(); + } + } + + await upstash.publishJSON({ + url: fullUrl, + body: options.body, + flowControl: options.flowControl, + }); + await deleteOutgoingRequest(request.id); + } + + type UpstashRequest = PublishBatchRequest; + + function buildUpstashRequest(request: OutgoingRequest): UpstashRequest { + // Prisma JsonValue doesn't carry a precise shape for this JSON blob. + const options = request.qstashOptions as any; + const baseUrl = getEnvVariable("NEXT_PUBLIC_STACK_API_URL"); + + let fullUrl = new URL(options.url, baseUrl).toString(); + + // In dev/test, QStash runs in Docker so "localhost" won't work. + // Replace with "host.docker.internal" to reach the host machine. + if (getNodeEnvironment().includes("development") || getNodeEnvironment().includes("test")) { + const url = new URL(fullUrl); + if (url.hostname === "localhost" || url.hostname === "127.0.0.1") { + url.hostname = "host.docker.internal"; + fullUrl = url.toString(); + } + } + + const flowControl = options.flowControl as UpstashRequest["flowControl"]; + + return { + url: fullUrl, + body: options.body, + ...(flowControl ? { flowControl } : {}), + }; + } + + async function processRequests(requests: OutgoingRequest[]): Promise { + return await traceSpan({ + description: "external-db-sync.poller.processRequests", + attributes: { + "stack.external-db-sync.pending-count": requests.length, + "stack.external-db-sync.direct-sync": directSyncEnabled(), + }, + }, async (processSpan) => { + let processed = 0; + + if (directSyncEnabled()) { + for (const request of requests) { + try { + await processRequest(request); + processed++; + } catch (error) { + processSpan.setAttribute("stack.external-db-sync.iteration-error", true); + captureError("poller-iteration-error", error); + } + } + processSpan.setAttribute("stack.external-db-sync.processed-count", processed); + return processed; + } + + if (requests.length === 0) { + processSpan.setAttribute("stack.external-db-sync.processed-count", 0); + return 0; + } + + try { + const batchPayload = requests.map(buildUpstashRequest); + await upstash.batchJSON(batchPayload); + await deleteOutgoingRequests(requests.map((request) => request.id)); + processSpan.setAttribute("stack.external-db-sync.processed-count", requests.length); + return requests.length; + } catch (error) { + processSpan.setAttribute("stack.external-db-sync.iteration-error", true); + captureError("poller-iteration-error", error); + processSpan.setAttribute("stack.external-db-sync.processed-count", 0); + return 0; + } + }); + } + + type PollerIterationResult = { + stopReason: "disabled" | "idle" | null, + processed: number, + }; + + while (performance.now() - startTime < maxDurationMs) { + const iterationResult = await traceSpan({ + description: "external-db-sync.poller.iteration", + attributes: { + "stack.external-db-sync.iteration": iterationCount + 1, + }, + }, async (iterationSpan) => { + const fusebox = await getExternalDbSyncFusebox(); + iterationSpan.setAttribute("stack.external-db-sync.poller-enabled", fusebox.pollerEnabled); + if (!fusebox.pollerEnabled) { + return { stopReason: "disabled", processed: 0 }; + } + + const pendingRequests = await claimPendingRequests(); + iterationSpan.setAttribute("stack.external-db-sync.pending-count", pendingRequests.length); + + if (stopWhenIdle && pendingRequests.length === 0) { + return { stopReason: "idle", processed: 0 }; + } + + const processed = await processRequests(pendingRequests); + iterationSpan.setAttribute("stack.external-db-sync.processed-count", processed); + return { stopReason: null, processed }; + }); + + iterationCount++; + totalRequestsProcessed += iterationResult.processed; + + await wait(pollIntervalMs); + } + + span.setAttribute("stack.external-db-sync.requests-processed", totalRequestsProcessed); + span.setAttribute("stack.external-db-sync.iterations", iterationCount); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { + ok: true, + requests_processed: totalRequestsProcessed, + }, + }; + }); + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts new file mode 100644 index 0000000000..2a40b3e0ef --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -0,0 +1,253 @@ +import { getExternalDbSyncFusebox } from "@/lib/external-db-sync-metadata"; +import { enqueueExternalDbSyncBatch } from "@/lib/external-db-sync-queue"; +import { globalPrismaClient } from "@/prisma-client"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { traceSpan } from "@/utils/telemetry"; +import { + yupBoolean, + yupNumber, + yupObject, + yupString, + yupTuple, +} from "@stackframe/stack-shared/dist/schema-fields"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError, StackAssertionError, StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { wait } from "@stackframe/stack-shared/dist/utils/promises"; + +const DEFAULT_MAX_DURATION_MS = 3 * 60 * 1000; +const SEQUENCER_BATCH_SIZE_ENV = "STACK_EXTERNAL_DB_SYNC_SEQUENCER_BATCH_SIZE"; +const DEFAULT_BATCH_SIZE = 1000; + +function parseMaxDurationMs(value: string | undefined): number { + if (!value) return DEFAULT_MAX_DURATION_MS; + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new StatusError(400, "maxDurationMs must be a positive integer"); + } + return parsed; +} + +function parseStopWhenIdle(value: string | undefined): boolean { + if (!value) return false; + if (value === "true") return true; + if (value === "false") return false; + throw new StatusError(400, "stopWhenIdle must be 'true' or 'false'"); +} + +function getSequencerBatchSize(): number { + const rawValue = getEnvVariable(SEQUENCER_BATCH_SIZE_ENV, ""); + if (!rawValue) return DEFAULT_BATCH_SIZE; + const parsed = Number.parseInt(rawValue, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new StackAssertionError( + `${SEQUENCER_BATCH_SIZE_ENV} must be a positive integer. Received: ${JSON.stringify(rawValue)}` + ); + } + return parsed; +} + + +// Assigns sequence IDs to rows that need them and queues sync requests for affected tenants. +// Processes up to batchSize rows at a time from each table. +async function backfillSequenceIds(batchSize: number): Promise { + return await traceSpan({ + description: "external-db-sync.sequencer.backfill", + attributes: { + "stack.external-db-sync.batch-size": batchSize, + }, + }, async (span) => { + let didUpdate = false; + + const projectUserTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "projectUserId" + FROM "ProjectUser" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "ProjectUser" pu + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE pu."tenancyId" = r."tenancyId" + AND pu."projectUserId" = r."projectUserId" + RETURNING pu."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.project-user-tenants", projectUserTenants.length); + + // Enqueue sync for all affected tenants in a single batch query + if (projectUserTenants.length > 0) { + await enqueueExternalDbSyncBatch(projectUserTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const contactChannelTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "projectUserId", "id" + FROM "ContactChannel" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "ContactChannel" cc + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE cc."tenancyId" = r."tenancyId" + AND cc."projectUserId" = r."projectUserId" + AND cc."id" = r."id" + RETURNING cc."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.contact-channel-tenants", contactChannelTenants.length); + + if (contactChannelTenants.length > 0) { + await enqueueExternalDbSyncBatch(contactChannelTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const deletedRowTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "id", "tenancyId" + FROM "DeletedRow" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "DeletedRow" dr + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE dr."id" = r."id" + RETURNING dr."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.deleted-row-tenants", deletedRowTenants.length); + + if (deletedRowTenants.length > 0) { + await enqueueExternalDbSyncBatch(deletedRowTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + span.setAttribute("stack.external-db-sync.did-update", didUpdate); + + return didUpdate; + }); +} + +// TODO: If we ever need to support non-hosted source-of-truth tenancies again, +// we'll need to implement a scalable way to iterate over them (pagination, etc.) +// instead of loading all tenancies into memory at once. + +export const GET = createSmartRouteHandler({ + metadata: { + summary: "Run sequence ID backfill", + description: + "Internal endpoint invoked by Vercel Cron to backfill null sequence IDs.", + tags: ["External DB Sync"], + hidden: true, + }, + request: yupObject({ + auth: yupObject({}).nullable().optional(), + method: yupString().oneOf(["GET"]).defined(), + headers: yupObject({ + authorization: yupTuple([yupString().defined()]).defined(), + }).defined(), + query: yupObject({ + maxDurationMs: yupString().optional(), + stopWhenIdle: yupString().optional(), + }).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + ok: yupBoolean().defined(), + iterations: yupNumber().defined(), + }).defined(), + }), + handler: async ({ headers, query }) => { + const authHeader = headers.authorization[0]; + if (authHeader !== `Bearer ${getEnvVariable("CRON_SECRET")}`) { + throw new StatusError(401, "Unauthorized"); + } + + return await traceSpan("external-db-sync.sequencer", async (span) => { + const startTime = performance.now(); + const maxDurationMs = parseMaxDurationMs(query.maxDurationMs); + const stopWhenIdle = parseStopWhenIdle(query.stopWhenIdle); + const pollIntervalMs = 50; + const batchSize = getSequencerBatchSize(); + + span.setAttribute("stack.external-db-sync.max-duration-ms", maxDurationMs); + span.setAttribute("stack.external-db-sync.stop-when-idle", stopWhenIdle); + span.setAttribute("stack.external-db-sync.poll-interval-ms", pollIntervalMs); + span.setAttribute("stack.external-db-sync.batch-size", batchSize); + + let iterations = 0; + + type SequencerIterationResult = { + stopReason: "disabled" | "idle" | null, + }; + + while (performance.now() - startTime < maxDurationMs) { + const iterationResult = await traceSpan({ + description: "external-db-sync.sequencer.iteration", + attributes: { + "stack.external-db-sync.iteration": iterations + 1, + }, + }, async (iterationSpan) => { + const fusebox = await getExternalDbSyncFusebox(); + iterationSpan.setAttribute("stack.external-db-sync.sequencer-enabled", fusebox.sequencerEnabled); + if (!fusebox.sequencerEnabled) { + return { stopReason: "disabled" }; + } + + try { + const didUpdate = await backfillSequenceIds(batchSize); + iterationSpan.setAttribute("stack.external-db-sync.did-update", didUpdate); + if (stopWhenIdle && !didUpdate) { + return { stopReason: "idle" }; + } + } catch (error) { + iterationSpan.setAttribute("stack.external-db-sync.iteration-error", true); + captureError( + `sequencer-iteration-error`, + error, + ); + } + + return { stopReason: null }; + }); + + iterations++; + await wait(pollIntervalMs); + } + + span.setAttribute("stack.external-db-sync.iterations", iterations); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { + ok: true, + iterations, + }, + }; + }); + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts new file mode 100644 index 0000000000..cf1c891563 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts @@ -0,0 +1,817 @@ +import { globalPrismaClient } from "@/prisma-client"; +import { Prisma } from "@/generated/prisma/client"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import type { CompleteConfig } from "@stackframe/stack-shared/dist/config/schema"; +import { + adaptSchema, + adminAuthTypeSchema, + yupArray, + yupBoolean, + yupNumber, + yupObject, + yupString, +} from "@stackframe/stack-shared/dist/schema-fields"; +import { errorToNiceString, StackAssertionError, throwErr } from "@stackframe/stack-shared/dist/utils/errors"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; +import { Client } from "pg"; +import { KnownErrors } from "@stackframe/stack-shared"; +import { traceSpan } from "@/utils/telemetry"; + +const STALE_CLAIM_INTERVAL_MINUTES = 5; + +const sequenceStatsSchema = yupObject({ + total: yupString().defined(), + pending: yupString().defined(), + null_sequence_id: yupString().defined(), + min_sequence_id: yupString().nullable().defined(), + max_sequence_id: yupString().nullable().defined(), +}); + +const deletedRowByTableSchema = yupObject({ + table_name: yupString().defined(), + total: yupString().defined(), + pending: yupString().defined(), + null_sequence_id: yupString().defined(), + min_sequence_id: yupString().nullable().defined(), + max_sequence_id: yupString().nullable().defined(), +}); + +const externalDbMetadataSchema = yupObject({ + mapping_name: yupString().defined(), + last_synced_sequence_id: yupString().defined(), + updated_at_millis: yupNumber().nullable().defined(), +}); + +const externalDbMappingStatusSchema = yupObject({ + mapping_id: yupString().defined(), + internal_max_sequence_id: yupString().nullable().defined(), + last_synced_sequence_id: yupString().nullable().defined(), + updated_at_millis: yupNumber().nullable().defined(), + backlog: yupString().nullable().defined(), +}); + +const externalDbSchema = yupObject({ + id: yupString().defined(), + type: yupString().defined(), + connection: yupObject({ + redacted: yupString().nullable().defined(), + host: yupString().nullable().defined(), + port: yupNumber().nullable().defined(), + database: yupString().nullable().defined(), + user: yupString().nullable().defined(), + }).defined(), + status: yupString().oneOf(["ok", "error"]).defined(), + error: yupString().nullable().defined(), + metadata: yupArray(externalDbMetadataSchema).defined(), + users_table: yupObject({ + exists: yupBoolean().defined(), + total_rows: yupString().nullable().defined(), + min_signed_up_at_millis: yupNumber().nullable().defined(), + max_signed_up_at_millis: yupNumber().nullable().defined(), + }).defined(), + mapping_status: yupArray(externalDbMappingStatusSchema).defined(), +}); + +const mappingSchema = yupObject({ + mapping_id: yupString().defined(), + internal_min_sequence_id: yupString().nullable().defined(), + internal_max_sequence_id: yupString().nullable().defined(), + internal_pending_count: yupString().defined(), +}); + +const globalSchema = yupObject({ + tenancies_total: yupString().defined(), + tenancies_with_db_sync: yupString().defined(), + sequencer: yupObject({ + project_users: sequenceStatsSchema.defined(), + contact_channels: sequenceStatsSchema.defined(), + deleted_rows: sequenceStatsSchema.shape({ + by_table: yupArray(deletedRowByTableSchema).defined(), + }).defined(), + }).defined(), + poller: yupObject({ + total: yupString().defined(), + pending: yupString().defined(), + in_flight: yupString().defined(), + stale: yupString().defined(), + oldest_created_at_millis: yupNumber().nullable().defined(), + newest_created_at_millis: yupNumber().nullable().defined(), + }).defined(), + sync_engine: yupObject({ + mappings: yupArray(mappingSchema).defined(), + }).defined(), +}); + +const responseSchema = yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + ok: yupBoolean().defined(), + generated_at_millis: yupNumber().defined(), + global: globalSchema.nullable().defined(), + tenancy: yupObject({ + id: yupString().defined(), + project_id: yupString().defined(), + branch_id: yupString().defined(), + }).defined(), + sequencer: yupObject({ + project_users: sequenceStatsSchema.defined(), + contact_channels: sequenceStatsSchema.defined(), + deleted_rows: sequenceStatsSchema.shape({ + by_table: yupArray(deletedRowByTableSchema).defined(), + }).defined(), + }).defined(), + poller: yupObject({ + total: yupString().defined(), + pending: yupString().defined(), + in_flight: yupString().defined(), + stale: yupString().defined(), + oldest_created_at_millis: yupNumber().nullable().defined(), + newest_created_at_millis: yupNumber().nullable().defined(), + }).defined(), + sync_engine: yupObject({ + mappings: yupArray(mappingSchema).defined(), + external_databases: yupArray(externalDbSchema).defined(), + }).defined(), + }).defined(), +}); + +type SequenceStatsRow = { + total: unknown, + pending: unknown, + null_sequence_id: unknown, + min_sequence_id: unknown, + max_sequence_id: unknown, +}; + +type DeletedRowStatsRow = SequenceStatsRow & { + table_name: string, +}; + +type OutgoingStatsRow = { + total: unknown, + pending: unknown, + in_flight: unknown, + stale: unknown, + oldest_created_at: unknown, + newest_created_at: unknown, +}; + +type ExternalDbMetadataRow = { + mapping_name: string, + last_synced_sequence_id: unknown, + updated_at: unknown, +}; + +type UsersTableStatsRow = { + total_rows: unknown, + min_signed_up_at: unknown, + max_signed_up_at: unknown, +}; + +type CountRow = { + total: unknown, +}; + +type SequenceStats = ReturnType; +type DeletedRowSummary = SequenceStats & { table_name: string }; + +function toBigIntString(value: unknown): string | null { + if (value === null || value === undefined) return null; + if (typeof value === "bigint") return value.toString(); + if (typeof value === "number" && Number.isFinite(value)) return Math.trunc(value).toString(); + if (typeof value === "string") return value; + return null; +} + +function toBigIntStringOrThrow(value: unknown, label: string): string { + return toBigIntString(value) ?? throwErr(`Expected ${label} to be a bigint-compatible value.`, { value }); +} + +function toMillis(value: unknown): number | null { + if (value === null || value === undefined) return null; + if (value instanceof Date) return value.getTime(); + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string") { + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed.getTime(); + } + return null; +} + +function addBigIntStrings(a: string | null | undefined, b: string | null | undefined): string { + const first = a ? BigInt(a) : 0n; + const second = b ? BigInt(b) : 0n; + return (first + second).toString(); +} + +function minBigIntString(values: Array): string | null { + let minValue: bigint | null = null; + for (const value of values) { + if (!value) continue; + const parsed = BigInt(value); + if (minValue === null || parsed < minValue) { + minValue = parsed; + } + } + return minValue === null ? null : minValue.toString(); +} + +function maxBigIntString(values: Array): string | null { + let maxValue: bigint | null = null; + for (const value of values) { + if (!value) continue; + const parsed = BigInt(value); + if (maxValue === null || parsed > maxValue) { + maxValue = parsed; + } + } + return maxValue === null ? null : maxValue.toString(); +} + +function buildMappingInternalStats( + projectUsersStats: SequenceStats, + deletedRowsByTable: DeletedRowSummary[], +) { + const deletedProjectUserStats = deletedRowsByTable.find((row) => row.table_name === "ProjectUser") ?? null; + + const mappingInternalStats = new Map(); + + const usersMappingMin = minBigIntString([ + projectUsersStats.min_sequence_id, + deletedProjectUserStats?.min_sequence_id, + ]); + const usersMappingMax = maxBigIntString([ + projectUsersStats.max_sequence_id, + deletedProjectUserStats?.max_sequence_id, + ]); + const usersMappingPending = addBigIntStrings( + projectUsersStats.pending, + deletedProjectUserStats?.pending, + ); + + mappingInternalStats.set("users", { + mapping_id: "users", + internal_min_sequence_id: usersMappingMin, + internal_max_sequence_id: usersMappingMax, + internal_pending_count: usersMappingPending, + }); + + const mappings = Array.from(mappingInternalStats.values()); + const mappingStatuses = mappings.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + })); + + return { mappings, mappingStatuses }; +} + +async function fetchInternalStats(tenancyId: string | null) { + const tenancyWhere = tenancyId + ? Prisma.sql`WHERE "tenancyId" = ${tenancyId}::uuid` + : Prisma.sql``; + + const projectUserStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "ProjectUser" + ${tenancyWhere} + `).at(0) ?? throwErr("Project user stats query returned no rows."); + + const contactChannelStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "ContactChannel" + ${tenancyWhere} + `).at(0) ?? throwErr("Contact channel stats query returned no rows."); + + const deletedRowStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "DeletedRow" + ${tenancyWhere} + `).at(0) ?? throwErr("Deleted row stats query returned no rows."); + + const deletedRowsByTableRows = await globalPrismaClient.$queryRaw` + SELECT + "tableName" AS "table_name", + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "DeletedRow" + ${tenancyWhere} + GROUP BY "tableName" + ORDER BY "tableName" ASC + `; + + const outgoingTenancyFilter = tenancyId + ? Prisma.sql`AND ("qstashOptions"->'body'->>'tenancyId') = ${tenancyId}` + : Prisma.sql``; + + const outgoingStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "startedFulfillingAt" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "startedFulfillingAt" IS NOT NULL)::bigint AS "in_flight", + COUNT(*) FILTER ( + WHERE "startedFulfillingAt" < NOW() - (${STALE_CLAIM_INTERVAL_MINUTES} * INTERVAL '1 minute') + )::bigint AS "stale", + MIN("createdAt") AS "oldest_created_at", + MAX("createdAt") AS "newest_created_at" + FROM "OutgoingRequest" + WHERE ("qstashOptions"->>'url') = '/api/latest/internal/external-db-sync/sync-engine' + ${outgoingTenancyFilter} + `).at(0) ?? throwErr("Outgoing request stats query returned no rows."); + + const projectUsersStats = formatSequenceStats(projectUserStatsRow); + const contactChannelStats = formatSequenceStats(contactChannelStatsRow); + const deletedRowStats = formatSequenceStats(deletedRowStatsRow); + + const deletedRowsByTable = deletedRowsByTableRows.map((row) => ({ + table_name: row.table_name, + ...formatSequenceStats(row), + })); + + const { mappings, mappingStatuses } = buildMappingInternalStats(projectUsersStats, deletedRowsByTable); + + return { + projectUsersStats, + contactChannelStats, + deletedRowStats, + deletedRowsByTable, + outgoingStatsRow, + mappings, + mappingStatuses, + }; +} + +function formatPollerStats(outgoingStats: OutgoingStatsRow) { + return { + total: toBigIntStringOrThrow(outgoingStats.total, "outgoing total"), + pending: toBigIntStringOrThrow(outgoingStats.pending, "outgoing pending"), + in_flight: toBigIntStringOrThrow(outgoingStats.in_flight, "outgoing in_flight"), + stale: toBigIntStringOrThrow(outgoingStats.stale, "outgoing stale"), + oldest_created_at_millis: toMillis(outgoingStats.oldest_created_at), + newest_created_at_millis: toMillis(outgoingStats.newest_created_at), + }; +} + +function formatSequenceStats(row: SequenceStatsRow) { + return { + total: toBigIntStringOrThrow(row.total, "sequence stats total"), + pending: toBigIntStringOrThrow(row.pending, "sequence stats pending"), + null_sequence_id: toBigIntStringOrThrow(row.null_sequence_id, "sequence stats null_sequence_id"), + min_sequence_id: toBigIntString(row.min_sequence_id), + max_sequence_id: toBigIntString(row.max_sequence_id), + }; +} + +function formatError(error: unknown): string { + return errorToNiceString(error); +} + +function parseConnectionString(connectionString: string | null | undefined) { + if (!connectionString) { + return { + redacted: null, + host: null, + port: null, + database: null, + user: null, + }; + } + + const parsed = Result.fromThrowing(() => new URL(connectionString)); + if (parsed.status === "error") { + return { + redacted: null, + host: null, + port: null, + database: null, + user: null, + }; + } + + const url = parsed.data; + const user = url.username ? decodeURIComponent(url.username) : null; + const host = url.hostname || null; + const port = url.port ? Number.parseInt(url.port, 10) : null; + const database = url.pathname ? url.pathname.replace(/^\//, "") : null; + const redacted = `${url.protocol}//${url.username ? encodeURIComponent(url.username) : ""}${url.username ? ":" : ""}${url.password ? "***" : ""}${url.username ? "@" : ""}${url.hostname}${url.port ? ":" + url.port : ""}${url.pathname}${url.search}`; + + return { + redacted, + host, + port: Number.isFinite(port ?? NaN) ? port : null, + database, + user, + }; +} + +async function fetchExternalDatabaseStatus( + dbId: string, + dbConfig: CompleteConfig["dbSync"]["externalDatabases"][string], + mappingStatuses: Array<{ + mapping_id: string, + internal_max_sequence_id: string | null, + }>, +) { + const connection = parseConnectionString(dbConfig.connectionString ?? null); + + if (dbConfig.type !== "postgres") { + return { + id: dbId, + type: String(dbConfig.type), + connection, + status: "error" as const, + error: `Unsupported database type: ${String(dbConfig.type)}`, + metadata: [], + users_table: { + exists: false, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + + if (!dbConfig.connectionString) { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: "Missing connection string", + metadata: [], + users_table: { + exists: false, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + + const client = new Client({ connectionString: dbConfig.connectionString }); + const connectResult = await Result.fromPromise(client.connect()); + if (connectResult.status === "error") { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: formatError(connectResult.error), + metadata: [], + users_table: { + exists: false, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + + let metadata: ExternalDbMetadataRow[] = []; + let metadataExists = false; + let usersExists = false; + let usersStats: UsersTableStatsRow | null = null; + + try { + const metadataExistsResult = await Result.fromPromise(client.query<{ exists: boolean }>(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = '_stack_sync_metadata' + ) AS "exists"; + `)); + if (metadataExistsResult.status === "error") { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: formatError(metadataExistsResult.error), + metadata: [], + users_table: { + exists: false, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + metadataExists = metadataExistsResult.data.rows[0]?.exists === true; + + const usersExistsResult = await Result.fromPromise(client.query<{ exists: boolean }>(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'users' + ) AS "exists"; + `)); + if (usersExistsResult.status === "error") { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: formatError(usersExistsResult.error), + metadata: [], + users_table: { + exists: false, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + usersExists = usersExistsResult.data.rows[0]?.exists === true; + + if (metadataExists) { + const metadataResult = await Result.fromPromise(client.query(` + SELECT "mapping_name", "last_synced_sequence_id", "updated_at" + FROM "_stack_sync_metadata" + ORDER BY "mapping_name" ASC; + `)); + if (metadataResult.status === "error") { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: formatError(metadataResult.error), + metadata: [], + users_table: { + exists: usersExists, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + metadata = metadataResult.data.rows; + } + + if (usersExists) { + const usersStatsResult = await Result.fromPromise(client.query(` + SELECT + COUNT(*)::bigint AS "total_rows", + MIN("signed_up_at") AS "min_signed_up_at", + MAX("signed_up_at") AS "max_signed_up_at" + FROM "users"; + `)); + if (usersStatsResult.status === "error") { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: formatError(usersStatsResult.error), + metadata: metadata.map((row) => ({ + mapping_name: row.mapping_name, + last_synced_sequence_id: toBigIntString(row.last_synced_sequence_id) ?? "-1", + updated_at_millis: toMillis(row.updated_at), + })), + users_table: { + exists: usersExists, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + usersStats = usersStatsResult.data.rows[0] ?? null; + } + } finally { + await Result.fromPromise(client.end()); + } + + const metadataMap = new Map(); + const formattedMetadata = metadata.map((row) => { + const lastSynced = toBigIntString(row.last_synced_sequence_id) ?? "-1"; + const updatedAt = toMillis(row.updated_at); + metadataMap.set(row.mapping_name, { last_synced_sequence_id: lastSynced, updated_at_millis: updatedAt }); + return { + mapping_name: row.mapping_name, + last_synced_sequence_id: lastSynced, + updated_at_millis: updatedAt, + }; + }); + + const mappingStatus = mappingStatuses.map((mapping) => { + const external = metadataMap.get(mapping.mapping_id); + const lastSynced = external?.last_synced_sequence_id ?? null; + const updatedAt = external?.updated_at_millis ?? null; + let backlog: string | null = null; + if (mapping.internal_max_sequence_id && lastSynced) { + backlog = (BigInt(mapping.internal_max_sequence_id) - BigInt(lastSynced)).toString(); + } + return { + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: lastSynced, + updated_at_millis: updatedAt, + backlog, + }; + }); + + return { + id: dbId, + type: dbConfig.type, + connection, + status: "ok" as const, + error: null, + metadata: formattedMetadata, + users_table: { + exists: usersExists, + total_rows: toBigIntString(usersStats?.total_rows ?? null), + min_signed_up_at_millis: toMillis(usersStats?.min_signed_up_at ?? null), + max_signed_up_at_millis: toMillis(usersStats?.max_signed_up_at ?? null), + }, + mapping_status: mappingStatus, + }; +} + +export const GET = createSmartRouteHandler({ + metadata: { + summary: "External DB sync status", + description: "Returns sequencing, queue, and external sync progress for the current tenancy. Optional global aggregate when scope=all.", + tags: ["External DB Sync"], + hidden: true, + }, + request: yupObject({ + auth: yupObject({ + type: adminAuthTypeSchema, + tenancy: adaptSchema, + }).defined(), + query: yupObject({ + scope: yupString().oneOf(["tenancy", "all"]).default("tenancy"), + }).defined(), + method: yupString().oneOf(["GET"]).defined(), + }), + response: responseSchema, + handler: async ({ auth, query }) => { + return await traceSpan({ + description: "external-db-sync.status", + attributes: { + "stack.external-db-sync.scope": query.scope, + "stack.external-db-sync.tenancy-id": auth.tenancy.id, + }, + }, async (span) => { + if (auth.tenancy.project.id !== "internal") { + throw new KnownErrors.ExpectedInternalProject(); + } + const tenancyId = auth.tenancy.id; + + const shouldIncludeGlobal = query.scope === "all"; + span.setAttribute("stack.external-db-sync.include-global", shouldIncludeGlobal); + + const currentStats = await traceSpan({ + description: "external-db-sync.status.fetchInternalStats", + attributes: { + "stack.external-db-sync.scope": shouldIncludeGlobal ? "all" : "tenancy", + }, + }, async () => shouldIncludeGlobal ? await fetchInternalStats(null) : await fetchInternalStats(tenancyId)); + + const globalStats = shouldIncludeGlobal ? currentStats : null; + const globalTenanciesCount = shouldIncludeGlobal + ? (await globalPrismaClient.$queryRaw` + SELECT COUNT(*)::bigint AS "total" + FROM "Tenancy" + `).at(0) ?? throwErr("Tenancy count query returned no rows.") + : null; + const globalDbSyncCount = shouldIncludeGlobal + ? (await globalPrismaClient.$queryRaw` + SELECT COUNT(*)::bigint AS "total" + FROM "EnvironmentConfigOverride" + WHERE ("config"->'dbSync'->'externalDatabases') IS NOT NULL + `).at(0) ?? throwErr("DB sync config count query returned no rows.") + : null; + + const externalDbStatuses = shouldIncludeGlobal + ? [] + : await traceSpan("external-db-sync.status.fetchExternalDatabaseStatuses", async (externalSpan) => { + const statuses = await Promise.all( + Object.entries( + auth.tenancy.config.dbSync.externalDatabases as CompleteConfig["dbSync"]["externalDatabases"], + ).map(([dbId, dbConfig]) => fetchExternalDatabaseStatus(dbId, dbConfig, currentStats.mappingStatuses)), + ); + externalSpan.setAttribute("stack.external-db-sync.external-db-count", statuses.length); + return statuses; + }); + + const outgoingStats = currentStats.outgoingStatsRow; + + return { + statusCode: 200 as const, + bodyType: "json" as const, + body: { + ok: true, + generated_at_millis: Date.now(), + global: shouldIncludeGlobal && globalStats && globalTenanciesCount && globalDbSyncCount ? { + tenancies_total: toBigIntStringOrThrow(globalTenanciesCount.total, "tenancies total"), + tenancies_with_db_sync: toBigIntStringOrThrow(globalDbSyncCount.total, "tenancies with db sync"), + sequencer: { + project_users: globalStats.projectUsersStats, + contact_channels: globalStats.contactChannelStats, + deleted_rows: { + ...globalStats.deletedRowStats, + by_table: globalStats.deletedRowsByTable, + }, + }, + poller: formatPollerStats(globalStats.outgoingStatsRow), + sync_engine: { + mappings: globalStats.mappings, + }, + } : null, + tenancy: { + id: tenancyId, + project_id: auth.tenancy.project.id, + branch_id: auth.tenancy.branchId, + }, + sequencer: { + project_users: currentStats.projectUsersStats, + contact_channels: currentStats.contactChannelStats, + deleted_rows: { + ...currentStats.deletedRowStats, + by_table: currentStats.deletedRowsByTable, + }, + }, + poller: formatPollerStats(outgoingStats), + sync_engine: { + mappings: currentStats.mappings, + external_databases: externalDbStatuses, + }, + }, + }; + }); + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sync-engine/route.tsx b/apps/backend/src/app/api/latest/internal/external-db-sync/sync-engine/route.tsx new file mode 100644 index 0000000000..444cca3708 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sync-engine/route.tsx @@ -0,0 +1,68 @@ +import { syncExternalDatabases } from "@/lib/external-db-sync"; +import { enqueueExternalDbSync } from "@/lib/external-db-sync-queue"; +import { getTenancy } from "@/lib/tenancies"; +import { ensureUpstashSignature } from "@/lib/upstash"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { getExternalDbSyncFusebox } from "@/lib/external-db-sync-metadata"; +import { yupNumber, yupObject, yupString, yupTuple } from "@stackframe/stack-shared/dist/schema-fields"; +import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { traceSpan } from "@/utils/telemetry"; + +export const POST = createSmartRouteHandler({ + metadata: { + summary: "Sync engine webhook endpoint", + description: "Receives webhook from QStash to trigger external database sync for a tenant", + tags: ["External DB Sync"], + hidden: true, + }, + request: yupObject({ + headers: yupObject({ + "upstash-signature": yupTuple([yupString()]).defined(), + }).defined(), + body: yupObject({ + tenancyId: yupString().defined(), + }).defined(), + method: yupString().oneOf(["POST"]).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["success"]).defined(), + }), + handler: async ({ body }, fullReq) => { + return await traceSpan({ + description: "external-db-sync.sync-engine", + attributes: { + "stack.external-db-sync.tenancy-id": body.tenancyId, + }, + }, async (span) => { + await ensureUpstashSignature(fullReq); + const { tenancyId } = body; + + const tenancy = await traceSpan("external-db-sync.sync-engine.loadTenancy", async (tenancySpan) => { + const foundTenancy = await getTenancy(tenancyId); + tenancySpan.setAttribute("stack.external-db-sync.tenancy-found", !!foundTenancy); + return foundTenancy; + }); + if (!tenancy) { + console.warn(`[sync-engine] Tenancy ${tenancyId} in queue but not found, assuming it was deleted.`); + throw new StatusError(400, `Tenancy ${tenancyId} not found.`); + } + + const needsResync = await traceSpan("external-db-sync.sync-engine.syncExternalDatabases", async (syncSpan) => { + const resync = await syncExternalDatabases(tenancy); + syncSpan.setAttribute("stack.external-db-sync.needs-resync", resync); + return resync; + }); + if (needsResync) { + await traceSpan("external-db-sync.sync-engine.enqueueResync", async () => { + await enqueueExternalDbSync(tenancy.id); + }); + } + + return { + statusCode: 200, + bodyType: "success", + }; + }); + }, +}); diff --git a/apps/backend/src/app/api/latest/users/crud.tsx b/apps/backend/src/app/api/latest/users/crud.tsx index 49f4641b0f..d76b80e625 100644 --- a/apps/backend/src/app/api/latest/users/crud.tsx +++ b/apps/backend/src/app/api/latest/users/crud.tsx @@ -2,6 +2,7 @@ import { BooleanTrue, Prisma } from "@/generated/prisma/client"; import { getRenderedOrganizationConfigQuery, getRenderedProjectConfigQuery } from "@/lib/config"; import { demoteAllContactChannelsToNonPrimary, setContactChannelAsPrimaryByValue } from "@/lib/contact-channel"; import { normalizeEmail } from "@/lib/emails"; +import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { grantDefaultProjectPermissions } from "@/lib/permissions"; import { ensureTeamMembershipExists, ensureUserExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; @@ -934,9 +935,9 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC isPrimary: "TRUE", }, }, - data: { + data: withExternalDbSyncUpdate({ isVerified: data.primary_email_verified, - }, + }), }); } @@ -952,9 +953,9 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC isPrimary: "TRUE", }, }, - data: { + data: withExternalDbSyncUpdate({ usedForAuth: primaryEmailAuthEnabled ? BooleanTrue.TRUE : null, - }, + }), }); } @@ -1130,7 +1131,7 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC projectUserId: params.user_id, }, }, - data: { + data: withExternalDbSyncUpdate({ displayName: data.display_name === undefined ? undefined : (data.display_name || null), clientMetadata: data.client_metadata === null ? Prisma.JsonNull : data.client_metadata, clientReadOnlyMetadata: data.client_read_only_metadata === null ? Prisma.JsonNull : data.client_read_only_metadata, @@ -1142,7 +1143,7 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC restrictedByAdmin: data.restricted_by_admin ?? undefined, restrictedByAdminReason: restrictedByAdminReason, restrictedByAdminPrivateDetails: restrictedByAdminPrivateDetails, - }, + }), include: userFullInclude, }); @@ -1189,6 +1190,17 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC }, }); + await recordExternalDbSyncDeletion(tx, { + tableName: "ProjectUser", + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + + await recordExternalDbSyncContactChannelDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + await tx.projectUser.delete({ where: { tenancyId_projectUserId: { diff --git a/apps/backend/src/lib/contact-channel.tsx b/apps/backend/src/lib/contact-channel.tsx index 7b90457a4c..1a8836a1e8 100644 --- a/apps/backend/src/lib/contact-channel.tsx +++ b/apps/backend/src/lib/contact-channel.tsx @@ -1,4 +1,5 @@ import { BooleanTrue, ContactChannelType } from "@/generated/prisma/client"; +import { markProjectUserForExternalDbSync, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; import { normalizeEmail } from "./emails"; import { PrismaTransaction } from "./types"; @@ -34,9 +35,13 @@ export async function demoteAllContactChannelsToNonPrimary( type: options.type, isPrimary: BooleanTrue.TRUE, }, - data: { + data: withExternalDbSyncUpdate({ isPrimary: null, - }, + }), + }); + await markProjectUserForExternalDbSync(tx, { + tenancyId: options.tenancyId, + projectUserId: options.projectUserId, }); } @@ -100,10 +105,14 @@ export async function setContactChannelAsPrimaryById( id: options.contactChannelId, }, }, - data: { + data: withExternalDbSyncUpdate({ isPrimary: BooleanTrue.TRUE, ...options.additionalUpdates, - }, + }), + }); + await markProjectUserForExternalDbSync(tx, { + tenancyId: options.tenancyId, + projectUserId: options.projectUserId, }); } @@ -141,10 +150,14 @@ export async function setContactChannelAsPrimaryByValue( value: options.value, }, }, - data: { + data: withExternalDbSyncUpdate({ isPrimary: BooleanTrue.TRUE, ...options.additionalUpdates, - }, + }), + }); + await markProjectUserForExternalDbSync(tx, { + tenancyId: options.tenancyId, + projectUserId: options.projectUserId, }); } diff --git a/apps/backend/src/lib/external-db-sync-metadata.ts b/apps/backend/src/lib/external-db-sync-metadata.ts new file mode 100644 index 0000000000..dd64cb039e --- /dev/null +++ b/apps/backend/src/lib/external-db-sync-metadata.ts @@ -0,0 +1,32 @@ +import { BooleanTrue } from "@/generated/prisma/client"; +import { globalPrismaClient } from "@/prisma-client"; + +export type ExternalDbSyncFusebox = { + sequencerEnabled: boolean, + pollerEnabled: boolean, +}; + +const fuseboxSelect = { + sequencerEnabled: true, + pollerEnabled: true, +}; + +export async function getExternalDbSyncFusebox(): Promise { + return await globalPrismaClient.externalDbSyncMetadata.upsert({ + where: { singleton: BooleanTrue.TRUE }, + create: { singleton: BooleanTrue.TRUE }, + update: {}, + select: fuseboxSelect, + }); +} + +export async function updateExternalDbSyncFusebox( + updates: ExternalDbSyncFusebox, +): Promise { + return await globalPrismaClient.externalDbSyncMetadata.upsert({ + where: { singleton: BooleanTrue.TRUE }, + create: { singleton: BooleanTrue.TRUE, ...updates }, + update: updates, + select: fuseboxSelect, + }); +} diff --git a/apps/backend/src/lib/external-db-sync-queue.ts b/apps/backend/src/lib/external-db-sync-queue.ts new file mode 100644 index 0000000000..ab666893eb --- /dev/null +++ b/apps/backend/src/lib/external-db-sync-queue.ts @@ -0,0 +1,43 @@ +import { globalPrismaClient } from "@/prisma-client"; +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; + +const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +function assertUuid(value: unknown, label: string): asserts value is string { + if (typeof value !== "string" || value.trim().length === 0 || !UUID_REGEX.test(value)) { + throw new StackAssertionError(`${label} must be a valid UUID. Received: ${JSON.stringify(value)}`); + } +} + +// Queues a sync request for a specific tenant if one isn't already pending. +export async function enqueueExternalDbSync(tenancyId: string): Promise { + assertUuid(tenancyId, "tenancyId"); + await enqueueExternalDbSyncBatch([tenancyId]); +} + +// Queues sync requests for multiple tenants in a single query. +// Only inserts for tenants that don't already have a pending request. +export async function enqueueExternalDbSyncBatch(tenancyIds: string[]): Promise { + if (tenancyIds.length === 0) return; + + for (const id of tenancyIds) { + assertUuid(id, "tenancyId"); + } + + // Use unnest to pass array of UUIDs and insert all in one query + await globalPrismaClient.$executeRaw` + INSERT INTO "OutgoingRequest" ("id", "createdAt", "qstashOptions", "startedFulfillingAt", "deduplicationKey") + SELECT + gen_random_uuid(), + NOW(), + json_build_object( + 'url', '/api/latest/internal/external-db-sync/sync-engine', + 'body', json_build_object('tenancyId', t.tenancy_id), + 'flowControl', json_build_object('key', 'sentinel-sync-key', 'parallelism', 20) + ), + NULL, + 'sentinel-sync-key-' || t.tenancy_id + FROM unnest(${tenancyIds}::uuid[]) AS t(tenancy_id) + ON CONFLICT ("deduplicationKey") DO NOTHING + `; +} diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts new file mode 100644 index 0000000000..57f89127a4 --- /dev/null +++ b/apps/backend/src/lib/external-db-sync.ts @@ -0,0 +1,498 @@ +import { Tenancy } from "@/lib/tenancies"; +import type { PrismaTransaction } from "@/lib/types"; +import { getPrismaClientForTenancy, PrismaClientWithReplica } from "@/prisma-client"; +import { Prisma } from "@/generated/prisma/client"; +import { DEFAULT_DB_SYNC_MAPPINGS } from "@stackframe/stack-shared/dist/config/db-sync-mappings"; +import type { CompleteConfig } from "@stackframe/stack-shared/dist/config/schema"; +import { captureError, StackAssertionError, throwErr } from "@stackframe/stack-shared/dist/utils/errors"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { omit } from "@stackframe/stack-shared/dist/utils/objects"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; +import { Client } from 'pg'; + +const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const MAX_BATCHES_PER_MAPPING_ENV = "STACK_EXTERNAL_DB_SYNC_MAX_BATCHES_PER_MAPPING"; + +function assertNonEmptyString(value: unknown, label: string): asserts value is string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new StackAssertionError(`${label} must be a non-empty string.`); + } +} + +function assertUuid(value: unknown, label: string): asserts value is string { + assertNonEmptyString(value, label); + if (!UUID_REGEX.test(value)) { + throw new StackAssertionError(`${label} must be a valid UUID. Received: ${JSON.stringify(value)}`); + } +} + +type ExternalDbSyncClient = PrismaTransaction | PrismaClientWithReplica; + +type ExternalDbSyncTarget = + | { + tableName: "ProjectUser", + tenancyId: string, + projectUserId: string, + } + | { + tableName: "ContactChannel", + tenancyId: string, + projectUserId: string, + contactChannelId: string, + }; + +export function withExternalDbSyncUpdate(data: T): T & { shouldUpdateSequenceId: true } { + return { + ...data, + shouldUpdateSequenceId: true, + }; +} + +export async function markProjectUserForExternalDbSync( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + } +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + await tx.projectUser.update({ + where: { + tenancyId_projectUserId: { + tenancyId: options.tenancyId, + projectUserId: options.projectUserId, + }, + }, + data: { + shouldUpdateSequenceId: true, + }, + }); +} + +export async function recordExternalDbSyncDeletion( + tx: ExternalDbSyncClient, + target: ExternalDbSyncTarget, +): Promise { + assertUuid(target.tenancyId, "tenancyId"); + assertUuid(target.projectUserId, "projectUserId"); + + if (target.tableName === "ProjectUser") { + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUser', + jsonb_build_object('tenancyId', "tenancyId", 'projectUserId', "projectUserId"), + to_jsonb("ProjectUser".*), + NOW(), + TRUE + FROM "ProjectUser" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "projectUserId" = ${target.projectUserId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ProjectUser, got ${insertedCount}.` + ); + } + return; + } + + assertUuid(target.contactChannelId, "contactChannelId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ContactChannel', + jsonb_build_object( + 'tenancyId', + "tenancyId", + 'projectUserId', + "projectUserId", + 'id', + "id" + ), + to_jsonb("ContactChannel".*), + NOW(), + TRUE + FROM "ContactChannel" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "projectUserId" = ${target.projectUserId}::uuid + AND "id" = ${target.contactChannelId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ContactChannel, got ${insertedCount}.` + ); + } +} + +export async function recordExternalDbSyncContactChannelDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ContactChannel', + jsonb_build_object( + 'tenancyId', + "tenancyId", + 'projectUserId', + "projectUserId", + 'id', + "id" + ), + to_jsonb("ContactChannel".*), + NOW(), + TRUE + FROM "ContactChannel" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + FOR UPDATE + `); +} + +type PgErrorLike = { + code?: string, + constraint?: string, + message?: string, +}; + +function isDuplicateTypeError(error: unknown): error is PgErrorLike { + if (!error || typeof error !== "object") return false; + const pgError = error as PgErrorLike; + return pgError.code === "23505" && pgError.constraint === "pg_type_typname_nsp_index"; +} + +function isConcurrentUpdateError(error: unknown): error is PgErrorLike { + if (!error || typeof error !== "object") return false; + const pgError = error as PgErrorLike; + // "tuple concurrently updated" occurs when multiple transactions race to modify + // the same system catalog row (e.g., during concurrent CREATE TABLE IF NOT EXISTS) + return typeof pgError.message === "string" && pgError.message.includes("tuple concurrently updated"); +} + +function getMaxBatchesPerMapping(): number | null { + const rawValue = getEnvVariable(MAX_BATCHES_PER_MAPPING_ENV, ""); + if (!rawValue) return null; + const parsed = Number.parseInt(rawValue, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new StackAssertionError( + `${MAX_BATCHES_PER_MAPPING_ENV} must be a positive integer. Received: ${JSON.stringify(rawValue)}` + ); + } + return parsed; +} + +async function ensureExternalSchema( + externalClient: Client, + tableSchemaSql: string, + tableName: string, +) { + try { + await externalClient.query(tableSchemaSql); + } catch (error) { + // Concurrent CREATE TABLE can race and cause various errors: + // - duplicate type error (23505 on pg_type_typname_nsp_index) + // - tuple concurrently updated (system catalog row modified by another transaction) + // If the table now exists, we can safely continue. + if (!isDuplicateTypeError(error) && !isConcurrentUpdateError(error)) { + throw error; + } + + const existsResult = await externalClient.query(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = $1 + ); + `, [tableName]); + if (existsResult.rows[0]?.exists === true) { + return; + } + + throw new StackAssertionError( + `Schema creation error while creating table ${JSON.stringify(tableName)}, but table does not exist.` + ); + } +} + +async function pushRowsToExternalDb( + externalClient: Client, + tableName: string, + newRows: any[], + upsertQuery: string, + expectedTenancyId: string, + mappingId: string, +) { + assertNonEmptyString(tableName, "tableName"); + assertNonEmptyString(mappingId, "mappingId"); + assertUuid(expectedTenancyId, "expectedTenancyId"); + if (!Array.isArray(newRows)) { + throw new StackAssertionError(`newRows must be an array for table ${JSON.stringify(tableName)}.`); + } + if (newRows.length === 0) return; + // Just for our own sanity, make sure that we have the right number of positional parameters + // The last parameter is mapping_name for metadata tracking + const placeholderMatches = upsertQuery.match(/\$\d+/g) ?? throwErr(`Could not find any positional parameters ($1, $2, ...) in the update SQL query.`); + const expectedParamCount = Math.max(...placeholderMatches.map((m: string) => Number(m.slice(1)))); + const sampleRow = newRows[0]; + const orderedKeys = Object.keys(omit(sampleRow, ["tenancyId"])); + // +1 for mapping_name parameter which is appended + if (orderedKeys.length + 1 !== expectedParamCount) { + throw new StackAssertionError(` + Column count mismatch for table ${JSON.stringify(tableName)} + → upsertQuery expects ${expectedParamCount} parameters (last one should be mapping_name). + → internalDbFetchQuery returned ${orderedKeys.length} columns (excluding tenancyId) + 1 for mapping_name = ${orderedKeys.length + 1}. + Fix your SELECT column order or your SQL parameter order. + `); + } + + for (const row of newRows) { + const { tenancyId, ...rest } = row; + + // Validate that all rows belong to the expected tenant + if (tenancyId !== expectedTenancyId) { + throw new StackAssertionError( + `Row has unexpected tenancyId. Expected ${expectedTenancyId}, got ${tenancyId}. ` + + `This indicates a bug in the internalDbFetchQuery.` + ); + } + + const rowKeys = Object.keys(rest); + + const validShape = + rowKeys.length === orderedKeys.length && + rowKeys.every((k, i) => k === orderedKeys[i]); + + if (!validShape) { + throw new StackAssertionError( + ` Row shape mismatch for table "${tableName}".\n` + + `Expected column order: [${orderedKeys.join(", ")}]\n` + + `Received column order: [${rowKeys.join(", ")}]\n` + + `Your SELECT must be explicit, ordered, and NEVER use SELECT *.\n` + + `Fix the SELECT in internalDbFetchQuery immediately.` + ); + } + + // Append mapping_name as the last parameter for metadata tracking + await externalClient.query(upsertQuery, [...Object.values(rest), mappingId]); + } +} + + +async function syncMapping( + externalClient: Client, + mappingId: string, + mapping: typeof DEFAULT_DB_SYNC_MAPPINGS[keyof typeof DEFAULT_DB_SYNC_MAPPINGS], + internalPrisma: PrismaClientWithReplica, + dbId: string, + tenancyId: string, + dbType: 'postgres', +): Promise { + assertNonEmptyString(mappingId, "mappingId"); + assertNonEmptyString(mapping.targetTable, "mapping.targetTable"); + assertUuid(tenancyId, "tenancyId"); + const fetchQuery = mapping.internalDbFetchQuery; + const updateQuery = mapping.externalDbUpdateQueries[dbType]; + const tableName = mapping.targetTable; + assertNonEmptyString(fetchQuery, "internalDbFetchQuery"); + assertNonEmptyString(updateQuery, "externalDbUpdateQueries"); + if (!fetchQuery.includes("$1") || !fetchQuery.includes("$2")) { + throw new StackAssertionError( + `internalDbFetchQuery must reference $1 (tenancyId) and $2 (lastSequenceId). Mapping: ${mappingId}` + ); + } + + const tableSchema = mapping.targetTableSchemas[dbType]; + await ensureExternalSchema(externalClient, tableSchema, tableName); + + let lastSequenceId = -1; + const metadataResult = await externalClient.query( + `SELECT "last_synced_sequence_id" FROM "_stack_sync_metadata" WHERE "mapping_name" = $1`, + [mappingId] + ); + if (metadataResult.rows.length > 0) { + lastSequenceId = Number(metadataResult.rows[0].last_synced_sequence_id); + } + if (!Number.isFinite(lastSequenceId)) { + throw new StackAssertionError( + `Invalid last_synced_sequence_id for mapping ${mappingId}: ${JSON.stringify(metadataResult.rows[0]?.last_synced_sequence_id)}` + ); + } + + const BATCH_LIMIT = 1000; + const maxBatchesPerMapping = getMaxBatchesPerMapping(); + let batchesProcessed = 0; + let throttled = false; + + while (true) { + assertUuid(tenancyId, "tenancyId"); + if (!Number.isFinite(lastSequenceId)) { + throw new StackAssertionError(`lastSequenceId must be a finite number for mapping ${mappingId}.`); + } + const rows = await internalPrisma.$replica().$queryRawUnsafe(fetchQuery, tenancyId, lastSequenceId); + + if (rows.length === 0) { + break; + } + + await pushRowsToExternalDb( + externalClient, + tableName, + rows, + updateQuery, + tenancyId, + mappingId, + ); + + let maxSeqInBatch = lastSequenceId; + for (const row of rows) { + const seq = row.sequence_id; + if (seq != null) { + const seqNum = typeof seq === 'bigint' ? Number(seq) : Number(seq); + if (seqNum > maxSeqInBatch) { + maxSeqInBatch = seqNum; + } + } + } + lastSequenceId = maxSeqInBatch; + + if (rows.length < BATCH_LIMIT) { + break; + } + + batchesProcessed++; + if (maxBatchesPerMapping !== null && batchesProcessed >= maxBatchesPerMapping) { + throttled = true; + break; + } + } + + return throttled; +} + + +async function syncDatabase( + dbId: string, + dbConfig: CompleteConfig["dbSync"]["externalDatabases"][string], + internalPrisma: PrismaClientWithReplica, + tenancyId: string, +): Promise { + assertNonEmptyString(dbId, "dbId"); + assertUuid(tenancyId, "tenancyId"); + const dbType = dbConfig.type; + if (dbType !== 'postgres') { + throw new StackAssertionError( + `Unsupported database type '${String(dbType)}' for external DB ${dbId}. Only 'postgres' is currently supported.` + ); + } + + if (!dbConfig.connectionString) { + throw new StackAssertionError( + `Invalid configuration for external DB ${dbId}: 'connectionString' is missing.` + ); + } + assertNonEmptyString(dbConfig.connectionString, `external DB ${dbId} connectionString`); + + const externalClient = new Client({ + connectionString: dbConfig.connectionString, + }); + + let needsResync = false; + const syncResult = await Result.fromPromise((async () => { + await externalClient.connect(); + + // Always use DEFAULT_DB_SYNC_MAPPINGS - users cannot customize mappings + // because internalDbFetchQuery runs against Stack Auth's internal DB + for (const [mappingId, mapping] of Object.entries(DEFAULT_DB_SYNC_MAPPINGS)) { + const mappingThrottled = await syncMapping( + externalClient, + mappingId, + mapping, + internalPrisma, + dbId, + tenancyId, + dbType, + ); + if (mappingThrottled) { + needsResync = true; + } + } + })()); + + const closeResult = await Result.fromPromise(externalClient.end()); + if (closeResult.status === "error") { + captureError(`external-db-sync-${dbId}-close`, closeResult.error); + } + + if (syncResult.status === "error") { + captureError(`external-db-sync-${dbId}`, syncResult.error); + return false; + } + + return needsResync; +} + + +export async function syncExternalDatabases(tenancy: Tenancy): Promise { + assertUuid(tenancy.id, "tenancy.id"); + const externalDatabases = tenancy.config.dbSync.externalDatabases; + const internalPrisma = await getPrismaClientForTenancy(tenancy); + let needsResync = false; + + for (const [dbId, dbConfig] of Object.entries(externalDatabases)) { + try { + const databaseThrottled = await syncDatabase(dbId, dbConfig, internalPrisma, tenancy.id); + if (databaseThrottled) { + needsResync = true; + } + } catch (error) { + // Log the error but continue syncing other databases + // This ensures one bad database config doesn't block successful syncs to other databases + captureError(`external-db-sync-${dbId}`, error); + } + } + + return needsResync; +} diff --git a/apps/backend/src/lib/tokens.tsx b/apps/backend/src/lib/tokens.tsx index deda644dd4..0f5f96eea5 100644 --- a/apps/backend/src/lib/tokens.tsx +++ b/apps/backend/src/lib/tokens.tsx @@ -1,5 +1,6 @@ import { usersCrudHandlers } from '@/app/api/latest/users/crud'; import { getPrismaClientForTenancy, globalPrismaClient } from '@/prisma-client'; +import { withExternalDbSyncUpdate } from '@/lib/external-db-sync'; import { KnownErrors } from '@stackframe/stack-shared'; import { yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; import { AccessTokenPayload } from '@stackframe/stack-shared/dist/sessions'; @@ -244,9 +245,9 @@ export async function generateAccessTokenFromRefreshTokenIfValid(options: Refres projectUserId: options.refreshTokenObj.projectUserId, }, }, - data: { + data: withExternalDbSyncUpdate({ lastActiveAt: now, - }, + }), }), globalPrismaClient.projectUserRefreshToken.update({ where: { diff --git a/apps/backend/src/proxy.tsx b/apps/backend/src/proxy.tsx index a413d29524..1171e73426 100644 --- a/apps/backend/src/proxy.tsx +++ b/apps/backend/src/proxy.tsx @@ -77,7 +77,7 @@ export async function proxy(request: NextRequest) { } : undefined; // ensure our clients can handle 429 responses - if (isApiRequest && !request.headers.get('x-stack-disable-artificial-development-delay') && getNodeEnvironment() === 'development' && request.method !== 'OPTIONS' && !request.url.includes(".well-known")) { + if (isApiRequest && !request.headers.get('x-stack-disable-artificial-development-delay') && getNodeEnvironment() === 'development' && request.method !== 'OPTIONS' && !request.url.includes(".well-known") && !request.url.includes("/api/latest/internal/external-db-sync/")) { const now = Date.now(); while (devRateLimitTimestamps.length > 0 && now - devRateLimitTimestamps[0] > DEV_RATE_LIMIT_WINDOW_MS) { devRateLimitTimestamps.shift(); diff --git a/apps/backend/vercel.json b/apps/backend/vercel.json index c90d9cdc3e..f40117b3e1 100644 --- a/apps/backend/vercel.json +++ b/apps/backend/vercel.json @@ -4,6 +4,14 @@ { "path": "/api/latest/internal/email-queue-step", "schedule": "* * * * *" + }, + { + "path": "/api/latest/internal/external-db-sync/poller", + "schedule": "* * * * *" + }, + { + "path": "/api/latest/internal/external-db-sync/sequencer", + "schedule": "* * * * *" } ], "github": { diff --git a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx new file mode 100644 index 0000000000..3a68d8d9d6 --- /dev/null +++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx @@ -0,0 +1,732 @@ +"use client"; + +import { PageLayout } from "../page-layout"; +import { useAdminApp } from "../use-admin-app"; +import { + Alert, + Button, + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, + Skeleton, + Switch, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, + Typography, +} from "@/components/ui"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; +import { runAsynchronously, runAsynchronouslyWithAlert } from "@stackframe/stack-shared/dist/utils/promises"; +import { urlString } from "@stackframe/stack-shared/dist/utils/urls"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { notFound } from "next/navigation"; + +const stackAppInternalsSymbol = Symbol.for("StackAuth--DO-NOT-USE-OR-YOU-WILL-BE-FIRED--StackAppInternals"); +const AUTO_REFRESH_INTERVAL_MS = 5000; + +type SequenceStats = { + total: string, + pending: string, + null_sequence_id: string, + min_sequence_id: string | null, + max_sequence_id: string | null, +}; + +type DeletedRowStats = SequenceStats & { + by_table: Array, +}; + +type PollerStats = { + total: string, + pending: string, + in_flight: string, + stale: string, + oldest_created_at_millis: number | null, + newest_created_at_millis: number | null, +}; + +type MappingStats = { + mapping_id: string, + internal_min_sequence_id: string | null, + internal_max_sequence_id: string | null, + internal_pending_count: string, +}; + +type ExternalDbMetadata = { + mapping_name: string, + last_synced_sequence_id: string, + updated_at_millis: number | null, +}; + +type ExternalDbMappingStatus = { + mapping_id: string, + internal_max_sequence_id: string | null, + last_synced_sequence_id: string | null, + updated_at_millis: number | null, + backlog: string | null, +}; + +type ExternalDbSyncStatus = { + ok: true, + generated_at_millis: number, + global: { + tenancies_total: string, + tenancies_with_db_sync: string, + sequencer: { + project_users: SequenceStats, + contact_channels: SequenceStats, + deleted_rows: DeletedRowStats, + }, + poller: PollerStats, + sync_engine: { + mappings: MappingStats[], + }, + } | null, + tenancy: { + id: string, + project_id: string, + branch_id: string, + }, + sequencer: { + project_users: SequenceStats, + contact_channels: SequenceStats, + deleted_rows: DeletedRowStats, + }, + poller: PollerStats, + sync_engine: { + mappings: MappingStats[], + external_databases: Array<{ + id: string, + type: string, + connection: { + redacted: string | null, + host: string | null, + port: number | null, + database: string | null, + user: string | null, + }, + status: "ok" | "error", + error: string | null, + metadata: ExternalDbMetadata[], + users_table: { + exists: boolean, + total_rows: string | null, + min_signed_up_at_millis: number | null, + max_signed_up_at_millis: number | null, + }, + mapping_status: ExternalDbMappingStatus[], + }>, + }, +}; + +type ExternalDbSyncFusebox = { + sequencerEnabled: boolean, + pollerEnabled: boolean, +}; + +type ExternalDbSyncFuseboxResponse = { + ok: true, + sequencer_enabled: boolean, + poller_enabled: boolean, +}; + +type AdminAppInternals = { + sendRequest: (path: string, requestOptions: RequestInit, requestType?: "client" | "server" | "admin") => Promise, +}; + +type AdminAppWithInternals = ReturnType & { + [stackAppInternalsSymbol]: AdminAppInternals, +}; + +function formatBigInt(value: string | null) { + if (value === null) return "—"; + if (value.length > 15) return value; + const asNumber = Number(value); + return Number.isFinite(asNumber) ? new Intl.NumberFormat().format(asNumber) : value; +} + +function formatMillis(value: number | null) { + if (!value) return "—"; + return new Date(value).toLocaleString(); +} + +function sumBigIntStrings(values: Array) { + let total = BigInt(0); + for (const value of values) { + if (!value) continue; + if (!/^[-]?\d+$/.test(value)) continue; + total += BigInt(value); + } + return total.toString(); +} + +function parseBigIntString(value: string | null | undefined) { + if (value === null || value === undefined) return null; + if (!/^[-]?\d+$/.test(value)) return null; + return BigInt(value); +} + +const BIGINT_ZERO = BigInt(0); +const BIGINT_HUNDRED = BigInt(100); +const BIGINT_THOUSAND = BigInt(1000); + +function formatThroughput(value: bigint | number | null) { + if (value === null) return "—"; + if (typeof value === "number") { + if (!Number.isFinite(value)) return "—"; + if (value === 0) return "0/s"; + const sign = value > 0 ? "+" : ""; + const abs = Math.abs(value); + const display = abs >= 100 ? abs.toFixed(0) : abs >= 10 ? abs.toFixed(1) : abs.toFixed(2); + return `${sign}${display}/s`; + } + if (value === BIGINT_ZERO) return "0/s"; + const sign = value > BIGINT_ZERO ? "+" : ""; + const abs = value > BIGINT_ZERO ? value : -value; + const intPart = abs / BIGINT_HUNDRED; + const fracPart = abs % BIGINT_HUNDRED; + const intDisplay = new Intl.NumberFormat().format(intPart); + const fracDisplay = fracPart.toString().padStart(2, "0"); + const display = intPart === BIGINT_ZERO ? `0.${fracDisplay}` : `${intDisplay}.${fracDisplay}`; + return `${sign}${display}/s`; +} + +function calculateThroughputScaled(prev: bigint | null, current: bigint | null, deltaMillis: number) { + if (prev === null || current === null) return null; + if (deltaMillis <= 0) return null; + const deltaMillisBigInt = BigInt(deltaMillis); + return (prev - current) * BIGINT_THOUSAND * BIGINT_HUNDRED / deltaMillisBigInt; +} + +function DataValue(props: { value: string | null | undefined, loading: boolean }) { + if (props.loading) { + return ; + } + return {formatBigInt(props.value ?? null)}; +} + +function DataDate(props: { value: number | null | undefined, loading: boolean }) { + if (props.loading) { + return ; + } + return {formatMillis(props.value ?? null)}; +} + +export default function PageClient() { + const adminApp = useAdminApp() as AdminAppWithInternals; + const [status, setStatus] = useState(null); + const [fusebox, setFusebox] = useState(null); + const [savedFusebox, setSavedFusebox] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(false); + const [autoRefresh, setAutoRefresh] = useState(true); + const [savingFusebox, setSavingFusebox] = useState(false); + const inFlightRef = useRef(false); + const summarySamplesRef = useRef>([]); + + const loadStatus = useCallback(async () => { + if (inFlightRef.current) return; + inFlightRef.current = true; + setLoading(true); + + const result = await Result.fromPromise((async () => { + const response = await adminApp[stackAppInternalsSymbol].sendRequest( + "/internal/external-db-sync/status?scope=all", + { method: "GET" }, + "admin", + ); + const body = await response.json(); + if (!response.ok) { + const message = typeof body?.error === "string" ? body.error : "Failed to load external DB sync status."; + throw new Error(message); + } + return body as ExternalDbSyncStatus; + })()); + + if (result.status === "error") { + const message = result.error instanceof Error ? result.error.message : String(result.error); + setError(message); + setLoading(false); + inFlightRef.current = false; + return; + } + + setStatus(result.data); + setError(null); + setLoading(false); + inFlightRef.current = false; + }, [adminApp]); + + const loadFusebox = useCallback(async () => { + const result = await Result.fromPromise((async () => { + const response = await adminApp[stackAppInternalsSymbol].sendRequest( + urlString`/internal/external-db-sync/fusebox`, + { method: "GET" }, + "admin", + ); + const body = await response.json(); + if (!response.ok) { + const message = typeof body?.error === "string" ? body.error : "Failed to load external DB sync fusebox."; + throw new Error(message); + } + return body as ExternalDbSyncFuseboxResponse; + })()); + + if (result.status === "error") { + const message = result.error instanceof Error ? result.error.message : String(result.error); + setError(message); + return; + } + + const nextFusebox = { + sequencerEnabled: result.data.sequencer_enabled, + pollerEnabled: result.data.poller_enabled, + }; + setFusebox(nextFusebox); + setSavedFusebox(nextFusebox); + setError(null); + }, [adminApp]); + + const saveFusebox = useCallback(async () => { + if (!fusebox) return; + setSavingFusebox(true); + const result = await Result.fromPromise((async () => { + const response = await adminApp[stackAppInternalsSymbol].sendRequest( + urlString`/internal/external-db-sync/fusebox`, + { + method: "POST", + body: JSON.stringify({ + sequencer_enabled: fusebox.sequencerEnabled, + poller_enabled: fusebox.pollerEnabled, + }), + headers: { "content-type": "application/json" }, + }, + "admin", + ); + const body = await response.json(); + if (!response.ok) { + const message = typeof body?.error === "string" ? body.error : "Failed to update external DB sync fusebox."; + throw new Error(message); + } + return body as ExternalDbSyncFuseboxResponse; + })()); + setSavingFusebox(false); + + if (result.status === "error") { + const message = result.error instanceof Error ? result.error.message : String(result.error); + setError(message); + return; + } + + const nextFusebox = { + sequencerEnabled: result.data.sequencer_enabled, + pollerEnabled: result.data.poller_enabled, + }; + setFusebox(nextFusebox); + setSavedFusebox(nextFusebox); + setError(null); + }, [adminApp, fusebox]); + + const refreshWithAlert = useCallback(() => { + runAsynchronouslyWithAlert(loadStatus); + }, [loadStatus]); + + useEffect(() => { + runAsynchronously(loadStatus); + }, [loadStatus]); + + useEffect(() => { + runAsynchronously(loadFusebox); + }, [loadFusebox]); + + useEffect(() => { + if (!autoRefresh) return undefined; + const interval = setInterval(() => { + runAsynchronously(loadStatus); + }, AUTO_REFRESH_INTERVAL_MS); + return () => clearInterval(interval); + }, [autoRefresh, loadStatus]); + + const summaryStats = useMemo(() => { + if (!status) return null; + const summarySource = status.global ?? status; + const sequencerPending = sumBigIntStrings([ + summarySource.sequencer.project_users.pending, + summarySource.sequencer.contact_channels.pending, + summarySource.sequencer.deleted_rows.pending, + ]); + const mappingPending = sumBigIntStrings( + summarySource.sync_engine.mappings.map((mapping) => mapping.internal_pending_count), + ); + + return { + sequencerPending, + pollerPending: summarySource.poller.pending, + mappingPending, + isGlobal: Boolean(status.global), + }; + }, [status]); + + const throughputStats = useMemo(() => { + if (!status || !summaryStats) return null; + const currentSample = { + timestampMillis: status.generated_at_millis, + sequencerPending: summaryStats.sequencerPending, + pollerPending: summaryStats.pollerPending, + mappingPending: summaryStats.mappingPending, + }; + const samples = summarySamplesRef.current; + const samplesWithCurrent = samples.length === 0 || samples[samples.length - 1].timestampMillis !== currentSample.timestampMillis + ? [...samples, currentSample] + : samples; + const windowStart = status.generated_at_millis - 20000; + const windowedSamples = samplesWithCurrent.filter((sample) => sample.timestampMillis >= windowStart); + if (windowedSamples.length < 2) return null; + const oldest = windowedSamples[0]; + const deltaMillis = status.generated_at_millis - oldest.timestampMillis; + if (deltaMillis <= 0) return null; + + return { + sequencer: calculateThroughputScaled( + parseBigIntString(oldest.sequencerPending), + parseBigIntString(summaryStats.sequencerPending), + deltaMillis, + ), + poller: calculateThroughputScaled( + parseBigIntString(oldest.pollerPending), + parseBigIntString(summaryStats.pollerPending), + deltaMillis, + ), + mapping: calculateThroughputScaled( + parseBigIntString(oldest.mappingPending), + parseBigIntString(summaryStats.mappingPending), + deltaMillis, + ), + }; + }, [status, summaryStats]); + + useEffect(() => { + if (!status || !summaryStats) return; + const nextSamples = [...summarySamplesRef.current, { + timestampMillis: status.generated_at_millis, + sequencerPending: summaryStats.sequencerPending, + pollerPending: summaryStats.pollerPending, + mappingPending: summaryStats.mappingPending, + }]; + const windowStart = status.generated_at_millis - 20000; + summarySamplesRef.current = nextSamples.filter((sample) => sample.timestampMillis >= windowStart); + }, [status, summaryStats]); + + const loadingState = loading && !status; + const globalStatus = status?.global ?? null; + const deletedRowsByTable = status?.sequencer.deleted_rows.by_table ?? []; + const mappingRows = status?.sync_engine.mappings ?? []; + const fuseboxDirty = useMemo(() => { + if (!fusebox || !savedFusebox) return false; + return fusebox.sequencerEnabled !== savedFusebox.sequencerEnabled + || fusebox.pollerEnabled !== savedFusebox.pollerEnabled; + }, [fusebox, savedFusebox]); + + if (adminApp.projectId !== "internal") { + return notFound(); + } + + return ( + +
+ + Auto refresh +
+ + + } + fillWidth + > + {error && {error}} + +
+ Scope: {globalStatus ? "All tenancies" : "Current tenancy"} + {globalStatus && ( + <> + Tenancies: {formatBigInt(globalStatus.tenancies_total)} + DB sync configs: {formatBigInt(globalStatus.tenancies_with_db_sync)} + + )} + Last updated: {status ? formatMillis(status.generated_at_millis) : "—"} +
+ + +
+ + + Sequencer pending rows + + {loadingState ? : formatBigInt(summaryStats?.sequencerPending ?? null)} + + + +
ProjectUser + ContactChannel + DeletedRow rows waiting for sequence IDs.
+
+ Throughput + {loadingState ? "—" : formatThroughput(throughputStats?.sequencer ?? null)} +
+ + + + + + Outgoing sync requests + + {loadingState ? : formatBigInt(summaryStats?.pollerPending ?? null)} + + + +
Requests still queued for the poller to dispatch.
+
+ Throughput + {loadingState ? "—" : formatThroughput(throughputStats?.poller ?? null)} +
+ + + + + + Mapping pending rows + + {loadingState ? : formatBigInt(summaryStats?.mappingPending ?? null)} + + + +
Pending internal rows waiting for sync across mappings.
+
+ Throughput + {loadingState ? "—" : formatThroughput(throughputStats?.mapping ?? null)} +
+ + +
+ +
+ + + Sequencer + Rows awaiting sequence ID backfill per table. + + + + + + Table + Total + Pending + Null Seq + Min Seq + Max Seq + + + + + ProjectUser + + + + + + + + ContactChannel + + + + + + + + DeletedRow + + + + + + + +
+ +
+ + Deleted rows by table + + + + + Table + Total + Pending + Null Seq + Min Seq + Max Seq + + + + {deletedRowsByTable.map((row) => ( + + {row.table_name} + + + + + + + ))} + {!loadingState && deletedRowsByTable.length === 0 && ( + + + No deleted rows recorded yet. + + + )} + +
+
+ + + + + + Poller + OutgoingRequest queue and processing overview. + + + + + + Total + Pending + In Flight + Stale + + + + + + + + + + +
+ +
+
+ Oldest request + +
+
+ Newest request + +
+
+ + +
+ + + + Sync Engine + Internal mapping checkpoints before external sync. + + + + + + Mapping + Min Seq + Max Seq + Pending Rows + + + + {mappingRows.map((mapping) => ( + + {mapping.mapping_id} + + + + + ))} + {!loadingState && mappingRows.length === 0 && ( + + + No mappings configured. + + + )} + +
+ + + + + + Fusebox + + + {!fusebox ? ( +
+ + + + +
+ ) : ( + <> +
+
+ Sequencer + Assigns sequence IDs and queues sync work. +
+ setFusebox((current) => current ? { ...current, sequencerEnabled: checked } : current)} + /> +
+
+
+ Poller + Dispatches queued sync jobs to QStash. +
+ setFusebox((current) => current ? { ...current, pollerEnabled: checked } : current)} + /> +
+ +
+ +
+ + )} + + + + + ); +} diff --git a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page.tsx b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page.tsx new file mode 100644 index 0000000000..60d3832f9e --- /dev/null +++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page.tsx @@ -0,0 +1,9 @@ +import PageClient from "./page-client"; + +export const metadata = { + title: "External DB Sync", +}; + +export default function Page() { + return ; +} diff --git a/apps/e2e/.env.development b/apps/e2e/.env.development index 331666f8c0..42b681a546 100644 --- a/apps/e2e/.env.development +++ b/apps/e2e/.env.development @@ -10,3 +10,5 @@ STACK_INBUCKET_API_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}05 STACK_SVIX_SERVER_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}13 STACK_EMAIL_MONITOR_SECRET_TOKEN=this-secret-token-is-for-local-development-only + +CRON_SECRET=mock_cron_secret diff --git a/apps/e2e/package.json b/apps/e2e/package.json index 2993b8cb1e..29481d0f22 100644 --- a/apps/e2e/package.json +++ b/apps/e2e/package.json @@ -20,8 +20,10 @@ "js-beautify": "^1.15.4" }, "devDependencies": { + "@types/pg": "^8.15.6", "@types/js-beautify": "^1.14.3", - "jose": "^5.6.3" + "jose": "^5.6.3", + "pg": "^8.16.3" }, "packageManager": "pnpm@10.23.0" } diff --git a/apps/e2e/tests/backend/endpoints/api/v1/auth/sessions/index.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/auth/sessions/index.test.ts index 3e0d88b4bd..5766ecef34 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/auth/sessions/index.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/auth/sessions/index.test.ts @@ -100,8 +100,8 @@ it("creates sessions that expire", async ({ expect }) => { await Auth.expectToBeSignedIn(); } finally { const timeSinceBeginDate = new Date().getTime() - beginDate.getTime(); - if (timeSinceBeginDate > 4_000) { - throw new StackAssertionError(`Timeout error: Requests were too slow (${timeSinceBeginDate}ms > 4000ms); try again or try to understand why they were slow.`); + if (timeSinceBeginDate > 6_000) { + throw new StackAssertionError(`Timeout error: Requests were too slow (${timeSinceBeginDate}ms > 6000ms); try again or try to understand why they were slow.`); } } await waitPromise; diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-advanced.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-advanced.test.ts new file mode 100644 index 0000000000..34b31f601e --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-advanced.test.ts @@ -0,0 +1,1124 @@ +import { Client } from 'pg'; +import { afterAll, beforeAll, describe, expect } from 'vitest'; +import { test } from '../../../../helpers'; +import { InternalApiKey, User, backendContext, niceBackendFetch } from '../../../backend-helpers'; +import { + HIGH_VOLUME_TIMEOUT, + POSTGRES_HOST, + POSTGRES_PASSWORD, + POSTGRES_USER, + TEST_TIMEOUT, + TestDbManager, + createProjectWithExternalDb as createProjectWithExternalDbRaw, + verifyNotInExternalDb, + waitForCondition, + waitForSyncedData, + waitForSyncedDeletion, + waitForTable +} from './external-db-sync-utils'; + +const COMPLEX_SEQUENCE_TIMEOUT = TEST_TIMEOUT * 2 + 30_000; + +describe.sequential('External DB Sync - Advanced Tests', () => { + let dbManager: TestDbManager; + const createProjectWithExternalDb = ( + externalDatabases: any, + projectOptions?: { display_name?: string, description?: string } + ) => { + return createProjectWithExternalDbRaw( + externalDatabases, + projectOptions, + { projectTracker: dbManager.createdProjects } + ); + }; + + beforeAll(async () => { + dbManager = new TestDbManager(); + await dbManager.init(); + }); + + afterAll(async () => { + await dbManager.cleanup(); + }); + + /** + * What it does: + * - Creates two separate projects with different external DB lists, one user per project, and triggers sync. + * - Queries every database to confirm each tenant’s user only appears in its own configured targets. + * + * Why it matters: + * - Prevents tenant data leakage by proving cross-project isolation at the sync layer. + */ + test('Multi-Tenant Isolation: User 1 -> 2 DBs, User 2 -> 3 DBs', async () => { + await InternalApiKey.createAndSetProjectKeys(); + + const db_a1 = await dbManager.createDatabase('tenant_a_db1'); + const db_a2 = await dbManager.createDatabase('tenant_a_db2'); + const db_b1 = await dbManager.createDatabase('tenant_b_db1'); + const db_b2 = await dbManager.createDatabase('tenant_b_db2'); + const db_b3 = await dbManager.createDatabase('tenant_b_db3'); + + await createProjectWithExternalDb({ + main_a1: { + type: 'postgres', + connectionString: db_a1, + }, + main_a2: { + type: 'postgres', + connectionString: db_a2, + } + }); + + const userA = await User.create({ primary_email: 'user-a@example.com' }); + await niceBackendFetch(`/api/v1/users/${userA.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User A' } + }); + + await createProjectWithExternalDb({ + main_b1: { + type: 'postgres', + connectionString: db_b1, + }, + main_b2: { + type: 'postgres', + connectionString: db_b2, + }, + main_b3: { + type: 'postgres', + connectionString: db_b3, + } + }); + + const userB = await User.create({ primary_email: 'user-b@example.com' }); + await niceBackendFetch(`/api/v1/users/${userB.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User B' } + }); + + const clientA1 = dbManager.getClient('tenant_a_db1'); + const clientA2 = dbManager.getClient('tenant_a_db2'); + const clientB1 = dbManager.getClient('tenant_b_db1'); + const clientB2 = dbManager.getClient('tenant_b_db2'); + const clientB3 = dbManager.getClient('tenant_b_db3'); + + await waitForCondition( + async () => { + try { + const res1 = await clientA1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + const res2 = await clientA2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + return res1.rows.length === 1 && res2.rows.length === 1; + } catch (err: any) { + if (err.code === '42P01') return false; + throw err; + } + }, + { description: 'User A to appear in both Project A databases', timeoutMs: 120000 } + ); + + await waitForCondition( + async () => { + try { + const res1 = await clientB1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + const res2 = await clientB2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + const res3 = await clientB3.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + return res1.rows.length === 1 && res2.rows.length === 1 && res3.rows.length === 1; + } catch (err: any) { + if (err.code === '42P01') return false; + throw err; + } + }, + { description: 'User B to appear in all three Project B databases', timeoutMs: 120000 } + ); + + const resA1 = await clientA1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + expect(resA1.rows.length).toBe(1); + expect(resA1.rows[0].display_name).toBe('User A'); + + const resA2 = await clientA2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + expect(resA2.rows.length).toBe(1); + expect(resA2.rows[0].display_name).toBe('User A'); + + const resB1_A = await clientB1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + expect(resB1_A.rows.length).toBe(0); + + const resB2_A = await clientB2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + expect(resB2_A.rows.length).toBe(0); + + const resB3_A = await clientB3.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + expect(resB3_A.rows.length).toBe(0); + + const resB1 = await clientB1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + expect(resB1.rows.length).toBe(1); + expect(resB1.rows[0].display_name).toBe('User B'); + + const resB2 = await clientB2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + expect(resB2.rows.length).toBe(1); + expect(resB2.rows[0].display_name).toBe('User B'); + + const resB3 = await clientB3.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + expect(resB3.rows.length).toBe(1); + expect(resB3.rows[0].display_name).toBe('User B'); + + const resA1_B = await clientA1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + expect(resA1_B.rows.length).toBe(0); + + const resA2_B = await clientA2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + expect(resA2_B.rows.length).toBe(0); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Syncs three baseline users to capture their sequence ordering, then exports a fourth user. + * - Compares sequenceIds to ensure the newest export exceeds the previous maximum. + * + * Why it matters: + * - Verifies metadata table tracks progress correctly for incremental sync. + */ + test('Metadata Tracking: Verify sync progress is tracked in metadata table', async () => { + const dbName = 'metadata_tracking_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user1 = await User.create({ primary_email: 'seq1@example.com' }); + const user2 = await User.create({ primary_email: 'seq2@example.com' }); + const user3 = await User.create({ primary_email: 'seq3@example.com' }); + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 1' } + }); + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 2' } + }); + await niceBackendFetch(`/api/v1/users/${user3.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 3' } + }); + + await waitForTable(client, 'users'); + + await waitForCondition( + async () => { + const res = await client.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(res.rows[0].count) === 3; + }, + { description: 'all 3 users to be synced' } + ); + + const res1 = await client.query(`SELECT * FROM "users" ORDER BY "primary_email"`); + expect(res1.rows.length).toBe(3); + + // Check metadata table tracks progress + const metadata1 = await client.query( + `SELECT "last_synced_sequence_id" FROM "_stack_sync_metadata" WHERE "mapping_name" = 'users'` + ); + expect(metadata1.rows.length).toBe(1); + const seq1 = Number(metadata1.rows[0].last_synced_sequence_id); + expect(seq1).toBeGreaterThan(0); + + const user4 = await User.create({ primary_email: 'seq4@example.com' }); + await niceBackendFetch(`/api/v1/users/${user4.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 4' } + }); + + await waitForSyncedData(client, 'seq4@example.com', 'User 4'); + const res2 = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['seq4@example.com']); + expect(res2.rows.length).toBe(1); + + // Metadata should have advanced + const metadata2 = await client.query( + `SELECT "last_synced_sequence_id" FROM "_stack_sync_metadata" WHERE "mapping_name" = 'users'` + ); + const seq2 = Number(metadata2.rows[0].last_synced_sequence_id); + expect(seq2).toBeGreaterThan(seq1); + + const finalRes = await client.query(`SELECT COUNT(*) as count FROM "users"`); + expect(parseInt(finalRes.rows[0].count)).toBe(4); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Exports a single user, then syncs again after adding a second user. + * - Ensures the first user's data stays untouched and both users exist. + * + * Why it matters: + * - Confirms repeated sync runs don't duplicate or rewrite already exported rows. + */ + test('Idempotency & Resume: Multiple syncs should not duplicate', async () => { + const dbName = 'idempotency_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const user1 = await User.create({ primary_email: 'user1@example.com' }); + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 1' } + }); + + const client = dbManager.getClient(dbName); + + await waitForSyncedData(client, 'user1@example.com', 'User 1'); + + let res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user1@example.com']); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('User 1'); + const user1Id = res.rows[0].id; + + const user2 = await User.create({ primary_email: 'user2@example.com' }); + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 2' } + }); + + await waitForSyncedData(client, 'user2@example.com', 'User 2'); + + const user1Row = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user1@example.com']); + const user2Row = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user2@example.com']); + + expect(user1Row.rows.length).toBe(1); + expect(user2Row.rows.length).toBe(1); + expect(user1Row.rows[0].display_name).toBe('User 1'); + expect(user2Row.rows[0].display_name).toBe('User 2'); + // User 1's ID should be unchanged + expect(user1Row.rows[0].id).toBe(user1Id); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Exports a user whose display name contains quotes, emoji, and non-Latin characters. + * - Queries users to confirm the string survives unchanged. + * + * Why it matters: + * - Ensures text encoding and escaping don’t corrupt data during sync. + */ + test('Special Characters: Emojis, quotes, international symbols', async () => { + const dbName = 'special_chars_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const specialName = "O'Connor 🚀 用户 \"Test\""; + const user = await User.create({ primary_email: 'special@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: specialName } + }); + + await waitForSyncedData(dbManager.getClient(dbName), 'special@example.com', specialName); + + const client = dbManager.getClient(dbName); + const res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['special@example.com']); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe(specialName); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates 200 users directly in the internal database using SQL (much faster than API). + * - Waits for all of them to sync to the external database. + * + * Why it matters: + * - Exercises batching code paths to ensure high volumes eventually flush completely. + */ + test('High Volume: 200+ users to test batching', async () => { + const dbName = 'high_volume_test'; + const externalConnectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString: externalConnectionString, + } + }); + + const projectKeys = backendContext.value.projectKeys; + if (projectKeys === "no-project") throw new Error("No project keys found"); + const projectId = projectKeys.projectId; + const externalClient = dbManager.getClient(dbName); + + // Connect to internal database to insert users directly + const internalClient = new Client({ + connectionString: `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/stackframe`, + }); + await internalClient.connect(); + + const userCount = 200; + + try { + // Get the tenancy ID for this project + const tenancyRes = await internalClient.query( + `SELECT id FROM "Tenancy" WHERE "projectId" = $1 AND "branchId" = 'main' LIMIT 1`, + [projectId] + ); + if (tenancyRes.rows.length === 0) { + throw new Error(`Tenancy not found for project ${projectId}`); + } + const tenancyId = tenancyRes.rows[0].id; + + // Insert all 200 users in a single batch + await internalClient.query(` + WITH generated AS ( + SELECT + $1::uuid AS tenancy_id, + $2::uuid AS project_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS contact_id, + gs AS idx, + now() AS ts + FROM generate_series(1, $3::int) AS gs + ), + insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", + "displayName", "createdAt", "updatedAt", "isAnonymous") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'HV User ' || idx, + ts, + ts, + false + FROM generated + RETURNING "tenancyId", "projectUserId" + ) + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", + "isVerified", "value", "createdAt", "updatedAt") + SELECT + g.tenancy_id, + g.project_user_id, + g.contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'hv-user-' || g.idx || '@test.example.com', + g.ts, + g.ts + FROM generated g + `, [tenancyId, projectId, userCount]); + + await waitForTable(externalClient, 'users'); + + await waitForCondition( + async () => { + const res = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(res.rows[0].count) >= userCount; + }, + { description: `all ${userCount} users to be synced`, timeoutMs: 120000 } + ); + + const res = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + const finalCount = parseInt(res.rows[0].count); + expect(finalCount).toBeGreaterThanOrEqual(userCount); + } finally { + await internalClient.end(); + } + }, HIGH_VOLUME_TIMEOUT); + + /** + * What it does: + * - Starts with three users, then mixes updates, deletes, and inserts before re-syncing. + * - Validates the external table reflects the final expected set. + * + * Why it matters: + * - Proves sequencing rules handle interleaved operations correctly. + */ + test('Complex Sequence: Multiple operations in different orders', async () => { + const dbName = 'complex_sequence_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const user1 = await User.create({ primary_email: 'seq1@example.com' }); + const user2 = await User.create({ primary_email: 'seq2@example.com' }); + const user3 = await User.create({ primary_email: 'seq3@example.com' }); + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 1' } + }); + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 2' } + }); + await niceBackendFetch(`/api/v1/users/${user3.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 3' } + }); + + const client = dbManager.getClient(dbName); + + await waitForCondition( + async () => { + try { + const res = await client.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(res.rows[0].count) === 3; + } catch (err: any) { + if (err.code === '42P01') return false; + throw err; + } + }, + { description: 'initial 3 users sync', timeoutMs: 120000 } + ); + + let res = await client.query(`SELECT COUNT(*) as count FROM "users"`); + expect(parseInt(res.rows[0].count)).toBe(3); + + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 2 Updated' } + }); + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + const user4 = await User.create({ primary_email: 'seq4@example.com' }); + await niceBackendFetch(`/api/v1/users/${user4.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 4' } + }); + + await waitForCondition( + async () => { + try { + const res = await client.query(`SELECT * FROM "users" ORDER BY "primary_email"`); + if (res.rows.length !== 3) return false; + + const emails = res.rows.map(r => r.primary_email); + if (emails.includes('seq1@example.com')) return false; + if (!emails.includes('seq2@example.com')) return false; + if (!emails.includes('seq3@example.com')) return false; + if (!emails.includes('seq4@example.com')) return false; + + const user2Row = res.rows.find(r => r.primary_email === 'seq2@example.com'); + return user2Row.display_name === 'User 2 Updated'; + } catch (err: any) { + if (err.code === '42P01') return false; + throw err; + } + }, + { description: 'final sync state correct', timeoutMs: 120000 } + ); + + res = await client.query(`SELECT * FROM "users" ORDER BY "primary_email"`); + expect(res.rows.length).toBe(3); + + const emails = res.rows.map(r => r.primary_email); + expect(emails).not.toContain('seq1@example.com'); + expect(emails).toContain('seq2@example.com'); + expect(emails).toContain('seq3@example.com'); + expect(emails).toContain('seq4@example.com'); + + const user2Row = res.rows.find(r => r.primary_email === 'seq2@example.com'); + expect(user2Row.display_name).toBe('User 2 Updated'); + }, COMPLEX_SEQUENCE_TIMEOUT); + + /** + * What it does: + * - Creates a readonly database role, grants SELECT on users, and tests SELECT/INSERT/UPDATE/DELETE commands. + * - Expects reads to succeed while writes fail. + * + * Why it matters: + * - Protects external tables from being mutated by consumers using readonly credentials. + */ + test('External write protection: readonly client cannot modify users', async () => { + const dbName = 'write_protection_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const superClient = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'write-protect@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Write Protect User' }, + }); + await waitForTable(superClient, 'users'); + await waitForSyncedData(superClient, 'write-protect@example.com', 'Write Protect User'); + + const readonlyUser = 'readonly_partialusers'; + const readonlyPassword = 'readonly_password'; + await superClient.query(`DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${readonlyUser}') THEN + CREATE ROLE ${readonlyUser} LOGIN PASSWORD '${readonlyPassword}'; + END IF; +END +$$;`); + + const url = new URL(connectionString); + url.username = readonlyUser; + url.password = readonlyPassword; + const readonlyClient = new Client({ connectionString: url.toString() }); + await readonlyClient.connect(); + + try { + const selectRes = await readonlyClient.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['write-protect@example.com'], + ); + expect(selectRes.rows.length).toBe(1); + await expect( + readonlyClient.query( + `INSERT INTO "users" ("id", "primary_email") VALUES (gen_random_uuid(), $1)`, + ['should-not-insert@example.com'], + ), + ).rejects.toThrow(); + + await expect( + readonlyClient.query( + `UPDATE "users" SET "display_name" = 'Hacked' WHERE "primary_email" = $1`, + ['write-protect@example.com'], + ), + ).rejects.toThrow(); + + await expect( + readonlyClient.query( + `DELETE FROM "users" WHERE "primary_email" = $1`, + ['write-protect@example.com'], + ), + ).rejects.toThrow(); + } finally { + await readonlyClient.end(); + } + }, TEST_TIMEOUT); + + /** + * What it does: + * - Patches the same user three times without syncing, then syncs once. + * - Checks users to confirm only the final name persists. + * + * Why it matters: + * - Verifies we export the latest snapshot instead of intermediate states. + */ + test('Multiple updates before sync: last update wins', async () => { + const dbName = 'multi_update_before_sync_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'multi-update@example.com' }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Name v1' }, + }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Name v2' }, + }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Name v3' }, + }); + + await waitForTable(client, 'users'); + await waitForSyncedData(client, 'multi-update@example.com', 'Name v3'); + + const row = await client.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['multi-update@example.com'], + ); + expect(row.rows.length).toBe(1); + expect(row.rows[0].display_name).toBe('Name v3'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates then deletes a user before the first sync happens. + * - Runs sync and checks that users never receives the email. + * + * Why it matters: + * - Ensures we don’t leak records that were deleted before the initial export cycle. + */ + test('Delete before first sync: row is never exported', async () => { + const dbName = 'delete_before_first_sync_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'delete-before-sync@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'To Be Deleted' }, + }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForTable(client, 'users'); + + await waitForCondition( + async () => { + const res = await client.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['delete-before-sync@example.com'], + ); + return res.rows.length === 0; + }, + { description: 'deleted user should never appear', timeoutMs: 120000 } + ); + + const res = await client.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['delete-before-sync@example.com'], + ); + expect(res.rows.length).toBe(0); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Syncs a user, deletes it, recreates the same email, and syncs again. + * - Compares IDs and sequenceIds to confirm the new row is distinct and persistent. + * + * Why it matters: + * - Proves a previous delete doesn’t block future users with the same email. + */ + test('Re-create same email after delete exports fresh contact channel', async () => { + const dbName = 'recreate_email_after_delete_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + const email = 'recreate-after-delete@example.com'; + + const firstUser = await User.create({ primary_email: email }); + await niceBackendFetch(`/api/v1/users/${firstUser.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Original Export' }, + }); + + await waitForSyncedData(client, email, 'Original Export'); + + let res = await client.query( + `SELECT "id" FROM "users" WHERE "primary_email" = $1`, + [email], + ); + expect(res.rows.length).toBe(1); + const firstId = res.rows[0].id; + + await niceBackendFetch(`/api/v1/users/${firstUser.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedDeletion(client, email); + await verifyNotInExternalDb(client, email); + + const secondUser = await User.create({ primary_email: email }); + await niceBackendFetch(`/api/v1/users/${secondUser.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Recreated Export' }, + }); + + await waitForSyncedData(client, email, 'Recreated Export'); + + res = await client.query( + `SELECT "id", "display_name" FROM "users" WHERE "primary_email" = $1`, + [email], + ); + expect(res.rows.length).toBe(1); + + const recreatedRow = res.rows[0]; + expect(recreatedRow.display_name).toBe('Recreated Export'); + expect(recreatedRow.id).not.toBe(firstId); + + await waitForCondition( + async () => { + const followUp = await client.query( + `SELECT "display_name" FROM "users" WHERE "primary_email" = $1`, + [email], + ); + return followUp.rows.length === 1 && followUp.rows[0].display_name === 'Recreated Export'; + }, + { description: 'recreated row persists after extra sync', timeoutMs: 120000 }, + ); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Performs a complex sequence: create → update → update → delete → create (same email) → update + * - Syncs after each phase and verifies the external DB reflects the correct state. + * + * Why it matters: + * - Proves the sync engine handles rapid lifecycle transitions on the same email correctly. + */ + test('Complex lifecycle: create → update → update → delete → create → update', async () => { + const dbName = 'complex_lifecycle_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + const email = 'lifecycle-test@example.com'; + + const user1 = await User.create({ primary_email: email }); + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Initial Name' }, + }); + + await waitForSyncedData(client, email, 'Initial Name'); + + let res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Initial Name'); + const firstId = res.rows[0].id; + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Updated Once' }, + }); + + await waitForSyncedData(client, email, 'Updated Once'); + + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Updated Once'); + expect(res.rows[0].id).toBe(firstId); + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Updated Twice' }, + }); + + await waitForSyncedData(client, email, 'Updated Twice'); + + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Updated Twice'); + expect(res.rows[0].id).toBe(firstId); + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedDeletion(client, email); + + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(0); + + const user2 = await User.create({ primary_email: email }); + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Recreated User' }, + }); + + await waitForSyncedData(client, email, 'Recreated User'); + + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Recreated User'); + expect(res.rows[0].id).not.toBe(firstId); + const newId = res.rows[0].id; + + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Final Name' }, + }); + + await waitForSyncedData(client, email, 'Final Name'); + + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Final Name'); + expect(res.rows[0].id).toBe(newId); + }, COMPLEX_SEQUENCE_TIMEOUT); + + /** + * What it does: + * - Exports 50 users, deletes 10, inserts 10 replacements, and syncs again. + * - Validates the final users dataset contains the remaining 40 originals plus 10 replacements (total 50). + * + * Why it matters: + * - Proves high-volume batches stay accurate even when deletes and inserts interleave. + */ + test('High volume with deletes interleaved retains the expected dataset', async () => { + const dbName = 'high_volume_delete_mix_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const projectKeys = backendContext.value.projectKeys; + if (projectKeys === "no-project") throw new Error("No project keys found"); + const projectId = projectKeys.projectId; + + const externalClient = dbManager.getClient(dbName); + const initialUserCount = 50; + const deletions = 10; + const replacements = 10; + + // Connect to internal database to insert users directly + const internalClient = new Client({ + connectionString: `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/stackframe`, + }); + await internalClient.connect(); + + let initialUsers: { projectUserId: string, email: string }[] = []; + + try { + // Get the tenancy ID for this project + const tenancyRes = await internalClient.query( + `SELECT id FROM "Tenancy" WHERE "projectId" = $1 AND "branchId" = 'main' LIMIT 1`, + [projectId] + ); + if (tenancyRes.rows.length === 0) { + throw new Error(`Tenancy not found for project ${projectId}`); + } + const tenancyId = tenancyRes.rows[0].id; + const testRunId = `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + + // Insert initial users and get their IDs back + const insertResult = await internalClient.query(` + WITH generated AS ( + SELECT + $1::uuid AS tenancy_id, + $2::uuid AS project_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS contact_id, + gs AS idx, + now() AS ts + FROM generate_series(1, $3::int) AS gs + ), + insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", + "displayName", "createdAt", "updatedAt", "isAnonymous") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'Interleave User ' || idx, + ts, + ts, + false + FROM generated + RETURNING "projectUserId" + ), + insert_contacts AS ( + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", + "isVerified", "value", "createdAt", "updatedAt") + SELECT + g.tenancy_id, + g.project_user_id, + g.contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'interleave-' || g.idx || '-' || $4 || '@example.com', + g.ts, + g.ts + FROM generated g + RETURNING "projectUserId", "value" AS email + ) + SELECT "projectUserId"::text, email FROM insert_contacts ORDER BY email + `, [tenancyId, projectId, initialUserCount, testRunId]); + + initialUsers = insertResult.rows.map(row => ({ + email: row.email, + projectUserId: row.projectUserId, + })); + + await waitForTable(externalClient, 'users'); + + await waitForCondition( + async () => { + const countRes = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(countRes.rows[0].count) === initialUserCount; + }, + { description: 'initial batch exported', timeoutMs: 60000 }, + ); + + // Delete first 10 users + const deletedUsers = initialUsers.slice(0, deletions); + for (const entry of deletedUsers) { + await niceBackendFetch(`/api/v1/users/${entry.projectUserId}`, { + accessType: 'admin', + method: 'DELETE', + }); + } + await waitForCondition( + async () => { + const countRes = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(countRes.rows[0].count) === (initialUserCount - deletions); + }, + { description: 'deletions synced to external DB', timeoutMs: 180000 }, + ); + + // Insert replacement users via direct SQL + const replacementResult = await internalClient.query(` + WITH generated AS ( + SELECT + $1::uuid AS tenancy_id, + $2::uuid AS project_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS contact_id, + gs AS idx, + now() AS ts + FROM generate_series(1, $3::int) AS gs + ), + insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", + "displayName", "createdAt", "updatedAt", "isAnonymous") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'Replacement ' || idx, + ts, + ts, + false + FROM generated + RETURNING "projectUserId" + ), + insert_contacts AS ( + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", + "isVerified", "value", "createdAt", "updatedAt") + SELECT + g.tenancy_id, + g.project_user_id, + g.contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'interleave-replacement-' || g.idx || '-' || $4 || '@example.com', + g.ts, + g.ts + FROM generated g + RETURNING "value" AS email + ) + SELECT email FROM insert_contacts + `, [tenancyId, projectId, replacements, testRunId]); + + const replacementEmails = replacementResult.rows.map(row => row.email); + + const expectedFinalCount = initialUserCount - deletions + replacements; + await waitForCondition( + async () => { + const countRes = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(countRes.rows[0].count) === expectedFinalCount; + }, + { description: 'final mixed batch exported', timeoutMs: 180000 }, + ); + + const finalRows = await externalClient.query(`SELECT "primary_email" FROM "users"`); + const finalEmails = new Set(finalRows.rows.map((row) => row.primary_email)); + expect(finalEmails.size).toBe(expectedFinalCount); + + for (const deleted of deletedUsers) { + expect(finalEmails.has(deleted.email)).toBe(false); + } + for (const survivor of initialUsers.slice(deletions)) { + expect(finalEmails.has(survivor.email)).toBe(true); + } + for (const replacement of replacementEmails) { + expect(finalEmails.has(replacement)).toBe(true); + } + } finally { + await internalClient.end(); + } + }, HIGH_VOLUME_TIMEOUT); +}); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts new file mode 100644 index 0000000000..acc149707a --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts @@ -0,0 +1,489 @@ +import { afterAll, beforeAll, describe, expect } from 'vitest'; +import { test } from '../../../../helpers'; +import { User, niceBackendFetch } from '../../../backend-helpers'; +import { + TEST_TIMEOUT, + TestDbManager, + createProjectWithExternalDb as createProjectWithExternalDbRaw, + verifyInExternalDb, + verifyNotInExternalDb, + waitForCondition, + waitForSyncedData, + waitForSyncedDeletion, + waitForTable +} from './external-db-sync-utils'; + +// Run tests sequentially to avoid concurrency issues with shared backend state +describe.sequential('External DB Sync - Basic Tests', () => { + let dbManager: TestDbManager; + const createProjectWithExternalDb = ( + externalDatabases: any, + projectOptions?: { display_name?: string, description?: string } + ) => { + return createProjectWithExternalDbRaw( + externalDatabases, + projectOptions, + { projectTracker: dbManager.createdProjects } + ); + }; + + beforeAll(async () => { + dbManager = new TestDbManager(); + await dbManager.init(); + }); + + afterAll(async () => { + await dbManager.cleanup(); + }); + + /** + * What it does: + * - Creates a user, patches the display name, and triggers the sync once. + * - Checks the users table for a matching row only after the sync completes. + * + * Why it matters: + * - Ensures inserts never appear externally until the sync pipeline runs. + */ + test('Insert: New user is synced to external DB', async () => { + const dbName = 'insert_only_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'insert-only@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Insert Only User' } + }); + + await waitForSyncedData(client, 'insert-only@example.com', 'Insert Only User'); + + await verifyInExternalDb(client, 'insert-only@example.com', 'Insert Only User'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Exports a baseline row, mutates the display name, runs another sync, and reads users table. + * - Compares the stored display name to guarantee it reflects the latest mutation. + * + * Why it matters: + * - Proves updates propagate to the external DB instead of leaving stale data. + */ + test('Update: Existing user changes are reflected in external DB', async () => { + const dbName = 'update_only_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'update-only@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Before Update' } + }); + + await waitForSyncedData(client, 'update-only@example.com', 'Before Update'); + + await verifyInExternalDb(client, 'update-only@example.com', 'Before Update'); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'After Update' } + }); + + await waitForSyncedData(client, 'update-only@example.com', 'After Update'); + + await verifyInExternalDb(client, 'update-only@example.com', 'After Update'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Syncs a user into the users table, deletes the user internally, and waits for the deletion helper. + * - Queries users table to ensure the row disappears. + * + * Why it matters: + * - Validates deletion events propagate and prevent orphaned rows in external DBs. + */ + test('Delete: Deleted user is removed from external DB', async () => { + const dbName = 'delete_only_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }, { + display_name: '🗑️ Delete Test Project', + description: 'Testing deletion sync to external database' + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'delete-only@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Delete Only User' } + }); + + await waitForSyncedData(client, 'delete-only@example.com', 'Delete Only User'); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + const deletedUserResponse = await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'GET', + }); + expect(deletedUserResponse.status).toBe(404); + + await waitForSyncedDeletion(client, 'delete-only@example.com'); + await verifyNotInExternalDb(client, 'delete-only@example.com'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user while verifying the users table is absent before sync. + * - Triggers sync, waits for table creation, and confirms the row appears afterward. + * + * Why it matters: + * - Demonstrates that syncs control both table provisioning and data export timing. + */ + test('Sync Mechanism Verification: Data appears ONLY after sync', async () => { + const dbName = 'sync_verification_test'; + const connectionString = await dbManager.createDatabase(dbName); + + const client = dbManager.getClient(dbName); + + // Verify the fresh database has no users table BEFORE we configure sync + const tableCheckBefore = await client.query(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'users' + ); + `); + expect(tableCheckBefore.rows[0].exists).toBe(false); + + // Now configure the external DB - this will trigger sync + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }, { + display_name: '🔄 Sync Verification Test Project', + description: 'Testing that data only appears after sync is triggered' + }); + + const user = await User.create({ primary_email: 'sync-verify@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Sync Verify User' } + }); + + // Wait for sync to create the table and populate data + await waitForTable(client, 'users'); + + await waitForSyncedData(client, 'sync-verify@example.com', 'Sync Verify User'); + await verifyInExternalDb(client, 'sync-verify@example.com', 'Sync Verify User'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Runs create, update, and delete actions in order while syncing between each step. + * - Verifies the users table reflects each intermediate state. + * + * Why it matters: + * - Confirms the sync handles the entire lifecycle without leaving stale records. + */ + test('Full CRUD Lifecycle: Create, Update, Delete', async () => { + const dbName = 'crud_lifecycle_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'crud-test@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Original Name' } + }); + + await waitForSyncedData(client, 'crud-test@example.com', 'Original Name'); + + await verifyInExternalDb(client, 'crud-test@example.com', 'Original Name'); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Updated Name' } + }); + + await waitForSyncedData(client, 'crud-test@example.com', 'Updated Name'); + await verifyInExternalDb(client, 'crud-test@example.com', 'Updated Name'); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedDeletion(client, 'crud-test@example.com'); + + await verifyNotInExternalDb(client, 'crud-test@example.com'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Syncs a user into an empty database to trigger table auto-creation. + * - Queries `information_schema` and users table to confirm the table and row exist. + * + * Why it matters: + * - Ensures mappings can provision their own schema without manual migrations. + */ + test('Automatic Table Creation', async () => { + const dbName = 'auto_table_creation_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const user = await User.create({ primary_email: 'auto-create@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Auto Create User' } + }); + + const client = dbManager.getClient(dbName); + + await waitForSyncedData(client, 'auto-create@example.com', 'Auto Create User'); + + const tableCheck = await client.query(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'users' + ); + `); + expect(tableCheck.rows[0].exists).toBe(true); + await verifyInExternalDb(client, 'auto-create@example.com', 'Auto Create User'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Configures one valid and one invalid external DB mapping for the same project. + * - Runs sync and verifies the healthy DB still receives the exported row. + * + * Why it matters: + * - Shows a failing database connection does not block successful targets. + */ + test('Resilience: One bad DB should not crash the sync', async () => { + const goodDbName = 'resilience_good_db'; + const goodConnectionString = await dbManager.createDatabase(goodDbName); + const badConnectionString = 'postgresql://invalid:invalid@invalid:5432/invalid'; + + await createProjectWithExternalDb({ + good_db: { + type: 'postgres', + connectionString: goodConnectionString, + }, + bad_db: { + type: 'postgres', + connectionString: badConnectionString, + } + }); + + const user = await User.create({ primary_email: 'resilience@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Resilience User' } + }); + + await waitForSyncedData(dbManager.getClient(goodDbName), 'resilience@example.com', 'Resilience User'); + + const client = dbManager.getClient(goodDbName); + const res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['resilience@example.com']); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Resilience User'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user with a primary email and adds a secondary email. + * - Verifies only one user row exists (the new schema is user-centric, not channel-centric). + * - Confirms the primary_email field contains the primary email. + * + * Why it matters: + * - Validates that the new user-centric schema syncs users, not individual contact channels. + */ + test('User with multiple emails: Only one row synced with primary email', async () => { + const dbName = 'multi_email_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'primary@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Multi Email User' } + }); + + // Add a secondary email + const secondEmailResponse = await niceBackendFetch(`/api/v1/contact-channels`, { + accessType: 'admin', + method: 'POST', + body: { + user_id: user.userId, + type: 'email', + value: 'secondary@example.com', + is_verified: false, + used_for_auth: false, + } + }); + expect(secondEmailResponse.status).toBe(201); + + await waitForSyncedData(client, 'primary@example.com', 'Multi Email User'); + + // Should only have ONE row per user (the new schema is user-centric) + const allRows = await client.query(`SELECT * FROM "users"`); + expect(allRows.rows.length).toBe(1); + + // The row should have the primary email + expect(allRows.rows[0].primary_email).toBe('primary@example.com'); + expect(allRows.rows[0].display_name).toBe('Multi Email User'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user, updates it multiple times, verifies each update is reflected. + * - Checks that the metadata table tracks the last synced sequence_id. + * + * Why it matters: + * - Demonstrates that updates are properly synced and metadata tracking works. + */ + test('Updates are synced correctly and metadata tracks progress', async () => { + const dbName = 'update_tracking_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'update-test@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Original Name' } + }); + + await waitForSyncedData(client, 'update-test@example.com', 'Original Name'); + await verifyInExternalDb(client, 'update-test@example.com', 'Original Name'); + + // Check metadata table exists and has a positive sequence_id + const metadata1 = await client.query( + `SELECT "last_synced_sequence_id" FROM "_stack_sync_metadata" WHERE "mapping_name" = 'users'` + ); + expect(metadata1.rows.length).toBe(1); + const seq1 = Number(metadata1.rows[0].last_synced_sequence_id); + expect(seq1).toBeGreaterThan(0); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Updated Name' } + }); + + await waitForSyncedData(client, 'update-test@example.com', 'Updated Name'); + await verifyInExternalDb(client, 'update-test@example.com', 'Updated Name'); + + // Metadata should have advanced + const metadata2 = await client.query( + `SELECT "last_synced_sequence_id" FROM "_stack_sync_metadata" WHERE "mapping_name" = 'users'` + ); + const seq2 = Number(metadata2.rows[0].last_synced_sequence_id); + expect(seq2).toBeGreaterThan(seq1); + }, TEST_TIMEOUT); + + + /** + * What it does: + * - Reads the external DB sync fusebox settings. + * - Writes the same values back to confirm the update endpoint. + * + * Why it matters: + * - Ensures internal fusebox controls are reachable and validated. + */ + test('Fusebox endpoint returns and accepts enablement flags', async () => { + const getResponse = await niceBackendFetch('/api/latest/internal/external-db-sync/fusebox', { + accessType: 'admin', + }); + + expect(getResponse.status).toBe(200); + expect(getResponse.body).toMatchObject({ + ok: true, + sequencer_enabled: expect.any(Boolean), + poller_enabled: expect.any(Boolean), + }); + + const postResponse = await niceBackendFetch('/api/latest/internal/external-db-sync/fusebox', { + accessType: 'admin', + method: 'POST', + body: { + sequencer_enabled: getResponse.body.sequencer_enabled, + poller_enabled: getResponse.body.poller_enabled, + }, + }); + + expect(postResponse.status).toBe(200); + expect(postResponse.body).toMatchObject({ + ok: true, + sequencer_enabled: getResponse.body.sequencer_enabled, + poller_enabled: getResponse.body.poller_enabled, + }); + }, TEST_TIMEOUT); +}); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-high-volume.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-high-volume.test.ts new file mode 100644 index 0000000000..fbd71db00d --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-high-volume.test.ts @@ -0,0 +1,187 @@ +import { Client } from 'pg'; +import { afterAll, beforeAll, describe, expect } from 'vitest'; +import { test } from '../../../../helpers'; +import { backendContext } from '../../../backend-helpers'; +import { + HIGH_VOLUME_TIMEOUT, + POSTGRES_HOST, + POSTGRES_PASSWORD, + POSTGRES_USER, + TestDbManager, + createProjectWithExternalDb as createProjectWithExternalDbRaw, + waitForCondition, + waitForTable, +} from './external-db-sync-utils'; + +// Run tests sequentially to avoid concurrency issues with shared backend state +describe.sequential('External DB Sync - High Volume Tests', () => { + let dbManager: TestDbManager; + const createProjectWithExternalDb = ( + externalDatabases: any, + projectOptions?: { display_name?: string, description?: string } + ) => { + return createProjectWithExternalDbRaw( + externalDatabases, + projectOptions, + { projectTracker: dbManager.createdProjects } + ); + }; + + beforeAll(async () => { + dbManager = new TestDbManager(); + await dbManager.init(); + }); + + afterAll(async () => { + await dbManager.cleanup(); + }, 60000); // 60 second timeout for cleanup + + /** + * What it does: + * - Creates 1500 users directly in the internal database using SQL (much faster than API) + * - Waits for all of them to sync to the external database + * + * Why it matters: + * - Ensures that when more than 1000 rows accumulate (e.g., external DB was down), + * the sync process loops and syncs all rows, not just the first 1000. + * - This tests the pagination logic in syncMapping() + */ + test('High Volume: Syncs more than 1000 users', async () => { + const dbName = 'high_volume_test'; + const externalConnectionString = await dbManager.createDatabase(dbName); + + // Create project with external DB config (this also tracks for cleanup) + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString: externalConnectionString, + } + }); + + const projectKeys = backendContext.value.projectKeys; + if (projectKeys === "no-project") throw new Error("No project keys found"); + const projectId = projectKeys.projectId; + const externalClient = dbManager.getClient(dbName); + + // Connect to internal database to insert users directly + const internalClient = new Client({ + connectionString: `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/stackframe`, + }); + await internalClient.connect(); + + const userCount = 1500; + console.log(`Inserting ${userCount} users directly into internal database...`); + + try { + // First, get the tenancy ID for this project + const tenancyRes = await internalClient.query( + `SELECT id FROM "Tenancy" WHERE "projectId" = $1 AND "branchId" = 'main' LIMIT 1`, + [projectId] + ); + if (tenancyRes.rows.length === 0) { + throw new Error(`Tenancy not found for project ${projectId}`); + } + const tenancyId = tenancyRes.rows[0].id; + console.log(`Found tenancy ID: ${tenancyId}`); + + // Insert users in batches using SQL + // This mimics what the users/crud.tsx does but without password hashing + const batchSize = 500; + for (let batch = 0; batch < userCount; batch += batchSize) { + const batchCount = Math.min(batchSize, userCount - batch); + const startIdx = batch + 1; + + await internalClient.query(` + WITH generated AS ( + SELECT + $1::uuid AS tenancy_id, + $2::uuid AS project_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS contact_id, + (gs + $3::int - 1) AS idx, + now() AS ts + FROM generate_series(1, $4::int) AS gs + ), + insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", + "displayName", "createdAt", "updatedAt", "isAnonymous") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'HV User ' || idx, + ts, + ts, + false + FROM generated + RETURNING "tenancyId", "projectUserId" + ) + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", + "isVerified", "value", "createdAt", "updatedAt") + SELECT + g.tenancy_id, + g.project_user_id, + g.contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'hv-user-' || g.idx || '@test.example.com', + g.ts, + g.ts + FROM generated g + `, [tenancyId, projectId, startIdx, batchCount]); + + console.log(`Inserted batch ${batch / batchSize + 1}: users ${startIdx} to ${startIdx + batchCount - 1}`); + } + + // Verify users were actually inserted + const verifyRes = await internalClient.query( + `SELECT COUNT(*) as count FROM "ProjectUser" WHERE "tenancyId" = $1::uuid`, + [tenancyId] + ); + console.log(`Verified ${verifyRes.rows[0].count} users in internal DB`); + + console.log(`Waiting for sync...`); + + await waitForTable(externalClient, 'users'); + + // Wait for all users to appear in the external DB + await waitForCondition( + async () => { + const res = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + const count = parseInt(res.rows[0].count, 10); + console.log(`Synced ${count}/${userCount} users`); + return count >= userCount; + }, + { + description: `all ${userCount} users to sync to external DB`, + timeoutMs: 480000, // 8 minutes + intervalMs: 5000, // Check every 5 seconds + } + ); + + // Verify the final count + const finalRes = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + const finalCount = parseInt(finalRes.rows[0].count, 10); + expect(finalCount).toBeGreaterThanOrEqual(userCount); + + // Spot-check a few specific users exist + const firstUser = await externalClient.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['hv-user-1@test.example.com']); + expect(firstUser.rows).toHaveLength(1); + + const middleUser = await externalClient.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['hv-user-750@test.example.com']); + expect(middleUser.rows).toHaveLength(1); + + const lastUser = await externalClient.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [`hv-user-${userCount}@test.example.com`]); + expect(lastUser.rows).toHaveLength(1); + + console.log(`Successfully synced all ${userCount} users!`); + } finally { + await internalClient.end(); + } + }, HIGH_VOLUME_TIMEOUT); +}); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts new file mode 100644 index 0000000000..2cb93a4912 --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts @@ -0,0 +1,410 @@ +import { Client } from 'pg'; +import { afterAll, beforeAll, describe, expect } from 'vitest'; +import { test } from '../../../../helpers'; +import { User, backendContext, niceBackendFetch } from '../../../backend-helpers'; +import { + HIGH_VOLUME_TIMEOUT, + POSTGRES_HOST, + POSTGRES_PASSWORD, + POSTGRES_USER, + TEST_TIMEOUT, + TestDbManager, + createProjectWithExternalDb as createProjectWithExternalDbRaw, + forceExternalDbSync, + waitForCondition, + waitForSyncedDeletion, + waitForTable +} from './external-db-sync-utils'; + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +describe.sequential('External DB Sync - Race Condition Tests', () => { + let dbManager: TestDbManager; + const createProjectWithExternalDb = ( + externalDatabases: any, + projectOptions?: { display_name?: string, description?: string } + ) => { + return createProjectWithExternalDbRaw( + externalDatabases, + projectOptions, + { projectTracker: dbManager.createdProjects } + ); + }; + + beforeAll(async () => { + dbManager = new TestDbManager(); + await dbManager.init(); + }); + + afterAll(async () => { + await dbManager.cleanup(); + }); + + /** + * What it does: + * - Updates a user, triggers two sync cycles concurrently, and waits for users table to show the last value. + * - Confirms only a single row exists with the final display name. + * + * Why it matters: + * - Demonstrates overlapping pollers remain idempotent instead of duplicating or reverting data. + */ + test('Concurrent sync triggers produce a single consistent export', async () => { + const dbName = 'race_parallel_sync_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + const user = await User.create({ primary_email: 'parallel-sync@example.com' }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Initial Name' }, + }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Final Name' }, + }); + + await waitForTable(client, 'users'); + + await waitForCondition( + async () => { + const res = await client.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['parallel-sync@example.com'], + ); + return res.rows.length === 1 && res.rows[0].display_name === 'Final Name'; + }, + { description: 'sync to converge on final state', timeoutMs: 90000 }, + ); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Issues a final update, deletes the user immediately afterward, and runs the deletion helper. + * - Confirms users table has zero rows for that value. + * + * Why it matters: + * - Shows delete events win over closely preceding updates, preventing stale data resurrection. + */ + test('Immediate delete after update removes the contact channel', async () => { + const dbName = 'race_update_delete_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + const user = await User.create({ primary_email: 'update-delete@example.com' }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Before Delete' }, + }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Should Be Deleted' }, + }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForTable(client, 'users'); + await waitForSyncedDeletion(client, 'update-delete@example.com'); + + const res = await client.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['update-delete@example.com'], + ); + expect(res.rows.length).toBe(0); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Exports 300 users (forcing multi-page fetches), deletes a low-sequence contact channel, and syncs again. + * - Checks the deleted row is gone and the total count drops by exactly one. + * + * Why it matters: + * - Prevents pagination LIMIT boundaries from causing delete events to be skipped. + */ + test('Deletes near pagination boundaries are honored', async () => { + const dbName = 'race_pagination_delete_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const projectKeys = backendContext.value.projectKeys; + if (projectKeys === "no-project") throw new Error("No project keys found"); + const projectId = projectKeys.projectId; + + const externalClient = dbManager.getClient(dbName); + const totalUsers = 300; + + // Connect to internal database to insert users directly + const internalClient = new Client({ + connectionString: `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/stackframe`, + }); + await internalClient.connect(); + + let users: { email: string, projectUserId: string }[] = []; + + try { + // Get the tenancy ID for this project + const tenancyRes = await internalClient.query( + `SELECT id FROM "Tenancy" WHERE "projectId" = $1 AND "branchId" = 'main' LIMIT 1`, + [projectId] + ); + if (tenancyRes.rows.length === 0) { + throw new Error(`Tenancy not found for project ${projectId}`); + } + const tenancyId = tenancyRes.rows[0].id; + + // Insert all users and get their IDs back + const insertResult = await internalClient.query(` + WITH generated AS ( + SELECT + $1::uuid AS tenancy_id, + $2::uuid AS project_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS contact_id, + gs AS idx, + now() AS ts + FROM generate_series(1, $3::int) AS gs + ), + insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", + "displayName", "createdAt", "updatedAt", "isAnonymous") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'Paged User ' || idx, + ts, + ts, + false + FROM generated + RETURNING "projectUserId" + ), + insert_contacts AS ( + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", + "isVerified", "value", "createdAt", "updatedAt") + SELECT + g.tenancy_id, + g.project_user_id, + g.contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'page-user-' || g.idx || '@example.com', + g.ts, + g.ts + FROM generated g + RETURNING "projectUserId", "value" AS email + ) + SELECT "projectUserId"::text, email FROM insert_contacts ORDER BY email + `, [tenancyId, projectId, totalUsers]); + + users = insertResult.rows.map(row => ({ + email: row.email, + projectUserId: row.projectUserId, + })); + + await waitForTable(externalClient, 'users'); + + await waitForCondition( + async () => { + const res = await externalClient.query(`SELECT COUNT(*) AS count FROM "users"`); + return parseInt(res.rows[0].count, 10) === totalUsers; + }, + { description: 'initial >300 users exported', timeoutMs: 120000 }, + ); + + // Delete user at index 1 (low sequence ID) + const deletedUser = users[1]; + await niceBackendFetch(`/api/v1/users/${deletedUser.projectUserId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForCondition( + async () => { + const res = await externalClient.query(`SELECT COUNT(*) AS count FROM "users"`); + return parseInt(res.rows[0].count, 10) === totalUsers - 1; + }, + { description: 'pagination delete reflected', timeoutMs: 180000 }, + ); + + const deletedRow = await externalClient.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + [deletedUser.email], + ); + expect(deletedRow.rows.length).toBe(0); + } finally { + await internalClient.end(); + } + }, HIGH_VOLUME_TIMEOUT); + + /** + * What it does: + * - Creates overlapping database transactions that update the same row + * - Commits them at different times while sync is happening + * - Verifies that the highest sequence ID wins in the external DB + * + * Why it matters: + * - Proves true database-level race conditions are handled correctly + * - Tests that sync captures all committed changes eventually + */ + describe('Race conditions with overlapping transactions', () => { + const LOCAL_TEST_TIMEOUT = TEST_TIMEOUT + 60_000; // Must cover baseline sync + fallback sleep on slow CI + + async function setupExternalDbWithBaseline(dbName: string) { + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const externalClient = dbManager.getClient(dbName); + const user = await User.create({ primary_email: `${dbName}@example.com` }); + + // Make sure the users row exists + await waitForTable(externalClient, 'users'); + + await waitForCondition( + async () => { + const res = await externalClient.query<{ + display_name: string | null, + }>( + ` + SELECT "display_name" + FROM "users" + WHERE "primary_email" = $1 + `, + [`${dbName}@example.com`], + ); + return res.rows.length === 1; + }, + { description: `baseline row for ${dbName}`, timeoutMs: 60000 }, + ); + + const baseline = await externalClient.query<{ + display_name: string | null, + }>( + ` + SELECT "display_name" + FROM "users" + WHERE "primary_email" = $1 + `, + [`${dbName}@example.com`], + ); + + if (baseline.rows.length !== 1) { + throw new Error(`Expected baseline row for ${dbName}, got ${baseline.rows.length}`); + } + + const baselineRow = baseline.rows[0]; + const baselineDisplayName = baselineRow.display_name; + + return { + externalClient, + user, + baselineDisplayName, + }; + } + + function makeInternalDbUrl() { + const portPrefix = process.env.NEXT_PUBLIC_STACK_PORT_PREFIX || '81'; + return `postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:${portPrefix}28/stackframe`; + } + + /** + * Scenario 1: + * Poller runs while a transaction is in-flight and uncommitted. + * Only the baseline committed value should be visible. + * + */ + test( + 'Poller ignores uncommitted overlapping updates', + async () => { + const dbName = 'race_uncommitted_poll_test'; + const { externalClient, user, baselineDisplayName } = + await setupExternalDbWithBaseline(dbName); + + const internalDbUrl = makeInternalDbUrl(); + const internalClient = new Client({ connectionString: internalDbUrl }); + + await internalClient.connect(); + + try { + await internalClient.query('BEGIN'); + await internalClient.query( + ` + UPDATE "ProjectUser" + SET "displayName" = 'Transaction 1', "updatedAt" = NOW() + WHERE "projectUserId" = $1 + `, + [user.userId], + ); + + const forced = await forceExternalDbSync(); + if (!forced) { + await sleep(70000); + } + + const during = await externalClient.query<{ + display_name: string | null, + }>( + ` + SELECT "display_name" + FROM "users" + WHERE "primary_email" = $1 + `, + [`${dbName}@example.com`], + ); + + expect(during.rows.length).toBe(1); + const row = during.rows[0]; + + // Uncommitted transaction should not be visible + expect(row.display_name).not.toBe('Transaction 1'); + expect(row.display_name).toBe(baselineDisplayName); + + await internalClient.query('ROLLBACK'); + } finally { + await internalClient.end(); + } + }, + LOCAL_TEST_TIMEOUT, + ); + }); +}); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts new file mode 100644 index 0000000000..2e588e61f7 --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -0,0 +1,387 @@ +import { Client, ClientConfig } from 'pg'; +import { expect } from 'vitest'; +import { niceFetch, STACK_BACKEND_BASE_URL } from '../../../../helpers'; +import { InternalApiKey, Project } from '../../../backend-helpers'; + + +const PORT_PREFIX = process.env.NEXT_PUBLIC_STACK_PORT_PREFIX || '81'; +export const POSTGRES_HOST = process.env.EXTERNAL_DB_TEST_HOST || `localhost:${PORT_PREFIX}28`; +export const POSTGRES_USER = process.env.EXTERNAL_DB_TEST_USER || 'postgres'; +export const POSTGRES_PASSWORD = process.env.EXTERNAL_DB_TEST_PASSWORD || 'PASSWORD-PLACEHOLDER--uqfEC1hmmv'; +export const TEST_TIMEOUT = 240000; +export const HIGH_VOLUME_TIMEOUT = 600000; // 10 minutes for 1500+ users +const SHOULD_FORCE_EXTERNAL_DB_SYNC = process.env.STACK_FORCE_EXTERNAL_DB_SYNC === 'true'; +const FORCE_SYNC_MAX_DURATION_MS = (() => { + const raw = process.env.STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS; + if (!raw) return 5000; + const parsed = Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error('STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS must be a positive integer'); + } + return parsed; +})(); +const FORCE_SYNC_INTERVAL_MS = 2000; +let lastForcedSyncAt = -Infinity; + +// Connection settings to prevent connection leaks +const CLIENT_CONFIG: Partial = { + // Timeout for connecting (10 seconds) + connectionTimeoutMillis: 10000, + // Timeout for queries (30 seconds) + query_timeout: 30000, + // Timeout for idle connections (60 seconds) + idle_in_transaction_session_timeout: 60000, +}; + +// Track all projects created with external DB configs for cleanup +export type ProjectContext = { + projectId: string, + superSecretAdminKey: string, +}; + +/** + * Helper class to manage external test databases + */ +export class TestDbManager { + private setupClient: Client | null = null; + private databases: Map = new Map(); + private databaseNames: Set = new Set(); + public readonly createdProjects: ProjectContext[] = []; + + async init() { + this.setupClient = new Client({ + connectionString: `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/postgres`, + ...CLIENT_CONFIG, + }); + await this.setupClient.connect(); + } + + async createDatabase(dbName: string): Promise { + if (!this.setupClient) throw new Error('TestDbManager not initialized'); + + const uniqueDbName = `${dbName}_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + await this.setupClient.query(`CREATE DATABASE "${uniqueDbName}"`); + const connectionString = `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${uniqueDbName}`; + const client = new Client({ + connectionString, + ...CLIENT_CONFIG, + }); + await client.connect(); + + this.databases.set(dbName, client); + this.databaseNames.add(uniqueDbName); + return connectionString; + } + + getClient(dbName: string): Client { + const client = this.databases.get(dbName); + if (!client) throw new Error(`Database ${dbName} not found`); + return client; + } + + async cleanup() { + // First, clean up all project configs to stop the sync cron from trying to connect + await cleanupProjectConfigs(this.createdProjects); + this.createdProjects.length = 0; + + // Close all tracked database clients + const closePromises = Array.from(this.databases.values()).map(async (client) => { + try { + await Promise.race([ + client.end(), + new Promise((_, reject) => setTimeout(() => reject(new Error('Client close timeout')), 5000)), + ]); + } catch (err) { + // Ignore errors when closing clients - they may already be closed or timed out + } + }); + await Promise.all(closePromises); + this.databases.clear(); + + if (this.setupClient) { + // Terminate all connections and drop databases + for (const dbName of this.databaseNames) { + try { + // Forcefully terminate ALL connections to this database + await this.setupClient.query(` + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = $1 + AND pid <> pg_backend_pid() + `, [dbName]); + + // Small delay to ensure connections are terminated + await new Promise(r => setTimeout(r, 100)); + + await this.setupClient.query(`DROP DATABASE IF EXISTS "${dbName}"`); + } catch (err) { + console.warn(`Failed to drop database ${dbName}:`, err); + } + } + this.databaseNames.clear(); + + try { + await this.setupClient.end(); + } catch (err) { + // Ignore errors when closing setup client + } + this.setupClient = null; + } + } +} + + +/** + * Wait for a condition to be true by polling, with timeout + */ +export async function waitForCondition( + checkFn: () => Promise, + options: { timeoutMs?: number, intervalMs?: number, description?: string } = {} +): Promise { + const { timeoutMs = 10000, intervalMs = 100, description = 'condition' } = options; + const startTime = performance.now(); + + while (performance.now() - startTime < timeoutMs) { + try { + await maybeForceExternalDbSync(); + if (await checkFn()) { + return; + } + } catch (err: any) { + // If the error is a connection error, wait and retry + if (err?.code === '57P01' || err?.code === '08006' || err?.code === '53300') { + // Connection terminated, connection failure, or too many clients + await new Promise(r => setTimeout(r, intervalMs)); + continue; + } + throw err; + } + await new Promise(r => setTimeout(r, intervalMs)); + } + + throw new Error(`Timeout waiting for ${description} after ${timeoutMs}ms`); +} + +export async function forceExternalDbSync(): Promise { + if (!SHOULD_FORCE_EXTERNAL_DB_SYNC) return false; + + const cronSecret = process.env.CRON_SECRET; + if (!cronSecret) { + throw new Error('CRON_SECRET is required when STACK_FORCE_EXTERNAL_DB_SYNC=true'); + } + + lastForcedSyncAt = performance.now(); + + await niceFetch(new URL('/api/latest/internal/external-db-sync/sequencer', STACK_BACKEND_BASE_URL), { + query: { + maxDurationMs: String(FORCE_SYNC_MAX_DURATION_MS), + stopWhenIdle: "true", + }, + headers: { + Authorization: `Bearer ${cronSecret}`, + }, + }); + await niceFetch(new URL('/api/latest/internal/external-db-sync/poller', STACK_BACKEND_BASE_URL), { + query: { + maxDurationMs: String(FORCE_SYNC_MAX_DURATION_MS), + stopWhenIdle: "true", + }, + headers: { + Authorization: `Bearer ${cronSecret}`, + }, + }); + return true; +} + +async function maybeForceExternalDbSync() { + if (!SHOULD_FORCE_EXTERNAL_DB_SYNC) return; + + const now = performance.now(); + if (now - lastForcedSyncAt < FORCE_SYNC_INTERVAL_MS) return; + + await forceExternalDbSync(); +} + +/** + * Wait for data to appear in external DB (relies on automatic cron job) + */ +export async function waitForSyncedData(client: Client, email: string, expectedName?: string) { + + await waitForCondition( + async () => { + let res; + try { + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + } catch (err: any) { + if (err && err.code === '42P01') { + return false; + } + throw err; + } + if (res.rows.length === 0) { + return false; + } + if (expectedName && res.rows[0].display_name !== expectedName) { + return false; + } + return true; + }, + { + description: `data for ${email} to appear in external DB`, + timeoutMs: 120000, + intervalMs: 500, + } + ); +} + +/** + * Wait for data to be removed from external DB (relies on automatic cron job) + */ +export async function waitForSyncedDeletion(client: Client, email: string) { + await waitForCondition( + async () => { + let res; + try { + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + } catch (err: any) { + if (err && err.code === '42P01') { + return false; + } + throw err; + } + return res.rows.length === 0; + }, + { + description: `data for ${email} to be removed from external DB`, + timeoutMs: 120000, + intervalMs: 500, + } + ); +} + +/** + * Wait for table to be created (relies on automatic cron job) + */ +export async function waitForTable(client: Client, tableName: string) { + await waitForCondition( + async () => { + const res = await client.query(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = $1 + ); + `, [tableName]); + const exists = res.rows[0].exists; + return exists; + }, + { + description: `table ${tableName} to be created`, + timeoutMs: 120000, + intervalMs: 500, + } + ); +} + +/** + * Helper to verify data does NOT exist in external DB + */ +export async function verifyNotInExternalDb(client: Client, email: string) { + const res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(0); +} + +/** + * Helper to verify data DOES exist in external DB + */ +export async function verifyInExternalDb(client: Client, email: string, expectedName?: string) { + const res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + if (expectedName) { + expect(res.rows[0].display_name).toBe(expectedName); + } + return res.rows[0]; +} + +/** + * Helper to count total users in external DB + */ +export async function countUsersInExternalDb(client: Client): Promise { + try { + const res = await client.query(`SELECT COUNT(*) FROM "users"`); + return parseInt(res.rows[0].count, 10); + } catch (err: any) { + if (err && err.code === '42P01') { + return 0; + } + throw err; + } +} + +/** + * Helper to create a project and update its config with external DB settings. + * Tracks the project for cleanup later. + */ +export async function createProjectWithExternalDb( + externalDatabases: any, + projectOptions?: { display_name?: string, description?: string }, + options?: { projectTracker?: ProjectContext[] } +) { + const project = await Project.createAndSwitch(projectOptions); + const { projectKeys } = await InternalApiKey.createAndSetProjectKeys(project.adminAccessToken); + if (!projectKeys.superSecretAdminKey) { + throw new Error('Expected super secret admin key to be present for external DB sync tests.'); + } + await Project.updateConfig({ + "dbSync.externalDatabases": externalDatabases + }); + + // Track this project for cleanup + if (options?.projectTracker) { + options.projectTracker.push({ + projectId: project.projectId, + superSecretAdminKey: projectKeys.superSecretAdminKey, + }); + } + + return project; +} + +/** + * Helper to remove external DB config from current project + */ +export async function cleanupProjectExternalDb() { + await Project.updateConfig({ + "dbSync.externalDatabases": {} + }); +} + +/** + * Clean up external DB configs for all tracked projects. + * This prevents the sync cron from trying to connect to deleted databases. + * + * Note: This function makes direct HTTP calls instead of using backendContext + * because it runs in afterAll, which is outside the test context. + */ +export async function cleanupProjectConfigs(projects: ProjectContext[]) { + for (const project of projects) { + try { + // Make direct HTTP call to clear the external DB config + await niceFetch(new URL('/api/latest/internal/config/override', STACK_BACKEND_BASE_URL), { + method: 'PATCH', + headers: { + 'Content-Type': 'application/json', + 'x-stack-access-type': 'admin', + 'x-stack-project-id': project.projectId, + 'x-stack-super-secret-admin-key': project.superSecretAdminKey, + }, + body: JSON.stringify({ + config_override_string: JSON.stringify({ "dbSync.externalDatabases": {} }) + }), + }); + } catch (err) { + // Ignore errors - project might have been deleted or config update might fail + console.warn(`Failed to cleanup project ${project.projectId}:`, err); + } + } +} diff --git a/apps/e2e/tests/backend/performance/mock-external-db-sync-projects.sql b/apps/e2e/tests/backend/performance/mock-external-db-sync-projects.sql new file mode 100644 index 0000000000..269d061797 --- /dev/null +++ b/apps/e2e/tests/backend/performance/mock-external-db-sync-projects.sql @@ -0,0 +1,262 @@ +--set -a; source apps/backend/.env.development; set +a; psql "$STACK_DATABASE_CONNECTION_STRING" -v ON_ERROR_STOP=1 -f apps/e2e/tests/backend/performance/mock-external-db-sync-projects.sql + +BEGIN; + +CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + +-- NOTE: +-- - This script is intentionally heavy (1,000,000 projects + 3,000,000 users). +-- - Update BOTH settings blocks if you need a different external DB connection string. +-- - The external DB should be reachable from the backend (default uses docker postgres on port 8128). + +-- ===================================================================================== +-- 1) One million projects, one user each +-- ===================================================================================== +WITH settings AS ( + SELECT + 'postgresql://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/loadtest'::text AS external_connection_string, + 1000000::int AS project_count +), +config AS ( + SELECT jsonb_build_object( + 'dbSync', + jsonb_build_object( + 'externalDatabases', + jsonb_build_object( + 'main', + jsonb_build_object( + 'type', 'postgres', + 'connectionString', external_connection_string + ) + ) + ) + ) AS config_json + FROM settings +), +small_projects AS ( + SELECT + gen_random_uuid() AS project_id, + gen_random_uuid() AS tenancy_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS auth_method_id, + gen_random_uuid() AS contact_id, + gs AS idx, + lpad(gs::text, 7, '0') AS padded_idx, + now() AS ts + FROM settings + CROSS JOIN generate_series(1, settings.project_count) AS gs +), +insert_projects AS ( + INSERT INTO "Project" ("id", "displayName", "description", "isProductionMode", "ownerTeamId", "createdAt", "updatedAt") + SELECT + project_id, + 'External DB Sync Project ' || padded_idx, + 'External DB sync load test project', + FALSE, + NULL, + ts, + ts + FROM small_projects + RETURNING "id" +), +insert_tenancies AS ( + INSERT INTO "Tenancy" ("id", "projectId", "branchId", "organizationId", "hasNoOrganization", "createdAt", "updatedAt") + SELECT + tenancy_id, + project_id, + 'main', + NULL, + 'TRUE'::"BooleanTrue", + ts, + ts + FROM small_projects + RETURNING "id" +), +insert_env_config AS ( + INSERT INTO "EnvironmentConfigOverride" ("projectId", "branchId", "config", "createdAt", "updatedAt") + SELECT + project_id, + 'main', + (SELECT config_json FROM config), + ts, + ts + FROM small_projects + ON CONFLICT ("projectId", "branchId") DO UPDATE SET + "config" = EXCLUDED."config", + "updatedAt" = EXCLUDED."updatedAt" + RETURNING "projectId" +), +insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", "displayName", "projectId", "createdAt", "updatedAt") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'External Sync User ' || padded_idx, + project_id, + ts, + ts + FROM small_projects + RETURNING "tenancyId", "projectUserId" +), +insert_contacts AS ( + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", "isVerified", "value", "createdAt", "updatedAt") + SELECT + tenancy_id, + project_user_id, + contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'external-sync-user-' || padded_idx || '@load.local', + ts, + ts + FROM small_projects + RETURNING "tenancyId", "projectUserId" +), +insert_auth_methods AS ( + INSERT INTO "AuthMethod" + ("tenancyId", "id", "projectUserId", "createdAt", "updatedAt") + SELECT + tenancy_id, + auth_method_id, + project_user_id, + ts, + ts + FROM small_projects + RETURNING "tenancyId", "id", "projectUserId" +) +INSERT INTO "PasswordAuthMethod" + ("tenancyId", "authMethodId", "projectUserId", "passwordHash", "createdAt", "updatedAt") +SELECT + tenancy_id, + auth_method_id, + project_user_id, + '$2a$13$TVyY/gpw9Db/w1fBeJkCgeNg2Rae2JfNqrPnSACtj.ufAO5cVF13.', + ts, + ts +FROM small_projects; + +COMMIT; + +BEGIN; + +-- ===================================================================================== +-- 2) Three projects, one million users each +-- ===================================================================================== +SET LOCAL synchronous_commit = off; + +CREATE TEMP TABLE tmp_large_projects AS +SELECT + gen_random_uuid() AS project_id, + gen_random_uuid() AS tenancy_id, + gs AS project_idx, + lpad(gs::text, 2, '0') AS padded_project_idx, + now() AS ts +FROM generate_series(1, 3) AS gs; + +INSERT INTO "Project" ("id", "displayName", "description", "isProductionMode", "ownerTeamId", "createdAt", "updatedAt") +SELECT + project_id, + 'External DB Sync Mega Project ' || padded_project_idx, + 'External DB sync load test project (mega)', + FALSE, + NULL, + ts, + ts +FROM tmp_large_projects; + +INSERT INTO "Tenancy" ("id", "projectId", "branchId", "organizationId", "hasNoOrganization", "createdAt", "updatedAt") +SELECT + tenancy_id, + project_id, + 'main', + NULL, + 'TRUE'::"BooleanTrue", + ts, + ts +FROM tmp_large_projects; + +WITH settings AS ( + SELECT + 'postgresql://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/loadtest'::text AS external_connection_string +), +config AS ( + SELECT jsonb_build_object( + 'dbSync', + jsonb_build_object( + 'externalDatabases', + jsonb_build_object( + 'main', + jsonb_build_object( + 'type', 'postgres', + 'connectionString', external_connection_string + ) + ) + ) + ) AS config_json + FROM settings +) +INSERT INTO "EnvironmentConfigOverride" ("projectId", "branchId", "config", "createdAt", "updatedAt") +SELECT + project_id, + 'main', + (SELECT config_json FROM config), + ts, + ts +FROM tmp_large_projects +ON CONFLICT ("projectId", "branchId") DO UPDATE SET + "config" = EXCLUDED."config", + "updatedAt" = EXCLUDED."updatedAt"; + +-- ALTER TABLE "ProjectUser" DISABLE TRIGGER project_user_insert_trigger; + +DO $$ +DECLARE + users_per_project int := 1000000; + batch_size int := 10000; + batch_start int := 1; + batch_end int; +BEGIN + WHILE batch_start <= users_per_project LOOP + batch_end := LEAST(batch_start + batch_size - 1, users_per_project); + + WITH mega_users AS ( + SELECT + lp.project_id, + lp.tenancy_id, + lp.project_idx, + lp.padded_project_idx, + gs AS user_idx, + lpad(gs::text, 7, '0') AS padded_user_idx, + gen_random_uuid() AS project_user_id, + lp.ts AS ts + FROM tmp_large_projects lp + CROSS JOIN generate_series(batch_start, batch_end) AS gs + ) + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", "displayName", "projectId", "createdAt", "updatedAt") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'Mega User ' || padded_project_idx || '-' || padded_user_idx, + project_id, + ts, + ts + FROM mega_users; + + RAISE NOTICE 'Inserted users %-% of % per project', batch_start, batch_end, users_per_project; + + batch_start := batch_end + 1; + END LOOP; +END $$; + +-- ALTER TABLE "ProjectUser" ENABLE TRIGGER project_user_insert_trigger; + +COMMIT; diff --git a/apps/e2e/tests/global-setup.ts b/apps/e2e/tests/global-setup.ts index f15e5b4727..99be2b1a8e 100644 --- a/apps/e2e/tests/global-setup.ts +++ b/apps/e2e/tests/global-setup.ts @@ -4,6 +4,8 @@ import path from "path"; export default function globalSetup() { dotenv.config({ path: [ + ".env.test.local", + ".env.test", ".env.development.local", ".env.local", ".env.development", diff --git a/apps/e2e/tests/helpers.ts b/apps/e2e/tests/helpers.ts index 2ff135a36a..bff855132d 100644 --- a/apps/e2e/tests/helpers.ts +++ b/apps/e2e/tests/helpers.ts @@ -241,7 +241,7 @@ export class Mailbox { }; this.waitForMessagesWithSubjectCount = async (subject: string, minCount: number, options?: { noBody?: boolean }) => { - const maxRetries = 20; + const maxRetries = 25; let messages: MailboxMessage[] = []; for (let i = 0; i < maxRetries; i++) { messages = await this.fetchMessages(options); diff --git a/docker/dependencies/docker.compose.yaml b/docker/dependencies/docker.compose.yaml index b58ff352ea..3cebc981ad 100644 --- a/docker/dependencies/docker.compose.yaml +++ b/docker/dependencies/docker.compose.yaml @@ -12,6 +12,8 @@ services: POSTGRES_DB: stackframe POSTGRES_DELAY_MS: ${POSTGRES_DELAY_MS:-0} POSTGRES_INITDB_ARGS: --nosync + # Increase max_connections for E2E tests that create many databases + command: postgres -c max_connections=500 ports: - "${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}28:5432" volumes: diff --git a/docker/dev-postgres-replica/entrypoint.sh b/docker/dev-postgres-replica/entrypoint.sh index 748c2ccd0c..d96a4dc641 100644 --- a/docker/dev-postgres-replica/entrypoint.sh +++ b/docker/dev-postgres-replica/entrypoint.sh @@ -42,6 +42,10 @@ if [ -z "$(ls -A ${PGDATA} 2>/dev/null)" ]; then primary_conninfo = 'host=${PRIMARY_HOST} port=${PRIMARY_PORT} user=${REPLICATOR_USER} password=${REPLICATOR_PASSWORD}' recovery_min_apply_delay = ${RECOVERY_MIN_APPLY_DELAY} hot_standby = on + +# pg_stat_statements for query stats +shared_preload_libraries = 'pg_stat_statements' +pg_stat_statements.track = all EOF # Create standby.signal to indicate this is a standby diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts new file mode 100644 index 0000000000..e73113590a --- /dev/null +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -0,0 +1,165 @@ +export const DEFAULT_DB_SYNC_MAPPINGS = { + "users": { + sourceTables: { "ProjectUser": "ProjectUser" }, + targetTable: "users", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "users" ( + "id" uuid PRIMARY KEY NOT NULL, + "display_name" text, + "profile_image_url" text, + "primary_email" text, + "primary_email_verified" boolean NOT NULL DEFAULT false, + "signed_up_at" timestamp without time zone NOT NULL, + "client_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "client_read_only_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "server_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "is_anonymous" boolean NOT NULL DEFAULT false + ); + REVOKE ALL ON "users" FROM PUBLIC; + GRANT SELECT ON "users" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "ProjectUser"."projectUserId" AS "id", + "ProjectUser"."displayName" AS "display_name", + "ProjectUser"."profileImageUrl" AS "profile_image_url", + ( + SELECT "ContactChannel"."value" + FROM "ContactChannel" + WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" + AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" + AND "ContactChannel"."type" = 'EMAIL' + AND "ContactChannel"."isPrimary" = 'TRUE' + LIMIT 1 + ) AS "primary_email", + COALESCE( + ( + SELECT "ContactChannel"."isVerified" + FROM "ContactChannel" + WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" + AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" + AND "ContactChannel"."type" = 'EMAIL' + AND "ContactChannel"."isPrimary" = 'TRUE' + LIMIT 1 + ), + false + ) AS "primary_email_verified", + "ProjectUser"."createdAt" AS "signed_up_at", + COALESCE("ProjectUser"."clientMetadata", '{}'::jsonb) AS "client_metadata", + COALESCE("ProjectUser"."clientReadOnlyMetadata", '{}'::jsonb) AS "client_read_only_metadata", + COALESCE("ProjectUser"."serverMetadata", '{}'::jsonb) AS "server_metadata", + "ProjectUser"."isAnonymous" AS "is_anonymous", + "ProjectUser"."sequenceId" AS "sequence_id", + "ProjectUser"."tenancyId", + false AS "is_deleted" + FROM "ProjectUser" + WHERE "ProjectUser"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "id", + NULL::text AS "display_name", + NULL::text AS "profile_image_url", + NULL::text AS "primary_email", + false AS "primary_email_verified", + "DeletedRow"."deletedAt"::timestamp without time zone AS "signed_up_at", + '{}'::jsonb AS "client_metadata", + '{}'::jsonb AS "client_read_only_metadata", + '{}'::jsonb AS "server_metadata", + false AS "is_anonymous", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUser' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + // Last parameter = mapping_name (for metadata tracking) + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::text AS "display_name", + $3::text AS "profile_image_url", + $4::text AS "primary_email", + $5::boolean AS "primary_email_verified", + $6::timestamp without time zone AS "signed_up_at", + $7::jsonb AS "client_metadata", + $8::jsonb AS "client_read_only_metadata", + $9::jsonb AS "server_metadata", + $10::boolean AS "is_anonymous", + $11::bigint AS "sequence_id", + $12::boolean AS "is_deleted", + $13::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "users" u + USING params p + WHERE p."is_deleted" = true AND u."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "users" ( + "id", + "display_name", + "profile_image_url", + "primary_email", + "primary_email_verified", + "signed_up_at", + "client_metadata", + "client_read_only_metadata", + "server_metadata", + "is_anonymous" + ) + SELECT + p."id", + p."display_name", + p."profile_image_url", + p."primary_email", + p."primary_email_verified", + p."signed_up_at", + p."client_metadata", + p."client_read_only_metadata", + p."server_metadata", + p."is_anonymous" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "display_name" = EXCLUDED."display_name", + "profile_image_url" = EXCLUDED."profile_image_url", + "primary_email" = EXCLUDED."primary_email", + "primary_email_verified" = EXCLUDED."primary_email_verified", + "signed_up_at" = EXCLUDED."signed_up_at", + "client_metadata" = EXCLUDED."client_metadata", + "client_read_only_metadata" = EXCLUDED."client_read_only_metadata", + "server_metadata" = EXCLUDED."server_metadata", + "is_anonymous" = EXCLUDED."is_anonymous" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, +} as const; diff --git a/packages/stack-shared/src/config/schema-fuzzer.test.ts b/packages/stack-shared/src/config/schema-fuzzer.test.ts index 7b2e4ba3f5..30ce228b94 100644 --- a/packages/stack-shared/src/config/schema-fuzzer.test.ts +++ b/packages/stack-shared/src/config/schema-fuzzer.test.ts @@ -62,6 +62,17 @@ const branchSchemaFuzzerConfig = [{ }], signUpRulesDefaultAction: ["allow", "reject"], }], + dbSync: [{ + externalDatabases: [{ + "some-external-db-id": [{ + type: ["postgres"] as const, + connectionString: [ + "postgres://user:password@host:port/database", + "some-connection-string", + ], + }], + }], + }], dataVault: [{ stores: [{ "some-store-id": [{ diff --git a/packages/stack-shared/src/config/schema.ts b/packages/stack-shared/src/config/schema.ts index 032919afa7..494dea24da 100644 --- a/packages/stack-shared/src/config/schema.ts +++ b/packages/stack-shared/src/config/schema.ts @@ -235,6 +235,16 @@ export const branchConfigSchema = canNoLongerBeOverridden(projectConfigSchema, [ payments: branchPaymentsSchema, + dbSync: yupObject({ + externalDatabases: yupRecord( + userSpecifiedIdSchema("externalDatabaseId"), + yupObject({ + type: yupString().oneOf(['postgres']).defined(), + connectionString: yupString().defined(), + }) + ), + }), + dataVault: yupObject({ stores: yupRecord( userSpecifiedIdSchema("storeId"), @@ -637,6 +647,14 @@ const organizationConfigDefaults = { } as const) }, + + dbSync: { + externalDatabases: (key: string) => ({ + type: undefined, + connectionString: undefined, + }), + }, + dataVault: { stores: (key: string) => ({ displayName: "Unnamed Vault", @@ -1001,7 +1019,7 @@ export async function getConfigOverrideErrors(schema: T // This is how the implementation would look like, but we don't support arrays in config JSON files (besides tuples) // const arraySchema = schema as yup.ArraySchema; // const innerType = arraySchema.innerType; - // return yupArray(innerType ? getRestrictedSchema(path + ".[]", innerType as any) : undefined); + // return yupArray(innerType ? getRestrictedSchema(path + ".[]", innerType as any) : undefined()); } case "tuple": { return yupTuple(schemaInfo.items.map((s, index) => getRestrictedSchema(path + `[${index}]`, s)) as any); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 21554297e0..64bc666232 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -182,7 +182,7 @@ importers: version: 1.2.1(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@sentry/nextjs': specifier: ^10.11.0 - version: 10.11.0(@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@16.1.5(@babel/core@7.28.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.24.2)) + version: 10.11.0(@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@16.1.5(@babel/core@7.26.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)) '@simplewebauthn/server': specifier: ^11.0.0 version: 11.0.0(encoding@0.1.13) @@ -233,7 +233,7 @@ importers: version: 1.0.6 next: specifier: 16.1.5 - version: 16.1.5(@babel/core@7.28.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + version: 16.1.5(@babel/core@7.26.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) nodemailer: specifier: ^6.9.10 version: 6.9.13 @@ -333,7 +333,7 @@ importers: version: 5.0.7 tsup: specifier: ^8.3.0 - version: 8.3.5(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(typescript@5.8.3)(yaml@2.4.5) + version: 8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(typescript@5.8.3)(yaml@2.4.5) tsx: specifier: ^4.7.2 version: 4.15.5 @@ -672,9 +672,15 @@ importers: '@types/js-beautify': specifier: ^1.14.3 version: 1.14.3 + '@types/pg': + specifier: ^8.15.6 + version: 8.16.0 jose: specifier: ^5.6.3 version: 5.6.3 + pg: + specifier: ^8.16.3 + version: 8.16.3 apps/mock-oauth-server: dependencies: @@ -1820,7 +1826,7 @@ importers: version: 3.4.14 tsup: specifier: ^8.0.2 - version: 8.1.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(postcss@8.4.47)(typescript@5.8.3) + version: 8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.4.47)(tsx@4.21.0)(typescript@5.8.3)(yaml@2.8.0) packages/stack-sc: dependencies: @@ -2284,7 +2290,6 @@ packages: '@assistant-ui/react-edge@0.2.12': resolution: {integrity: sha512-95Y912lW8ASMT52qZd6ZHRiF+T7WxbeJ1yb2z/I0lCKegPt0q3spGy92YnO7mwz0uJaNjqu4/oZZybYfeIDzJg==} - deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: '@assistant-ui/react': '*' '@types/react': ^18.2.0 @@ -2583,10 +2588,6 @@ packages: resolution: {integrity: sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==} engines: {node: '>=6.9.0'} - '@babel/core@7.28.0': - resolution: {integrity: sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==} - engines: {node: '>=6.9.0'} - '@babel/core@7.28.5': resolution: {integrity: sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==} engines: {node: '>=6.9.0'} @@ -2599,10 +2600,6 @@ packages: resolution: {integrity: sha512-kEWdzjOAUMW4hAyrzJ0ZaTOu9OmpyDIQicIh0zg0EEcEkYXZb2TjtBhnHi2ViX7PKwZqF4xwqfAm299/QMP3lg==} engines: {node: '>=6.9.0'} - '@babel/generator@7.28.0': - resolution: {integrity: sha512-lJjzvrbEeWrhB4P3QBsH7tey117PjLZnDbLiQEKjQ/fNJTjuq4HSqgFA+UNSwZT8D7dxxbnuSBMsa1lrWzKlQg==} - engines: {node: '>=6.9.0'} - '@babel/generator@7.28.5': resolution: {integrity: sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ==} engines: {node: '>=6.9.0'} @@ -2651,12 +2648,6 @@ packages: peerDependencies: '@babel/core': ^7.0.0 - '@babel/helper-module-transforms@7.27.3': - resolution: {integrity: sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg==} - engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.0.0 - '@babel/helper-module-transforms@7.28.3': resolution: {integrity: sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==} engines: {node: '>=6.9.0'} @@ -2667,10 +2658,6 @@ packages: resolution: {integrity: sha512-jKiTsW2xmWwxT1ixIdfXUZp+P5yURx2suzLZr5Hi64rURpDYdMW0pv+Uf17EYk2Rd428Lx4tLsnjGJzYKDM/6A==} engines: {node: '>=6.9.0'} - '@babel/helper-plugin-utils@7.24.8': - resolution: {integrity: sha512-FFWx5142D8h2Mgr/iPVGH5G7w6jDn4jUSpZTyDnQO0Yn7Ks2Kuz6Pci8H6MPCoUJegd/UZQ3tAvfLCxQSnWWwg==} - engines: {node: '>=6.9.0'} - '@babel/helper-plugin-utils@7.27.1': resolution: {integrity: sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==} engines: {node: '>=6.9.0'} @@ -2725,10 +2712,6 @@ packages: resolution: {integrity: sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==} engines: {node: '>=6.9.0'} - '@babel/helpers@7.27.6': - resolution: {integrity: sha512-muE8Tt8M22638HU31A3CgfSUciwz1fhATfoVai05aPXGor//CdWDCbnlY1yvBPo07njuVOCNGCSp/GTt12lIug==} - engines: {node: '>=6.9.0'} - '@babel/helpers@7.28.4': resolution: {integrity: sha512-HFN59MmQXGHVyYadKLVumYsA9dBFun/ldYxipEjzA4196jpLZd8UjEEBLkbEkvfYreDqJhZxYAWFPtrfhNpj4w==} engines: {node: '>=6.9.0'} @@ -2839,10 +2822,6 @@ packages: resolution: {integrity: sha512-ZYW7L+pL8ahU5fXmNbPF+iZFHCv5scFak7MZ9bwaRPLUhHh7QQEMjZUg0HevihoqCM5iSYHN61EyCoZvqC+bxg==} engines: {node: '>=6.9.0'} - '@babel/traverse@7.28.0': - resolution: {integrity: sha512-mGe7UK5wWyh0bKRfupsUchrQGqvDbZDbKJw+kcRGSmdHVYrv+ltd0pnpDTVpiTqnaBru9iEvA8pz8W46v0Amwg==} - engines: {node: '>=6.9.0'} - '@babel/traverse@7.28.5': resolution: {integrity: sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ==} engines: {node: '>=6.9.0'} @@ -2863,10 +2842,6 @@ packages: resolution: {integrity: sha512-jYnje+JyZG5YThjHiF28oT4SIZLnYOcSBb6+SDaFIyzDVSkXQmQQYclJ2R+YxcdmK0AX6x1E5OQNtuh3jHDrUg==} engines: {node: '>=6.9.0'} - '@babel/types@7.28.1': - resolution: {integrity: sha512-x0LvFTekgSX+83TI28Y9wYPUfzrnl2aT5+5QLnO6v7mSJYtEEevuDRN0F0uSHRk1G1IWZC43o00Y0xDDrpBGPQ==} - engines: {node: '>=6.9.0'} - '@babel/types@7.28.5': resolution: {integrity: sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==} engines: {node: '>=6.9.0'} @@ -4639,16 +4614,9 @@ packages: resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} - '@jridgewell/gen-mapping@0.3.12': - resolution: {integrity: sha512-OuLGC46TjB5BbN1dH8JULVVZY4WTdkF7tV9Ys6wLL1rubZnCMstOhNHueU5bLCrnRuDhKPDM4g6sw4Bel5Gzqg==} - '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} - '@jridgewell/gen-mapping@0.3.5': - resolution: {integrity: sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==} - engines: {node: '>=6.0.0'} - '@jridgewell/remapping@2.3.5': resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} @@ -4656,10 +4624,6 @@ packages: resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} engines: {node: '>=6.0.0'} - '@jridgewell/set-array@1.2.1': - resolution: {integrity: sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==} - engines: {node: '>=6.0.0'} - '@jridgewell/source-map@0.3.11': resolution: {integrity: sha512-ZMp1V8ZFcPG5dIWnQLr3NSI1MiCU7UETdS/A0G8V/XWHvJv3ZsFqutJn1Y5RPmAPX6F3BiE397OqveU/9NCuIA==} @@ -4672,9 +4636,6 @@ packages: '@jridgewell/trace-mapping@0.3.25': resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} - '@jridgewell/trace-mapping@0.3.29': - resolution: {integrity: sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==} - '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} @@ -7317,11 +7278,6 @@ packages: rollup: optional: true - '@rollup/rollup-android-arm-eabi@4.18.0': - resolution: {integrity: sha512-Tya6xypR10giZV1XzxmH5wr25VcZSncG0pZIjfePT0OVBvqNEurzValetGNarVrGiq66EBVAFn15iYX4w6FKgQ==} - cpu: [arm] - os: [android] - '@rollup/rollup-android-arm-eabi@4.24.4': resolution: {integrity: sha512-jfUJrFct/hTA0XDM5p/htWKoNNTbDLY0KRwEt6pyOA6k2fmk0WVwl65PdUdJZgzGEHWx+49LilkcSaumQRyNQw==} cpu: [arm] @@ -7337,11 +7293,6 @@ packages: cpu: [arm] os: [android] - '@rollup/rollup-android-arm64@4.18.0': - resolution: {integrity: sha512-avCea0RAP03lTsDhEyfy+hpfr85KfyTctMADqHVhLAF3MlIkq83CP8UfAHUssgXTYd+6er6PaAhx/QGv4L1EiA==} - cpu: [arm64] - os: [android] - '@rollup/rollup-android-arm64@4.24.4': resolution: {integrity: sha512-j4nrEO6nHU1nZUuCfRKoCcvh7PIywQPUCBa2UsootTHvTHIoIu2BzueInGJhhvQO/2FTRdNYpf63xsgEqH9IhA==} cpu: [arm64] @@ -7357,11 +7308,6 @@ packages: cpu: [arm64] os: [android] - '@rollup/rollup-darwin-arm64@4.18.0': - resolution: {integrity: sha512-IWfdwU7KDSm07Ty0PuA/W2JYoZ4iTj3TUQjkVsO/6U+4I1jN5lcR71ZEvRh52sDOERdnNhhHU57UITXz5jC1/w==} - cpu: [arm64] - os: [darwin] - '@rollup/rollup-darwin-arm64@4.24.4': resolution: {integrity: sha512-GmU/QgGtBTeraKyldC7cDVVvAJEOr3dFLKneez/n7BvX57UdhOqDsVwzU7UOnYA7AAOt+Xb26lk79PldDHgMIQ==} cpu: [arm64] @@ -7377,11 +7323,6 @@ packages: cpu: [arm64] os: [darwin] - '@rollup/rollup-darwin-x64@4.18.0': - resolution: {integrity: sha512-n2LMsUz7Ynu7DoQrSQkBf8iNrjOGyPLrdSg802vk6XT3FtsgX6JbE8IHRvposskFm9SNxzkLYGSq9QdpLYpRNA==} - cpu: [x64] - os: [darwin] - '@rollup/rollup-darwin-x64@4.24.4': resolution: {integrity: sha512-N6oDBiZCBKlwYcsEPXGDE4g9RoxZLK6vT98M8111cW7VsVJFpNEqvJeIPfsCzbf0XEakPslh72X0gnlMi4Ddgg==} cpu: [x64] @@ -7427,12 +7368,6 @@ packages: cpu: [x64] os: [freebsd] - '@rollup/rollup-linux-arm-gnueabihf@4.18.0': - resolution: {integrity: sha512-C/zbRYRXFjWvz9Z4haRxcTdnkPt1BtCkz+7RtBSuNmKzMzp3ZxdM28Mpccn6pt28/UWUCTXa+b0Mx1k3g6NOMA==} - cpu: [arm] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-arm-gnueabihf@4.24.4': resolution: {integrity: sha512-10ICosOwYChROdQoQo589N5idQIisxjaFE/PAnX2i0Zr84mY0k9zul1ArH0rnJ/fpgiqfu13TFZR5A5YJLOYZA==} cpu: [arm] @@ -7451,12 +7386,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-arm-musleabihf@4.18.0': - resolution: {integrity: sha512-l3m9ewPgjQSXrUMHg93vt0hYCGnrMOcUpTz6FLtbwljo2HluS4zTXFy2571YQbisTnfTKPZ01u/ukJdQTLGh9A==} - cpu: [arm] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-arm-musleabihf@4.24.4': resolution: {integrity: sha512-ySAfWs69LYC7QhRDZNKqNhz2UKN8LDfbKSMAEtoEI0jitwfAG2iZwVqGACJT+kfYvvz3/JgsLlcBP+WWoKCLcw==} cpu: [arm] @@ -7475,12 +7404,6 @@ packages: os: [linux] libc: [musl] - '@rollup/rollup-linux-arm64-gnu@4.18.0': - resolution: {integrity: sha512-rJ5D47d8WD7J+7STKdCUAgmQk49xuFrRi9pZkWoRD1UeSMakbcepWXPF8ycChBoAqs1pb2wzvbY6Q33WmN2ftw==} - cpu: [arm64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-arm64-gnu@4.24.4': resolution: {integrity: sha512-uHYJ0HNOI6pGEeZ/5mgm5arNVTI0nLlmrbdph+pGXpC9tFHFDQmDMOEqkmUObRfosJqpU8RliYoGz06qSdtcjg==} cpu: [arm64] @@ -7499,12 +7422,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-arm64-musl@4.18.0': - resolution: {integrity: sha512-be6Yx37b24ZwxQ+wOQXXLZqpq4jTckJhtGlWGZs68TgdKXJgw54lUUoFYrg6Zs/kjzAQwEwYbp8JxZVzZLRepQ==} - cpu: [arm64] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-arm64-musl@4.24.4': resolution: {integrity: sha512-38yiWLemQf7aLHDgTg85fh3hW9stJ0Muk7+s6tIkSUOMmi4Xbv5pH/5Bofnsb6spIwD5FJiR+jg71f0CH5OzoA==} cpu: [arm64] @@ -7535,12 +7452,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-powerpc64le-gnu@4.18.0': - resolution: {integrity: sha512-hNVMQK+qrA9Todu9+wqrXOHxFiD5YmdEi3paj6vP02Kx1hjd2LLYR2eaN7DsEshg09+9uzWi2W18MJDlG0cxJA==} - cpu: [ppc64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-powerpc64le-gnu@4.24.4': resolution: {integrity: sha512-q73XUPnkwt9ZNF2xRS4fvneSuaHw2BXuV5rI4cw0fWYVIWIBeDZX7c7FWhFQPNTnE24172K30I+dViWRVD9TwA==} cpu: [ppc64] @@ -7559,12 +7470,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-riscv64-gnu@4.18.0': - resolution: {integrity: sha512-ROCM7i+m1NfdrsmvwSzoxp9HFtmKGHEqu5NNDiZWQtXLA8S5HBCkVvKAxJ8U+CVctHwV2Gb5VUaK7UAkzhDjlg==} - cpu: [riscv64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-riscv64-gnu@4.24.4': resolution: {integrity: sha512-Aie/TbmQi6UXokJqDZdmTJuZBCU3QBDA8oTKRGtd4ABi/nHgXICulfg1KI6n9/koDsiDbvHAiQO3YAUNa/7BCw==} cpu: [riscv64] @@ -7589,12 +7494,6 @@ packages: os: [linux] libc: [musl] - '@rollup/rollup-linux-s390x-gnu@4.18.0': - resolution: {integrity: sha512-0UyyRHyDN42QL+NbqevXIIUnKA47A+45WyasO+y2bGJ1mhQrfrtXUpTxCOrfxCR4esV3/RLYyucGVPiUsO8xjg==} - cpu: [s390x] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-s390x-gnu@4.24.4': resolution: {integrity: sha512-P8MPErVO/y8ohWSP9JY7lLQ8+YMHfTI4bAdtCi3pC2hTeqFJco2jYspzOzTUB8hwUWIIu1xwOrJE11nP+0JFAQ==} cpu: [s390x] @@ -7613,12 +7512,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-x64-gnu@4.18.0': - resolution: {integrity: sha512-xuglR2rBVHA5UsI8h8UbX4VJ470PtGCf5Vpswh7p2ukaqBGFTnsfzxUBetoWBWymHMxbIG0Cmx7Y9qDZzr648w==} - cpu: [x64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-x64-gnu@4.24.4': resolution: {integrity: sha512-K03TljaaoPK5FOyNMZAAEmhlyO49LaE4qCsr0lYHUKyb6QacTNF9pnfPpXnFlFD3TXuFbFbz7tJ51FujUXkXYA==} cpu: [x64] @@ -7637,12 +7530,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-x64-musl@4.18.0': - resolution: {integrity: sha512-LKaqQL9osY/ir2geuLVvRRs+utWUNilzdE90TpyoX0eNqPzWjRm14oMEE+YLve4k/NAqCdPkGYDaDF5Sw+xBfg==} - cpu: [x64] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-x64-musl@4.24.4': resolution: {integrity: sha512-VJYl4xSl/wqG2D5xTYncVWW+26ICV4wubwN9Gs5NrqhJtayikwCXzPL8GDsLnaLU3WwhQ8W02IinYSFJfyo34Q==} cpu: [x64] @@ -7666,11 +7553,6 @@ packages: cpu: [arm64] os: [openharmony] - '@rollup/rollup-win32-arm64-msvc@4.18.0': - resolution: {integrity: sha512-7J6TkZQFGo9qBKH0pk2cEVSRhJbL6MtfWxth7Y5YmZs57Pi+4x6c2dStAUvaQkHQLnEQv1jzBUW43GvZW8OFqA==} - cpu: [arm64] - os: [win32] - '@rollup/rollup-win32-arm64-msvc@4.24.4': resolution: {integrity: sha512-ku2GvtPwQfCqoPFIJCqZ8o7bJcj+Y54cZSr43hHca6jLwAiCbZdBUOrqE6y29QFajNAzzpIOwsckaTFmN6/8TA==} cpu: [arm64] @@ -7686,11 +7568,6 @@ packages: cpu: [arm64] os: [win32] - '@rollup/rollup-win32-ia32-msvc@4.18.0': - resolution: {integrity: sha512-Txjh+IxBPbkUB9+SXZMpv+b/vnTEtFyfWZgJ6iyCmt2tdx0OF5WhFowLmnh8ENGNpfUlUZkdI//4IEmhwPieNg==} - cpu: [ia32] - os: [win32] - '@rollup/rollup-win32-ia32-msvc@4.24.4': resolution: {integrity: sha512-V3nCe+eTt/W6UYNr/wGvO1fLpHUrnlirlypZfKCT1fG6hWfqhPgQV/K/mRBXBpxc0eKLIF18pIOFVPh0mqHjlg==} cpu: [ia32] @@ -7706,11 +7583,6 @@ packages: cpu: [ia32] os: [win32] - '@rollup/rollup-win32-x64-msvc@4.18.0': - resolution: {integrity: sha512-UOo5FdvOL0+eIVTgS4tIdbW+TtnBLWg1YBCcU2KWM7nuNwRz9bksDX1bekJJCpu25N1DVWaCwnT39dVQxzqS8g==} - cpu: [x64] - os: [win32] - '@rollup/rollup-win32-x64-msvc@4.24.4': resolution: {integrity: sha512-LTw1Dfd0mBIEqUVCxbvTE/LLo+9ZxVC9k99v1v4ahg9Aak6FpqOfNu5kRkeTAn0wphoC4JU7No1/rL+bBCEwhg==} cpu: [x64] @@ -7940,7 +7812,6 @@ packages: '@simplewebauthn/types@11.0.0': resolution: {integrity: sha512-b2o0wC5u2rWts31dTgBkAtSNKGX0cvL6h8QedNsKmj8O4QoLFQFR3DBVBUlpyVEhYKA+mXGUaXbcOc4JdQ3HzA==} - deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. '@sinclair/typebox@0.27.8': resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} @@ -8650,9 +8521,6 @@ packages: '@types/estree-jsx@1.0.5': resolution: {integrity: sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==} - '@types/estree@1.0.5': - resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==} - '@types/estree@1.0.6': resolution: {integrity: sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==} @@ -9568,12 +9436,6 @@ packages: resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} engines: {node: '>=18'} - bundle-require@4.2.1: - resolution: {integrity: sha512-7Q/6vkyYAwOmQNRw75x+4yRtZCZJXUDmHHlFdkiV0wgv/reNjtJwpu1jPJ0w2kbEpIM0uoKI3S4/f39dU7AjSA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - peerDependencies: - esbuild: '>=0.17' - bundle-require@5.0.0: resolution: {integrity: sha512-GuziW3fSSmopcx4KRymQEJVbZUfqlCqcq7dvs6TYwKRZiegK/2buMxQTPs6MGlNv50wms1699qYO54R8XfRX4w==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -9992,10 +9854,6 @@ packages: resolution: {integrity: sha512-I5qxpzLv+sJhTVEoLYNcTW+bThDCPsit0vLNKShZx6rLtpilNpmmeTPaeqJb9ZE9dV3DGaeby6Vuhrw38WjeyQ==} engines: {node: ^14.18.0 || >=16.10.0} - consola@3.4.0: - resolution: {integrity: sha512-EiPU8G6dQG0GFHNR8ljnZFki/8a+cQwEQ+7wpxdChl02Q8HXlwEZWD5lqAF8vC2sEC3Tehr8hy7vErz88LHyUA==} - engines: {node: ^14.18.0 || >=16.10.0} - consola@3.4.2: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} @@ -10348,15 +10206,6 @@ packages: supports-color: optional: true - debug@4.3.5: - resolution: {integrity: sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - debug@4.3.7: resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} engines: {node: '>=6.0'} @@ -11228,22 +11077,6 @@ packages: picomatch: optional: true - fdir@6.4.3: - resolution: {integrity: sha512-PMXmW2y1hDDfTSRc9gaXIuCCRpuoz3Kaz8cUelp3smouvfT632ozg2vrT6lJsHKKOF59YLbOGfAWGUcKEfRMQw==} - peerDependencies: - picomatch: ^3 || ^4 - peerDependenciesMeta: - picomatch: - optional: true - - fdir@6.4.6: - resolution: {integrity: sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==} - peerDependencies: - picomatch: ^3 || ^4 - peerDependenciesMeta: - picomatch: - optional: true - fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -12410,7 +12243,6 @@ packages: keygrip@1.1.0: resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} engines: {node: '>= 0.6'} - deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -13069,9 +12901,6 @@ packages: ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} - ms@2.1.2: - resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -14521,11 +14350,6 @@ packages: robust-predicates@3.0.2: resolution: {integrity: sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==} - rollup@4.18.0: - resolution: {integrity: sha512-QmJz14PX3rzbJCN1SG4Xe/bAAX2a6NpCP8ab2vfu2GiUr8AQcr2nCV/oEO3yneFarB67zk8ShlIyWb2LGTb3Sg==} - engines: {node: '>=18.0.0', npm: '>=8.0.0'} - hasBin: true - rollup@4.24.4: resolution: {integrity: sha512-vGorVWIsWfX3xbcyAS+I047kFKapHYivmkaT63Smj77XwvLSJos6M1xGqZnBPFQFBRZDOcG1QnYEIxAvTr/HjA==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -15225,10 +15049,6 @@ packages: resolution: {integrity: sha512-Zc+8eJlFMvgatPZTl6A9L/yht8QqdmUNtURHaKZLmKBE12hNPSrqNkUp2cs3M/UKmNVVAMFQYSjYIVHDjW5zew==} engines: {node: '>=12.0.0'} - tinyglobby@0.2.12: - resolution: {integrity: sha512-qkf4trmKSIiMTs/E63cxH+ojC2unam7rJ0WrauAzpT3ECNTxGRMlaXxVbfxMUC/w0LaYk6jQ4y/nGR9uBO3tww==} - engines: {node: '>=12.0.0'} - tinyglobby@0.2.15: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} @@ -15330,25 +15150,6 @@ packages: resolution: {integrity: sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==} engines: {node: '>=0.6.x'} - tsup@8.1.0: - resolution: {integrity: sha512-UFdfCAXukax+U6KzeTNO2kAARHcWxmKsnvSPXUcfA1D+kU05XDccCrkffCQpFaWDsZfV0jMyTsxU39VfCp6EOg==} - engines: {node: '>=18'} - hasBin: true - peerDependencies: - '@microsoft/api-extractor': ^7.36.0 - '@swc/core': ^1 - postcss: ^8.4.12 - typescript: '>=4.5.0' - peerDependenciesMeta: - '@microsoft/api-extractor': - optional: true - '@swc/core': - optional: true - postcss: - optional: true - typescript: - optional: true - tsup@8.3.5: resolution: {integrity: sha512-Tunf6r6m6tnZsG9GYWndg0z8dEV7fD733VBFzFJ5Vcm1FtlXB8xBD/rtrBi2a3YKEV7hHtxiZtW5EAVADoe1pA==} engines: {node: '>=18'} @@ -15946,10 +15747,6 @@ packages: resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} engines: {node: '>=12'} - webpack-sources@3.2.3: - resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==} - engines: {node: '>=10.13.0'} - webpack-sources@3.3.3: resolution: {integrity: sha512-yd1RBzSGanHkitROoPFd6qsrxt+oFhg/129YzheDGqeustzX0vTZJZsSsQjVQC4yzBQ56K55XU8gaNCtIzOnTg==} engines: {node: '>=10.13.0'} @@ -16308,7 +16105,7 @@ snapshots: '@ampproject/remapping@2.3.0': dependencies: - '@jridgewell/gen-mapping': 0.3.5 + '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.25 '@antfu/install-pkg@1.1.0': @@ -17223,26 +17020,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/core@7.28.0': - dependencies: - '@ampproject/remapping': 2.3.0 - '@babel/code-frame': 7.27.1 - '@babel/generator': 7.28.0 - '@babel/helper-compilation-targets': 7.27.2 - '@babel/helper-module-transforms': 7.27.3(@babel/core@7.28.0) - '@babel/helpers': 7.27.6 - '@babel/parser': 7.28.0 - '@babel/template': 7.27.2 - '@babel/traverse': 7.28.0 - '@babel/types': 7.28.1 - convert-source-map: 2.0.0 - debug: 4.4.1 - gensync: 1.0.0-beta.2 - json5: 2.2.3 - semver: 6.3.1 - transitivePeerDependencies: - - supports-color - '@babel/core@7.28.5': dependencies: '@babel/code-frame': 7.27.1 @@ -17267,7 +17044,7 @@ snapshots: dependencies: '@babel/parser': 7.26.2 '@babel/types': 7.26.0 - '@jridgewell/gen-mapping': 0.3.5 + '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.25 jsesc: 3.0.2 @@ -17276,15 +17053,7 @@ snapshots: '@babel/parser': 7.28.5 '@babel/types': 7.28.5 '@jridgewell/gen-mapping': 0.3.13 - '@jridgewell/trace-mapping': 0.3.25 - jsesc: 3.0.2 - - '@babel/generator@7.28.0': - dependencies: - '@babel/parser': 7.28.5 - '@babel/types': 7.28.5 - '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 3.0.2 '@babel/generator@7.28.5': @@ -17323,7 +17092,7 @@ snapshots: '@babel/helper-optimise-call-expression': 7.24.7 '@babel/helper-replace-supers': 7.25.0(@babel/core@7.28.5) '@babel/helper-skip-transparent-expression-wrappers': 7.24.7 - '@babel/traverse': 7.26.9 + '@babel/traverse': 7.28.5 semver: 6.3.1 transitivePeerDependencies: - supports-color @@ -17367,15 +17136,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.27.3(@babel/core@7.28.0)': - dependencies: - '@babel/core': 7.28.0 - '@babel/helper-module-imports': 7.27.1 - '@babel/helper-validator-identifier': 7.28.5 - '@babel/traverse': 7.28.5 - transitivePeerDependencies: - - supports-color - '@babel/helper-module-transforms@7.28.3(@babel/core@7.28.5)': dependencies: '@babel/core': 7.28.5 @@ -17389,8 +17149,6 @@ snapshots: dependencies: '@babel/types': 7.28.5 - '@babel/helper-plugin-utils@7.24.8': {} - '@babel/helper-plugin-utils@7.27.1': {} '@babel/helper-replace-supers@7.25.0(@babel/core@7.28.5)': @@ -17432,11 +17190,6 @@ snapshots: '@babel/template': 7.25.9 '@babel/types': 7.26.0 - '@babel/helpers@7.27.6': - dependencies: - '@babel/template': 7.27.2 - '@babel/types': 7.28.5 - '@babel/helpers@7.28.4': dependencies: '@babel/template': 7.27.2 @@ -17474,7 +17227,7 @@ snapshots: '@babel/core': 7.28.5 '@babel/helper-annotate-as-pure': 7.24.7 '@babel/helper-create-class-features-plugin': 7.25.0(@babel/core@7.28.5) - '@babel/helper-plugin-utils': 7.24.8 + '@babel/helper-plugin-utils': 7.27.1 '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.28.5) transitivePeerDependencies: - supports-color @@ -17557,7 +17310,7 @@ snapshots: '@babel/parser': 7.26.9 '@babel/template': 7.26.9 '@babel/types': 7.26.9 - debug: 4.4.1 + debug: 4.4.3 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -17574,18 +17327,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/traverse@7.28.0': - dependencies: - '@babel/code-frame': 7.27.1 - '@babel/generator': 7.28.5 - '@babel/helper-globals': 7.28.0 - '@babel/parser': 7.28.5 - '@babel/template': 7.27.2 - '@babel/types': 7.28.5 - debug: 4.4.3 - transitivePeerDependencies: - - supports-color - '@babel/traverse@7.28.5': dependencies: '@babel/code-frame': 7.27.1 @@ -17619,11 +17360,6 @@ snapshots: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.27.1 - '@babel/types@7.28.1': - dependencies: - '@babel/helper-string-parser': 7.27.1 - '@babel/helper-validator-identifier': 7.28.5 - '@babel/types@7.28.5': dependencies: '@babel/helper-string-parser': 7.27.1 @@ -18641,7 +18377,7 @@ snapshots: '@eslint/eslintrc@1.4.1': dependencies: ajv: 6.12.6 - debug: 4.4.0 + debug: 4.4.3 espree: 9.6.1 globals: 13.24.0 ignore: 5.3.2 @@ -18820,7 +18556,7 @@ snapshots: '@humanwhocodes/config-array@0.11.14': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.4.0 + debug: 4.4.3 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -18840,7 +18576,7 @@ snapshots: '@antfu/install-pkg': 1.1.0 '@antfu/utils': 8.1.1 '@iconify/types': 2.0.0 - debug: 4.4.1 + debug: 4.4.3 globals: 15.15.0 kolorist: 1.8.0 local-pkg: 1.1.1 @@ -18961,22 +18697,11 @@ snapshots: dependencies: '@sinclair/typebox': 0.27.8 - '@jridgewell/gen-mapping@0.3.12': - dependencies: - '@jridgewell/sourcemap-codec': 1.5.0 - '@jridgewell/trace-mapping': 0.3.29 - '@jridgewell/gen-mapping@0.3.13': dependencies: - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@jridgewell/trace-mapping': 0.3.31 - '@jridgewell/gen-mapping@0.3.5': - dependencies: - '@jridgewell/set-array': 1.2.1 - '@jridgewell/sourcemap-codec': 1.5.0 - '@jridgewell/trace-mapping': 0.3.25 - '@jridgewell/remapping@2.3.5': dependencies: '@jridgewell/gen-mapping': 0.3.13 @@ -18984,8 +18709,6 @@ snapshots: '@jridgewell/resolve-uri@3.1.2': {} - '@jridgewell/set-array@1.2.1': {} - '@jridgewell/source-map@0.3.11': dependencies: '@jridgewell/gen-mapping': 0.3.13 @@ -18998,12 +18721,7 @@ snapshots: '@jridgewell/trace-mapping@0.3.25': dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.0 - - '@jridgewell/trace-mapping@0.3.29': - dependencies: - '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@jridgewell/trace-mapping@0.3.31': dependencies: @@ -19020,7 +18738,7 @@ snapshots: '@koa/router@12.0.1': dependencies: - debug: 4.4.0 + debug: 4.4.3 http-errors: 2.0.0 koa-compose: 4.1.0 methods: 1.1.2 @@ -23091,10 +22809,10 @@ snapshots: '@rollup/pluginutils': 5.1.0(rollup@4.50.1) commondir: 1.0.1 estree-walker: 2.0.2 - fdir: 6.4.6(picomatch@4.0.2) + fdir: 6.5.0(picomatch@4.0.3) is-reference: 1.2.1 magic-string: 0.30.17 - picomatch: 4.0.2 + picomatch: 4.0.3 optionalDependencies: rollup: 4.50.1 @@ -23106,9 +22824,6 @@ snapshots: optionalDependencies: rollup: 4.50.1 - '@rollup/rollup-android-arm-eabi@4.18.0': - optional: true - '@rollup/rollup-android-arm-eabi@4.24.4': optional: true @@ -23118,9 +22833,6 @@ snapshots: '@rollup/rollup-android-arm-eabi@4.50.1': optional: true - '@rollup/rollup-android-arm64@4.18.0': - optional: true - '@rollup/rollup-android-arm64@4.24.4': optional: true @@ -23130,9 +22842,6 @@ snapshots: '@rollup/rollup-android-arm64@4.50.1': optional: true - '@rollup/rollup-darwin-arm64@4.18.0': - optional: true - '@rollup/rollup-darwin-arm64@4.24.4': optional: true @@ -23142,9 +22851,6 @@ snapshots: '@rollup/rollup-darwin-arm64@4.50.1': optional: true - '@rollup/rollup-darwin-x64@4.18.0': - optional: true - '@rollup/rollup-darwin-x64@4.24.4': optional: true @@ -23172,9 +22878,6 @@ snapshots: '@rollup/rollup-freebsd-x64@4.50.1': optional: true - '@rollup/rollup-linux-arm-gnueabihf@4.18.0': - optional: true - '@rollup/rollup-linux-arm-gnueabihf@4.24.4': optional: true @@ -23184,9 +22887,6 @@ snapshots: '@rollup/rollup-linux-arm-gnueabihf@4.50.1': optional: true - '@rollup/rollup-linux-arm-musleabihf@4.18.0': - optional: true - '@rollup/rollup-linux-arm-musleabihf@4.24.4': optional: true @@ -23196,9 +22896,6 @@ snapshots: '@rollup/rollup-linux-arm-musleabihf@4.50.1': optional: true - '@rollup/rollup-linux-arm64-gnu@4.18.0': - optional: true - '@rollup/rollup-linux-arm64-gnu@4.24.4': optional: true @@ -23208,9 +22905,6 @@ snapshots: '@rollup/rollup-linux-arm64-gnu@4.50.1': optional: true - '@rollup/rollup-linux-arm64-musl@4.18.0': - optional: true - '@rollup/rollup-linux-arm64-musl@4.24.4': optional: true @@ -23226,9 +22920,6 @@ snapshots: '@rollup/rollup-linux-loongarch64-gnu@4.50.1': optional: true - '@rollup/rollup-linux-powerpc64le-gnu@4.18.0': - optional: true - '@rollup/rollup-linux-powerpc64le-gnu@4.24.4': optional: true @@ -23238,9 +22929,6 @@ snapshots: '@rollup/rollup-linux-ppc64-gnu@4.50.1': optional: true - '@rollup/rollup-linux-riscv64-gnu@4.18.0': - optional: true - '@rollup/rollup-linux-riscv64-gnu@4.24.4': optional: true @@ -23253,9 +22941,6 @@ snapshots: '@rollup/rollup-linux-riscv64-musl@4.50.1': optional: true - '@rollup/rollup-linux-s390x-gnu@4.18.0': - optional: true - '@rollup/rollup-linux-s390x-gnu@4.24.4': optional: true @@ -23265,9 +22950,6 @@ snapshots: '@rollup/rollup-linux-s390x-gnu@4.50.1': optional: true - '@rollup/rollup-linux-x64-gnu@4.18.0': - optional: true - '@rollup/rollup-linux-x64-gnu@4.24.4': optional: true @@ -23277,9 +22959,6 @@ snapshots: '@rollup/rollup-linux-x64-gnu@4.50.1': optional: true - '@rollup/rollup-linux-x64-musl@4.18.0': - optional: true - '@rollup/rollup-linux-x64-musl@4.24.4': optional: true @@ -23292,9 +22971,6 @@ snapshots: '@rollup/rollup-openharmony-arm64@4.50.1': optional: true - '@rollup/rollup-win32-arm64-msvc@4.18.0': - optional: true - '@rollup/rollup-win32-arm64-msvc@4.24.4': optional: true @@ -23304,9 +22980,6 @@ snapshots: '@rollup/rollup-win32-arm64-msvc@4.50.1': optional: true - '@rollup/rollup-win32-ia32-msvc@4.18.0': - optional: true - '@rollup/rollup-win32-ia32-msvc@4.24.4': optional: true @@ -23316,9 +22989,6 @@ snapshots: '@rollup/rollup-win32-ia32-msvc@4.50.1': optional: true - '@rollup/rollup-win32-x64-msvc@4.18.0': - optional: true - '@rollup/rollup-win32-x64-msvc@4.24.4': optional: true @@ -23378,7 +23048,7 @@ snapshots: '@sentry/bundler-plugin-core@4.3.0(encoding@0.1.13)': dependencies: - '@babel/core': 7.28.0 + '@babel/core': 7.28.5 '@sentry/babel-plugin-component-annotate': 4.3.0 '@sentry/cli': 2.53.0(encoding@0.1.13) dotenv: 16.6.1 @@ -23436,7 +23106,7 @@ snapshots: '@sentry/core@10.11.0': {} - '@sentry/nextjs@10.11.0(@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@16.1.5(@babel/core@7.28.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.24.2))': + '@sentry/nextjs@10.11.0(@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@16.1.5(@babel/core@7.26.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11))': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/semantic-conventions': 1.37.0 @@ -23448,9 +23118,9 @@ snapshots: '@sentry/opentelemetry': 10.11.0(@opentelemetry/api@1.9.0)(@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.37.0) '@sentry/react': 10.11.0(react@19.2.3) '@sentry/vercel-edge': 10.11.0 - '@sentry/webpack-plugin': 4.3.0(encoding@0.1.13)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.24.2)) + '@sentry/webpack-plugin': 4.3.0(encoding@0.1.13)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)) chalk: 3.0.0 - next: 16.1.5(@babel/core@7.28.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + next: 16.1.5(@babel/core@7.26.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) resolve: 1.22.8 rollup: 4.50.1 stacktrace-parser: 0.1.11 @@ -23627,6 +23297,16 @@ snapshots: - encoding - supports-color + '@sentry/webpack-plugin@4.3.0(encoding@0.1.13)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11))': + dependencies: + '@sentry/bundler-plugin-core': 4.3.0(encoding@0.1.13) + unplugin: 1.0.1 + uuid: 9.0.1 + webpack: 5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11) + transitivePeerDependencies: + - encoding + - supports-color + '@shikijs/core@3.14.0': dependencies: '@shikijs/types': 3.14.0 @@ -24602,8 +24282,6 @@ snapshots: dependencies: '@types/estree': 1.0.8 - '@types/estree@1.0.5': {} - '@types/estree@1.0.6': {} '@types/estree@1.0.8': {} @@ -25340,7 +25018,7 @@ snapshots: agent-base@7.1.1: dependencies: - debug: 4.4.1 + debug: 4.4.3 transitivePeerDependencies: - supports-color @@ -25745,7 +25423,7 @@ snapshots: dependencies: bytes: 3.1.2 content-type: 1.0.5 - debug: 4.4.1 + debug: 4.4.3 http-errors: 2.0.0 iconv-lite: 0.6.3 on-finished: 2.4.1 @@ -25815,7 +25493,7 @@ snapshots: browserslist@4.23.1: dependencies: - caniuse-lite: 1.0.30001751 + caniuse-lite: 1.0.30001696 electron-to-chromium: 1.4.803 node-releases: 2.0.14 update-browserslist-db: 1.0.16(browserslist@4.23.1) @@ -25851,19 +25529,14 @@ snapshots: dependencies: run-applescript: 7.0.0 - bundle-require@4.2.1(esbuild@0.21.5): - dependencies: - esbuild: 0.21.5 - load-tsconfig: 0.2.5 - bundle-require@5.0.0(esbuild@0.24.2): dependencies: esbuild: 0.24.2 load-tsconfig: 0.2.5 - bundle-require@5.1.0(esbuild@0.25.3): + bundle-require@5.1.0(esbuild@0.25.11): dependencies: - esbuild: 0.25.3 + esbuild: 0.25.11 load-tsconfig: 0.2.5 busboy@1.6.0: @@ -26093,7 +25766,7 @@ snapshots: citty@0.1.6: dependencies: - consola: 3.4.0 + consola: 3.4.2 cjs-module-lexer@1.4.0: {} @@ -26301,8 +25974,6 @@ snapshots: consola@3.2.3: {} - consola@3.4.0: {} - consola@3.4.2: {} console-control-strings@1.1.0: {} @@ -26674,10 +26345,6 @@ snapshots: dependencies: ms: 2.1.3 - debug@4.3.5: - dependencies: - ms: 2.1.2 - debug@4.3.7: dependencies: ms: 2.1.3 @@ -27407,7 +27074,7 @@ snapshots: eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint@8.30.0))(eslint@8.30.0): dependencies: - debug: 4.4.1 + debug: 4.4.3 enhanced-resolve: 5.17.0 eslint: 8.30.0 eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint@8.30.0))(eslint@8.30.0))(eslint@8.30.0) @@ -27425,7 +27092,7 @@ snapshots: eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@8.30.0): dependencies: '@nolyfill/is-core-module': 1.0.39 - debug: 4.4.0 + debug: 4.4.3 enhanced-resolve: 5.17.1 eslint: 8.30.0 eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.30.0) @@ -27444,7 +27111,7 @@ snapshots: eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.8.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.8.3))(eslint@8.30.0))(eslint@8.30.0): dependencies: '@nolyfill/is-core-module': 1.0.39 - debug: 4.4.0 + debug: 4.4.3 enhanced-resolve: 5.17.1 eslint: 8.30.0 eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.8.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.8.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.8.3))(eslint@8.30.0))(eslint@8.30.0))(eslint@8.30.0) @@ -27908,7 +27575,7 @@ snapshots: estree-walker@3.0.3: dependencies: - '@types/estree': 1.0.6 + '@types/estree': 1.0.8 esutils@2.0.3: {} @@ -28000,7 +27667,7 @@ snapshots: content-type: 1.0.5 cookie: 0.7.1 cookie-signature: 1.2.2 - debug: 4.4.1 + debug: 4.4.3 encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 @@ -28106,14 +27773,6 @@ snapshots: optionalDependencies: picomatch: 4.0.2 - fdir@6.4.3(picomatch@4.0.2): - optionalDependencies: - picomatch: 4.0.2 - - fdir@6.4.6(picomatch@4.0.2): - optionalDependencies: - picomatch: 4.0.2 - fdir@6.5.0(picomatch@4.0.3): optionalDependencies: picomatch: 4.0.3 @@ -28148,7 +27807,7 @@ snapshots: finalhandler@2.1.0: dependencies: - debug: 4.4.1 + debug: 4.4.3 encodeurl: 2.0.0 escape-html: 1.0.3 on-finished: 2.4.1 @@ -28581,7 +28240,7 @@ snapshots: giget@2.0.0: dependencies: citty: 0.1.6 - consola: 3.4.0 + consola: 3.4.2 defu: 6.1.4 node-fetch-native: 1.6.7 nypm: 0.6.2 @@ -28942,7 +28601,7 @@ snapshots: http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.1 - debug: 4.4.0 + debug: 4.4.3 transitivePeerDependencies: - supports-color @@ -28963,7 +28622,7 @@ snapshots: https-proxy-agent@7.0.5: dependencies: agent-base: 7.1.1 - debug: 4.4.1 + debug: 4.4.3 transitivePeerDependencies: - supports-color @@ -29548,7 +29207,7 @@ snapshots: content-disposition: 0.5.4 content-type: 1.0.5 cookies: 0.9.1 - debug: 4.4.0 + debug: 4.4.3 delegates: 1.0.0 depd: 2.0.0 destroy: 1.2.0 @@ -29790,7 +29449,7 @@ snapshots: magic-string@0.30.8: dependencies: - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 make-dir@3.1.0: dependencies: @@ -30409,8 +30068,6 @@ snapshots: ms@2.0.0: {} - ms@2.1.2: {} - ms@2.1.3: {} mute-stream@1.0.0: {} @@ -30684,7 +30341,7 @@ snapshots: - '@babel/core' - babel-plugin-macros - next@16.1.5(@babel/core@7.28.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3): + next@16.1.5(@babel/core@7.26.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3): dependencies: '@next/env': 16.1.5 '@swc/helpers': 0.5.15 @@ -30693,7 +30350,7 @@ snapshots: postcss: 8.4.31 react: 19.2.3 react-dom: 19.2.3(react@19.2.3) - styled-jsx: 5.1.6(@babel/core@7.28.0)(react@19.2.3) + styled-jsx: 5.1.6(@babel/core@7.26.0)(react@19.2.3) optionalDependencies: '@next/swc-darwin-arm64': 16.1.5 '@next/swc-darwin-x64': 16.1.5 @@ -31258,23 +30915,16 @@ snapshots: camelcase-css: 2.0.1 postcss: 8.5.6 - postcss-load-config@4.0.2(postcss@8.4.47): - dependencies: - lilconfig: 3.1.2 - yaml: 2.6.0 - optionalDependencies: - postcss: 8.4.47 - postcss-load-config@4.0.2(postcss@8.5.3): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 yaml: 2.6.0 optionalDependencies: postcss: 8.5.3 postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.6)(tsx@4.21.0)(yaml@2.8.0): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 1.21.7 postcss: 8.5.6 @@ -31283,7 +30933,7 @@ snapshots: postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.4.47)(tsx@4.21.0)(yaml@2.8.0): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 2.4.2 postcss: 8.4.47 @@ -31292,7 +30942,7 @@ snapshots: postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.5.2)(tsx@4.21.0)(yaml@2.8.0): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 2.4.2 postcss: 8.5.2 @@ -31301,7 +30951,7 @@ snapshots: postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(yaml@2.4.5): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 2.4.2 postcss: 8.5.6 @@ -31310,7 +30960,7 @@ snapshots: postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.19.3)(yaml@2.6.0): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 2.4.2 postcss: 8.5.6 @@ -31319,7 +30969,7 @@ snapshots: postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.21.0)(yaml@2.8.0): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 2.4.2 postcss: 8.5.6 @@ -32316,28 +31966,6 @@ snapshots: robust-predicates@3.0.2: {} - rollup@4.18.0: - dependencies: - '@types/estree': 1.0.5 - optionalDependencies: - '@rollup/rollup-android-arm-eabi': 4.18.0 - '@rollup/rollup-android-arm64': 4.18.0 - '@rollup/rollup-darwin-arm64': 4.18.0 - '@rollup/rollup-darwin-x64': 4.18.0 - '@rollup/rollup-linux-arm-gnueabihf': 4.18.0 - '@rollup/rollup-linux-arm-musleabihf': 4.18.0 - '@rollup/rollup-linux-arm64-gnu': 4.18.0 - '@rollup/rollup-linux-arm64-musl': 4.18.0 - '@rollup/rollup-linux-powerpc64le-gnu': 4.18.0 - '@rollup/rollup-linux-riscv64-gnu': 4.18.0 - '@rollup/rollup-linux-s390x-gnu': 4.18.0 - '@rollup/rollup-linux-x64-gnu': 4.18.0 - '@rollup/rollup-linux-x64-musl': 4.18.0 - '@rollup/rollup-win32-arm64-msvc': 4.18.0 - '@rollup/rollup-win32-ia32-msvc': 4.18.0 - '@rollup/rollup-win32-x64-msvc': 4.18.0 - fsevents: 2.3.3 - rollup@4.24.4: dependencies: '@types/estree': 1.0.6 @@ -32423,7 +32051,7 @@ snapshots: router@2.2.0: dependencies: - debug: 4.4.1 + debug: 4.4.3 depd: 2.0.0 is-promise: 4.0.0 parseurl: 1.3.3 @@ -32544,7 +32172,7 @@ snapshots: send@1.2.0: dependencies: - debug: 4.4.1 + debug: 4.4.3 encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 @@ -32994,12 +32622,12 @@ snapshots: optionalDependencies: '@babel/core': 7.26.0 - styled-jsx@5.1.6(@babel/core@7.28.0)(react@19.2.3): + styled-jsx@5.1.6(@babel/core@7.26.0)(react@19.2.3): dependencies: client-only: 0.0.1 react: 19.2.3 optionalDependencies: - '@babel/core': 7.28.0 + '@babel/core': 7.26.0 styled-jsx@5.1.6(@babel/core@7.28.5)(react@19.2.3): dependencies: @@ -33014,7 +32642,7 @@ snapshots: sucrase@3.35.0: dependencies: - '@jridgewell/gen-mapping': 0.3.5 + '@jridgewell/gen-mapping': 0.3.13 commander: 4.1.1 glob: 10.4.5 lines-and-columns: 1.2.4 @@ -33242,6 +32870,18 @@ snapshots: '@swc/core': 1.3.101(@swc/helpers@0.5.15) esbuild: 0.24.2 + terser-webpack-plugin@5.3.14(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)): + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + jest-worker: 27.5.1 + schema-utils: 4.3.3 + serialize-javascript: 6.0.2 + terser: 5.44.0 + webpack: 5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11) + optionalDependencies: + '@swc/core': 1.3.101(@swc/helpers@0.5.15) + esbuild: 0.25.11 + terser@5.44.0: dependencies: '@jridgewell/source-map': 0.3.11 @@ -33337,11 +32977,6 @@ snapshots: fdir: 6.4.2(picomatch@4.0.2) picomatch: 4.0.2 - tinyglobby@0.2.12: - dependencies: - fdir: 6.4.3(picomatch@4.0.2) - picomatch: 4.0.2 - tinyglobby@0.2.15: dependencies: fdir: 6.5.0(picomatch@4.0.3) @@ -33428,30 +33063,6 @@ snapshots: tsscmp@1.0.6: {} - tsup@8.1.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(postcss@8.4.47)(typescript@5.8.3): - dependencies: - bundle-require: 4.2.1(esbuild@0.21.5) - cac: 6.7.14 - chokidar: 3.6.0 - debug: 4.3.5 - esbuild: 0.21.5 - execa: 5.1.1 - globby: 11.1.0 - joycon: 3.1.1 - postcss-load-config: 4.0.2(postcss@8.4.47) - resolve-from: 5.0.0 - rollup: 4.18.0 - source-map: 0.8.0-beta.0 - sucrase: 3.35.0 - tree-kill: 1.2.2 - optionalDependencies: - '@swc/core': 1.3.101(@swc/helpers@0.5.15) - postcss: 8.4.47 - typescript: 5.8.3 - transitivePeerDependencies: - - supports-color - - ts-node - tsup@8.3.5(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.4.47)(tsx@4.21.0)(typescript@5.8.3)(yaml@2.8.0): dependencies: bundle-require: 5.0.0(esbuild@0.24.2) @@ -33480,7 +33091,7 @@ snapshots: - tsx - yaml - tsup@8.3.5(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(typescript@5.8.3)(yaml@2.4.5): + tsup@8.3.5(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.19.3)(typescript@5.3.3)(yaml@2.6.0): dependencies: bundle-require: 5.0.0(esbuild@0.24.2) cac: 6.7.14 @@ -33490,7 +33101,7 @@ snapshots: esbuild: 0.24.2 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(yaml@2.4.5) + postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.19.3)(yaml@2.6.0) resolve-from: 5.0.0 rollup: 4.24.4 source-map: 0.8.0-beta.0 @@ -33501,35 +33112,35 @@ snapshots: optionalDependencies: '@swc/core': 1.3.101(@swc/helpers@0.5.15) postcss: 8.5.6 - typescript: 5.8.3 + typescript: 5.3.3 transitivePeerDependencies: - jiti - supports-color - tsx - yaml - tsup@8.3.5(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.19.3)(typescript@5.3.3)(yaml@2.6.0): + tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.4.47)(tsx@4.21.0)(typescript@5.8.3)(yaml@2.8.0): dependencies: - bundle-require: 5.0.0(esbuild@0.24.2) + bundle-require: 5.1.0(esbuild@0.25.11) cac: 6.7.14 - chokidar: 4.0.1 - consola: 3.2.3 - debug: 4.3.7 - esbuild: 0.24.2 + chokidar: 4.0.3 + consola: 3.4.2 + debug: 4.4.3 + esbuild: 0.25.11 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.19.3)(yaml@2.6.0) + postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.4.47)(tsx@4.21.0)(yaml@2.8.0) resolve-from: 5.0.0 - rollup: 4.24.4 + rollup: 4.50.1 source-map: 0.8.0-beta.0 sucrase: 3.35.0 - tinyexec: 0.3.1 - tinyglobby: 0.2.10 + tinyexec: 0.3.2 + tinyglobby: 0.2.15 tree-kill: 1.2.2 optionalDependencies: '@swc/core': 1.3.101(@swc/helpers@0.5.15) - postcss: 8.5.6 - typescript: 5.3.3 + postcss: 8.4.47 + typescript: 5.8.3 transitivePeerDependencies: - jiti - supports-color @@ -33538,21 +33149,21 @@ snapshots: tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.2)(tsx@4.21.0)(typescript@5.8.3)(yaml@2.8.0): dependencies: - bundle-require: 5.1.0(esbuild@0.25.3) + bundle-require: 5.1.0(esbuild@0.25.11) cac: 6.7.14 chokidar: 4.0.3 - consola: 3.4.0 - debug: 4.4.0 - esbuild: 0.25.3 + consola: 3.4.2 + debug: 4.4.3 + esbuild: 0.25.11 joycon: 3.1.1 picocolors: 1.1.1 postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.2)(tsx@4.21.0)(yaml@2.8.0) resolve-from: 5.0.0 - rollup: 4.34.8 + rollup: 4.50.1 source-map: 0.8.0-beta.0 sucrase: 3.35.0 tinyexec: 0.3.2 - tinyglobby: 0.2.12 + tinyglobby: 0.2.15 tree-kill: 1.2.2 optionalDependencies: '@swc/core': 1.3.101(@swc/helpers@0.5.15) @@ -33564,23 +33175,51 @@ snapshots: - tsx - yaml + tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(typescript@5.8.3)(yaml@2.4.5): + dependencies: + bundle-require: 5.1.0(esbuild@0.25.11) + cac: 6.7.14 + chokidar: 4.0.3 + consola: 3.4.2 + debug: 4.4.3 + esbuild: 0.25.11 + joycon: 3.1.1 + picocolors: 1.1.1 + postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(yaml@2.4.5) + resolve-from: 5.0.0 + rollup: 4.50.1 + source-map: 0.8.0-beta.0 + sucrase: 3.35.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.15 + tree-kill: 1.2.2 + optionalDependencies: + '@swc/core': 1.3.101(@swc/helpers@0.5.15) + postcss: 8.5.6 + typescript: 5.8.3 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.21.0)(typescript@5.3.3)(yaml@2.8.0): dependencies: - bundle-require: 5.1.0(esbuild@0.25.3) + bundle-require: 5.1.0(esbuild@0.25.11) cac: 6.7.14 chokidar: 4.0.3 - consola: 3.4.0 - debug: 4.4.0 - esbuild: 0.25.3 + consola: 3.4.2 + debug: 4.4.3 + esbuild: 0.25.11 joycon: 3.1.1 picocolors: 1.1.1 postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.21.0)(yaml@2.8.0) resolve-from: 5.0.0 - rollup: 4.34.8 + rollup: 4.50.1 source-map: 0.8.0-beta.0 sucrase: 3.35.0 tinyexec: 0.3.2 - tinyglobby: 0.2.12 + tinyglobby: 0.2.15 tree-kill: 1.2.2 optionalDependencies: '@swc/core': 1.3.101(@swc/helpers@0.5.15) @@ -33800,7 +33439,7 @@ snapshots: dependencies: acorn: 8.15.0 chokidar: 3.6.0 - webpack-sources: 3.2.3 + webpack-sources: 3.3.3 webpack-virtual-modules: 0.5.0 update-browserslist-db@1.0.16(browserslist@4.23.1): @@ -34026,7 +33665,7 @@ snapshots: vite-node@1.6.0(@types/node@20.17.6)(lightningcss@1.30.1)(terser@5.44.0): dependencies: cac: 6.7.14 - debug: 4.4.0 + debug: 4.4.3 pathe: 1.1.2 picocolors: 1.1.1 vite: 5.4.21(@types/node@20.17.6)(lightningcss@1.30.1)(terser@5.44.0) @@ -34056,7 +33695,7 @@ snapshots: dependencies: esbuild: 0.21.5 postcss: 8.5.6 - rollup: 4.34.8 + rollup: 4.50.1 optionalDependencies: '@types/node': 20.17.6 fsevents: 2.3.3 @@ -34221,8 +33860,6 @@ snapshots: webidl-conversions@7.0.0: {} - webpack-sources@3.2.3: {} - webpack-sources@3.3.3: {} webpack-virtual-modules@0.5.0: {} @@ -34258,6 +33895,37 @@ snapshots: - esbuild - uglify-js + webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11): + dependencies: + '@types/eslint-scope': 3.7.7 + '@types/estree': 1.0.8 + '@webassemblyjs/ast': 1.14.1 + '@webassemblyjs/wasm-edit': 1.14.1 + '@webassemblyjs/wasm-parser': 1.14.1 + acorn: 8.15.0 + acorn-import-attributes: 1.9.5(acorn@8.15.0) + browserslist: 4.27.0 + chrome-trace-event: 1.0.4 + enhanced-resolve: 5.18.3 + es-module-lexer: 1.7.0 + eslint-scope: 5.1.1 + events: 3.3.0 + glob-to-regexp: 0.4.1 + graceful-fs: 4.2.11 + json-parse-even-better-errors: 2.3.1 + loader-runner: 4.3.1 + mime-types: 2.1.35 + neo-async: 2.6.2 + schema-utils: 3.3.0 + tapable: 2.3.0 + terser-webpack-plugin: 5.3.14(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)) + watchpack: 2.4.4 + webpack-sources: 3.3.3 + transitivePeerDependencies: + - '@swc/core' + - esbuild + - uglify-js + whatwg-encoding@3.1.1: dependencies: iconv-lite: 0.6.3