External db sync

.github/workflows/db-migration-backwards-compatibility.yaml

@@ -193,6 +193,17 @@ jobs:
           wait-for: 30s
           log-output-if: true
 
+      - name: Start run-cron-jobs in background
+        uses: JarvusInnovations/background-action@v1.0.7
+        if: ${{ hashFiles('apps/backend/scripts/run-cron-jobs.ts') != '' }}
+        with:
+          run: pnpm -C apps/backend run with-env:dev tsx scripts/run-cron-jobs.ts --log-order=stream &
+          wait-on: |
+            http://localhost:8102
+          tail: true
+          wait-for: 30s
+          log-output-if: true
+
       - name: Wait 10 seconds
         run: sleep 10
 
@@ -230,4 +241,3 @@ jobs:
     steps:
       - name: No migration changes detected
         run: echo "No changes to migrations folder detected. Skipping backwards compatibility test."
-

.github/workflows/e2e-api-tests.yaml

@@ -19,6 +19,9 @@ jobs:
       NODE_ENV: test
       STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes
       STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe"
+      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
+      STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
+      STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
     strategy:
       matrix:
@@ -100,6 +103,9 @@ jobs:
 
       - name: Wait on Svix
         run: pnpx wait-on tcp:localhost:8113
+
+      - name: Wait on QStash
+        run: pnpx wait-on tcp:localhost:8125
 
       - name: Initialize database
         run: pnpm run db:init
@@ -140,20 +146,29 @@ jobs:
           tail: true
           wait-for: 30s
           log-output-if: true
+      - name: Start run-cron-jobs in background
+        uses: JarvusInnovations/background-action@v1.0.7
+        with:
+          run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream &
+          wait-on: |
+            http://localhost:8102
+          tail: true
+          wait-for: 30s
+          log-output-if: true
 
       - name: Wait 10 seconds
         run: sleep 10
 
       - name: Run tests
-        run: pnpm test ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }}
+        run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }}
 
-      - name: Run tests again, to make sure they are stable (attempt 1)
+      - name: Run tests again (attempt 1)
         if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }}
+        run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }}
 
-      - name: Run tests again, to make sure they are stable (attempt 2)
+      - name: Run tests again (attempt 2)
         if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }}
+        run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }}
 
       - name: Verify data integrity
         run: pnpm run verify-data-integrity --no-bail

.github/workflows/e2e-custom-base-port-api-tests.yaml

@@ -19,6 +19,9 @@ jobs:
       STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes
       STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:6728/stackframe"
       NEXT_PUBLIC_STACK_PORT_PREFIX: "67"
+      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
+      STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
+      STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
     strategy:
       matrix:
@@ -94,6 +97,9 @@ jobs:
 
       - name: Wait on Svix
         run: pnpx wait-on tcp:localhost:6713
+
+      - name: Wait on QStash
+        run: pnpx wait-on tcp:localhost:6725
 
       - name: Initialize database
         run: pnpm run db:init
@@ -134,20 +140,29 @@ jobs:
           tail: true
           wait-for: 30s
           log-output-if: true
+      - name: Start run-cron-jobs in background
+        uses: JarvusInnovations/background-action@v1.0.7
+        with:
+          run: pnpm -C apps/backend run run-cron-jobs --log-order=stream &
+          wait-on: |
+            http://localhost:6702
+          tail: true
+          wait-for: 30s
+          log-output-if: true
 
       - name: Wait 10 seconds
         run: sleep 10
 
       - name: Run tests
-        run: pnpm test
+        run: pnpm test run
 
-      - name: Run tests again, to make sure they are stable (attempt 1)
+      - name: Run tests again (attempt 1)
         if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test
+        run: pnpm test run
 
-      - name: Run tests again, to make sure they are stable (attempt 2)
+      - name: Run tests again (attempt 2)
         if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test
+        run: pnpm test run
 
       - name: Verify data integrity
         run: pnpm run verify-data-integrity --no-bail

.github/workflows/e2e-source-of-truth-api-tests.yaml

@@ -17,9 +17,13 @@ jobs:
     env:
       NODE_ENV: test
       STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes
+      STACK_ACCESS_TOKEN_EXPIRATION_TIME: 30m
       STACK_OVERRIDE_SOURCE_OF_TRUTH: '{"type": "postgres", "connectionString": "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/source-of-truth-db?schema=sot-schema"}'
       STACK_TEST_SOURCE_OF_TRUTH: true
       STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe"
+      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
+      STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
+      STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
     strategy:
       matrix:
@@ -95,6 +99,9 @@ jobs:
 
       - name: Wait on Svix
         run: pnpx wait-on tcp:localhost:8113
+
+      - name: Wait on QStash
+        run: pnpx wait-on tcp:localhost:8125
 
       - name: Create source-of-truth database and schema
         run: |
@@ -140,20 +147,29 @@ jobs:
           tail: true
           wait-for: 30s
           log-output-if: true
+      - name: Start run-cron-jobs in background
+        uses: JarvusInnovations/background-action@v1.0.7
+        with:
+          run: pnpm -C apps/backend run run-cron-jobs --log-order=stream &
+          wait-on: |
+            http://localhost:8102
+          tail: true
+          wait-for: 30s
+          log-output-if: true
 
       - name: Wait 10 seconds
         run: sleep 10
 
       - name: Run tests
-        run: pnpm test
+        run: pnpm test run --exclude "**/external-db-sync*.test.ts" # external-db-sync does not support external sot
 
-      - name: Run tests again, to make sure they are stable (attempt 1)
+      - name: Run tests again (attempt 1)
         if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test
+        run: pnpm test run --exclude "**/external-db-sync*.test.ts"
 
-      - name: Run tests again, to make sure they are stable (attempt 2)
+      - name: Run tests again (attempt 2)
         if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test
+        run: pnpm test run --exclude "**/external-db-sync*.test.ts"
 
       - name: Verify data integrity
         run: pnpm run verify-data-integrity --no-bail

.github/workflows/restart-dev-and-test-with-custom-base-port.yaml

@@ -19,6 +19,9 @@ jobs:
     runs-on: ubicloud-standard-16
     env:
       NEXT_PUBLIC_STACK_PORT_PREFIX: "69"
+      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
+      STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
+      STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
     steps:
       - uses: actions/checkout@v6
@@ -38,7 +41,7 @@ jobs:
         run: pnpm run restart-dev-environment
 
       - name: Run tests
-        run: pnpm run test --reporter=verbose
+        run: pnpm run test run --reporter=verbose
 
       - name: Print dev server logs
         run: cat dev-server.log.untracked.txt

.github/workflows/restart-dev-and-test.yaml

@@ -17,6 +17,10 @@ env:
 jobs:
   restart-dev-and-test:
     runs-on: ubicloud-standard-16
+    env:
+      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
+      STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
+      STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
     steps:
       - uses: actions/checkout@v6
@@ -36,7 +40,7 @@ jobs:
         run: pnpm run restart-dev-environment
 
       - name: Run tests
-        run: pnpm run test --reporter=verbose
+        run: pnpm run test run --reporter=verbose
 
       - name: Print dev server logs
         run: cat dev-server.log.untracked.txt

.github/workflows/setup-tests-with-custom-base-port.yaml

@@ -19,6 +19,9 @@ jobs:
     runs-on: ubicloud-standard-16
     env:
       NEXT_PUBLIC_STACK_PORT_PREFIX: "69"
+      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
+      STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
+      STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
 
     steps:
       - uses: actions/checkout@v6
@@ -46,4 +49,5 @@ jobs:
           tail: true
           wait-for: 120s
           log-output-if: true
-      - run: pnpm run test --reporter=verbose
+      - name: Run tests
+        run: pnpm run test run --reporter=verbose

.github/workflows/setup-tests.yaml

@@ -17,6 +17,10 @@ env:
 jobs:
   setup-tests:
     runs-on: ubicloud-standard-16
+    env:
+      STACK_FORCE_EXTERNAL_DB_SYNC: "true"
+      STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000"
+      STACK_EXTERNAL_DB_SYNC_DIRECT: "false"
     steps:
       - uses: actions/checkout@v6
 
@@ -43,4 +47,5 @@ jobs:
           tail: true
           wait-for: 120s
           log-output-if: true
-      - run: pnpm run test --reporter=verbose
+      - name: Run tests
+        run: pnpm run test run --reporter=verbose

apps/backend/.env.development

@@ -35,8 +35,8 @@ STACK_DATABASE_REPLICATION_WAIT_STRATEGY=pg-stat-replication
 STACK_EMAIL_HOST=127.0.0.1
 STACK_EMAIL_PORT=${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}29
 STACK_EMAIL_SECURE=false
-STACK_EMAIL_USERNAME=does not matter, ignored by Inbucket
-STACK_EMAIL_PASSWORD=does not matter, ignored by Inbucket
+STACK_EMAIL_USERNAME="does not matter, ignored by Inbucket"
+STACK_EMAIL_PASSWORD="does not matter, ignored by Inbucket"
 STACK_EMAIL_SENDER=noreply@example.com
 
 STACK_ACCESS_TOKEN_EXPIRATION_TIME=60s
@@ -50,7 +50,7 @@ STACK_ARTIFICIAL_DEVELOPMENT_DELAY_MS=500
 
 STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING=yes
 
-STACK_INTEGRATION_CLIENTS_CONFIG=[{"client_id": "neon-local", "client_secret": "neon-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}, {"client_id": "custom-local", "client_secret": "custom-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}]
+STACK_INTEGRATION_CLIENTS_CONFIG='[{"client_id": "neon-local", "client_secret": "neon-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}, {"client_id": "custom-local", "client_secret": "custom-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}]'
 CRON_SECRET=mock_cron_secret
 STACK_FREESTYLE_API_KEY=mock_stack_freestyle_key
 STACK_VERCEL_SANDBOX_TOKEN=vercel_sandbox_disabled_for_local_development

apps/backend/package.json

@@ -10,7 +10,8 @@
     "with-env": "dotenv -c --",
     "with-env:dev": "dotenv -c development --",
     "with-env:prod": "dotenv -c production --",
-    "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\"",
+    "with-env:test": "dotenv -c test --",
+    "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue,cron-jobs\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\" \"pnpm run run-cron-jobs\"",
     "dev:inspect": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--inspect\" pnpm run dev",
     "dev:profile": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--experimental-cpu-prof\" pnpm run dev",
     "build": "pnpm run codegen && next build",
@@ -42,6 +43,8 @@
     "codegen-docs:watch": "pnpm run with-env tsx watch --exclude '**/node_modules/**' --clear-screen=false scripts/generate-openapi-fumadocs.ts",
     "generate-keys": "pnpm run with-env tsx scripts/generate-keys.ts",
     "db-seed-script": "pnpm run db:seed",
+    "run-cron-jobs": "pnpm run with-env:dev tsx scripts/run-cron-jobs.ts",
+    "run-cron-jobs:test": "pnpm run with-env:test tsx scripts/run-cron-jobs.ts",
     "verify-data-integrity": "pnpm run with-env:dev tsx scripts/verify-data-integrity/index.ts",
     "run-email-queue": "pnpm run with-env:dev tsx scripts/run-email-queue.ts"
   },