fix: add matrix into acceptance tests

.env.acceptance.sample

@@ -6,6 +6,7 @@ ACCEPTANCE_PROFILE=smoke
 ACCEPTANCE_BASE_URL=http://127.0.0.1:5000
 ACCEPTANCE_S3_ENDPOINT=http://127.0.0.1:5000/s3
 ACCEPTANCE_TUS_ENDPOINT=http://127.0.0.1:5000/upload/resumable
+ACCEPTANCE_X_FORWARDED_HOST=
 ACCEPTANCE_REGION=us-east-1
 ACCEPTANCE_RESOURCE_PREFIX=acc-local
 # Matches the default dummy-data tenant; change this for non-default seeds or remote targets.

.github/workflows/acceptance.yml

@@ -40,10 +40,22 @@ concurrency:
 
 jobs:
   acceptance_local:
-    name: Local
+    name: Local / ${{ matrix.storage_backend }} / ${{ matrix.database }} / ${{ matrix.tenancy }}
     if: ${{ github.event_name != 'workflow_dispatch' || inputs.acceptance_environment == 'local' }}
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    timeout-minutes: 35
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        storage_backend:
+          - s3
+          - file
+        database:
+          - pg
+          - oriole
+        tenancy:
+          - single
+          - multitenant
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
@@ -67,7 +79,16 @@ jobs:
         run: npm run acceptance:typecheck
       - name: Run local acceptance profile
         env:
+          ACCEPTANCE_ADMIN_URL: ${{ matrix.tenancy == 'multitenant' && 'http://127.0.0.1:5001' || '' }}
+          ACCEPTANCE_ENABLE_ADMIN: ${{ matrix.tenancy == 'multitenant' && 'true' || 'false' }}
+          ACCEPTANCE_ENABLE_PATH_EDGES: ${{ matrix.storage_backend == 'file' && 'true' || 'false' }}
+          ACCEPTANCE_INFRA_RESTART_SCRIPT: ${{ matrix.database == 'oriole' && 'infra:restart:ci:oriole' || 'infra:restart:ci' }}
           ACCEPTANCE_PROFILE: ${{ inputs.profile || 'smoke' }}
+          ACCEPTANCE_X_FORWARDED_HOST: ${{ matrix.tenancy == 'multitenant' && 'bjhaohmqunupljrqypxz.local.dev' || '' }}
+          MULTI_TENANT: ${{ matrix.tenancy == 'multitenant' && 'true' || 'false' }}
+          REQUEST_X_FORWARDED_HOST_REGEXP: ${{ matrix.tenancy == 'multitenant' && '^([a-z]{20})[.]local[.](?:com|dev)$' || '' }}
+          STORAGE_PUBLIC_URL: ${{ matrix.tenancy == 'multitenant' && 'http://127.0.0.1:5000' || '' }}
+          STORAGE_BACKEND: ${{ matrix.storage_backend }}
         run: |
           mkdir -p data coverage/acceptance
           chmod -R 777 data
@@ -76,7 +97,7 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
-          name: acceptance-local
+          name: acceptance-local-${{ matrix.storage_backend }}-${{ matrix.database }}-${{ matrix.tenancy }}
           path: coverage/acceptance
           if-no-files-found: ignore
 
@@ -126,6 +147,7 @@ jobs:
           ACCEPTANCE_TLS_REJECT_UNAUTHORIZED: ${{ vars.ACCEPTANCE_TLS_REJECT_UNAUTHORIZED }}
           ACCEPTANCE_TENANT_ID: ${{ vars.ACCEPTANCE_TENANT_ID || secrets.ACCEPTANCE_TENANT_ID }}
           ACCEPTANCE_TUS_ENDPOINT: ${{ vars.ACCEPTANCE_TUS_ENDPOINT }}
+          ACCEPTANCE_X_FORWARDED_HOST: ${{ vars.ACCEPTANCE_X_FORWARDED_HOST }}
           ACCEPTANCE_ADMIN_URL: ${{ vars.ACCEPTANCE_ADMIN_URL || secrets.ACCEPTANCE_ADMIN_URL }}
           ACCEPTANCE_ADMIN_API_KEY: ${{ secrets.ACCEPTANCE_ADMIN_API_KEY }}
           ACCEPTANCE_ANON_KEY: ${{ secrets.ACCEPTANCE_ANON_KEY }}

acceptance/README.md

@@ -65,6 +65,7 @@ secrets as environment secrets.
 | `ACCEPTANCE_BASE_URL`             | REST base URL. Defaults to `http://127.0.0.1:5000`.                          |
 | `ACCEPTANCE_S3_ENDPOINT`          | S3 endpoint. Defaults to `$ACCEPTANCE_BASE_URL/s3`.                          |
 | `ACCEPTANCE_TUS_ENDPOINT`         | TUS endpoint. Defaults to `$ACCEPTANCE_BASE_URL/upload/resumable`.           |
+| `ACCEPTANCE_X_FORWARDED_HOST`     | Optional tenant-routing host header for multitenant targets.                 |
 | `ACCEPTANCE_ADMIN_URL`            | Admin API base URL for admin tests.                                          |
 | `ACCEPTANCE_SERVICE_KEY`          | Service role JWT for REST tests.                                             |
 | `ACCEPTANCE_S3_ACCESS_KEY_ID`     | S3 protocol access key.                                                      |

acceptance/scripts/run-managed-local.ts

@@ -13,7 +13,7 @@ const profile = readArg('profile') ?? acceptanceEnv('ACCEPTANCE_PROFILE') ?? 'sm
 const serverEnv = loadServerEnvFiles(inheritedEnv)
 const serverPort = serverEnv.SERVER_PORT || serverEnv.PORT || '5000'
 const baseUrl = acceptanceEnv('ACCEPTANCE_BASE_URL') ?? `http://127.0.0.1:${serverPort}`
-const acceptanceRunEnv = {
+const acceptanceRunEnv: NodeJS.ProcessEnv = {
   ...process.env,
   ACCEPTANCE_BASE_URL: baseUrl,
   ACCEPTANCE_PROFILE: profile,
@@ -23,6 +23,7 @@ const acceptanceRunEnv = {
 }
 
 let server: ChildProcess | undefined
+let provisionedS3Credential: ProvisionedS3Credential | undefined
 
 main().catch((error) => {
   console.error(error)
@@ -32,7 +33,7 @@ main().catch((error) => {
 async function main() {
   try {
     if (process.env.ACCEPTANCE_SKIP_INFRA !== 'true') {
-      await run('npm', ['run', 'infra:restart:ci'], serverEnv)
+      await run('npm', ['run', resolveInfraRestartScript()], serverEnv)
       await run('npm', ['run', 'test:dummy-data'], serverEnv)
     }
 
@@ -45,14 +46,215 @@ async function main() {
     prefixOutput(server.stderr, '[storage] ')
 
     await waitForStatus(`${baseUrl}/status`, 60_000)
+
+    if (isTruthy(serverEnv.MULTI_TENANT) || isTruthy(serverEnv.IS_MULTITENANT)) {
+      provisionedS3Credential = await provisionLocalMultitenantTenant(serverEnv)
+      acceptanceRunEnv.ACCEPTANCE_TENANT_ID = provisionedS3Credential.tenantId
+      acceptanceRunEnv.ACCEPTANCE_S3_ACCESS_KEY_ID = provisionedS3Credential.accessKey
+      acceptanceRunEnv.ACCEPTANCE_S3_SECRET_ACCESS_KEY = provisionedS3Credential.secretKey
+    }
+
     await run('npm', ['run', 'acceptance:run', '--', ...args], acceptanceRunEnv)
   } finally {
+    if (provisionedS3Credential) {
+      await deleteProvisionedS3Credential(provisionedS3Credential).catch((error) => {
+        process.stderr.write(
+          `[acceptance] failed to delete local S3 credential: ${String(error)}\n`
+        )
+      })
+    }
+
     if (server) {
       await stopServer(server)
     }
   }
 }
 
+interface ProvisionedS3Credential {
+  accessKey: string
+  adminApiKey: string
+  adminUrl: string
+  id: string
+  secretKey: string
+  tenantId: string
+}
+
+interface S3CredentialResponse {
+  access_key?: string
+  id?: string
+  secret_key?: string
+}
+
+async function provisionLocalMultitenantTenant(
+  env: NodeJS.ProcessEnv
+): Promise<ProvisionedS3Credential> {
+  const adminPort = env.SERVER_ADMIN_PORT || '5001'
+  const adminUrl = `http://127.0.0.1:${adminPort}`
+  const adminApiKey = firstCsvValue(requiredEnv(env, 'SERVER_ADMIN_API_KEYS', 'ADMIN_API_KEYS'))
+  const tenantId = requiredEnv(env, 'TENANT_ID')
+
+  await waitForStatus(`${adminUrl}/status`, 60_000)
+
+  await requestAdmin(adminUrl, adminApiKey, 'PUT', `/tenants/${encodeURIComponent(tenantId)}`, {
+    anonKey: requiredEnv(env, 'ANON_KEY'),
+    databasePoolUrl: env.DATABASE_POOL_URL || undefined,
+    databaseUrl: requiredEnv(env, 'DATABASE_URL'),
+    features: {
+      icebergCatalog: {
+        enabled: isTruthy(env.ICEBERG_ENABLED),
+        maxCatalogs: envNumber(env.ICEBERG_MAX_CATALOGS, 2),
+        maxNamespaces: envNumber(env.ICEBERG_MAX_NAMESPACES, 25),
+        maxTables: envNumber(env.ICEBERG_MAX_TABLES, 10),
+      },
+      imageTransformation: {
+        enabled: isTruthy(env.IMAGE_TRANSFORMATION_ENABLED),
+        maxResolution: envNumber(env.IMAGE_TRANSFORMATION_LIMIT_MAX_SIZE, 2000),
+      },
+      purgeCache: {
+        enabled: false,
+      },
+      s3Protocol: {
+        enabled: true,
+      },
+      vectorBuckets: {
+        enabled: isTruthy(env.VECTOR_ENABLED),
+        maxBuckets: envNumber(env.VECTOR_MAX_BUCKETS, 10),
+        maxIndexes: envNumber(env.VECTOR_MAX_INDEXES, 20),
+      },
+    },
+    fileSizeLimit: envNumber(env.UPLOAD_FILE_SIZE_LIMIT, 524288000),
+    jwtSecret: requiredEnv(env, 'AUTH_JWT_SECRET', 'PGRST_JWT_SECRET'),
+    serviceKey: requiredEnv(env, 'SERVICE_KEY'),
+  })
+
+  const credential = await requestAdmin<S3CredentialResponse>(
+    adminUrl,
+    adminApiKey,
+    'POST',
+    `/s3/${encodeURIComponent(tenantId)}/credentials`,
+    {
+      claims: {
+        role: env.DB_SERVICE_ROLE || 'service_role',
+        sub: 'local-acceptance',
+      },
+      description: `local-acceptance-${Date.now()}`,
+    },
+    201
+  )
+
+  if (!credential?.id || !credential.access_key || !credential.secret_key) {
+    throw new Error('Local multitenant S3 credential response was incomplete')
+  }
+
+  return {
+    accessKey: credential.access_key,
+    adminApiKey,
+    adminUrl,
+    id: credential.id,
+    secretKey: credential.secret_key,
+    tenantId,
+  }
+}
+
+async function deleteProvisionedS3Credential(credential: ProvisionedS3Credential) {
+  await requestAdmin(
+    credential.adminUrl,
+    credential.adminApiKey,
+    'DELETE',
+    `/s3/${encodeURIComponent(credential.tenantId)}/credentials`,
+    {
+      id: credential.id,
+    },
+    [200, 204]
+  )
+}
+
+async function requestAdmin<T = unknown>(
+  adminUrl: string,
+  adminApiKey: string,
+  method: string,
+  route: string,
+  body?: Record<string, unknown>,
+  expectedStatus: number | number[] = 204
+): Promise<T | undefined> {
+  const response = await fetch(new URL(route.replace(/^\/+/, ''), `${adminUrl}/`), {
+    body: body ? JSON.stringify(body) : undefined,
+    headers: {
+      apikey: adminApiKey,
+      ...(body ? { 'content-type': 'application/json' } : undefined),
+    },
+    method,
+  })
+  const text = await response.text()
+
+  if (!statusMatches(response.status, expectedStatus)) {
+    throw new Error(
+      [
+        `Unexpected admin status for ${method} ${route}`,
+        `expected: ${Array.isArray(expectedStatus) ? expectedStatus.join(', ') : expectedStatus}`,
+        `received: ${response.status}`,
+        `body: ${text}`,
+      ].join('\n')
+    )
+  }
+
+  return parseJson<T>(text)
+}
+
+function resolveInfraRestartScript() {
+  const script = acceptanceEnv('ACCEPTANCE_INFRA_RESTART_SCRIPT') ?? 'infra:restart:ci'
+  const allowed = new Set(['infra:restart:ci', 'infra:restart:ci:oriole'])
+
+  if (!allowed.has(script)) {
+    throw new Error(`Unsupported ACCEPTANCE_INFRA_RESTART_SCRIPT: ${script}`)
+  }
+
+  return script
+}
+
+function requiredEnv(env: NodeJS.ProcessEnv, name: string, fallbackName?: string): string {
+  const value = env[name] || (fallbackName ? env[fallbackName] : undefined)
+
+  if (!value) {
+    throw new Error(
+      `Missing required local acceptance environment variable: ${
+        fallbackName ? `${name} or ${fallbackName}` : name
+      }`
+    )
+  }
+
+  return value
+}
+
+function envNumber(value: string | undefined, fallback: number): number {
+  if (!value) {
+    return fallback
+  }
+
+  const number = Number(value)
+  return Number.isFinite(number) ? number : fallback
+}
+
+function firstCsvValue(value: string): string {
+  return value.split(',')[0]?.trim() || value
+}
+
+function isTruthy(value: string | undefined): boolean {
+  return value === '1' || value === 'true' || value === 'yes'
+}
+
+function statusMatches(status: number, expected: number | number[]) {
+  return Array.isArray(expected) ? expected.includes(status) : status === expected
+}
+
+function parseJson<T>(text: string): T | undefined {
+  if (!text) {
+    return undefined
+  }
+
+  return JSON.parse(text) as T
+}
+
 function run(cmd: string, runArgs: string[], runEnv: NodeJS.ProcessEnv): Promise<void> {
   return new Promise((resolve, reject) => {
     const child = spawn(command(cmd), runArgs, {

acceptance/specs/cdn-render.test.ts

@@ -4,7 +4,7 @@ import {
   getAcceptanceConfig,
   joinUrl,
 } from '../support/config'
-import { createRestClient } from '../support/http'
+import { createAcceptanceHeaders, createRestClient } from '../support/http'
 import {
   cleanupRestResources,
   createRestBucket,
@@ -140,11 +140,13 @@ async function expectRenderedImage(url: string, token?: string) {
   let response: Response | undefined
   try {
     response = await fetch(url, {
-      headers: token
-        ? {
-            authorization: `Bearer ${token}`,
-          }
-        : undefined,
+      headers: createAcceptanceHeaders(
+        token
+          ? {
+              authorization: `Bearer ${token}`,
+            }
+          : undefined
+      ),
     })
 
     expect(response.status).toBe(200)

acceptance/specs/rest-extended.test.ts

@@ -4,7 +4,7 @@ import {
   getAcceptanceConfig,
   joinUrl,
 } from '../support/config'
-import { createRestClient } from '../support/http'
+import { createAcceptanceHeaders, createRestClient } from '../support/http'
 import {
   cleanupRestResources,
   createRestBucket,
@@ -209,9 +209,9 @@ describeAcceptance(
             joinUrl(config.baseUrl, signedUpload.json?.url ?? ''),
             {
               body: signedPayload,
-              headers: {
+              headers: createAcceptanceHeaders({
                 'content-type': 'text/plain',
-              },
+              }),
               method: 'PUT',
             }
           )

acceptance/specs/rest-object.test.ts

@@ -4,7 +4,7 @@ import {
   getAcceptanceConfig,
   joinUrl,
 } from '../support/config'
-import { createRestClient } from '../support/http'
+import { createAcceptanceHeaders, createRestClient } from '../support/http'
 import {
   cleanupRestResources,
   createRestBucket,
@@ -106,7 +106,9 @@ describeAcceptance(
         const signedUrl = joinUrl(config.baseUrl, signed.json?.signedURL ?? '')
         let signedResponse: Response | undefined
         try {
-          signedResponse = await fetch(signedUrl)
+          signedResponse = await fetch(signedUrl, {
+            headers: createAcceptanceHeaders(),
+          })
           expect(signedResponse.status).toBe(200)
           expect(await signedResponse.text()).toBe(payload)
         } finally {