diff --git a/.dockerignore b/.dockerignore index 836abeef03..893ad8ac96 100644 --- a/.dockerignore +++ b/.dockerignore @@ -61,6 +61,8 @@ dist .docusaurus .cache-loader **.tsbuildinfo +docker/local-emulator/qemu/images +docker/local-emulator/qemu/run .xata* @@ -149,4 +151,3 @@ packages/stack/* !packages/react/package.json !packages/next/package.json !packages/stack/package.json - diff --git a/.github/workflows/qemu-emulator-build.yaml b/.github/workflows/qemu-emulator-build.yaml new file mode 100644 index 0000000000..e4a42207ca --- /dev/null +++ b/.github/workflows/qemu-emulator-build.yaml @@ -0,0 +1,255 @@ +name: Build & Publish QEMU Emulator Images + +on: + push: + branches: + - main + - dev + pull_request: + paths: + - 'docker/local-emulator/**' + - '.github/workflows/qemu-emulator-build.yaml' + workflow_dispatch: + inputs: + publish: + description: 'Publish images to GitHub Releases' + type: boolean + default: false + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }} + +env: + EMULATOR_IMAGE_NAME: stack-local-emulator + +jobs: + build: + name: Build QEMU Image (${{ matrix.arch }}) + runs-on: ubicloud-standard-8 + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + - arch: arm64 + + steps: + - uses: actions/checkout@v6 + + - name: Set up QEMU user-mode emulation + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Install QEMU dependencies + run: | + sudo apt-get update + sudo apt-get install -y qemu-system-x86 qemu-system-arm qemu-utils genisoimage socat qemu-efi-aarch64 + + - name: Build QEMU image + run: | + chmod +x docker/local-emulator/qemu/build-image.sh + EMULATOR_PROVISION_TIMEOUT=6000 \ + docker/local-emulator/qemu/build-image.sh ${{ matrix.arch }} + + - name: Generate emulator env + run: node docker/local-emulator/generate-env-development.mjs + + - name: Start emulator and verify + run: | + chmod +x docker/local-emulator/qemu/run-emulator.sh + EMULATOR_ARCH=${{ matrix.arch }} \ + EMULATOR_READY_TIMEOUT=3200 \ + docker/local-emulator/qemu/run-emulator.sh start + + - name: Verify services are healthy + run: | + EMULATOR_ARCH=${{ matrix.arch }} \ + docker/local-emulator/qemu/run-emulator.sh status + + - name: Stop emulator + if: always() + run: | + EMULATOR_ARCH=${{ matrix.arch }} \ + docker/local-emulator/qemu/run-emulator.sh stop + + - name: Package image + run: | + BASE_IMG="docker/local-emulator/qemu/images/stack-emulator-${{ matrix.arch }}.qcow2" + cp "$BASE_IMG" "stack-emulator-${{ matrix.arch }}.qcow2" + + - name: Upload image artifact + uses: actions/upload-artifact@v4 + with: + name: qemu-emulator-${{ matrix.arch }} + path: stack-emulator-${{ matrix.arch }}.qcow2 + retention-days: 30 + compression-level: 0 + + test: + name: Smoke Test (${{ matrix.arch }}) + needs: build + runs-on: ubicloud-standard-8 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + + steps: + - uses: actions/checkout@v6 + + - name: Install QEMU dependencies + run: | + sudo apt-get update + sudo apt-get install -y qemu-system-x86 qemu-utils genisoimage socat + + - name: Download built image + uses: actions/download-artifact@v4 + with: + name: qemu-emulator-${{ matrix.arch }} + path: docker/local-emulator/qemu/images/ + + - name: Generate emulator env + run: node docker/local-emulator/generate-env-development.mjs + + - name: Start emulator from artifact + run: | + chmod +x docker/local-emulator/qemu/run-emulator.sh docker/local-emulator/qemu/common.sh + EMULATOR_ARCH=${{ matrix.arch }} \ + EMULATOR_READY_TIMEOUT=600 \ + docker/local-emulator/qemu/run-emulator.sh start + + - name: Verify services are healthy + run: | + EMULATOR_ARCH=${{ matrix.arch }} \ + docker/local-emulator/qemu/run-emulator.sh status + + - name: Smoke test — backend health + run: curl -sf http://localhost:26701/health?db=1 + + - name: Smoke test — dashboard reachable + run: curl -sf -o /dev/null -w "HTTP %{http_code}\n" http://localhost:26700/handler/sign-in + + - name: Smoke test — MinIO health + run: curl -sf http://localhost:26702/minio/health/live + + - name: Smoke test — Inbucket reachable + run: curl -sf -o /dev/null -w "HTTP %{http_code}\n" http://localhost:26703/ + + - name: Stop emulator + if: always() + run: | + EMULATOR_ARCH=${{ matrix.arch }} \ + docker/local-emulator/qemu/run-emulator.sh stop + + - name: Print serial log on failure + if: failure() + run: tail -100 docker/local-emulator/qemu/run/vm/serial.log 2>/dev/null || true + + publish: + name: Publish to GitHub Releases + needs: [build, test] + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || (github.event_name == 'workflow_dispatch' && inputs.publish) + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v6 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Prepare release assets + run: | + mkdir -p release + SHORT_SHA="${GITHUB_SHA:0:8}" + BRANCH="${GITHUB_REF_NAME}" + DATE="$(date -u +%Y%m%d)" + TAG="emulator-${BRANCH}-${DATE}-${SHORT_SHA}" + echo "RELEASE_TAG=${TAG}" >> "$GITHUB_ENV" + echo "SHORT_SHA=${SHORT_SHA}" >> "$GITHUB_ENV" + + for f in artifacts/qemu-emulator-*/*.qcow2; do + cp "$f" release/ + done + + cat > release-notes.md </dev/null 2>&1; then + gh release edit "$RELEASE_TAG" \ + --title "$TITLE" \ + --notes-file release-notes.md \ + --prerelease + gh release upload "$RELEASE_TAG" release/* --clobber + else + gh release create "$RELEASE_TAG" \ + --title "$TITLE" \ + --notes-file release-notes.md \ + --prerelease \ + release/* + fi + + - name: Update latest tag for branch + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + LATEST_TAG="emulator-${{ github.ref_name }}-latest" + TITLE="QEMU Emulator — ${{ github.ref_name }} (latest)" + NOTES="Latest emulator images from \`${{ github.ref_name }}\`. Auto-updated on each build." + + if gh release view "$LATEST_TAG" >/dev/null 2>&1; then + gh release edit "$LATEST_TAG" \ + --draft \ + --prerelease \ + --target "${{ github.sha }}" \ + --title "$TITLE" \ + --notes "$NOTES" + else + gh release create "$LATEST_TAG" \ + --draft \ + --prerelease \ + --target "${{ github.sha }}" \ + --title "$TITLE" \ + --notes "$NOTES" \ + || gh release edit "$LATEST_TAG" \ + --draft \ + --prerelease \ + --target "${{ github.sha }}" \ + --title "$TITLE" \ + --notes "$NOTES" + fi + + gh release upload "$LATEST_TAG" release/* --clobber + gh release edit "$LATEST_TAG" --draft=false --prerelease diff --git a/.gitignore b/.gitignore index 592215d698..a8264f9e7d 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ vite.config.ts.timestamp-* .eslintcache .env.local .env.*.local +docker/local-emulator/.env.development scratch/ npm-debug.log* diff --git a/apps/backend/prisma/seed.ts b/apps/backend/prisma/seed.ts index 8dd8a33b52..b6ce20cef5 100644 --- a/apps/backend/prisma/seed.ts +++ b/apps/backend/prisma/seed.ts @@ -502,6 +502,13 @@ export async function seed() { } else { console.log('Ensured emulator user is a member of emulator team'); } + + await grantTeamPermission(internalPrisma, { + tenancy: internalTenancy, + teamId: LOCAL_EMULATOR_OWNER_TEAM_ID, + userId: LOCAL_EMULATOR_ADMIN_USER_ID, + permissionId: "team_admin", + }); } console.log('Seeding complete!'); diff --git a/apps/backend/src/app/api/latest/internal/local-emulator/project/route.tsx b/apps/backend/src/app/api/latest/internal/local-emulator/project/route.tsx index 0ad1ea4f8d..e660c21c75 100644 --- a/apps/backend/src/app/api/latest/internal/local-emulator/project/route.tsx +++ b/apps/backend/src/app/api/latest/internal/local-emulator/project/route.tsx @@ -5,6 +5,7 @@ import { LOCAL_EMULATOR_OWNER_TEAM_ID, isLocalEmulatorEnabled, readConfigFromFile, + resolveEmulatorPath, writeConfigToFile, } from "@/lib/local-emulator"; import { DEFAULT_BRANCH_ID, getSoleTenancyFromProjectBranch } from "@/lib/tenancies"; @@ -192,11 +193,12 @@ export const POST = createSmartRouteHandler({ } const absoluteFilePath = path.resolve(req.body.absolute_file_path); + const resolvedFilePath = resolveEmulatorPath(absoluteFilePath); // Validate file exists before creating a project let fileExists: boolean; try { - await fs.access(absoluteFilePath); + await fs.access(resolvedFilePath); fileExists = true; } catch { fileExists = false; @@ -206,7 +208,7 @@ export const POST = createSmartRouteHandler({ } // If the file is empty, write a default config - const fileContent = await fs.readFile(absoluteFilePath, "utf-8"); + const fileContent = await fs.readFile(resolvedFilePath, "utf-8"); if (fileContent.trim() === "") { await writeConfigToFile(absoluteFilePath, {}); } diff --git a/apps/backend/src/lib/local-emulator.test.ts b/apps/backend/src/lib/local-emulator.test.ts new file mode 100644 index 0000000000..e5519cae31 --- /dev/null +++ b/apps/backend/src/lib/local-emulator.test.ts @@ -0,0 +1,92 @@ +import fs from "fs/promises"; +import os from "os"; +import path from "path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + LOCAL_EMULATOR_HOST_MOUNT_ROOT_ENV, + readConfigFromFile, + writeConfigToFile, +} from "./local-emulator"; + +describe("local emulator config", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("reads config from STACK_LOCAL_EMULATOR_CONFIG_CONTENT env var when set", async () => { + const content = `export const config = { auth: { allowLocalhost: true } };\n`; + vi.stubEnv("STACK_LOCAL_EMULATOR_CONFIG_CONTENT", Buffer.from(content).toString("base64")); + + await expect(readConfigFromFile("/irrelevant/path/stack.config.ts")).resolves.toMatchInlineSnapshot(` + { + "auth": { + "allowLocalhost": true, + }, + } + `); + }); + + it("returns empty object when env var is not set and file does not exist", async () => { + await expect(readConfigFromFile("/nonexistent/stack.config.ts")).resolves.toEqual({}); + }); + + it("returns empty object when env var content is empty", async () => { + const content = ``; + vi.stubEnv("STACK_LOCAL_EMULATOR_CONFIG_CONTENT", Buffer.from(content).toString("base64")); + + await expect(readConfigFromFile("/irrelevant/path/stack.config.ts")).resolves.toEqual({}); + }); + + it("throws when the config module does not export config", async () => { + const content = `export default { auth: { allowLocalhost: true } };\n`; + vi.stubEnv("STACK_LOCAL_EMULATOR_CONFIG_CONTENT", Buffer.from(content).toString("base64")); + + await expect(readConfigFromFile("/irrelevant/path/stack.config.ts")).rejects.toThrow( + "Invalid config in /irrelevant/path/stack.config.ts. The file must export a 'config' object." + ); + }); + + it("reads config files from the host mount when configured", async () => { + const hostMountRoot = await fs.mkdtemp(path.join(os.tmpdir(), "stack-host-mount-")); + const absoluteFilePath = "/Users/foo/project/stack.config.ts"; + const mountedFilePath = path.join(hostMountRoot, absoluteFilePath); + await fs.mkdir(path.dirname(mountedFilePath), { recursive: true }); + await fs.writeFile(mountedFilePath, `export const config = { auth: { allowLocalhost: true } };\n`, "utf-8"); + + vi.stubEnv(LOCAL_EMULATOR_HOST_MOUNT_ROOT_ENV, hostMountRoot); + + await expect(readConfigFromFile(absoluteFilePath)).resolves.toMatchInlineSnapshot(` + { + "auth": { + "allowLocalhost": true, + }, + } + `); + }); + + it("writes new config files to the host mount when the mounted parent directory exists", async () => { + const hostMountRoot = await fs.mkdtemp(path.join(os.tmpdir(), "stack-host-mount-")); + const absoluteFilePath = "/Users/foo/project/stack.config.ts"; + const mountedParentPath = path.join(hostMountRoot, "/Users/foo/project"); + const mountedFilePath = path.join(hostMountRoot, absoluteFilePath); + await fs.mkdir(mountedParentPath, { recursive: true }); + + vi.stubEnv(LOCAL_EMULATOR_HOST_MOUNT_ROOT_ENV, hostMountRoot); + + await writeConfigToFile(absoluteFilePath, { auth: { allowLocalhost: true } }); + + await expect(fs.readFile(mountedFilePath, "utf-8")).resolves.toBe( + `export const config = {\n "auth": {\n "allowLocalhost": true\n }\n};\n` + ); + }); + + it("fails loudly when the QEMU host mount root is configured but unavailable", async () => { + const hostMountRoot = await fs.mkdtemp(path.join(os.tmpdir(), "stack-host-mount-")); + vi.stubEnv(LOCAL_EMULATOR_HOST_MOUNT_ROOT_ENV, hostMountRoot); + + await expect(writeConfigToFile("/Users/foo/project/stack.config.ts", { auth: { allowLocalhost: true } })).rejects.toThrow( + `Local emulator host mount root ${hostMountRoot} is configured` + ); + }); +}); diff --git a/apps/backend/src/lib/local-emulator.ts b/apps/backend/src/lib/local-emulator.ts index ec9b27f29e..8b433d1d03 100644 --- a/apps/backend/src/lib/local-emulator.ts +++ b/apps/backend/src/lib/local-emulator.ts @@ -1,10 +1,10 @@ -import fs from "fs/promises"; -import path from "path"; -import { createJiti } from "jiti"; +import { globalPrismaClient } from "@/prisma-client"; import { isValidConfig } from "@stackframe/stack-shared/dist/config/format"; import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; -import { globalPrismaClient } from "@/prisma-client"; +import fs from "fs/promises"; +import { createJiti } from "jiti"; +import path from "path"; export const LOCAL_EMULATOR_ADMIN_USER_ID = "63abbc96-5329-454a-ba56-e0460173c6c1"; export const LOCAL_EMULATOR_OWNER_TEAM_ID = "5a0c858b-d9e9-49d4-9943-8ce385d86428"; @@ -15,6 +15,7 @@ export const LOCAL_EMULATOR_ENV_CONFIG_BLOCKED_MESSAGE = "Environment configuration overrides cannot be changed in the local emulator. Update this in your production deployment instead."; export const LOCAL_EMULATOR_ONLY_ENDPOINT_MESSAGE = "This endpoint is only available in local emulator mode (set NEXT_PUBLIC_STACK_IS_LOCAL_EMULATOR=true)."; +export const LOCAL_EMULATOR_HOST_MOUNT_ROOT_ENV = "STACK_LOCAL_EMULATOR_HOST_MOUNT_ROOT"; export function isLocalEmulatorEnabled() { return getEnvVariable("NEXT_PUBLIC_STACK_IS_LOCAL_EMULATOR", "") === "true"; @@ -44,16 +45,36 @@ export async function getLocalEmulatorFilePath(projectId: string): Promise> { + // Check for base64-encoded config content override from env var + const envContent = getEnvVariable("STACK_LOCAL_EMULATOR_CONFIG_CONTENT", ""); let content: string; - try { - content = await fs.readFile(filePath, "utf-8"); - } catch (e: any) { - if (e?.code === "ENOENT") { - throw new StatusError(StatusError.BadRequest, `Config file not found: ${filePath}`); + if (envContent) { + content = Buffer.from(envContent, "base64").toString("utf-8"); + } else { + const resolvedPath = resolveEmulatorPath(filePath); + try { + content = await fs.readFile(resolvedPath, "utf-8"); + } catch (e: any) { + if (e?.code === "ENOENT") { + return {}; + } + throw e; } - throw e; } + + if (content.trim() === "") { + return {}; + } + const jiti = createJiti(import.meta.url, { cache: false }); const mod = jiti.evalModule(content, { filename: filePath }) as Record; const config = mod.config; @@ -64,8 +85,18 @@ export async function readConfigFromFile(filePath: string): Promise): Promise { - const dir = path.dirname(filePath); - await fs.mkdir(dir, { recursive: true }); + const resolvedPath = resolveEmulatorPath(filePath); + const dir = path.dirname(resolvedPath); + const hostMountRoot = getEnvVariable(LOCAL_EMULATOR_HOST_MOUNT_ROOT_ENV, ""); + if (hostMountRoot) { + try { + await fs.access(dir); + } catch { + throw new Error(`Local emulator host mount root ${hostMountRoot} is configured but the parent directory for ${filePath} is not available at ${dir}. Ensure the host filesystem is mounted correctly.`); + } + } else { + await fs.mkdir(dir, { recursive: true }); + } const content = `export const config = ${JSON.stringify(config, null, 2)};\n`; - await fs.writeFile(filePath, content, "utf-8"); + await fs.writeFile(resolvedPath, content, "utf-8"); } diff --git a/apps/dashboard/src/app/(main)/(protected)/(outside-dashboard)/projects/page-client.tsx b/apps/dashboard/src/app/(main)/(protected)/(outside-dashboard)/projects/page-client.tsx index f546388030..7398c60865 100644 --- a/apps/dashboard/src/app/(main)/(protected)/(outside-dashboard)/projects/page-client.tsx +++ b/apps/dashboard/src/app/(main)/(protected)/(outside-dashboard)/projects/page-client.tsx @@ -8,7 +8,7 @@ import { getPublicEnvVar } from "@/lib/env"; import { stackAppInternalsSymbol } from "@/lib/stack-app-internals"; import { GearIcon } from "@phosphor-icons/react"; import { AdminOwnedProject, Team, useStackApp, useUser } from "@stackframe/stack"; -import { projectOnboardingStatusValues, strictEmailSchema, type ProjectOnboardingStatus, yupObject } from "@stackframe/stack-shared/dist/schema-fields"; +import { projectOnboardingStatusValues, strictEmailSchema, yupObject, type ProjectOnboardingStatus } from "@stackframe/stack-shared/dist/schema-fields"; import { groupBy } from "@stackframe/stack-shared/dist/utils/arrays"; import { runAsynchronously, runAsynchronouslyWithAlert, wait } from "@stackframe/stack-shared/dist/utils/promises"; import { useQueryState } from "@stackframe/stack-shared/dist/utils/react"; diff --git a/apps/e2e/tests/general/cli.test.ts b/apps/e2e/tests/general/cli.test.ts index c2f1176d61..cd23a6da73 100644 --- a/apps/e2e/tests/general/cli.test.ts +++ b/apps/e2e/tests/general/cli.test.ts @@ -8,6 +8,7 @@ import { describe, beforeAll, afterAll } from "vitest"; import { it, niceFetch, STACK_BACKEND_BASE_URL, STACK_INTERNAL_PROJECT_CLIENT_KEY, STACK_INTERNAL_PROJECT_SERVER_KEY, STACK_INTERNAL_PROJECT_ADMIN_KEY } from "../helpers"; const CLI_BIN = path.resolve("packages/stack-cli/dist/index.js"); +const CLI_SRC_BIN = path.resolve("packages/stack-cli/src/index.ts"); function runCli( args: string[], @@ -464,3 +465,72 @@ describe("Stack CLI", () => { expect(stdout).toContain("STACK AUTH SETUP INSTRUCTIONS"); }); }); + +// Emulator CLI tests — no backend required, just validates help/arg parsing +describe("Stack CLI — Emulator", () => { + function runCliBare( + args: string[], + ): Promise<{ stdout: string, stderr: string, exitCode: number | null }> { + return new Promise((resolve) => { + execFile("node", [CLI_BIN, ...args], { + env: { PATH: process.env.PATH ?? "", HOME: process.env.HOME ?? "", CI: "1" }, + timeout: 15_000, + }, (error, stdout, stderr) => { + resolve({ + stdout: stdout.toString(), + stderr: stderr.toString(), + exitCode: error ? (error as any).code ?? 1 : 0, + }); + }); + }); + } + + function runCliBareFromSource( + args: string[], + ): Promise<{ stdout: string, stderr: string, exitCode: number | null }> { + return new Promise((resolve) => { + execFile("node", ["--import", "tsx", CLI_SRC_BIN, ...args], { + env: { PATH: process.env.PATH ?? "", HOME: process.env.HOME ?? "", CI: "1" }, + timeout: 15_000, + }, (error, stdout, stderr) => { + resolve({ + stdout: stdout.toString(), + stderr: stderr.toString(), + exitCode: error ? (error as any).code ?? 1 : 0, + }); + }); + }); + } + + it("emulator help shows subcommands", async ({ expect }) => { + const { stdout, exitCode } = await runCliBare(["emulator", "--help"]); + expect(exitCode).toBe(0); + expect(stdout).toContain("pull"); + expect(stdout).toContain("start"); + expect(stdout).toContain("stop"); + expect(stdout).toContain("reset"); + expect(stdout).toContain("status"); + expect(stdout).toContain("list-releases"); + }); + + it("emulator pull help shows options", async ({ expect }) => { + const { stdout, exitCode } = await runCliBare(["emulator", "pull", "--help"]); + expect(exitCode).toBe(0); + expect(stdout).toContain("--arch"); + expect(stdout).toContain("--branch"); + expect(stdout).toContain("--tag"); + expect(stdout).toContain("--repo"); + }); + + it("emulator pull rejects invalid arch values", async ({ expect }) => { + const { stderr, exitCode } = await runCliBareFromSource(["emulator", "pull", "--arch", "sparc"]); + expect(exitCode).toBe(1); + expect(stderr).toContain("Invalid architecture: sparc. Expected arm64 or amd64."); + }); + + it("emulator list-releases help shows repo option", async ({ expect }) => { + const { stdout, exitCode } = await runCliBare(["emulator", "list-releases", "--help"]); + expect(exitCode).toBe(0); + expect(stdout).toContain("--repo"); + }); +}); diff --git a/claude/CLAUDE-KNOWLEDGE.md b/claude/CLAUDE-KNOWLEDGE.md index 0f5fa27dad..320b100e94 100644 --- a/claude/CLAUDE-KNOWLEDGE.md +++ b/claude/CLAUDE-KNOWLEDGE.md @@ -139,6 +139,8 @@ A: Update affected inline snapshots in `apps/e2e/tests/backend/endpoints/api/v1/ Q: How should `createOrUpdateProjectWithLegacyConfig` handle `onboardingStatus` for forward-compat checks? A: Only write `onboardingStatus` when the `Project.onboardingStatus` column exists (for example by checking `information_schema.columns` in-transaction) so current code can still run against older schemas where that column is absent. +Q: What caused the March 19, 2026 QEMU local emulator deps startup regression? +A: The QEMU runtime path regressed when it switched from mounting `docker/local-emulator/base.env` into the runtime ISO to mounting the generated hidden file `docker/local-emulator/.env.development` instead. In testing, the `.env.development` QEMU path left cold boot stuck with only PostgreSQL healthy, while restoring the runtime ISO back to `base.env` brought deps startup back to about 12-13 seconds. The env payloads were effectively the same, so the likely issue was the QEMU runtime bundle/path handling for `.env.development`, not the actual env values. Q: Where is the private sign-up risk engine generated entrypoint in backend now? A: The generator script writes `apps/backend/src/private/implementation.generated.ts` (not `src/generated/private-sign-up-risk-engine.ts`), and backend runtime imports should target `@/private/implementation.generated`. diff --git a/docker/local-emulator/Dockerfile b/docker/local-emulator/Dockerfile new file mode 100644 index 0000000000..7f9e6d45a3 --- /dev/null +++ b/docker/local-emulator/Dockerfile @@ -0,0 +1,199 @@ +# Stack Auth Local Emulator — All-in-One Image +# Packages: PostgreSQL 16, Redis 7, Inbucket, Svix, ClickHouse, MinIO, QStash +# + built Stack Auth backend and dashboard + +ARG NODE_VERSION=22.21.1 + +# ── Node.js build stages ────────────────────────────────────────────────────── + +FROM node:${NODE_VERSION} AS node-base + +WORKDIR /app + +RUN apt-get update && \ + apt-get upgrade -y && \ + rm -rf /var/lib/apt/lists + +ENV PNPM_HOME=/pnpm +ENV PATH=$PNPM_HOME:$PATH + +RUN corepack enable +RUN corepack prepare pnpm@10.23.0 --activate +RUN pnpm add -g turbo +RUN pnpm add -g tsx + + +FROM node-base AS pruner + +COPY . . + +RUN tsx ./scripts/generate-sdks.ts + +# https://turbo.build/repo/docs/guides/tools/docker +RUN turbo prune --scope=@stackframe/backend --scope=@stackframe/dashboard --docker + + +FROM node-base AS builder + +# copy over package.json files and install dependencies +COPY --from=pruner /app/out/json/ . +COPY --from=pruner /app/out/pnpm-lock.yaml . +COPY .gitignore . +COPY pnpm-workspace.yaml . +COPY turbo.json . +COPY configs ./configs +RUN --mount=type=cache,id=pnpm,target=/pnpm/store STACK_SKIP_TEMPLATE_GENERATION=true pnpm install --frozen-lockfile + +# copy over the rest of the code for the build +COPY --from=pruner /app/out/full/ . + +# docs are currently required for the NextJS backend build, but won't exist in the final image +COPY docs ./docs + +# https://nextjs.org/docs/pages/api-reference/next-config-js/output +ENV NEXT_CONFIG_OUTPUT=standalone + +# Build the backend NextJS app +RUN pnpm turbo run docker-build --filter=@stackframe/backend... --filter=@stackframe/dashboard... + +# Build the self-host seed script +RUN cd apps/backend && pnpm build-self-host-migration-script + + +# Prune node_modules for runtime: remove dev tools, heavy UI packages, +# duplicate framework copies, and native binaries not needed by the +# migration script or server at runtime. +FROM builder AS migration-pruner +RUN cp -a /app/node_modules /pruned-node_modules && \ + cd /pruned-node_modules/.pnpm && \ + rm -rf \ + # Dev tools (never needed at runtime) + typescript@* eslint@* eslint-*@* @typescript-eslint+*@* \ + prettier@* vitest@* jsdom@* turbo@* turbo-*@* \ + tsdown@* @changesets+*@* codebuff@* \ + @testing-library+*@* vite@* vite-*@* @vitejs+*@* \ + # Heavy UI packages (already traced into Next.js standalone bundles) + monaco-editor@* \ + three@* three-globe@* globe.gl@* react-globe*@* \ + react-icons@* lucide-react@* @phosphor-icons+*@* \ + # Large optional packages not needed by migration script + posthog-js@* \ + @prisma+studio-core@* @prisma+dev@* @prisma+query-plan-executor@* \ + convex@* @electric-sql+*@* \ + # Duplicate Next.js copies (keep only one for next/headers.js resolution) + 'next@16.1.5_@babel+core@7.29.0*' 'next@16.1.5_@babel+core@7.28.5*' \ + next@14* @next+swc-*@14* \ + # Native build binaries not needed at runtime + @esbuild+*@* esbuild@* @rolldown+*@* \ + # Duplicate date-fns versions (keep v4 only) + date-fns@2* date-fns@3* + + +# ── Service binary stages ───────────────────────────────────────────────────── + +FROM inbucket/inbucket:3.1.0 AS inbucket-bin +FROM svix/svix-server:v1.88.0 AS svix-bin +FROM clickhouse/clickhouse-server:25.10 AS clickhouse-bin +FROM minio/minio:RELEASE.2025-09-07T16-13-09Z AS minio-bin +FROM minio/mc:RELEASE.2025-02-21T16-00-46Z AS mc-bin + +FROM bgodil/qstash:latest AS qstash-bin +RUN cp $(which qstash) /qstash-binary 2>/dev/null || \ + cp $(find / -name 'qstash' -type f -executable 2>/dev/null | head -1) /qstash-binary || \ + { echo "ERROR: qstash binary not found" >&2; exit 1; } + + +# ── Final image ─────────────────────────────────────────────────────────────── + +FROM debian:trixie-slim + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gnupg2 \ + lsb-release \ + curl \ + ca-certificates \ + && echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ + > /etc/apt/sources.list.d/pgdg.list \ + && curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-16 \ + postgresql-client-16 \ + redis-server \ + supervisor \ + gosu \ + procps \ + libssl3 \ + openssl \ + socat \ + && apt-get purge -y --auto-remove gnupg2 lsb-release \ + && rm -rf /var/lib/apt/lists/* /usr/share/doc /usr/share/man /usr/share/i18n + +# Node.js runtime (binary only — app bundles include all JS dependencies) +COPY --from=node-base /usr/local/bin/node /usr/local/bin/node + +# Inbucket +COPY --from=inbucket-bin /opt/inbucket /opt/inbucket + +# Svix +COPY --from=svix-bin /usr/local/bin/svix-server /usr/local/bin/svix-server + +# ClickHouse +COPY --from=clickhouse-bin /usr/bin/clickhouse /usr/bin/clickhouse +RUN ln -sf /usr/bin/clickhouse /usr/bin/clickhouse-server && \ + ln -sf /usr/bin/clickhouse /usr/bin/clickhouse-client + +# MinIO +COPY --from=minio-bin /usr/bin/minio /usr/local/bin/minio +COPY --from=mc-bin /usr/bin/mc /usr/local/bin/mc + +# QStash +COPY --from=qstash-bin --chmod=755 /qstash-binary /usr/local/bin/qstash + +# App +WORKDIR /app +COPY --from=builder /app/apps/backend/.next/standalone ./ +COPY --from=builder /app/apps/backend/.next/static ./apps/backend/.next/static +COPY --from=builder /app/apps/backend/prisma ./apps/backend/prisma +COPY --from=builder /app/apps/backend/dist ./apps/backend/dist +COPY --from=builder /app/apps/backend/node_modules ./apps/backend/node_modules +COPY --from=builder /app/apps/dashboard/.next/standalone ./ +COPY --from=builder /app/apps/dashboard/.next/static ./apps/dashboard/.next/static +COPY --from=builder /app/apps/dashboard/public ./apps/dashboard/public +COPY --from=migration-pruner /pruned-node_modules ./node_modules +COPY --from=builder /app/packages ./packages + +RUN mkdir -p \ + /data/postgres \ + /data/redis \ + /data/clickhouse \ + /data/clickhouse/access \ + /data/clickhouse/tmp \ + /data/clickhouse/user_files \ + /data/clickhouse/format_schemas \ + /data/minio \ + /data/inbucket \ + /var/log/supervisor \ + /var/log/clickhouse \ + /etc/clickhouse-server \ + && chown -R postgres:postgres /data/postgres + +COPY docker/local-emulator/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY docker/local-emulator/entrypoint.sh /entrypoint.sh +COPY docker/local-emulator/init-services.sh /init-services.sh +COPY docker/local-emulator/start-app.sh /start-app.sh +COPY docker/local-emulator/clickhouse-config.xml /etc/clickhouse-server/config.xml +COPY docker/local-emulator/clickhouse-users.xml /etc/clickhouse-server/users.xml +COPY docker/server/entrypoint.sh /app-entrypoint.sh +RUN chmod +x /entrypoint.sh /init-services.sh /start-app.sh /app-entrypoint.sh + +# PostgreSQL: 5432, Redis: 6379, Inbucket: 2500/9001/1100, +# Svix: 8071, ClickHouse: 8123/9009, MinIO: 9090, QStash: 8080 +# Backend: 8102, Dashboard: 8101 +EXPOSE 5432 6379 2500 9001 1100 8071 8123 9009 9090 8080 8101 8102 + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/local-emulator/clickhouse-config.xml b/docker/local-emulator/clickhouse-config.xml new file mode 100644 index 0000000000..31aa71922c --- /dev/null +++ b/docker/local-emulator/clickhouse-config.xml @@ -0,0 +1,26 @@ + + + warning + 1 + + + 8123 + 9009 + 0.0.0.0 + + /data/clickhouse/ + /data/clickhouse/tmp/ + /data/clickhouse/user_files/ + /data/clickhouse/format_schemas/ + + 0.5 + + + + users.xml + + + /data/clickhouse/access/ + + + diff --git a/docker/local-emulator/clickhouse-users.xml b/docker/local-emulator/clickhouse-users.xml new file mode 100644 index 0000000000..3f1e67f1f9 --- /dev/null +++ b/docker/local-emulator/clickhouse-users.xml @@ -0,0 +1,35 @@ + + + + + ::/0 + default + default + 1 + + + PASSWORD-PLACEHOLDER--9gKyMxJeMx + ::/0 + default + default + 1 + + + + + 1000000000 + + + + + + 3600 + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/docker/local-emulator/entrypoint.sh b/docker/local-emulator/entrypoint.sh new file mode 100644 index 0000000000..daa9854653 --- /dev/null +++ b/docker/local-emulator/entrypoint.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e + +PGDATA=/data/postgres +PG_BIN=/usr/lib/postgresql/16/bin + +if [ -z "$(ls -A "$PGDATA" 2>/dev/null)" ]; then + gosu postgres "$PG_BIN/initdb" -D "$PGDATA" --no-sync --auth-local=trust --auth-host=md5 + + { + echo "host all all 0.0.0.0/0 md5" + echo "host all all ::/0 md5" + } >> "$PGDATA/pg_hba.conf" + + echo "shared_preload_libraries = 'pg_stat_statements'" >> "$PGDATA/postgresql.conf" + echo "pg_stat_statements.track = all" >> "$PGDATA/postgresql.conf" + + gosu postgres "$PG_BIN/pg_ctl" -D "$PGDATA" start -w \ + -o "-c listen_addresses=127.0.0.1 -c shared_preload_libraries=pg_stat_statements" + + gosu postgres psql -c "ALTER USER postgres PASSWORD 'PASSWORD-PLACEHOLDER--uqfEC1hmmv';" + gosu postgres psql -c "CREATE DATABASE stackframe;" + gosu postgres psql -c "CREATE DATABASE svix;" + gosu postgres psql -d stackframe -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;" + gosu postgres psql -d stackframe -c "CREATE ROLE anon NOLOGIN;" + gosu postgres psql -d stackframe -c "CREATE ROLE authenticated NOLOGIN;" + + gosu postgres "$PG_BIN/pg_ctl" -D "$PGDATA" stop -w +fi + +exec /usr/bin/supervisord -n -c /etc/supervisor/conf.d/supervisord.conf diff --git a/docker/local-emulator/generate-env-development.mjs b/docker/local-emulator/generate-env-development.mjs new file mode 100644 index 0000000000..f0b0b20d23 --- /dev/null +++ b/docker/local-emulator/generate-env-development.mjs @@ -0,0 +1,203 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const rootDir = path.resolve(scriptDir, "..", ".."); + +const outputPath = path.join(scriptDir, ".env.development"); +const backendEnvPath = path.join(rootDir, "apps", "backend", ".env.development"); +const dashboardEnvPath = path.join(rootDir, "apps", "dashboard", ".env.development"); + +const args = process.argv.slice(2); +if (args.length > 1 || (args[0] != null && args[0] !== "--check")) { + throw new Error("Usage: node docker/local-emulator/generate-env-development.mjs [--check]"); +} + +const parseEnvFile = (filePath) => { + const env = new Map(); + + for (const rawLine of fs.readFileSync(filePath, "utf8").split(/\r?\n/)) { + const trimmedLine = rawLine.trim(); + if (trimmedLine === "" || trimmedLine.startsWith("#")) { + continue; + } + + const separatorIndex = rawLine.indexOf("="); + if (separatorIndex < 0) { + throw new Error(`Invalid env line in ${filePath}: ${rawLine}`); + } + + const key = rawLine.slice(0, separatorIndex).trim(); + const value = rawLine.slice(separatorIndex + 1); + env.set(key, value); + } + + return env; +}; + +const backendEnv = parseEnvFile(backendEnvPath); +const dashboardEnv = parseEnvFile(dashboardEnvPath); + +const getRequiredEnvValue = (sourceName, envMap, key) => { + const value = envMap.get(key); + if (value == null) { + throw new Error(`Missing ${key} in ${sourceName}; update the generator or source env file.`); + } + return value; +}; + +const fromSource = (sourceName, envMap, key) => ({ + type: "entry", + key, + value: getRequiredEnvValue(sourceName, envMap, key), +}); + +const literal = (key, value) => ({ + type: "entry", + key, + value, +}); + +const comment = (value) => ({ + type: "comment", + value, +}); + +const blank = () => ({ + type: "blank", +}); + +const entries = [ + comment("# Generated by docker/local-emulator/generate-env-development.mjs"), + comment("# Do not edit manually; update apps/backend/.env.development, apps/dashboard/.env.development, or this generator."), + blank(), + comment("# Public emulator/app credentials"), + literal("NEXT_PUBLIC_STACK_DOCS_BASE_URL", "https://docs.stack-auth.com"), + literal("NEXT_PUBLIC_STACK_IS_LOCAL_EMULATOR", "true"), + fromSource("apps/dashboard/.env.development", dashboardEnv, "NEXT_PUBLIC_STACK_PROJECT_ID"), + fromSource("apps/dashboard/.env.development", dashboardEnv, "NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY"), + fromSource("apps/dashboard/.env.development", dashboardEnv, "STACK_SECRET_SERVER_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SERVER_SECRET"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_CHANGELOG_URL"), + blank(), + comment("# Seed/project defaults"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SEED_ENABLE_DUMMY_PROJECT"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SEED_INTERNAL_PROJECT_SIGN_UP_ENABLED"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SEED_INTERNAL_PROJECT_OTP_ENABLED"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SEED_INTERNAL_PROJECT_ALLOW_LOCALHOST"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SEED_INTERNAL_PROJECT_OAUTH_PROVIDERS"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SEED_INTERNAL_PROJECT_USER_INTERNAL_ACCESS"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SEED_INTERNAL_PROJECT_PUBLISHABLE_CLIENT_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SEED_INTERNAL_PROJECT_SECRET_SERVER_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"), + blank(), + comment("# Third-party/test integrations"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SVIX_API_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_OPENAI_API_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_OPENROUTER_API_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_STRIPE_SECRET_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_STRIPE_WEBHOOK_SECRET"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_RESEND_API_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_RESEND_WEBHOOK_SECRET"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_DNSIMPLE_API_TOKEN"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_DNSIMPLE_ACCOUNT_ID"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_DNSIMPLE_API_BASE_URL"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_FREESTYLE_API_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_VERCEL_SANDBOX_TOKEN"), + fromSource("apps/backend/.env.development", backendEnv, "CRON_SECRET"), + blank(), + comment("# Storage, queueing, and analytics"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_S3_REGION"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_S3_ACCESS_KEY_ID"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_S3_SECRET_ACCESS_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_S3_BUCKET"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_S3_PRIVATE_BUCKET"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_AWS_REGION"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_AWS_ACCESS_KEY_ID"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_AWS_SECRET_ACCESS_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_QSTASH_TOKEN"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_QSTASH_CURRENT_SIGNING_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_QSTASH_NEXT_SIGNING_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_CLICKHOUSE_ADMIN_USER"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_CLICKHOUSE_ADMIN_PASSWORD"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_CLICKHOUSE_EXTERNAL_PASSWORD"), + blank(), + comment("# Email and dashboard integration"), + literal("STACK_EMAIL_PORT", "2500"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_EMAIL_SECURE"), + literal("STACK_EMAIL_USERNAME", "does-not-matter"), + literal("STACK_EMAIL_PASSWORD", "does-not-matter"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_EMAIL_SENDER"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_DEFAULT_EMAIL_CAPACITY_PER_HOUR"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_EMAIL_MONITOR_PROJECT_ID"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_EMAIL_MONITOR_PUBLISHABLE_CLIENT_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_EMAIL_MONITOR_RESEND_EMAIL_DOMAIN"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_EMAIL_MONITOR_RESEND_EMAIL_API_KEY"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_EMAIL_MONITOR_SECRET_TOKEN"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_EMAIL_MONITOR_USE_INBUCKET"), + fromSource("apps/dashboard/.env.development", dashboardEnv, "STACK_FEATUREBASE_JWT_SECRET"), + blank(), + comment("# Mock OAuth defaults"), + literal("STACK_FORWARD_MOCK_OAUTH_SERVER", "false"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_GITHUB_CLIENT_ID"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_GITHUB_CLIENT_SECRET"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_GOOGLE_CLIENT_ID"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_GOOGLE_CLIENT_SECRET"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_MICROSOFT_CLIENT_ID"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_MICROSOFT_CLIENT_SECRET"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SPOTIFY_CLIENT_ID"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_SPOTIFY_CLIENT_SECRET"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_ALLOW_SHARED_OAUTH_ACCESS_TOKENS"), + blank(), + comment("# Internal service endpoints (defaults for docker-compose; overridden in QEMU)"), + literal("STACK_DATABASE_CONNECTION_STRING", "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@127.0.0.1:5432/stackframe"), + fromSource("apps/backend/.env.development", backendEnv, "STACK_EMAIL_HOST"), + literal("STACK_SVIX_SERVER_URL", "http://127.0.0.1:8071"), + literal("STACK_S3_ENDPOINT", "http://127.0.0.1:9090"), + literal("STACK_QSTASH_URL", "http://127.0.0.1:8080"), + literal("STACK_CLICKHOUSE_URL", "http://127.0.0.1:8123"), + literal("STACK_CLICKHOUSE_DATABASE", "analytics"), + literal("STACK_EMAIL_MONITOR_INBUCKET_API_URL", "http://127.0.0.1:9001"), + literal("BACKEND_PORT", "8102"), + literal("DASHBOARD_PORT", "8101"), +]; + +const seenKeys = new Set(); +for (const entry of entries) { + if (entry.type !== "entry") { + continue; + } + + if (seenKeys.has(entry.key)) { + throw new Error(`Duplicate env key in generator: ${entry.key}`); + } + + seenKeys.add(entry.key); +} + +const content = `${entries.map((entry) => { + if (entry.type === "blank") { + return ""; + } + + if (entry.type === "comment") { + return entry.value; + } + + return `${entry.key}=${entry.value}`; +}).join("\n")}\n`; + +if (args[0] === "--check") { + const currentContent = fs.readFileSync(outputPath, "utf8"); + if (currentContent !== content) { + throw new Error(`${path.relative(rootDir, outputPath)} is out of date. Run pnpm run emulator:generate-env.`); + } + + console.log(`${path.relative(rootDir, outputPath)} is up to date.`); +} else { + fs.writeFileSync(outputPath, content); + console.log(`Wrote ${path.relative(rootDir, outputPath)}.`); +} diff --git a/docker/local-emulator/init-services.sh b/docker/local-emulator/init-services.sh new file mode 100644 index 0000000000..8fd1f7de32 --- /dev/null +++ b/docker/local-emulator/init-services.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + +INIT_SERVICES_DONE_FILE=/var/run/stack-local-init-services.done +INIT_SERVICES_FAILED_FILE=/var/run/stack-local-init-services.failed + +rm -f "$INIT_SERVICES_DONE_FILE" "$INIT_SERVICES_FAILED_FILE" +trap 'touch "$INIT_SERVICES_FAILED_FILE"' ERR + +wait_for_http() { + local url="$1" attempts=0 + while [ "$attempts" -lt 60 ]; do + if curl -sf "$url" > /dev/null 2>&1; then return 0; fi + sleep 1 + attempts=$((attempts + 1)) + done + echo "Timed out waiting for $url" >&2 + exit 1 +} + +wait_for_http http://127.0.0.1:9090/minio/health/live +mc alias set local http://127.0.0.1:9090 s3mockroot s3mockroot --api S3v4 +mc mb --ignore-existing local/stack-storage +mc mb --ignore-existing local/stack-storage-private + +wait_for_http http://127.0.0.1:8123/ping +curl -s "http://127.0.0.1:8123/?user=default" --data "CREATE DATABASE IF NOT EXISTS analytics" + +rm -f "$INIT_SERVICES_FAILED_FILE" +touch "$INIT_SERVICES_DONE_FILE" diff --git a/docker/local-emulator/qemu/.gitignore b/docker/local-emulator/qemu/.gitignore new file mode 100644 index 0000000000..6f1a609192 --- /dev/null +++ b/docker/local-emulator/qemu/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +run/ diff --git a/docker/local-emulator/qemu/README.md b/docker/local-emulator/qemu/README.md new file mode 100644 index 0000000000..57ed713149 --- /dev/null +++ b/docker/local-emulator/qemu/README.md @@ -0,0 +1,121 @@ +# QEMU Local Emulator + +The local emulator packages the entire Stack Auth backend (PostgreSQL, Redis, ClickHouse, MinIO, Inbucket, Svix, QStash, Dashboard, and Backend) into a single QEMU virtual machine image. Users run it via the `stack emulator` CLI commands. + +## Architecture + +``` +Host machine + └─ QEMU VM (Debian 13 cloud image) + └─ Docker container (all-in-one image from ../Dockerfile) + ├─ PostgreSQL 16 + ├─ Redis 7 + ├─ ClickHouse + ├─ MinIO + ├─ Inbucket + ├─ Svix + ├─ QStash + ├─ Stack Dashboard (→ host:26700) + └─ Stack Backend (→ host:26701) +``` + +Only four services are exposed to the host via port forwarding: + +| Service | Host Port | Description | +|-----------|-----------|--------------------------| +| Dashboard | 26700 | Stack Auth dashboard UI | +| Backend | 26701 | Stack Auth API server | +| MinIO | 26702 | S3-compatible storage | +| Inbucket | 26703 | Email testing interface | + +All other services (PostgreSQL, Redis, ClickHouse, Svix, QStash) remain internal to the VM. + +## Scripts + +| Script | Purpose | +|--------------------|----------------------------------------------------------------| +| `build-image.sh` | Builds a QEMU disk image for a target architecture | +| `run-emulator.sh` | Manages the VM lifecycle: `start`, `stop`, `reset`, `status`, `bench` | +| `common.sh` | Shared helpers: host detection, QEMU binary selection, firmware lookup, ISO creation | + +## Building an Image + +```bash +# Build for current architecture +./docker/local-emulator/qemu/build-image.sh + +# Build for a specific architecture (arm64 or amd64) +./docker/local-emulator/qemu/build-image.sh arm64 + +# Build both +./docker/local-emulator/qemu/build-image.sh both +``` + +The build process: +1. Builds the all-in-one Docker image from `../Dockerfile` and exports it as a tarball +2. Downloads a Debian 13 cloud base image +3. Boots a QEMU VM with cloud-init provisioning (`cloud-init/emulator/user-data`) +4. Cloud-init loads the Docker image and runs a full startup cycle to warm caches +5. Shuts down and compresses the disk image to `images/stack-emulator-.qcow2` + +Default resources: 4 CPUs, 4096 MB RAM. Override with `EMULATOR_CPUS` / `EMULATOR_RAM_MB`. + +### Why a single Docker image? + +The `../Dockerfile` bundles all services into one image rather than using separate containers. This keeps the QEMU disk image size small — separate images would each carry their own base layers, significantly inflating the final qcow2. + +## Running the Emulator + +```bash +# Via CLI (recommended) +stack emulator start +stack emulator stop +stack emulator reset # wipe data +stack emulator status + +# Via script directly +EMULATOR_ARCH=arm64 ./docker/local-emulator/qemu/run-emulator.sh start +``` + +The VM uses an overlay disk (`run/vm/disk.qcow2`) on top of the base image, so data persists across stop/start cycles. Use `reset` to wipe the overlay and start fresh. + +### Hardware acceleration + +- **macOS**: Uses HVF (Hypervisor.framework) for native-arch VMs +- **Linux**: Uses KVM when available +- **Cross-arch**: Falls back to TCG (software emulation) — significantly slower + +## Optimizations Taken + +- **Single bundled Docker image** to minimize qcow2 size +- **Cloud-init provisioning** pre-warms all services during build so first boot is fast +- **Overlay disks** avoid copying the multi-GB base image on each start +- **Compressed qcow2** images (`-c` flag) reduce download size +- **Only 4 ports forwarded** to minimize host-side surface area + +## Possible Future Optimizations + +- External server for reads and writes to relative dir instead of full host access allowing snapshots + - Or copying the config file on start with --config-file enforced and writing the config file to host directory on stop + +## Updating the Image + +1. Make changes to the `../Dockerfile`, `../entrypoint.sh`, or cloud-init config +2. Rebuild: `./docker/local-emulator/qemu/build-image.sh ` +3. The CI workflow (`.github/workflows/qemu-emulator-build.yaml`) builds and publishes images on push to `main`/`dev` +4. Users pull the latest via `stack emulator pull` + +## Directory Layout + +``` +qemu/ +├── build-image.sh # Image builder +├── run-emulator.sh # VM lifecycle manager +├── common.sh # Shared utilities +├── cloud-init/ +│ └── emulator/ +│ ├── meta-data # VM instance metadata +│ └── user-data # Provisioning script +├── images/ # Built qcow2 images (gitignored) +└── run/ # Runtime state: overlay disk, PID, logs (gitignored) +``` diff --git a/docker/local-emulator/qemu/build-image.sh b/docker/local-emulator/qemu/build-image.sh new file mode 100755 index 0000000000..8071fb5012 --- /dev/null +++ b/docker/local-emulator/qemu/build-image.sh @@ -0,0 +1,290 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=common.sh +source "$SCRIPT_DIR/common.sh" + +IMAGE_DIR="$SCRIPT_DIR/images" +CLOUD_INIT_ROOT="$SCRIPT_DIR/cloud-init" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +DEBIAN_VERSION="${DEBIAN_VERSION:-13}" +DISK_SIZE="${EMULATOR_DISK_SIZE:-12G}" +RAM="${EMULATOR_BUILD_RAM:-4096}" +CPUS="${EMULATOR_BUILD_CPUS:-$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)}" +PROVISION_TIMEOUT="${EMULATOR_PROVISION_TIMEOUT:-3200}" +EMULATOR_IMAGE_NAME="${EMULATOR_IMAGE_NAME:-stack-local-emulator}" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +NC='\033[0m' + +log() { echo -e "${GREEN}[build]${NC} $*"; } +warn() { echo -e "${YELLOW}[build]${NC} $*"; } +err() { echo -e "${RED}[build]${NC} $*" >&2; } + +detect_host +TARGET_ARCH="${1:-$HOST_ARCH}" + +TARGET_ARCHS=() +case "$TARGET_ARCH" in + arm64) TARGET_ARCHS=(arm64) ;; + amd64) TARGET_ARCHS=(amd64) ;; + both) TARGET_ARCHS=(arm64 amd64) ;; + *) err "Usage: $0 [arm64|amd64|both]"; exit 1 ;; +esac + +DOCKER_IMAGES=("$EMULATOR_IMAGE_NAME") + +check_deps() { + local missing=() + local arch qemu_bin + + for arch in "${TARGET_ARCHS[@]}"; do + qemu_bin="$(qemu_binary_for_arch "$arch")" + command -v "$qemu_bin" >/dev/null 2>&1 || missing+=("$qemu_bin") + done + + for cmd in qemu-img curl docker gzip; do + command -v "$cmd" >/dev/null 2>&1 || missing+=("$cmd") + done + + if ! command -v mkisofs >/dev/null 2>&1 && ! command -v genisoimage >/dev/null 2>&1 && ! command -v hdiutil >/dev/null 2>&1; then + missing+=("mkisofs/genisoimage/hdiutil") + fi + + if [ "${#missing[@]}" -gt 0 ]; then + err "Missing build dependencies: ${missing[*]}" + exit 1 + fi +} + +check_deps +mkdir -p "$IMAGE_DIR" + +download_cloud_image() { + local arch="$1" + local dest="$2" + local deb_arch + + case "$arch" in + arm64) deb_arch="arm64" ;; + amd64) deb_arch="amd64" ;; + *) err "Unsupported target arch: $arch"; exit 1 ;; + esac + + local url="https://cloud.debian.org/images/cloud/trixie/daily/latest/debian-${DEBIAN_VERSION}-generic-${deb_arch}-daily.qcow2" + if [ -f "$dest" ]; then + log "Base image already cached: $dest" + return 0 + fi + + log "Downloading Debian ${DEBIAN_VERSION} cloud image for ${arch}..." + curl -fSL --progress-bar -o "$dest" "$url" +} + +docker_platform_for_arch() { + case "$1" in + arm64) echo "linux/arm64" ;; + amd64) echo "linux/amd64" ;; + *) err "Unsupported target arch: $1"; exit 1 ;; + esac +} + +build_local_emulator_image() { + local arch="$1" + local platform + platform="$(docker_platform_for_arch "$arch")" + + log "Building Docker emulator image (${arch})..." + docker buildx build \ + --platform "$platform" \ + --tag "$EMULATOR_IMAGE_NAME" \ + --load \ + -f "$REPO_ROOT/docker/local-emulator/Dockerfile" \ + "$REPO_ROOT" +} + +qemu_cmd_prefix_for_arch() { + local arch="$1" + case "$arch" in + arm64) + local accel="tcg" + if [ "$HOST_ARCH" = "arm64" ]; then + case "$HOST_OS" in + darwin) accel="hvf" ;; + linux) [ -w /dev/kvm ] && accel="kvm" ;; + esac + fi + local firmware + firmware="$(find_aarch64_firmware)" + echo "qemu-system-aarch64 -machine virt -accel $accel -cpu max -bios $firmware" + ;; + amd64) + local accel="tcg" + local cpu="max" + if [ "$HOST_ARCH" = "amd64" ]; then + case "$HOST_OS" in + darwin) accel="hvf" ;; + linux) [ -w /dev/kvm ] && accel="kvm" ;; + esac + else + cpu="qemu64" + fi + echo "qemu-system-x86_64 -machine q35 -accel $accel -cpu $cpu" + ;; + esac +} + +final_image_name() { + echo "$IMAGE_DIR/stack-emulator-$1.qcow2" +} + +prepare_bundle_artifacts() { + local arch="$1" + local bundle_tgz="$IMAGE_DIR/emulator-${arch}-docker-images.tar.gz" + local bundle_meta="$bundle_tgz.image-ids" + + local current_ids="" + for img in "${DOCKER_IMAGES[@]}"; do + current_ids+="$(docker image inspect --format '{{.ID}}' "$img")"$'\n' + done + + local cached_ids="" + if [ -f "$bundle_meta" ]; then + cached_ids="$(cat "$bundle_meta")" + fi + + if [ -f "$bundle_tgz" ] && [ "$cached_ids" = "$current_ids" ]; then + log "Reusing bundle: $bundle_tgz" + return 0 + fi + + log "Creating Docker image bundle (${arch})..." + for img in "${DOCKER_IMAGES[@]}"; do + if ! docker image inspect "$img" >/dev/null 2>&1; then + err "Missing Docker image: $img. Build the local emulator images first, then rerun the QEMU image build." + exit 1 + fi + done + local tmp_bundle="${bundle_tgz}.tmp" + rm -f "$tmp_bundle" + docker save "${DOCKER_IMAGES[@]}" | gzip -c > "$tmp_bundle" + mv "$tmp_bundle" "$bundle_tgz" + printf "%s" "$current_ids" > "$bundle_meta" +} + +build_one() { + local arch="$1" + local base_img="$IMAGE_DIR/debian-${DEBIAN_VERSION}-base-${arch}.qcow2" + local bundle_tgz="$IMAGE_DIR/emulator-${arch}-docker-images.tar.gz" + local final_img + final_img="$(final_image_name "$arch")" + + log "━━━ Building emulator image (${arch}) ━━━" + + local tmp_dir + tmp_dir="$(mktemp -d /tmp/stack-qemu-build-${arch}-XXXXXX)" + local tmp_img="$tmp_dir/disk.qcow2" + local seed_iso="$tmp_dir/seed.iso" + local bundle_iso="$tmp_dir/bundle.iso" + local bundle_dir="$tmp_dir/bundle" + local serial_log="$tmp_dir/serial.log" + local pidfile="$tmp_dir/qemu.pid" + local qemu_base pid elapsed + local start_time=$SECONDS + + cp "$base_img" "$tmp_img" + qemu-img resize "$tmp_img" "$DISK_SIZE" >/dev/null 2>&1 || true + + local seed_dir + seed_dir="$(mktemp -d)" + mkdir -p "$seed_dir" + cp "$CLOUD_INIT_ROOT/emulator/meta-data" "$seed_dir/meta-data" + cp "$CLOUD_INIT_ROOT/emulator/user-data" "$seed_dir/user-data" + make_iso_from_dir "$seed_iso" "cidata" "$seed_dir" + rm -rf "$seed_dir" + + mkdir -p "$bundle_dir" + cp "$bundle_tgz" "$bundle_dir/img.tgz" + make_iso_from_dir "$bundle_iso" "STACKBUNDLE" "$bundle_dir" + + : > "$serial_log" + qemu_base="$(qemu_cmd_prefix_for_arch "$arch")" + + # shellcheck disable=SC2086 + $qemu_base \ + -boot order=c \ + -m "$RAM" \ + -smp "$CPUS" \ + -drive "file=$tmp_img,format=qcow2,if=virtio" \ + -drive "file=$seed_iso,format=raw,if=virtio,readonly=on" \ + -drive "file=$bundle_iso,format=raw,if=virtio,readonly=on" \ + -netdev user,id=net0 \ + -device virtio-net-pci,netdev=net0 \ + -serial "file:$serial_log" \ + -display none \ + -daemonize \ + -pidfile "$pidfile" + + pid="$(cat "$pidfile")" + elapsed=0 + while [ "$elapsed" -lt "$PROVISION_TIMEOUT" ]; do + if grep -q "STACK_CLOUD_INIT_DONE" "$serial_log" 2>/dev/null; then + break + fi + sleep 5 + elapsed=$((SECONDS - start_time)) + printf "\r [%3ds / %ds] provisioning emulator..." "$elapsed" "$PROVISION_TIMEOUT" + done + echo "" + + if ! grep -q "STACK_CLOUD_INIT_DONE" "$serial_log" 2>/dev/null; then + err "Provisioning timed out for emulator (${arch})" + tail -50 "$serial_log" >&2 || true + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi + rm -rf "$tmp_dir" + exit 1 + fi + + local shutdown_wait=0 + while [ "$shutdown_wait" -lt 90 ] && kill -0 "$pid" 2>/dev/null; do + sleep 1 + shutdown_wait=$((shutdown_wait + 1)) + done + + if kill -0 "$pid" 2>/dev/null; then + warn "Guest did not power off cleanly; forcing shutdown." + kill "$pid" 2>/dev/null || true + sleep 2 + kill -9 "$pid" 2>/dev/null || true + fi + + cp "$tmp_img" "$final_img" + cp "$serial_log" "$IMAGE_DIR/provision-emulator-${arch}.log" + rm -rf "$tmp_dir" + + log "Compressing final image (this may take several minutes)..." + qemu-img convert -p -O qcow2 -c "$final_img" "$final_img.tmp" + mv "$final_img.tmp" "$final_img" + + local size + size="$(du -h "$final_img" | cut -f1)" + log "━━━ Emulator image ready: $final_img (${size}) ━━━" +} + +for arch in "${TARGET_ARCHS[@]}"; do + local_base="$IMAGE_DIR/debian-${DEBIAN_VERSION}-base-${arch}.qcow2" + download_cloud_image "$arch" "$local_base" + build_local_emulator_image "$arch" + prepare_bundle_artifacts "$arch" + build_one "$arch" +done + +log "Done. Start with: docker/local-emulator/qemu/run-emulator.sh start" diff --git a/docker/local-emulator/qemu/cloud-init/emulator/meta-data b/docker/local-emulator/qemu/cloud-init/emulator/meta-data new file mode 100644 index 0000000000..4a17d62aed --- /dev/null +++ b/docker/local-emulator/qemu/cloud-init/emulator/meta-data @@ -0,0 +1,2 @@ +instance-id: stack-emulator-001 +local-hostname: stack-emulator diff --git a/docker/local-emulator/qemu/cloud-init/emulator/user-data b/docker/local-emulator/qemu/cloud-init/emulator/user-data new file mode 100644 index 0000000000..39b8c33cdb --- /dev/null +++ b/docker/local-emulator/qemu/cloud-init/emulator/user-data @@ -0,0 +1,185 @@ +#cloud-config + +hostname: stack-emulator +manage_etc_hosts: true + +users: + - name: stack + shell: /bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + lock_passwd: false + +chpasswd: + list: | + root:stack-emulator + stack:stack-emulator + expire: false + +ssh_pwauth: false + +package_update: true +package_upgrade: false + +packages: + - docker.io + - ca-certificates + - curl + - netcat-openbsd + - qemu-guest-agent + +write_files: + - path: /usr/local/bin/install-emulator-containers + permissions: '0755' + content: | + #!/bin/bash + set -euo pipefail + + mkdir -p /mnt/stack-bundle + bundle_device="$(readlink -f /dev/disk/by-label/STACKBUNDLE)" + mount -o ro "$bundle_device" /mnt/stack-bundle + + systemctl enable --now docker + until docker info >/dev/null 2>&1; do sleep 1; done + + gzip -dc /mnt/stack-bundle/img.tgz | docker load + + - path: /usr/local/bin/render-stack-env + permissions: '0755' + content: | + #!/bin/bash + set -euo pipefail + + mkdir -p /mnt/stack-runtime /run/stack-auth + runtime_device="$(readlink -f /dev/disk/by-label/STACKCFG)" + mountpoint -q /mnt/stack-runtime || mount -o ro "$runtime_device" /mnt/stack-runtime + + set -a + source /mnt/stack-runtime/runtime.env + source /mnt/stack-runtime/base.env + set +a + + # Container-local dependencies run on localhost. Host-only development + # services (such as the OAuth mock server) are reachable via the QEMU + # user-network host alias. + DEPS_HOST=127.0.0.1 + HOST_SERVICES_HOST=10.0.2.2 + P="$STACK_EMULATOR_PORT_PREFIX" + + { + # Static vars from base config and runtime (e.g. API keys, feature flags) + cat /mnt/stack-runtime/base.env + cat /mnt/stack-runtime/runtime.env + + # Computed vars — depend on port prefix or deps host + cat < /run/stack-auth/local-emulator.env + + - path: /usr/local/bin/mount-host-fs + permissions: '0755' + content: | + #!/bin/bash + set -euo pipefail + mkdir -p /host + if ! mountpoint -q /host; then + if ! mount -t 9p -o trans=virtio,version=9p2000.L hostfs /host; then + echo "Failed to mount host filesystem at /host" >&2 + exit 1 + fi + fi + + - path: /usr/local/bin/run-stack-container + permissions: '0755' + content: | + #!/bin/bash + set -euo pipefail + + /usr/local/bin/mount-host-fs + /usr/local/bin/render-stack-env + docker rm -f stack >/dev/null 2>&1 || true + exec docker run \ + --rm \ + --name stack \ + --network host \ + --add-host host.docker.internal:host-gateway \ + --env-file /run/stack-auth/local-emulator.env \ + -v stack-postgres-data:/data/postgres \ + -v stack-redis-data:/data/redis \ + -v stack-clickhouse-data:/data/clickhouse \ + -v stack-minio-data:/data/minio \ + -v stack-inbucket-data:/data/inbucket \ + -v /host:/host \ + stack-local-emulator + + - path: /usr/local/bin/wait-for-deps + permissions: '0755' + content: | + #!/bin/bash + set -euo pipefail + + until nc -z 127.0.0.1 5432 >/dev/null 2>&1; do sleep 1; done + until curl -sf http://127.0.0.1:8123/ping >/dev/null 2>&1; do sleep 1; done + until curl -sf http://127.0.0.1:8071/api/v1/health/ >/dev/null 2>&1; do sleep 1; done + until curl -sf http://127.0.0.1:9090/minio/health/live >/dev/null 2>&1; do sleep 1; done + until [ "$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8080/ 2>/dev/null || true)" = "401" ]; do sleep 1; done + + - path: /etc/systemd/system/stack.service + content: | + [Unit] + Description=Stack Auth local emulator + Wants=network-online.target docker.service + After=network-online.target docker.service + + [Service] + Restart=always + RestartSec=5 + TimeoutStartSec=0 + ExecStart=/usr/local/bin/run-stack-container + ExecStop=/usr/bin/docker stop stack + + [Install] + WantedBy=multi-user.target + +runcmd: + - systemctl disable --now ssh || true + - systemctl mask ssh || true + - bash /usr/local/bin/install-emulator-containers + - systemctl daemon-reload + - systemctl enable stack.service + - docker run --rm --name stack-build-init + --network host + -e STACK_DEPS_ONLY=true + -v stack-postgres-data:/data/postgres + -v stack-redis-data:/data/redis + -v stack-clickhouse-data:/data/clickhouse + -v stack-minio-data:/data/minio + -v stack-inbucket-data:/data/inbucket + -d stack-local-emulator + - bash /usr/local/bin/wait-for-deps + - docker stop stack-build-init || true + - echo "STACK_CLOUD_INIT_DONE" > /dev/console 2>/dev/null || true + - echo "STACK_CLOUD_INIT_DONE" > /dev/ttyAMA0 2>/dev/null || true + - echo "STACK_CLOUD_INIT_DONE" > /dev/ttyS0 2>/dev/null || true + - shutdown -P now diff --git a/docker/local-emulator/qemu/common.sh b/docker/local-emulator/qemu/common.sh new file mode 100755 index 0000000000..1e3374dad4 --- /dev/null +++ b/docker/local-emulator/qemu/common.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Shared helpers for QEMU emulator scripts. +# Source this file; do not execute it directly. + +AARCH64_FIRMWARE_PATHS=( + /opt/homebrew/share/qemu/edk2-aarch64-code.fd + /usr/share/qemu/edk2-aarch64-code.fd + /usr/share/AAVMF/AAVMF_CODE.fd + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd +) + +detect_host() { + case "$(uname -m)" in + arm64|aarch64) HOST_ARCH="arm64" ;; + x86_64|amd64) HOST_ARCH="amd64" ;; + *) echo "Unsupported host architecture: $(uname -m)" >&2; exit 1 ;; + esac + + case "$(uname -s)" in + Darwin) HOST_OS="darwin" ;; + Linux) HOST_OS="linux" ;; + MINGW*|MSYS*|CYGWIN*) HOST_OS="windows" ;; + *) HOST_OS="unknown" ;; + esac +} + +qemu_binary_for_arch() { + case "$1" in + arm64) echo "qemu-system-aarch64" ;; + amd64) echo "qemu-system-x86_64" ;; + *) return 1 ;; + esac +} + +find_aarch64_firmware() { + local p + for p in "${AARCH64_FIRMWARE_PATHS[@]}"; do + if [ -f "$p" ]; then + echo "$p" + return 0 + fi + done + echo "No aarch64 UEFI firmware found." >&2 + return 1 +} + +make_iso_from_dir() { + local iso_path="$1" + local volume_name="$2" + local source_dir="$3" + + rm -f "$iso_path" "${iso_path}.iso" + if command -v hdiutil >/dev/null 2>&1; then + local tmp_dir + tmp_dir="$(mktemp -d /tmp/stack-emulator-iso-XXXXXX)" + cp -R "$source_dir/." "$tmp_dir/" + hdiutil makehybrid -o "$iso_path" "$tmp_dir" -joliet -iso -default-volume-name "$volume_name" 2>/dev/null + if [ -f "${iso_path}.iso" ]; then + mv "${iso_path}.iso" "$iso_path" + fi + rm -rf "$tmp_dir" + elif command -v mkisofs >/dev/null 2>&1; then + mkisofs -output "$iso_path" -volid "$volume_name" -joliet -rock "$source_dir" >/dev/null 2>&1 + elif command -v genisoimage >/dev/null 2>&1; then + genisoimage -output "$iso_path" -volid "$volume_name" -joliet -rock "$source_dir" >/dev/null 2>&1 + else + echo "Missing ISO creation tool (need hdiutil, mkisofs, or genisoimage)" >&2 + exit 1 + fi +} diff --git a/docker/local-emulator/qemu/images/.gitignore b/docker/local-emulator/qemu/images/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/docker/local-emulator/qemu/images/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/docker/local-emulator/qemu/run-emulator.sh b/docker/local-emulator/qemu/run-emulator.sh new file mode 100755 index 0000000000..f2f3028ca6 --- /dev/null +++ b/docker/local-emulator/qemu/run-emulator.sh @@ -0,0 +1,391 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=common.sh +source "$SCRIPT_DIR/common.sh" + +IMAGE_DIR="$SCRIPT_DIR/images" +RUN_DIR="${EMULATOR_RUN_DIR:-$SCRIPT_DIR/run}" + +VM_RAM="${EMULATOR_RAM:-4096}" +VM_CPUS="${EMULATOR_CPUS:-4}" +PORT_PREFIX="${PORT_PREFIX:-${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}}" +READY_TIMEOUT="${EMULATOR_READY_TIMEOUT:-240}" + +# Fixed host-side ports for the QEMU emulator (267xx range). +# Only user-facing services are exposed; internal deps stay inside the VM. +EMULATOR_DASHBOARD_PORT="${EMULATOR_DASHBOARD_PORT:-26700}" +EMULATOR_BACKEND_PORT="${EMULATOR_BACKEND_PORT:-26701}" +EMULATOR_MINIO_PORT="${EMULATOR_MINIO_PORT:-26702}" +EMULATOR_INBUCKET_PORT="${EMULATOR_INBUCKET_PORT:-26703}" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +log() { echo -e "${GREEN}[emulator]${NC} $*"; } +warn() { echo -e "${YELLOW}[emulator]${NC} $*"; } +err() { echo -e "${RED}[emulator]${NC} $*" >&2; } +info() { echo -e "${CYAN}[emulator]${NC} $*"; } + + +detect_host +ARCH="${EMULATOR_ARCH:-$HOST_ARCH}" + +select_accelerator() { + local accel="tcg" + if [ "$ARCH" = "$HOST_ARCH" ]; then + case "$HOST_OS" in + darwin) + if "$(qemu_binary_for_arch "$ARCH")" -accel help 2>&1 | grep -q hvf; then + accel="hvf" + fi + ;; + linux) + if [ -w /dev/kvm ]; then + accel="kvm" + fi + ;; + esac + fi + ACCEL="$accel" +} + +select_accelerator + +VM_DIR="$RUN_DIR/vm" + +image_path() { + echo "$IMAGE_DIR/stack-emulator-$ARCH.qcow2" +} + +runtime_iso_path() { + echo "$VM_DIR/runtime-config.iso" +} + +# Returns a fast fingerprint (size:mtime) of the base QEMU image. +# Used to detect whether the image has changed since the overlay was created. +base_image_fingerprint() { + local img="$1" + case "$HOST_OS" in + darwin) stat -f "%z:%m" "$img" 2>/dev/null ;; + linux) stat -c "%s:%Y" "$img" 2>/dev/null ;; + *) stat -f "%z:%m" "$img" 2>/dev/null || stat -c "%s:%Y" "$img" 2>/dev/null ;; + esac +} + +prepare_runtime_config_iso() { + local cfg_dir="$VM_DIR/runtime-config" + local cfg_iso + cfg_iso="$(runtime_iso_path)" + rm -rf "$cfg_dir" + mkdir -p "$cfg_dir" + { + printf "STACK_EMULATOR_PORT_PREFIX=%s\n" "$PORT_PREFIX" + } > "$cfg_dir/runtime.env" + cp "$SCRIPT_DIR/../.env.development" "$cfg_dir/base.env" + make_iso_from_dir "$cfg_iso" "STACKCFG" "$cfg_dir" +} + +service_is_up() { + local port="$1" + local proto="$2" + local path="${3:-/}" + local expected_codes="${4:-200}" + + if [ "$proto" = "tcp" ]; then + nc -z -w2 127.0.0.1 "$port" 2>/dev/null + return $? + fi + + local code + code="$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "http://127.0.0.1:${port}${path}" 2>/dev/null || true)" + local expected + for expected in ${expected_codes//,/ }; do + if [ "$code" = "$expected" ]; then + return 0 + fi + done + return 1 +} + +deps_ready() { + service_is_up "$EMULATOR_MINIO_PORT" http /minio/health/live && + service_is_up "$EMULATOR_INBUCKET_PORT" http / +} + +app_ready() { + service_is_up "$EMULATOR_BACKEND_PORT" http "/health?db=1" && + service_is_up "$EMULATOR_DASHBOARD_PORT" http /handler/sign-in +} + +all_ready() { + deps_ready && app_ready +} + +wait_for_condition() { + local label="$1" + local timeout="$2" + local check_fn="$3" + local started=$SECONDS + local elapsed=0 + + log "Waiting for ${label}..." + while [ "$elapsed" -lt "$timeout" ]; do + if "$check_fn"; then + echo "" + log "${label} ready in ${elapsed}s" + return 0 + fi + sleep 1 + elapsed=$((SECONDS - started)) + printf "\r [%3ds] %s..." "$elapsed" "$label" + done + echo "" + return 1 +} + +build_qemu_cmd() { + local base_img + base_img="$(image_path)" + + if [ ! -f "$base_img" ]; then + err "Missing QEMU image: $base_img" + err "Run docker/local-emulator/qemu/build-image.sh $ARCH first." + exit 1 + fi + + mkdir -p "$VM_DIR" + local fingerprint_file="$VM_DIR/base-image.fingerprint" + local current_fp + current_fp="$(base_image_fingerprint "$base_img")" + if [ -f "$VM_DIR/disk.qcow2" ]; then + if [ -f "$fingerprint_file" ] && [ "$(cat "$fingerprint_file")" = "$current_fp" ]; then + log "Reusing existing overlay disk (changes persist)" + else + warn "QEMU base image has changed — recreating overlay." + rm -f "$VM_DIR/disk.qcow2" "$fingerprint_file" + fi + fi + if [ ! -f "$VM_DIR/disk.qcow2" ]; then + qemu-img create -f qcow2 -b "$base_img" -F qcow2 "$VM_DIR/disk.qcow2" >/dev/null + base_image_fingerprint "$base_img" > "$fingerprint_file" + fi + + local qemu_bin machine cpu firmware_args=() + qemu_bin="$(qemu_binary_for_arch "$ARCH")" + case "$ARCH" in + arm64) + machine="virt" + cpu="max" + local firmware + firmware="$(find_aarch64_firmware)" + firmware_args=(-bios "$firmware") + ;; + amd64) + machine="q35" + if [ "$ACCEL" = "tcg" ] && [ "$HOST_ARCH" != "amd64" ]; then + cpu="qemu64" + else + cpu="max" + fi + ;; + esac + + local netdev="user,id=net0" + # Only expose user-facing services; internal deps stay inside the VM. + netdev+=",hostfwd=tcp::${EMULATOR_DASHBOARD_PORT}-:${PORT_PREFIX}01" + netdev+=",hostfwd=tcp::${EMULATOR_BACKEND_PORT}-:${PORT_PREFIX}02" + netdev+=",hostfwd=tcp::${EMULATOR_MINIO_PORT}-:9090" + netdev+=",hostfwd=tcp::${EMULATOR_INBUCKET_PORT}-:9001" + + QEMU_CMD=( + "$qemu_bin" + -machine "$machine" + -accel "$ACCEL" + -cpu "$cpu" + "${firmware_args[@]}" + -boot order=c + -m "$VM_RAM" + -smp "$VM_CPUS" + -drive "file=$VM_DIR/disk.qcow2,format=qcow2,if=virtio" + -drive "file=$(runtime_iso_path),format=raw,if=virtio,readonly=on" + -netdev "$netdev" + -device virtio-net-pci,netdev=net0 + -device virtio-balloon-pci + -virtfs "local,path=/,mount_tag=hostfs,security_model=none" + -chardev "socket,id=monitor,path=$VM_DIR/monitor.sock,server=on,wait=off" + -mon "chardev=monitor,mode=control" + -serial "file:$VM_DIR/serial.log" + -display none + -daemonize + -pidfile "$VM_DIR/qemu.pid" + ) + +} + +is_running() { + if [ ! -f "$VM_DIR/qemu.pid" ]; then + return 1 + fi + local pid + pid="$(cat "$VM_DIR/qemu.pid")" + kill -0 "$pid" 2>/dev/null +} + +tail_vm_logs() { + if [ -f "$VM_DIR/serial.log" ]; then + echo "" + warn "Last serial log lines:" + tail -40 "$VM_DIR/serial.log" || true + fi +} + +ensure_ports_free() { + local ports=("$EMULATOR_DASHBOARD_PORT" "$EMULATOR_BACKEND_PORT" "$EMULATOR_MINIO_PORT" "$EMULATOR_INBUCKET_PORT") + local port + for port in "${ports[@]}"; do + if lsof -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then + err "Port $port is already in use. Stop any conflicting services first." + exit 1 + fi + done +} + +start_vm() { + mkdir -p "$VM_DIR" + : > "$VM_DIR/serial.log" + prepare_runtime_config_iso + build_qemu_cmd + "${QEMU_CMD[@]}" +} + +stop_vm() { + if [ ! -f "$VM_DIR/qemu.pid" ]; then + return 0 + fi + local pid + pid="$(cat "$VM_DIR/qemu.pid")" + if kill -0 "$pid" 2>/dev/null; then + if [ -S "$VM_DIR/monitor.sock" ]; then + echo '{"execute":"qmp_capabilities"}' | socat - UNIX-CONNECT:"$VM_DIR/monitor.sock" >/dev/null 2>&1 || true + echo '{"execute":"system_powerdown"}' | socat - UNIX-CONNECT:"$VM_DIR/monitor.sock" >/dev/null 2>&1 || true + sleep 3 + fi + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi + fi + rm -f "$VM_DIR/qemu.pid" "$VM_DIR/monitor.sock" "$VM_DIR/serial.log" + rm -rf "$VM_DIR/runtime-config" + rm -f "$VM_DIR/runtime-config.iso" +} + +cmd_start() { + ensure_ports_free + mkdir -p "$RUN_DIR" + + info "Starting QEMU local emulator" + info "Arch: $ARCH | Accel: $ACCEL" + info "Ports: Dashboard=$EMULATOR_DASHBOARD_PORT Backend=$EMULATOR_BACKEND_PORT MinIO=$EMULATOR_MINIO_PORT Inbucket=$EMULATOR_INBUCKET_PORT" + + start_vm + + info "VM: ${VM_RAM}MB / ${VM_CPUS} CPUs" + + if ! wait_for_condition "deps services" "$READY_TIMEOUT" deps_ready; then + tail_vm_logs + exit 1 + fi + + if ! wait_for_condition "dashboard/backend" "$READY_TIMEOUT" app_ready; then + tail_vm_logs + exit 1 + fi + + log "All services are green." + info "Dashboard: http://localhost:${EMULATOR_DASHBOARD_PORT}" + info "Backend: http://localhost:${EMULATOR_BACKEND_PORT}" +} + +cmd_stop() { + stop_vm + log "QEMU emulator stopped." +} + +cmd_reset() { + cmd_stop 2>/dev/null || true + rm -rf "$RUN_DIR" + log "Emulator state reset. Next start will be a fresh boot." +} + +STATUS_FAILED=0 + +print_service_status() { + local name="$1" + local port="$2" + local proto="$3" + local path="${4:-/}" + local expected_codes="${5:-200}" + if service_is_up "$port" "$proto" "$path" "$expected_codes"; then + echo -e " ${GREEN}●${NC} $name (:$port)" + else + echo -e " ${RED}●${NC} $name (:$port)" + STATUS_FAILED=1 + fi +} + +cmd_status() { + STATUS_FAILED=0 + echo "VM:" + if is_running; then + echo -e " ${GREEN}●${NC} emulator" + else + echo -e " ${RED}●${NC} emulator" + STATUS_FAILED=1 + fi + echo "" + echo "Services:" + print_service_status "Dashboard" "$EMULATOR_DASHBOARD_PORT" http /handler/sign-in + print_service_status "Backend" "$EMULATOR_BACKEND_PORT" http "/health?db=1" + print_service_status "MinIO" "$EMULATOR_MINIO_PORT" http /minio/health/live + print_service_status "Inbucket HTTP" "$EMULATOR_INBUCKET_PORT" http / + exit "$STATUS_FAILED" +} + +cmd_bench() { + local elapsed + cmd_stop >/dev/null 2>&1 || true + SECONDS=0 + cmd_start + elapsed="$SECONDS" + printf "Startup time: %.1fs\n" "$elapsed" +} + +ACTION="start" + +while [[ $# -gt 0 ]]; do + case "$1" in + start|stop|reset|status|bench) + ACTION="$1" + shift + ;; + *) + echo "Usage: $0 [start|stop|reset|status|bench]" + exit 1 + ;; + esac +done + +case "$ACTION" in + start) cmd_start ;; + stop) cmd_stop ;; + reset) cmd_reset ;; + status) cmd_status ;; + bench) cmd_bench ;; +esac diff --git a/docker/local-emulator/start-app.sh b/docker/local-emulator/start-app.sh new file mode 100644 index 0000000000..ad7472732d --- /dev/null +++ b/docker/local-emulator/start-app.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e + +# In deps-only mode (used during QEMU image build), skip app startup entirely. +# The build only needs the infrastructure services to initialize; the app +# requires runtime env vars that are not available at build time. +if [ "${STACK_DEPS_ONLY:-false}" = "true" ]; then + echo "Deps-only mode: app startup skipped." + while true; do sleep 3600; done +fi + +# Wait for all infrastructure services to be ready before running migrations +# and starting the backend/dashboard. +INIT_SERVICES_DONE_FILE=/var/run/stack-local-init-services.done +INIT_SERVICES_FAILED_FILE=/var/run/stack-local-init-services.failed + +until pg_isready -h 127.0.0.1 -p 5432 -U postgres >/dev/null 2>&1; do sleep 2; done +until curl -sf http://127.0.0.1:8123/ping >/dev/null 2>&1; do sleep 2; done +until curl -sf http://127.0.0.1:8071/api/v1/health/ >/dev/null 2>&1; do sleep 2; done +until curl -sf http://127.0.0.1:9090/minio/health/live >/dev/null 2>&1; do sleep 2; done +until [ "$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8080/ 2>/dev/null || true)" = "401" ]; do sleep 2; done + +until [ -f "$INIT_SERVICES_DONE_FILE" ]; do + if [ -f "$INIT_SERVICES_FAILED_FILE" ]; then + echo "init-services.sh failed; refusing to start the app." >&2 + exit 1 + fi + sleep 1 +done + +exec /app-entrypoint.sh diff --git a/docker/local-emulator/supervisord.conf b/docker/local-emulator/supervisord.conf new file mode 100644 index 0000000000..e8b1fc4782 --- /dev/null +++ b/docker/local-emulator/supervisord.conf @@ -0,0 +1,148 @@ +[supervisord] +nodaemon=true +logfile=/var/log/supervisor/supervisord.log +pidfile=/var/run/supervisord.pid +loglevel=info + +; --- PostgreSQL --- + +[program:postgres] +command=/usr/lib/postgresql/16/bin/postgres + -D /data/postgres + -c listen_addresses=* + -c max_connections=500 + -c shared_preload_libraries=pg_stat_statements + -c pg_stat_statements.track=all + -c statement_timeout=30s +user=postgres +autostart=true +autorestart=true +priority=10 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +; --- Redis --- + +[program:redis] +command=/usr/bin/redis-server + --port 6379 + --dir /data/redis + --save 60 500 + --appendonly yes + --appendfsync everysec + --requirepass PASSWORD-PLACEHOLDER--oVn8GSD6b9 +autostart=true +autorestart=true +priority=10 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +; --- Inbucket --- + +[program:inbucket] +command=/opt/inbucket/bin/inbucket +environment= + INBUCKET_SMTP_ADDR="0.0.0.0:2500", + INBUCKET_WEB_ADDR="0.0.0.0:9001", + INBUCKET_POP3_ADDR="0.0.0.0:1100", + INBUCKET_STORAGE_TYPE="file", + INBUCKET_STORAGE_PARAMS="path:/data/inbucket" +autostart=true +autorestart=true +priority=20 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +; --- ClickHouse --- + +[program:clickhouse] +command=/usr/bin/clickhouse-server --config-file=/etc/clickhouse-server/config.xml +autostart=true +autorestart=true +priority=20 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +; --- MinIO --- + +[program:minio] +command=/usr/local/bin/minio server /data/minio --address :9090 --console-address :9091 +environment= + MINIO_ROOT_USER="s3mockroot", + MINIO_ROOT_PASSWORD="s3mockroot" +autostart=true +autorestart=true +priority=20 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +; --- Svix --- + +[program:svix] +command=/usr/local/bin/svix-server +environment= + WAIT_FOR="true", + SVIX_DB_DSN="postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@127.0.0.1:5432/svix", + SVIX_REDIS_DSN="redis://:PASSWORD-PLACEHOLDER--oVn8GSD6b9@127.0.0.1:6379", + SVIX_CACHE_TYPE="memory", + SVIX_JWT_SECRET="secret", + SVIX_LOG_LEVEL="info", + SVIX_QUEUE_TYPE="redis" +autostart=true +autorestart=true +priority=30 +startsecs=5 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +; --- QStash --- + +[program:qstash] +command=/usr/local/bin/qstash dev +environment=HOST_ON_HOST="host.docker.internal" +autostart=true +autorestart=true +priority=30 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +; --- Post-startup init --- + +[program:init-services] +command=/init-services.sh +autostart=true +autorestart=false +startsecs=0 +exitcodes=0 +priority=50 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +; --- Stack Auth backend + dashboard --- + +[program:stack-app] +command=/start-app.sh +autostart=true +autorestart=unexpected +startsecs=0 +priority=60 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index 1b598b1c40..da7214a013 100644 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -67,13 +67,16 @@ fi # ============= ENV VARS ============= -# Create a working directory for our processed files -# This is necessary because we need to replace the env vars in all files and we might want to run the seed script multiple times with different env vars. -WORK_DIR="/tmp/processed" +# Create a working directory for our processed files. +# Keep this off /tmp so local-emulator config sharing can bind-mount /tmp +# without pushing the whole runtime copy step onto the host filesystem. +WORK_DIR="${STACK_RUNTIME_WORK_DIR:-/var/tmp/stack-runtime}" mkdir -p "$WORK_DIR" -echo "Copying files to working directory..." -cp -vr /app/. "$WORK_DIR"/. +if [ "$WORK_DIR" != "/app" ]; then + echo "Copying files to working directory..." + cp -r /app/. "$WORK_DIR"/. +fi # Find all files in the apps directory that contain a STACK_ENV_VAR_SENTINEL and extract the unique sentinel strings. echo "Finding unhandled sentinels..." diff --git a/examples/demo/src/app/api/emulator-status/route.ts b/examples/demo/src/app/api/emulator-status/route.ts new file mode 100644 index 0000000000..1cf488711c --- /dev/null +++ b/examples/demo/src/app/api/emulator-status/route.ts @@ -0,0 +1,80 @@ +import { NextResponse } from 'next/server'; + +export const dynamic = 'force-dynamic'; + +type ServiceCheck = { + name: string; + description: string; + port: number; + protocol: 'http' | 'tcp'; + httpPath?: string; +}; + +const SERVICES: ServiceCheck[] = [ + { + name: 'Stack Dashboard', + description: 'Dashboard UI', + port: 26700, + protocol: 'http', + httpPath: '/handler/sign-in', + }, + { + name: 'Stack Backend', + description: 'API server', + port: 26701, + protocol: 'http', + httpPath: '/health?db=1', + }, + { + name: 'MinIO (S3)', + description: 'Object storage', + port: 26702, + protocol: 'http', + httpPath: '/minio/health/live', + }, + { + name: 'Inbucket (HTTP)', + description: 'Email capture UI', + port: 26703, + protocol: 'http', + httpPath: '/', + }, +]; + +async function checkHttp(port: number, path: string, timeoutMs = 3000): Promise<{ up: boolean; latencyMs: number }> { + const start = performance.now(); + try { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const res = await fetch(`http://127.0.0.1:${port}${path}`, { signal: controller.signal }); + clearTimeout(timeout); + return { up: res.ok || res.status < 500, latencyMs: Math.round(performance.now() - start) }; + } catch { + return { up: false, latencyMs: Math.round(performance.now() - start) }; + } +} + +export async function GET() { + const results = await Promise.all( + SERVICES.map(async (svc) => { + const check = await checkHttp(svc.port, svc.httpPath ?? '/'); + return { + name: svc.name, + description: svc.description, + port: svc.port, + status: check.up ? 'up' as const : 'down' as const, + latencyMs: check.latencyMs, + }; + }) + ); + + return NextResponse.json({ + timestamp: new Date().toISOString(), + services: results, + summary: { + total: results.length, + up: results.filter((r) => r.status === 'up').length, + down: results.filter((r) => r.status === 'down').length, + }, + }); +} diff --git a/examples/demo/src/app/emulator-status/page.tsx b/examples/demo/src/app/emulator-status/page.tsx new file mode 100644 index 0000000000..61c57e22bb --- /dev/null +++ b/examples/demo/src/app/emulator-status/page.tsx @@ -0,0 +1,188 @@ +'use client'; + +import { runAsynchronously } from '@stackframe/stack-shared/dist/utils/promises'; +import { Card, CardContent, CardHeader, Typography } from '@stackframe/stack-ui'; +import { useCallback, useEffect, useState } from 'react'; + +type ServiceResult = { + name: string; + description: string; + port: number; + status: 'up' | 'down'; + latencyMs: number; +}; + +type StatusResponse = { + timestamp: string; + services: ServiceResult[]; + summary: { total: number; up: number; down: number }; +}; + +function StatusDot({ status }: { status: 'up' | 'down' | 'checking' }) { + const color = status === 'up' + ? 'bg-emerald-500' + : status === 'down' + ? 'bg-red-500' + : 'bg-yellow-400 animate-pulse'; + return ( + + ); +} + +function ServiceRow({ service }: { service: ServiceResult }) { + return ( +
+
+ +
+ {service.name} + {service.description} +
+
+
+ :{service.port} + {service.status === 'up' && ( + {service.latencyMs}ms + )} + + {service.status === 'up' ? 'Online' : 'Offline'} + +
+
+ ); +} + +export default function EmulatorStatusPage() { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [autoRefresh, setAutoRefresh] = useState(true); + + const fetchStatus = useCallback(async () => { + try { + const res = await fetch('/api/emulator-status', { cache: 'no-store' }); + const json = await res.json(); + setData(json as StatusResponse); + } catch { + // keep last known state + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + runAsynchronously(fetchStatus()); + if (!autoRefresh) return; + const interval = setInterval(() => { + runAsynchronously(fetchStatus()); + }, 5000); + return () => clearInterval(interval); + }, [fetchStatus, autoRefresh]); + + const summary = data?.summary; + const allUp = summary != null && summary.down === 0; + + return ( +
+
+
+
+ Local Emulator Status + + Monitoring services in the all-in-one dependencies container + +
+
+ + +
+
+ + + + {loading && !data ? ( +
+ + Checking services... +
+ ) : summary ? ( +
+
+ + + {allUp + ? 'All services operational' + : `${summary.down} of ${summary.total} services are offline`} + +
+
+ {summary.up} up + {summary.down > 0 && ( + {summary.down} down + )} + updated {new Date(data.timestamp).toLocaleTimeString()} +
+
+ ) : null} +
+
+ + + + Services + + + {data?.services.map((svc) => ( + + ))} + {!data && loading && ( +
Loading...
+ )} +
+
+ + + + Quick Start + + + Start the QEMU local emulator: +
+              {`# Pull the latest image and start the emulator
+pnpm run emulator:start
+
+# Check service health
+pnpm run emulator:status
+
+# Stop (data is preserved)
+pnpm run emulator:stop
+
+# Reset for a fresh boot
+pnpm run emulator:reset`}
+            
+ + Dashboard: localhost:26700 | Backend: localhost:26701 + +
+
+
+
+ ); +} diff --git a/package.json b/package.json index 9f93e85fe9..0aca2a60cc 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,14 @@ "codegen": "pnpm pre && turbo run codegen && pnpm run generate-sdks", "codegen:backend": "pnpm pre && turbo run codegen --filter=@stackframe/backend...", "deps-compose": "docker compose -p stack-dependencies-${NEXT_PUBLIC_STACK_PORT_PREFIX:-81} -f docker/dependencies/docker.compose.yaml", + "emulator:generate-env": "node ./docker/local-emulator/generate-env-development.mjs", + "emulator:check-env": "node ./docker/local-emulator/generate-env-development.mjs --check", + "emulator:start": "pnpm run emulator:generate-env && docker/local-emulator/qemu/run-emulator.sh start", + "emulator:stop": "docker/local-emulator/qemu/run-emulator.sh stop", + "emulator:reset": "docker/local-emulator/qemu/run-emulator.sh reset", + "emulator:status": "docker/local-emulator/qemu/run-emulator.sh status", + "emulator:build": "docker/local-emulator/qemu/build-image.sh", + "emulator:bench": "docker/local-emulator/qemu/run-emulator.sh bench", "stop-deps": "POSTGRES_DELAY_MS=0 pnpm run deps-compose kill && POSTGRES_DELAY_MS=0 pnpm run deps-compose down -v", "wait-until-postgres-is-ready:pg_isready": "until pg_isready -h localhost -p ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}28 && pg_isready -h localhost -p ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}34; do sleep 1; done", "wait-until-postgres-is-ready": "command -v pg_isready >/dev/null 2>&1 && pnpm run wait-until-postgres-is-ready:pg_isready || sleep 10 # not everyone has pg_isready installed, so we fallback to sleeping", diff --git a/packages/stack-cli/src/commands/emulator.ts b/packages/stack-cli/src/commands/emulator.ts new file mode 100644 index 0000000000..a4878b2371 --- /dev/null +++ b/packages/stack-cli/src/commands/emulator.ts @@ -0,0 +1,138 @@ +import { Command } from "commander"; +import { execFileSync, spawn } from "child_process"; +import { existsSync, mkdirSync, renameSync, unlinkSync } from "fs"; +import { join, resolve } from "path"; +import { CliError } from "../lib/errors.js"; + +function gh(args: string[]): string { + try { + return execFileSync("gh", args, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim(); + } catch (err: unknown) { + if (err instanceof Error && "stderr" in err && typeof err.stderr === "string") { + throw new CliError(`GitHub CLI error: ${err.stderr}`); + } + throw new CliError("GitHub CLI (gh) is required. Install: https://cli.github.com/"); + } +} + +function findQemuDir(): string { + for (const rel of ["docker/local-emulator/qemu", "../docker/local-emulator/qemu"]) { + const dir = resolve(process.cwd(), rel); + if (existsSync(join(dir, "run-emulator.sh"))) return dir; + } + throw new CliError("Could not find QEMU emulator directory. Run this from the stack-auth repo root."); +} + +function runEmulator(action: string, env?: Record): Promise { + const qemuDir = findQemuDir(); + return new Promise((resolve, reject) => { + const child = spawn(join(qemuDir, "run-emulator.sh"), [action], { + stdio: "inherit", + env: { ...process.env, ...env }, + cwd: qemuDir, + }); + child.on("close", (code) => code === 0 ? resolve() : reject(new CliError(`run-emulator.sh ${action} exited with code ${code}`))); + child.on("error", (err) => reject(new CliError(`Failed to run run-emulator.sh: ${err.message}`))); + }); +} + +function resolveArch(raw?: string): "arm64" | "amd64" { + const arch = raw ?? (process.arch === "arm64" ? "arm64" : process.arch === "x64" ? "amd64" : null); + if (arch === "arm64" || arch === "amd64") return arch; + throw new CliError(`Invalid architecture: ${raw ?? process.arch}. Expected arm64 or amd64.`); +} + +function pullRelease(arch: "arm64" | "amd64", opts: { repo?: string; branch?: string; tag?: string } = {}) { + const repo = opts.repo ?? "stack-auth/stack-auth"; + const branch = opts.branch ?? "dev"; + const tag = opts.tag ?? `emulator-${branch}-latest`; + const asset = `stack-emulator-${arch}.qcow2`; + const imageDir = join(findQemuDir(), "images"); + mkdirSync(imageDir, { recursive: true }); + const dest = join(imageDir, asset); + const tmpDest = `${dest}.download`; + + console.log(`Pulling ${asset} from release ${tag}...`); + try { + execFileSync("gh", ["release", "download", tag, "--repo", repo, "--pattern", asset, "--output", tmpDest, "--clobber"], { stdio: "inherit" }); + } catch (err) { + if (existsSync(tmpDest)) unlinkSync(tmpDest); + throw new CliError(`Failed to download ${asset} from release ${tag}: ${err instanceof Error ? err.message : err}\nRun 'stack emulator list-releases' to see available releases.`); + } + renameSync(tmpDest, dest); + console.log(`Downloaded: ${dest}`); +} + +export function registerEmulatorCommand(program: Command) { + const emulator = program.command("emulator").description("Manage the QEMU local emulator"); + + emulator + .command("pull") + .description("Download an emulator image from GitHub Releases or a PR build") + .option("--arch ", "Target architecture (default: current system arch)") + .option("--branch ", "Release branch (default: dev)") + .option("--tag ", "Specific release tag (default: latest)") + .option("--repo ", "GitHub repository (default: stack-auth/stack-auth)") + .option("--pr ", "Pull from a PR's CI artifacts") + .option("--run ", "Pull from a specific workflow run's artifacts") + .action(async (opts) => { + const arch = resolveArch(opts.arch); + const repo = opts.repo ?? "stack-auth/stack-auth"; + + if (opts.run || opts.pr) { + let runId = opts.run as string | undefined; + if (!runId) { + console.log(`Finding latest successful build for PR #${opts.pr}...`); + const { headRefName } = JSON.parse(gh(["pr", "view", opts.pr, "--repo", repo, "--json", "headRefName"])); + const runs = JSON.parse(gh(["run", "list", "--repo", repo, "--workflow", "qemu-emulator-build.yaml", "--branch", headRefName, "--status", "success", "--limit", "1", "--json", "databaseId"])); + if (runs.length === 0) throw new CliError(`No successful build found for PR #${opts.pr} (branch: ${headRefName}).`); + runId = String(runs[0].databaseId); + } + + const imageDir = join(findQemuDir(), "images"); + mkdirSync(imageDir, { recursive: true }); + const dest = join(imageDir, `stack-emulator-${arch}.qcow2`); + if (existsSync(dest)) unlinkSync(dest); + console.log(`Downloading qemu-emulator-${arch} from workflow run ${runId}...`); + try { + execFileSync("gh", ["run", "download", runId, "--repo", repo, "--name", `qemu-emulator-${arch}`, "--dir", imageDir], { stdio: "inherit" }); + } catch (err) { + throw new CliError(`Failed to download artifact from run ${runId}: ${err instanceof Error ? err.message : err}`); + } + if (!existsSync(dest)) throw new CliError(`Expected image not found at ${dest} after download.`); + console.log(`Downloaded: ${dest}`); + } else { + pullRelease(arch, { repo, branch: opts.branch, tag: opts.tag }); + } + }); + + emulator + .command("start") + .description("Start the emulator in the background (auto-pulls the latest image if none exists)") + .option("--arch ", "Target architecture (default: current system arch). Non-native uses software emulation and is significantly slower.") + .action(async (opts) => { + const arch = resolveArch(opts.arch); + const img = join(findQemuDir(), "images", `stack-emulator-${arch}.qcow2`); + if (!existsSync(img)) { + console.log("No emulator image found. Pulling latest..."); + pullRelease(arch); + } + await runEmulator("start", { EMULATOR_ARCH: arch }); + }); + + emulator.command("stop").description("Stop the emulator (data preserved; use 'reset' to clear)").action(() => runEmulator("stop")); + emulator.command("reset").description("Reset emulator state for a fresh boot").action(() => runEmulator("reset")); + emulator.command("status").description("Show emulator and service health").action(() => runEmulator("status")); + + emulator + .command("list-releases") + .description("List available emulator releases") + .option("--repo ", "GitHub repository (default: stack-auth/stack-auth)") + .action((opts) => { + const repo = opts.repo ?? "stack-auth/stack-auth"; + console.log(`Available emulator releases from ${repo}:\n`); + const lines = gh(["release", "list", "--repo", repo, "--limit", "20"]).split("\n").filter((l) => l.toLowerCase().includes("emulator")); + if (lines.length === 0) console.log("No emulator releases found."); + else for (const line of lines) console.log(line); + }); +} diff --git a/packages/stack-cli/src/index.ts b/packages/stack-cli/src/index.ts index 08af5837ed..69f4ddc372 100644 --- a/packages/stack-cli/src/index.ts +++ b/packages/stack-cli/src/index.ts @@ -9,6 +9,7 @@ import { registerExecCommand } from "./commands/exec.js"; import { registerConfigCommand } from "./commands/config-file.js"; import { registerInitCommand } from "./commands/init.js"; import { registerProjectCommand } from "./commands/project.js"; +import { registerEmulatorCommand } from "./commands/emulator.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -29,6 +30,7 @@ registerExecCommand(program); registerConfigCommand(program); registerInitCommand(program); registerProjectCommand(program); +registerEmulatorCommand(program); async function main() { try { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 20a52c3b7c..f19de820e0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -95,7 +95,7 @@ importers: version: 0.20.3(typescript@5.9.3) turbo: specifier: ^2.8.15 - version: 2.8.15 + version: 2.8.17 typescript: specifier: 5.9.3 version: 5.9.3 @@ -4987,14 +4987,6 @@ packages: '@types/node': optional: true - '@isaacs/balanced-match@4.0.1': - resolution: {integrity: sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==} - engines: {node: 20 || >=22} - - '@isaacs/brace-expansion@5.0.0': - resolution: {integrity: sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==} - engines: {node: 20 || >=22} - '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -13816,10 +13808,6 @@ packages: resolution: {integrity: sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==} engines: {node: 20 || >=22} - minimatch@10.1.1: - resolution: {integrity: sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==} - engines: {node: 20 || >=22} - minimatch@10.2.4: resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==} engines: {node: 18 || 20 || >=22} @@ -16311,41 +16299,41 @@ packages: resolution: {integrity: sha512-axr3IdNuVIxnaK5XGEUFTu3YmAQ6lllgrvqfEoR16g/HGnYY/6We4oWENtAnzK6/LpJ2ur9PAb80RBt7/U4ugw==} engines: {node: '>= 6.0.0'} - turbo-darwin-64@2.8.15: - resolution: {integrity: sha512-EElCh+Ltxex9lXYrouV3hHjKP3HFP31G91KMghpNHR/V99CkFudRcHcnWaorPbzAZizH1m8o2JkLL8rptgb8WQ==} + turbo-darwin-64@2.8.17: + resolution: {integrity: sha512-ZFkv2hv7zHpAPEXBF6ouRRXshllOavYc+jjcrYyVHvxVTTwJWsBZwJ/gpPzmOKGvkSjsEyDO5V6aqqtZzwVF+Q==} cpu: [x64] os: [darwin] - turbo-darwin-arm64@2.8.15: - resolution: {integrity: sha512-ORmvtqHiHwvNynSWvLIleyU8dKtwQ4ILk39VsEwfKSEzSHWYWYxZhBmD9GAGRPlNl7l7S1irrziBlDEGVpq+vQ==} + turbo-darwin-arm64@2.8.17: + resolution: {integrity: sha512-5DXqhQUt24ycEryXDfMNKEkW5TBHs+QmU23a2qxXwwFDaJsWcPo2obEhBxxdEPOv7qmotjad+09RGeWCcJ9JDw==} cpu: [arm64] os: [darwin] - turbo-linux-64@2.8.15: - resolution: {integrity: sha512-Bk1E61a+PCWUTfhqfXFlhEJMLp6nak0J0Qt14IZX1og1zyaiBLkM6M1GQFbPpiWfbUcdLwRaYQhO0ySB07AJ8w==} + turbo-linux-64@2.8.17: + resolution: {integrity: sha512-KLUbz6w7F73D/Ihh51hVagrKR0/CTsPEbRkvXLXvoND014XJ4BCrQUqSxlQ4/hu+nqp1v5WlM85/h3ldeyujuA==} cpu: [x64] os: [linux] - turbo-linux-arm64@2.8.15: - resolution: {integrity: sha512-3BX0Vk+XkP0uiZc8pkjQGNsAWjk5ojC53bQEMp6iuhSdWpEScEFmcT6p7DL7bcJmhP2mZ1HlAu0A48wrTGCtvg==} + turbo-linux-arm64@2.8.17: + resolution: {integrity: sha512-pJK67XcNJH40lTAjFu7s/rUlobgVXyB3A3lDoq+/JccB3hf+SysmkpR4Itlc93s8LEaFAI4mamhFuTV17Z6wOg==} cpu: [arm64] os: [linux] turbo-stream@2.4.0: resolution: {integrity: sha512-FHncC10WpBd2eOmGwpmQsWLDoK4cqsA/UT/GqNoaKOQnT8uzhtCbg3EoUDMvqpOSAI0S26mr0rkjzbOO6S3v1g==} - turbo-windows-64@2.8.15: - resolution: {integrity: sha512-m14ogunMF+grHZ1jzxSCO6q0gEfF1tmr+0LU+j1QNd/M1X33tfKnQqmpkeUR/REsGjfUlkQlh6PAzqlT3cA3Pg==} + turbo-windows-64@2.8.17: + resolution: {integrity: sha512-EijeQ6zszDMmGZLP2vT2RXTs/GVi9rM0zv2/G4rNu2SSRSGFapgZdxgW4b5zUYLVaSkzmkpWlGfPfj76SW9yUg==} cpu: [x64] os: [win32] - turbo-windows-arm64@2.8.15: - resolution: {integrity: sha512-HWh6dnzhl7nu5gRwXeqP61xbyDBNmQ4UCeWNa+si4/6RAtHlKEcZTNs7jf4U+oqBnbtv4uxbKZZPf/kN0EK4+A==} + turbo-windows-arm64@2.8.17: + resolution: {integrity: sha512-crpfeMPkfECd4V1PQ/hMoiyVcOy04+bWedu/if89S15WhOalHZ2BYUi6DOJhZrszY+mTT99OwpOsj4wNfb/GHQ==} cpu: [arm64] os: [win32] - turbo@2.8.15: - resolution: {integrity: sha512-ERZf7pKOR155NKs/PZt1+83NrSEJfUL7+p9/TGZg/8xzDVMntXEFQlX4CsNJQTyu4h3j+dZYiQWOOlv5pssuHQ==} + turbo@2.8.17: + resolution: {integrity: sha512-YwPsNSqU2f/RXU/+Kcb7cPkPZARxom4+me7LKEdN5jsvy2tpfze3zDZ4EiGrJnvOm9Avu9rK0aaYsP7qZ3iz7A==} hasBin: true type-check@0.4.0: @@ -20152,12 +20140,6 @@ snapshots: optionalDependencies: '@types/node': 20.17.6 - '@isaacs/balanced-match@4.0.1': {} - - '@isaacs/brace-expansion@5.0.0': - dependencies: - '@isaacs/balanced-match': 4.0.1 - '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -26130,7 +26112,7 @@ snapshots: '@ts-morph/common@0.27.0': dependencies: fast-glob: 3.3.3 - minimatch: 10.1.1 + minimatch: 10.2.4 path-browserify: 1.0.1 '@turf/boolean-point-in-polygon@7.1.0': @@ -30370,7 +30352,7 @@ snapshots: glob@13.0.0: dependencies: - minimatch: 10.1.1 + minimatch: 10.2.4 minipass: 7.1.2 path-scurry: 2.0.0 @@ -32075,10 +32057,6 @@ snapshots: dependencies: brace-expansion: 2.0.1 - minimatch@10.1.1: - dependencies: - '@isaacs/brace-expansion': 5.0.0 - minimatch@10.2.4: dependencies: brace-expansion: 5.0.4 @@ -35324,34 +35302,34 @@ snapshots: dependencies: tslib: 1.14.1 - turbo-darwin-64@2.8.15: + turbo-darwin-64@2.8.17: optional: true - turbo-darwin-arm64@2.8.15: + turbo-darwin-arm64@2.8.17: optional: true - turbo-linux-64@2.8.15: + turbo-linux-64@2.8.17: optional: true - turbo-linux-arm64@2.8.15: + turbo-linux-arm64@2.8.17: optional: true turbo-stream@2.4.0: {} - turbo-windows-64@2.8.15: + turbo-windows-64@2.8.17: optional: true - turbo-windows-arm64@2.8.15: + turbo-windows-arm64@2.8.17: optional: true - turbo@2.8.15: + turbo@2.8.17: optionalDependencies: - turbo-darwin-64: 2.8.15 - turbo-darwin-arm64: 2.8.15 - turbo-linux-64: 2.8.15 - turbo-linux-arm64: 2.8.15 - turbo-windows-64: 2.8.15 - turbo-windows-arm64: 2.8.15 + turbo-darwin-64: 2.8.17 + turbo-darwin-arm64: 2.8.17 + turbo-linux-64: 2.8.17 + turbo-linux-arm64: 2.8.17 + turbo-windows-64: 2.8.17 + turbo-windows-arm64: 2.8.17 type-check@0.4.0: dependencies: