From 21183b5eb663aa1d94bbb2db8c128b60c8757f24 Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Thu, 7 May 2026 11:47:01 +0530 Subject: [PATCH 01/18] feat: support split-config layout for database remotes by separating secrets from public configuration --- cli/src/commands/init.ts | 82 +++++-- cli/src/common/config.ts | 119 +++++++-- cli/src/modules/db/commands/import.ts | 2 +- cli/src/modules/db/types/config.ts | 2 +- cli/src/modules/db/utils/remotes.ts | 228 ++++++++++-------- cli/test/common/config.test.ts | 53 +++- cli/test/e2e/helpers/test-project.ts | 23 +- .../e2e/workflows/remote-management.test.ts | 17 +- cli/test/modules/db/utils/remotes.test.ts | 6 +- 9 files changed, 368 insertions(+), 164 deletions(-) diff --git a/cli/src/commands/init.ts b/cli/src/commands/init.ts index 8d947bc..057a839 100644 --- a/cli/src/commands/init.ts +++ b/cli/src/commands/init.ts @@ -6,27 +6,42 @@ import {promptConfirm} from "../common/prompt"; import { projectRoot, POSTKIT_CONFIG_FILE, + POSTKIT_SECRETS_FILE, POSTKIT_DIR, getConfigFilePath, + getSecretsFilePath, getPostkitDir, getPostkitAuthDir, } from "../common/config"; import type {CommandOptions} from "../common/types"; -import type {PostkitConfig} from "../common/config"; +import type {PostkitPublicConfig, PostkitSecrets} from "../common/config"; +// Only .postkit/ and the secrets file are gitignored. +// postkit.config.json is safe to commit. const GITIGNORE_ENTRIES = [ "# Postkit", ".postkit/", - "postkit.config.json", + "postkit.secrets.json", ]; -const SCAFFOLD_CONFIG: PostkitConfig = { +// Non-sensitive settings committed to git +const SCAFFOLD_PUBLIC_CONFIG: PostkitPublicConfig = { db: { - localDbUrl: "", schemaPath: "schema", schema: "public", remotes: {}, }, + auth: { + configCliImage: "adorsys/keycloak-config-cli:6.4.0-24", + }, +}; + +// Sensitive credentials — gitignored +const SCAFFOLD_SECRETS: PostkitSecrets = { + db: { + localDbUrl: "", + remotes: {}, + }, auth: { source: { url: "", @@ -39,7 +54,31 @@ const SCAFFOLD_CONFIG: PostkitConfig = { adminUser: "", adminPass: "", }, - configCliImage: "adorsys/keycloak-config-cli:6.4.0-24", + }, +}; + +// Example secrets template committed alongside the public config +const SCAFFOLD_SECRETS_EXAMPLE: PostkitSecrets = { + db: { + localDbUrl: "postgres://user:pass@localhost:5432/mydb", + remotes: { + dev: { + url: "postgres://user:pass@dev-host:5432/mydb", + }, + }, + }, + auth: { + source: { + url: "http://keycloak-source:8080", + adminUser: "admin", + adminPass: "changeme", + realm: "myrealm", + }, + target: { + url: "http://keycloak-target:8080", + adminUser: "admin", + adminPass: "changeme", + }, }, }; @@ -80,8 +119,7 @@ export async function initCommand(options: CommandOptions): Promise { const spinner = ora("Creating .postkit/db/ directory...").start(); const postkitDbDir = path.join(postkitDir, "db"); fs.mkdirSync(postkitDbDir, {recursive: true}); - // Create runtime files with proper initial content - // session.json is intentionally excluded — it is created only when a session starts + // session.json is intentionally excluded — created only when a session starts const runtimeFiles: Record = { "committed.json": JSON.stringify({migrations: []}, null, 2), "plan.sql": "", @@ -93,7 +131,6 @@ export async function initCommand(options: CommandOptions): Promise { fs.writeFileSync(filePath, content); } } - // Create subdirectories for (const subdir of ["session", "migrations"]) { const subPath = path.join(postkitDbDir, subdir); if (!fs.existsSync(subPath)) { @@ -110,7 +147,6 @@ export async function initCommand(options: CommandOptions): Promise { } else { const spinner = ora("Creating .postkit/auth/ directory...").start(); const postkitAuthDir = getPostkitAuthDir(); - // Create subdirectories for (const subdir of ["raw", "realm"]) { const subPath = path.join(postkitAuthDir, subdir); if (!fs.existsSync(subPath)) { @@ -120,14 +156,23 @@ export async function initCommand(options: CommandOptions): Promise { spinner.succeed(".postkit/auth/ directory created"); } - // Step 3: Generate postkit.config.json - logger.step(3, totalSteps, "Generating postkit.config.json"); + // Step 3: Generate config and secrets files + logger.step(3, totalSteps, "Generating config and secrets files"); if (options.dryRun) { - logger.info(`Dry run: would create ${POSTKIT_CONFIG_FILE}`); + logger.info(`Dry run: would create ${POSTKIT_CONFIG_FILE} (committed) and ${POSTKIT_SECRETS_FILE} (gitignored)`); } else { - const spinner = ora("Writing postkit.config.json...").start(); - fs.writeFileSync(configFile, JSON.stringify(SCAFFOLD_CONFIG, null, 2) + "\n"); - spinner.succeed("postkit.config.json created"); + const spinner = ora("Writing config files...").start(); + + fs.writeFileSync(configFile, JSON.stringify(SCAFFOLD_PUBLIC_CONFIG, null, 2) + "\n"); + + const secretsFile = getSecretsFilePath(); + fs.writeFileSync(secretsFile, JSON.stringify(SCAFFOLD_SECRETS, null, 2) + "\n"); + + // Write example secrets template so teammates know the expected shape + const exampleFile = path.join(projectRoot, "postkit.secrets.example.json"); + fs.writeFileSync(exampleFile, JSON.stringify(SCAFFOLD_SECRETS_EXAMPLE, null, 2) + "\n"); + + spinner.succeed(`${POSTKIT_CONFIG_FILE}, ${POSTKIT_SECRETS_FILE}, and postkit.secrets.example.json created`); } // Step 4: Update .gitignore @@ -163,8 +208,13 @@ export async function initCommand(options: CommandOptions): Promise { logger.blank(); logger.success("Postkit project initialized!"); logger.blank(); + logger.info("Config split:"); + logger.info(` ${POSTKIT_CONFIG_FILE} — committed to git (schema paths, remote metadata)`); + logger.info(` ${POSTKIT_SECRETS_FILE} — gitignored (DB URLs, passwords)`); + logger.info(` postkit.secrets.example.json — committed template for teammates`); + logger.blank(); logger.info("Next steps:"); - logger.info(` 1. Edit ${POSTKIT_CONFIG_FILE} with your database settings`); + logger.info(` 1. Fill in ${POSTKIT_SECRETS_FILE} with your database credentials`); logger.info(" 2. Add remote databases:"); logger.info(" postkit db remote add staging \"postgres://...\""); logger.info(" 3. Run postkit db start to begin a migration session"); diff --git a/cli/src/common/config.ts b/cli/src/common/config.ts index a205fba..f0a1526 100644 --- a/cli/src/common/config.ts +++ b/cli/src/common/config.ts @@ -17,12 +17,17 @@ export const projectRoot = process.cwd(); // Postkit project paths export const POSTKIT_CONFIG_FILE = "postkit.config.json"; +export const POSTKIT_SECRETS_FILE = "postkit.secrets.json"; export const POSTKIT_DIR = ".postkit"; export function getConfigFilePath(): string { return path.join(projectRoot, POSTKIT_CONFIG_FILE); } +export function getSecretsFilePath(): string { + return path.join(projectRoot, POSTKIT_SECRETS_FILE); +} + export function getPostkitDir(): string { return path.join(projectRoot, POSTKIT_DIR); } @@ -58,6 +63,53 @@ export interface AuthInputConfig { configCliImage?: string; } +// ─── Public config (committed to git) ─────────────────────────────────────── +// Non-sensitive settings: schema paths, remote metadata, docker images, etc. + +export interface RemotePublicConfig { + default?: boolean; + addedAt?: string; +} + +export interface DbPublicConfig { + schemaPath?: string; + schema?: string; + remotes?: Record; +} + +export interface AuthPublicConfig { + configCliImage?: string; +} + +export interface PostkitPublicConfig { + db?: DbPublicConfig; + auth?: AuthPublicConfig; +} + +// ─── Secrets (gitignored) ───────────────────────────────────────────────────── +// Sensitive credentials: DB URLs, passwords, auth tokens. + +export interface RemoteSecretConfig { + url: string; +} + +export interface DbSecretsConfig { + localDbUrl?: string; + remotes?: Record; +} + +export interface AuthSecretsConfig { + source?: Partial; + target?: Partial; +} + +export interface PostkitSecrets { + db?: DbSecretsConfig; + auth?: AuthSecretsConfig; +} + +// ─── Merged runtime config ──────────────────────────────────────────────────── + // PostkitConfig interface matching the JSON structure export interface PostkitConfig { db: DbInputConfig; @@ -80,11 +132,38 @@ export function checkInitialized(): void { if (!fs.existsSync(configPath)) { throw new Error( "Postkit project is not initialized.\n" + - `Run \"postkit init\" to initialize your project first.`, + `Run "postkit init" to initialize your project first.`, ); } } +/** + * Deep-merge two plain objects. Values in `override` win over `base`. + * Only plain objects are recursed into; primitives and arrays are replaced wholesale. + */ +function deepMerge(base: T, override: Partial): T { + const result: T = {...base}; + for (const key of Object.keys(override) as (keyof T)[]) { + const overrideVal = override[key]; + const baseVal = base[key]; + if ( + overrideVal !== null && + overrideVal !== undefined && + typeof overrideVal === "object" && + !Array.isArray(overrideVal) && + baseVal !== null && + baseVal !== undefined && + typeof baseVal === "object" && + !Array.isArray(baseVal) + ) { + result[key] = deepMerge(baseVal as object, overrideVal as object) as T[keyof T]; + } else if (overrideVal !== undefined) { + result[key] = overrideVal as T[keyof T]; + } + } + return result; +} + export function loadPostkitConfig(): PostkitConfig { if (cachedConfig) { return cachedConfig; @@ -99,41 +178,47 @@ export function loadPostkitConfig(): PostkitConfig { } const raw = fs.readFileSync(configPath, "utf-8"); - const parsed = JSON.parse(raw); + let parsed: Record = JSON.parse(raw); // Auto-migrate from old config format (remoteDbUrl/environments to remotes) - if (parsed.db && (parsed.db.remoteDbUrl || parsed.db.environments)) { - if (!parsed.db.remotes || Object.keys(parsed.db.remotes).length === 0) { - // Migrate remoteDbUrl to default remote - if (parsed.db.remoteDbUrl) { - parsed.db.remotes = parsed.db.remotes || {}; - parsed.db.remotes.default = { - url: parsed.db.remoteDbUrl, + if (parsed.db && (parsed.db as Record).remoteDbUrl || parsed.db && (parsed.db as Record).environments) { + const db = parsed.db as Record; + if (!db.remotes || Object.keys(db.remotes as object).length === 0) { + if (db.remoteDbUrl) { + db.remotes = db.remotes || {}; + (db.remotes as Record).default = { + url: db.remoteDbUrl, default: true, addedAt: new Date().toISOString(), }; - delete parsed.db.remoteDbUrl; + delete db.remoteDbUrl; } - // Migrate environments to named remotes - if (parsed.db.environments) { - parsed.db.remotes = parsed.db.remotes || {}; - for (const [name, url] of Object.entries(parsed.db.environments)) { + if (db.environments) { + db.remotes = db.remotes || {}; + for (const [name, url] of Object.entries(db.environments as Record)) { if (name !== "default" && typeof url === "string") { - parsed.db.remotes[name] = { + (db.remotes as Record)[name] = { url, addedAt: new Date().toISOString(), }; } } - delete parsed.db.environments; + delete db.environments; } - // Save migrated config fs.writeFileSync(configPath, JSON.stringify(parsed, null, 2), "utf-8"); } } + // Load and merge secrets file if it exists + const secretsPath = getSecretsFilePath(); + if (fs.existsSync(secretsPath)) { + const secretsRaw = fs.readFileSync(secretsPath, "utf-8"); + const secrets: PostkitSecrets = JSON.parse(secretsRaw); + parsed = deepMerge(parsed as object, secrets as object) as Record; + } + cachedConfig = parsed as PostkitConfig; return cachedConfig; } diff --git a/cli/src/modules/db/commands/import.ts b/cli/src/modules/db/commands/import.ts index 3f1a7d0..f75442e 100644 --- a/cli/src/modules/db/commands/import.ts +++ b/cli/src/modules/db/commands/import.ts @@ -83,7 +83,7 @@ export async function importCommand(options: ImportOptions): Promise { if (!targetUrl) { throw new PostkitError( "No database URL provided.", - "Use --url flag or set localDbUrl in postkit.config.json.", + "Use --url flag or set localDbUrl in postkit.secrets.json.", ); } diff --git a/cli/src/modules/db/types/config.ts b/cli/src/modules/db/types/config.ts index aebd8ac..a189a28 100644 --- a/cli/src/modules/db/types/config.ts +++ b/cli/src/modules/db/types/config.ts @@ -13,7 +13,7 @@ export interface RemoteInputConfig { } export interface DbInputConfig { - localDbUrl: string; + localDbUrl?: string; schemaPath?: string; schema?: string; remotes?: Record; diff --git a/cli/src/modules/db/utils/remotes.ts b/cli/src/modules/db/utils/remotes.ts index 754c9f5..22e2174 100644 --- a/cli/src/modules/db/utils/remotes.ts +++ b/cli/src/modules/db/utils/remotes.ts @@ -1,6 +1,7 @@ import fs from "fs/promises"; +import {existsSync} from "fs"; import {logger} from "../../../common/logger"; -import {loadPostkitConfig, getConfigFilePath, invalidateConfig, projectRoot} from "../../../common/config"; +import {loadPostkitConfig, getConfigFilePath, getSecretsFilePath, invalidateConfig} from "../../../common/config"; import type {RemoteConfig} from "../../../common/config"; export interface RemoteInfo { @@ -11,7 +12,7 @@ export interface RemoteInfo { } /** - * Get all configured remotes from the config + * Get all configured remotes from the merged config * @throws Error if no remotes are configured */ export function getRemotes(): Record { @@ -61,7 +62,6 @@ export function getDefaultRemote(): string | null { const defaultName = Object.keys(remotes).find(name => remotes[name]?.default === true); if (!defaultName) { - // If no default is explicitly set, use the first remote const firstRemote = Object.keys(remotes)[0]; if (firstRemote) { return firstRemote; @@ -72,17 +72,35 @@ export function getDefaultRemote(): string | null { return defaultName; } +// ─── File read helpers ─────────────────────────────────────────────────────── + +async function readJsonFile(filePath: string): Promise> { + const raw = await fs.readFile(filePath, "utf-8"); + return JSON.parse(raw) as Record; +} + +async function writeJsonFile(filePath: string, data: Record): Promise { + await fs.writeFile(filePath, JSON.stringify(data, null, 2) + "\n", "utf-8"); +} + /** - * Add a new remote configuration - * @param name - Name of the remote - * @param url - Database connection URL - * @param setAsDefault - Whether to set this as the default remote + * Returns true when the project was initialized with the split-config layout + * (i.e. postkit.secrets.json exists next to postkit.config.json). */ -export async function addRemote(name: string, url: string, setAsDefault: boolean = false): Promise { - const configPath = getConfigFilePath(); - const raw = await fs.readFile(configPath, "utf-8"); - const config = JSON.parse(raw); +function hasSplitConfig(): boolean { + return existsSync(getSecretsFilePath()); +} + +// ─── Remote management ─────────────────────────────────────────────────────── +/** + * Add a new remote configuration. + * + * Split-config projects: metadata (default, addedAt) → postkit.config.json + * URL → postkit.secrets.json + * Legacy projects (no secrets file): everything → postkit.config.json + */ +export async function addRemote(name: string, url: string, setAsDefault: boolean = false): Promise { // Validate name — only letters, numbers, hyphens, underscores if (!name || name.trim().length === 0) { throw new Error("Remote name cannot be empty"); @@ -94,92 +112,99 @@ export async function addRemote(name: string, url: string, setAsDefault: boolean ); } - // Check if remote name already exists - if (config.db.remotes && config.db.remotes[name]) { - throw new Error(`Remote "${name}" already exists`); - } - - // Basic URL format validation if (!isValidDatabaseUrl(url)) { throw new Error( "Invalid database URL format. Expected format: postgres://user:pass@host:port/database", ); } - // Check if URL matches local database URL - if (config.db.localDbUrl && normalizeUrl(url) === normalizeUrl(config.db.localDbUrl)) { + const configPath = getConfigFilePath(); + const config = await readJsonFile(configPath); + const db = (config.db ?? {}) as Record; + const existingRemotes = (db.remotes ?? {}) as Record>; + + if (existingRemotes[name]) { + throw new Error(`Remote "${name}" already exists`); + } + + // Check for URL conflicts in the merged (runtime) config + const merged = loadPostkitConfig(); + if (merged.db.localDbUrl && normalizeUrl(url) === normalizeUrl(merged.db.localDbUrl)) { throw new Error( "Cannot add remote: URL matches local database URL.\n" + "The remote URL must be different from your local database." ); } - // Check if URL already exists in another remote - const existingRemote = findRemoteByUrl(config.db.remotes, url); - if (existingRemote) { + const existingByUrl = findRemoteByUrl(merged.db.remotes, url); + if (existingByUrl) { throw new Error( - `Cannot add remote: URL already used by remote "${existingRemote}".\n` + + `Cannot add remote: URL already used by remote "${existingByUrl}".\n` + "Each remote must have a unique URL." ); } - // Initialize remotes object if it doesn't exist - if (!config.db.remotes) { - config.db.remotes = {}; - } + const remoteCount = Object.keys(existingRemotes).length; + const makeDefault = setAsDefault || remoteCount === 0; - // If this is the first remote or setAsDefault is true, clear other defaults - const remoteCount = Object.keys(config.db.remotes).length; - - if (remoteCount === 0 || setAsDefault) { - for (const key of Object.keys(config.db.remotes)) { - delete config.db.remotes[key].default; + if (makeDefault) { + for (const key of Object.keys(existingRemotes)) { + delete existingRemotes[key].default; } } - // Add the new remote - config.db.remotes[name] = { - url, - addedAt: new Date().toISOString(), - }; - - // Set as default if requested or if it's the first remote - if (setAsDefault || remoteCount === 0) { - config.db.remotes[name].default = true; + const addedAt = new Date().toISOString(); + + if (hasSplitConfig()) { + // Write metadata to public config + existingRemotes[name] = {addedAt}; + if (makeDefault) existingRemotes[name].default = true; + db.remotes = existingRemotes; + config.db = db; + await writeJsonFile(configPath, config); + + // Write URL to secrets + const secretsPath = getSecretsFilePath(); + const secrets = await readJsonFile(secretsPath); + const secretsDb = (secrets.db ?? {}) as Record; + const secretsRemotes = (secretsDb.remotes ?? {}) as Record; + secretsRemotes[name] = {url}; + secretsDb.remotes = secretsRemotes; + secrets.db = secretsDb; + await writeJsonFile(secretsPath, secrets); + } else { + // Legacy: write everything to postkit.config.json + existingRemotes[name] = {url, addedAt}; + if (makeDefault) existingRemotes[name].default = true; + db.remotes = existingRemotes; + config.db = db; + await writeJsonFile(configPath, config); } - // Save the updated config - await fs.writeFile(configPath, JSON.stringify(config, null, 2), "utf-8"); invalidateConfig(); - logger.success(`Remote "${name}" added successfully`); } /** - * Remove a remote configuration - * @param name - Name of the remote to remove - * @param force - Skip confirmation (not used here, kept for API consistency) + * Remove a remote configuration from both config and secrets files. */ export async function removeRemote(name: string, force: boolean = false): Promise { - const configPath = getConfigFilePath(); - const raw = await fs.readFile(configPath, "utf-8"); - const config = JSON.parse(raw); + // Use merged config to validate existence and count + const merged = loadPostkitConfig(); + const remotes = merged.db.remotes ?? {}; - if (!config.db.remotes || !config.db.remotes[name]) { + if (!remotes[name]) { throw new Error(`Remote "${name}" not found`); } - const remotes = config.db.remotes; const remoteCount = Object.keys(remotes).length; - // Cannot remove the only remote if (remoteCount === 1) { throw new Error( "Cannot remove the only remaining remote. Add another remote first.", ); } - // Check if it's the default remote const isDefault = remotes[name].default === true; if (isDefault && !force) { @@ -190,57 +215,73 @@ export async function removeRemote(name: string, force: boolean = false): Promis ); } - // Remove the remote - delete remotes[name]; + // Remove from public config + const configPath = getConfigFilePath(); + const config = await readJsonFile(configPath); + const db = (config.db ?? {}) as Record; + const configRemotes = (db.remotes ?? {}) as Record>; + delete configRemotes[name]; - // If we removed the default, set the first remaining remote as default if (isDefault) { - const firstKey = Object.keys(remotes)[0]; - if (firstKey) { - remotes[firstKey].default = true; - } + const firstKey = Object.keys(configRemotes)[0]; + if (firstKey) configRemotes[firstKey].default = true; } - // Save the updated config - await fs.writeFile(configPath, JSON.stringify(config, null, 2), "utf-8"); - invalidateConfig(); + db.remotes = configRemotes; + config.db = db; + await writeJsonFile(configPath, config); + + // Remove from secrets if split-config layout is in use + if (hasSplitConfig()) { + const secretsPath = getSecretsFilePath(); + const secrets = await readJsonFile(secretsPath); + const secretsDb = (secrets.db ?? {}) as Record; + const secretsRemotes = (secretsDb.remotes ?? {}) as Record; + delete secretsRemotes[name]; + secretsDb.remotes = secretsRemotes; + secrets.db = secretsDb; + await writeJsonFile(secretsPath, secrets); + } + invalidateConfig(); logger.success(`Remote "${name}" removed successfully`); } /** - * Set a remote as the default - * @param name - Name of the remote to set as default + * Set a remote as the default. + * Only updates postkit.config.json — the `default` flag is not sensitive. */ export async function setDefaultRemote(name: string): Promise { + const merged = loadPostkitConfig(); + if (!merged.db.remotes || !merged.db.remotes[name]) { + throw new Error(`Remote "${name}" not found`); + } + const configPath = getConfigFilePath(); - const raw = await fs.readFile(configPath, "utf-8"); - const config = JSON.parse(raw); + const config = await readJsonFile(configPath); + const db = (config.db ?? {}) as Record; + const configRemotes = (db.remotes ?? {}) as Record>; - if (!config.db.remotes || !config.db.remotes[name]) { - throw new Error(`Remote "${name}" not found`); + for (const key of Object.keys(configRemotes)) { + delete configRemotes[key].default; } - // Clear default flag from all remotes - for (const key of Object.keys(config.db.remotes)) { - delete config.db.remotes[key].default; + // Ensure the entry exists in config (it may only exist in secrets for legacy remotes) + if (!configRemotes[name]) { + configRemotes[name] = {}; } + configRemotes[name].default = true; - // Set the new default - config.db.remotes[name].default = true; + db.remotes = configRemotes; + config.db = db; + await writeJsonFile(configPath, config); - // Save the updated config - await fs.writeFile(configPath, JSON.stringify(config, null, 2), "utf-8"); invalidateConfig(); - logger.success(`Remote "${name}" set as default`); } /** * Resolve the URL for a remote by name, or use the default - * @param remoteName - Optional name of the remote to use - * @returns The URL of the resolved remote - * @throws Error if no remotes are configured or named remote not found */ export function resolveRemoteUrl(remoteName?: string): string { const remotes = getRemotes(); @@ -256,7 +297,6 @@ export function resolveRemoteUrl(remoteName?: string): string { return remote.url; } - // Use default remote const defaultName = getDefaultRemote(); if (!defaultName) { throw new Error("No default remote configured."); @@ -269,9 +309,6 @@ export function resolveRemoteUrl(remoteName?: string): string { /** * Resolve the name and URL for a remote by name, or use the default - * @param remoteName - Optional name of the remote to use - * @returns Object with name and url of the resolved remote - * @throws Error if no remotes are configured or named remote not found */ export function resolveRemote(remoteName?: string): {name: string; url: string} { const remotes = getRemotes(); @@ -287,7 +324,6 @@ export function resolveRemote(remoteName?: string): {name: string; url: string} return {name: remoteName, url: remote.url}; } - // Use default remote const defaultName = getDefaultRemote(); if (!defaultName) { throw new Error("No default remote configured."); @@ -298,17 +334,12 @@ export function resolveRemote(remoteName?: string): {name: string; url: string} return {name: defaultName, url: remote.url}; } -/** - * Validate remote name — only alphanumeric, hyphens, underscores. - * Prevents shell metacharacter injection and path traversal. - */ +// ─── Utilities ─────────────────────────────────────────────────────────────── + function isValidRemoteName(name: string): boolean { return /^[a-zA-Z0-9_-]+$/.test(name); } -/** - * Validate database URL format (basic check) - */ function isValidDatabaseUrl(url: string): boolean { try { const parsed = new URL(url); @@ -321,9 +352,6 @@ function isValidDatabaseUrl(url: string): boolean { } } -/** - * Normalize URL for comparison (remove trailing slash, lowercase host) - */ export function normalizeUrl(url: string): string { try { const parsed = new URL(url); @@ -335,9 +363,6 @@ export function normalizeUrl(url: string): string { } } -/** - * Find a remote by URL (returns remote name or null) - */ function findRemoteByUrl( remotes: Record | undefined, url: string, @@ -352,9 +377,6 @@ function findRemoteByUrl( return null; } -/** - * Mask sensitive parts of a database URL for logging - */ export function maskRemoteUrl(url: string): string { try { const parsed = new URL(url); diff --git a/cli/test/common/config.test.ts b/cli/test/common/config.test.ts index 968f81c..308f968 100644 --- a/cli/test/common/config.test.ts +++ b/cli/test/common/config.test.ts @@ -16,6 +16,7 @@ import { checkInitialized, invalidateConfig, getConfigFilePath, + getSecretsFilePath, getVendorDir, } from "../../src/common/config"; @@ -30,6 +31,13 @@ const mockConfig = { }, }; +/** Mock existsSync: config exists, secrets does not (single-file / legacy mode). */ +function mockConfigOnly() { + vi.mocked(fs.existsSync) + .mockReturnValueOnce(true) // postkit.config.json + .mockReturnValueOnce(false); // postkit.secrets.json +} + describe("config", () => { beforeEach(() => { vi.clearAllMocks(); @@ -42,33 +50,38 @@ describe("config", () => { expect(() => loadPostkitConfig()).toThrow("Config file not found"); }); - it("returns parsed JSON when config exists", () => { - vi.mocked(fs.existsSync).mockReturnValue(true); + it("returns parsed JSON when config exists (no secrets file)", () => { + mockConfigOnly(); vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(mockConfig)); const config = loadPostkitConfig(); expect(config.db.localDbUrl).toBe("postgres://localhost:5432/test"); }); it("caches config (same reference on second call)", () => { - vi.mocked(fs.existsSync).mockReturnValue(true); + mockConfigOnly(); vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(mockConfig)); const first = loadPostkitConfig(); const second = loadPostkitConfig(); expect(first).toBe(second); + // readFileSync called once — only for the config file (secrets=false skips second read) expect(fs.readFileSync).toHaveBeenCalledTimes(1); }); it("invalidateConfig() clears cache", () => { - vi.mocked(fs.existsSync).mockReturnValue(true); + // Two loads, each with config=true / secrets=false + vi.mocked(fs.existsSync) + .mockReturnValueOnce(true).mockReturnValueOnce(false) + .mockReturnValueOnce(true).mockReturnValueOnce(false); vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(mockConfig)); loadPostkitConfig(); invalidateConfig(); loadPostkitConfig(); + // One read per load = 2 total expect(fs.readFileSync).toHaveBeenCalledTimes(2); }); it("auto-migrates remoteDbUrl to remotes.default", () => { - vi.mocked(fs.existsSync).mockReturnValue(true); + mockConfigOnly(); vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ db: {localDbUrl: "postgres://localhost:5432/test", remoteDbUrl: "postgres://remote:5432/test"}, })); @@ -79,7 +92,7 @@ describe("config", () => { }); it("auto-migrates environments to named remotes", () => { - vi.mocked(fs.existsSync).mockReturnValue(true); + mockConfigOnly(); vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ db: {localDbUrl: "postgres://localhost:5432/test", environments: {staging: "postgres://staging:5432/test"}}, })); @@ -89,11 +102,33 @@ describe("config", () => { }); it("does not re-migrate if remotes already exist", () => { - vi.mocked(fs.existsSync).mockReturnValue(true); + mockConfigOnly(); vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(mockConfig)); loadPostkitConfig(); expect(fs.writeFileSync).not.toHaveBeenCalled(); }); + + it("merges secrets file when both files exist", () => { + const publicConfig = { + db: {schemaPath: "schema", schema: "public", remotes: {dev: {default: true, addedAt: "2024-01-01"}}}, + auth: {configCliImage: "keycloak:latest"}, + }; + const secrets = { + db: {localDbUrl: "postgres://localhost:5432/test", remotes: {dev: {url: "postgres://dev:5432/test"}}}, + auth: {source: {url: "http://kc:8080", adminUser: "admin", adminPass: "pass", realm: "r"}}, + }; + vi.mocked(fs.existsSync).mockReturnValue(true); // both files exist + vi.mocked(fs.readFileSync) + .mockReturnValueOnce(JSON.stringify(publicConfig)) // config file + .mockReturnValueOnce(JSON.stringify(secrets)); // secrets file + const config = loadPostkitConfig(); + // Secrets values are merged in + expect(config.db.localDbUrl).toBe("postgres://localhost:5432/test"); + expect(config.db.remotes.dev.url).toBe("postgres://dev:5432/test"); + // Public config values are preserved + expect(config.db.remotes.dev.default).toBe(true); + expect((config.auth as any).configCliImage).toBe("keycloak:latest"); + }); }); describe("checkInitialized()", () => { @@ -113,6 +148,10 @@ describe("config", () => { expect(getConfigFilePath()).toMatch(/postkit\.config\.json$/); }); + it("getSecretsFilePath() ends with postkit.secrets.json", () => { + expect(getSecretsFilePath()).toMatch(/postkit\.secrets\.json$/); + }); + it("getVendorDir() ends with vendor", () => { expect(getVendorDir()).toMatch(/vendor$/); }); diff --git a/cli/test/e2e/helpers/test-project.ts b/cli/test/e2e/helpers/test-project.ts index fd806af..54755dd 100644 --- a/cli/test/e2e/helpers/test-project.ts +++ b/cli/test/e2e/helpers/test-project.ts @@ -46,22 +46,27 @@ export async function createTestProject( // Ensure schema directory exists (init doesn't create it) await fs.mkdir(schemaPath, {recursive: true}); - // Read the generated config and merge in test-specific values - const existingConfig = JSON.parse(await fs.readFile(configPath, "utf-8")); + // Patch the public config: add remote name/metadata (no URLs) + const secretsPath = path.join(rootDir, "postkit.secrets.json"); const remoteName = config.remoteName ?? "test-remote"; - existingConfig.db.localDbUrl = config.localDbUrl; if (config.remoteDbUrl) { + const existingConfig = JSON.parse(await fs.readFile(configPath, "utf-8")); existingConfig.db.remotes = { - [remoteName]: { - url: config.remoteDbUrl, - default: true, - addedAt: new Date().toISOString(), - }, + [remoteName]: {default: true, addedAt: new Date().toISOString()}, }; + await fs.writeFile(configPath, JSON.stringify(existingConfig, null, 2)); } - await fs.writeFile(configPath, JSON.stringify(existingConfig, null, 2)); + // Patch secrets: write localDbUrl and remote URLs + const existingSecrets = JSON.parse(await fs.readFile(secretsPath, "utf-8")); + existingSecrets.db.localDbUrl = config.localDbUrl; + if (config.remoteDbUrl) { + existingSecrets.db.remotes = { + [remoteName]: {url: config.remoteDbUrl}, + }; + } + await fs.writeFile(secretsPath, JSON.stringify(existingSecrets, null, 2)); return {rootDir, configPath, postkitDir, dbDir, schemaPath}; } diff --git a/cli/test/e2e/workflows/remote-management.test.ts b/cli/test/e2e/workflows/remote-management.test.ts index 98fd635..3552eb8 100644 --- a/cli/test/e2e/workflows/remote-management.test.ts +++ b/cli/test/e2e/workflows/remote-management.test.ts @@ -38,12 +38,12 @@ describe("Remote management", () => { ); expect(result.exitCode).toBe(0); - // Verify in config file - const config = await readJson<{ - db: {remotes: Record}; - }>(project, "postkit.config.json"); - expect(config.db.remotes.staging).toBeDefined(); - expect(config.db.remotes.staging.url).toBe("postgres://localhost:5432/staging"); + // URL is in secrets file; public config only has metadata + const secrets = await readJson<{ + db: {remotes: Record}; + }>(project, "postkit.secrets.json"); + expect(secrets.db.remotes.staging).toBeDefined(); + expect(secrets.db.remotes.staging?.url).toBe("postgres://localhost:5432/staging"); }); it("adds a remote with --default flag", async () => { @@ -64,7 +64,7 @@ describe("Remote management", () => { db: {remotes: Record}; }>(project, "postkit.config.json"); expect(config.db.remotes.prod).toBeDefined(); - expect(config.db.remotes.prod.default).toBe(true); + expect(config.db.remotes.prod?.default).toBe(true); }); it("sets default remote with 'use'", async () => { @@ -90,7 +90,8 @@ describe("Remote management", () => { const config = await readJson<{ db: {remotes: Record}; }>(project, "postkit.config.json"); - expect(config.db.remotes.staging.default).toBe(true); + expect(config.db.remotes.staging).toBeDefined(); + expect(config.db.remotes.staging?.default).toBe(true); }); it("removes a remote with --force", async () => { diff --git a/cli/test/modules/db/utils/remotes.test.ts b/cli/test/modules/db/utils/remotes.test.ts index ca5ab4a..bad6861 100644 --- a/cli/test/modules/db/utils/remotes.test.ts +++ b/cli/test/modules/db/utils/remotes.test.ts @@ -7,6 +7,7 @@ vi.mock("../../../../src/common/config", async () => { ...actual, loadPostkitConfig: vi.fn(), getConfigFilePath: vi.fn(() => "/project/postkit.config.json"), + getSecretsFilePath: vi.fn(() => "/project/postkit.secrets.json"), invalidateConfig: vi.fn(), }; }); @@ -168,9 +169,10 @@ describe("remotes", () => { }); it("throws when removing the only remote", async () => { - vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify({ + // Validation now uses the merged config, so mock loadPostkitConfig with a single remote + vi.mocked(loadPostkitConfig).mockReturnValueOnce({ db: {localDbUrl: "postgres://localhost/db", remotes: {dev: {url: "postgres://dev/db", default: true}}}, - })); + } as any); await expect(removeRemote("dev", true)).rejects.toThrow("only remaining remote"); }); }); From 1728f832ad35c214a94ab202d02da4873e0a651e Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Thu, 7 May 2026 18:38:00 +0530 Subject: [PATCH 02/18] feat: support ephemeral Postgres containers in db commands for local development sessions --- cli/src/modules/db/commands/abort.ts | 16 +++++ cli/src/modules/db/commands/deploy.ts | 87 +++++++++++++++--------- cli/src/modules/db/commands/start.ts | 52 ++++++++++---- cli/src/modules/db/services/container.ts | 87 ++++++++++++++++++++++++ cli/src/modules/db/types/session.ts | 1 + cli/src/modules/db/utils/db-config.ts | 2 +- cli/src/modules/db/utils/session.ts | 2 + 7 files changed, 199 insertions(+), 48 deletions(-) create mode 100644 cli/src/modules/db/services/container.ts diff --git a/cli/src/modules/db/commands/abort.ts b/cli/src/modules/db/commands/abort.ts index d1127ef..a47ef94 100644 --- a/cli/src/modules/db/commands/abort.ts +++ b/cli/src/modules/db/commands/abort.ts @@ -6,6 +6,7 @@ import {getSessionMigrationsPath} from "../utils/db-config"; import {deletePlanFile} from "../services/pgschema"; import {deleteGeneratedSchema} from "../services/schema-generator"; import {dropDatabase, parseConnectionUrl} from "../services/database"; +import {stopSessionContainer} from "../services/container"; import type {CommandOptions} from "../../../common/types"; export async function abortCommand(options: CommandOptions): Promise { @@ -87,6 +88,21 @@ export async function abortCommand(options: CommandOptions): Promise { options.verbose, ); } + + // Stop Docker container if the session used one + if (session.containerID) { + spinner.start("Stopping session container..."); + try { + await stopSessionContainer(session.containerID); + spinner.succeed("Session container stopped and removed"); + } catch (error) { + spinner.warn("Could not stop container (may already be removed)"); + logger.debug( + error instanceof Error ? error.message : String(error), + options.verbose, + ); + } + } } // Step 4: Delete session migrations folder diff --git a/cli/src/modules/db/commands/deploy.ts b/cli/src/modules/db/commands/deploy.ts index beacad9..f95d08c 100644 --- a/cli/src/modules/db/commands/deploy.ts +++ b/cli/src/modules/db/commands/deploy.ts @@ -2,6 +2,7 @@ import ora from "ora"; import {logger} from "../../../common/logger"; import {promptConfirm} from "../../../common/prompt"; import {getDbConfig} from "../utils/db-config"; +import type {DbConfig} from "../utils/db-config"; import {hasActiveSession, deleteSession} from "../utils/session"; import {deletePlanFile} from "../services/pgschema"; import {deleteGeneratedSchema} from "../services/schema-generator"; @@ -14,6 +15,7 @@ import { import {runCommittedMigrate, runDbmateStatus} from "../services/dbmate"; import {loadInfra, applyInfra} from "../services/infra-generator"; import {loadSeeds, applySeeds} from "../services/seed-generator"; +import {checkDockerAvailable, startSessionContainer, stopSessionContainer} from "../services/container"; import {getPendingCommittedMigrations} from "../utils/committed"; import {resolveRemote, maskRemoteUrl, normalizeUrl} from "../utils/remotes"; import type {CommandOptions} from "../../../common/types"; @@ -39,6 +41,21 @@ function resolveTargetUrl(options: DeployOptions): {url: string; label: string} return {url: resolved.url, label: `${resolved.name} (default)`}; } +async function resolveLocalDbUrl( + config: DbConfig, + spinner: ReturnType, +): Promise<{url: string; tempContainerID?: string}> { + if (config.localDbUrl) { + return {url: config.localDbUrl}; + } + spinner.start("Checking Docker availability..."); + await checkDockerAvailable(); + spinner.text = "Starting temporary Postgres container for dry-run..."; + const container = await startSessionContainer(); + spinner.succeed(`Temporary container started on port ${container.port}`); + return {url: container.localDbUrl, tempContainerID: container.containerID}; +} + async function confirmAndRemoveSession( spinner: ReturnType, options: DeployOptions, @@ -118,16 +135,18 @@ export async function deployCommand(options: DeployOptions): Promise { // Step 1: Resolve target URL const {url: targetUrl, label: targetLabel} = resolveTargetUrl(options); - // Validate: localDbUrl cannot equal target URL - const normalizedLocalUrl = normalizeUrl(config.localDbUrl); - const normalizedTargetUrl = normalizeUrl(targetUrl); - - if (normalizedLocalUrl === normalizedTargetUrl) { - throw new PostkitError( - `Cannot deploy: localDbUrl equals target URL (${targetLabel}).`, - "Your local database URL must be different from the target remote. " + - "Update your postkit.config.json or use a different remote.", - ); + // Validate: localDbUrl cannot equal target URL (skip check if localDbUrl is empty — container will be used) + if (config.localDbUrl) { + const normalizedLocalUrl = normalizeUrl(config.localDbUrl); + const normalizedTargetUrl = normalizeUrl(targetUrl); + + if (normalizedLocalUrl === normalizedTargetUrl) { + throw new PostkitError( + `Cannot deploy: localDbUrl equals target URL (${targetLabel}).`, + "Your local database URL must be different from the target remote. " + + "Update your postkit.config.json or use a different remote.", + ); + } } logger.heading("Deploy Migrations"); @@ -205,8 +224,15 @@ export async function deployCommand(options: DeployOptions): Promise { spinner.succeed("Target database up to date"); } - // Step 3: Clone target DB to local - const localDbUrl = config.localDbUrl; + // Step 3: Resolve local DB URL (spin up a container if localDbUrl is not configured) + const {url: localDbUrl, tempContainerID} = await resolveLocalDbUrl(config, spinner); + + const cleanupLocal = async () => { + try { await dropDatabase(localDbUrl); } catch { /* best effort */ } + if (tempContainerID) { + try { await stopSessionContainer(tempContainerID); } catch { /* best effort */ } + } + }; logger.step(3, totalSteps, "Cloning target database to local..."); spinner.start("Cloning target database to local for dry-run verification..."); @@ -227,11 +253,7 @@ export async function deployCommand(options: DeployOptions): Promise { // Clean up local clone logger.info("Cleaning up local clone..."); - try { - await dropDatabase(localDbUrl); - } catch { - // Best effort cleanup - } + await cleanupLocal(); throw new PostkitError( "Deployment aborted — dry run failed. No changes were made to the target database.", @@ -245,11 +267,7 @@ export async function deployCommand(options: DeployOptions): Promise { // If --dry-run, stop here — don't touch the target database if (options.dryRun) { logger.info("Dry run complete. Target database was not modified."); - try { - await dropDatabase(localDbUrl); - } catch { - // Best effort cleanup - } + await cleanupLocal(); return; } @@ -261,12 +279,7 @@ export async function deployCommand(options: DeployOptions): Promise { if (!confirmed) { logger.info("Deploy cancelled."); - // Clean up local clone - try { - await dropDatabase(localDbUrl); - } catch { - // Best effort cleanup - } + await cleanupLocal(); return; } @@ -279,11 +292,7 @@ export async function deployCommand(options: DeployOptions): Promise { } catch (error) { logger.error(error instanceof Error ? error.message : String(error)); logger.blank(); - try { - await dropDatabase(localDbUrl); - } catch { - // Best effort cleanup - } + await cleanupLocal(); throw new PostkitError( "Target deployment failed. The target database may be in a partial state.", @@ -291,7 +300,7 @@ export async function deployCommand(options: DeployOptions): Promise { ); } - // Step 10: Drop local clone + // Step 10: Drop local clone and stop temp container logger.blank(); logger.step(10, totalSteps, "Cleaning up local clone..."); spinner.start("Dropping local clone database..."); @@ -303,6 +312,16 @@ export async function deployCommand(options: DeployOptions): Promise { spinner.warn("Failed to drop local clone (non-fatal): " + (error instanceof Error ? error.message : String(error))); } + if (tempContainerID) { + spinner.start("Stopping temporary container..."); + try { + await stopSessionContainer(tempContainerID); + spinner.succeed("Temporary container stopped and removed"); + } catch { + spinner.warn("Could not stop temporary container (non-fatal)"); + } + } + // Report success logger.blank(); logger.success(`Deployment to ${targetLabel} completed successfully!`); diff --git a/cli/src/modules/db/commands/start.ts b/cli/src/modules/db/commands/start.ts index e65e871..d7b844e 100644 --- a/cli/src/modules/db/commands/start.ts +++ b/cli/src/modules/db/commands/start.ts @@ -14,6 +14,7 @@ import { } from "../services/database"; import {checkPgschemaInstalled} from "../services/pgschema"; import {checkDbmateInstalled, runDbmateStatus} from "../services/dbmate"; +import {checkDockerAvailable, startSessionContainer} from "../services/container"; import {getPendingCommittedMigrations} from "../utils/committed"; import type {CommandOptions} from "../../../common/types"; import {PostkitError} from "../../../common/errors"; @@ -63,6 +64,14 @@ export async function startCommand(options: StartOptions): Promise { const config = getDbConfig(); + // Determine whether we need an auto-container (localDbUrl is empty) + let localDbUrl = config.localDbUrl; + let containerID: string | undefined; + const needsContainer = !localDbUrl; + + // Total steps: 5 normally, 6 when auto-container is needed + const totalSteps = needsContainer ? 6 : 5; + // Resolve remote let targetRemoteName: string; let targetRemoteUrl: string; @@ -91,16 +100,18 @@ export async function startCommand(options: StartOptions): Promise { `Remote DB (${targetRemoteName}): ${maskRemoteUrl(targetRemoteUrl)}`, options.verbose, ); - logger.debug( - `Local DB: ${maskConnectionUrl(config.localDbUrl)}`, - options.verbose, - ); + if (localDbUrl) { + logger.debug( + `Local DB: ${maskConnectionUrl(localDbUrl)}`, + options.verbose, + ); + } // Ensure .pgschemaignore exists in schema directory await ensurePgschemaIgnore(config.schemaPath); // Step 3: Test remote connection - logger.step(3, 5, "Testing remote database connection..."); + logger.step(3, totalSteps, "Testing remote database connection..."); spinner.start("Connecting to remote database..."); const remoteConnected = await testConnection(targetRemoteUrl); @@ -119,7 +130,7 @@ export async function startCommand(options: StartOptions): Promise { logger.info(`Remote database has ${remoteTableCount} tables`); // Step 4: Verify database state - logger.step(4, 6, "Verifying database state..."); + logger.step(4, totalSteps, "Verifying database state..."); // Check 1: Pending committed migrations (check remote's schema_migrations table) const pendingCommitted = await getPendingCommittedMigrations(targetRemoteUrl); @@ -186,28 +197,43 @@ export async function startCommand(options: StartOptions): Promise { spinner.succeed("All migrations applied - database is in sync"); } - // Step 5: Clone database - logger.step(4, 5, "Cloning remote database to local..."); + // Step 5 (only when no localDbUrl): Start local Postgres container + if (needsContainer) { + logger.step(5, totalSteps, "Starting local Postgres container..."); + spinner.start("Checking Docker availability..."); + await checkDockerAvailable(); + spinner.text = "Starting postgres:16-alpine container..."; + const container = await startSessionContainer(); + containerID = container.containerID; + localDbUrl = container.localDbUrl; + spinner.succeed(`Postgres container started on port ${container.port}`); + logger.debug(`Local DB (container): ${maskConnectionUrl(localDbUrl)}`, options.verbose); + } + + // Step 5/6: Clone database + const cloneStep = needsContainer ? 6 : 5; + logger.step(cloneStep, totalSteps, "Cloning remote database to local..."); spinner.start("Cloning database (this may take a moment)..."); if (options.dryRun) { spinner.info("Dry run - skipping database clone"); } else { - await cloneDatabase(targetRemoteUrl, config.localDbUrl); + await cloneDatabase(targetRemoteUrl, localDbUrl); spinner.succeed("Database cloned successfully"); - const localTableCount = await getTableCount(config.localDbUrl); + const localTableCount = await getTableCount(localDbUrl); logger.info(`Local clone has ${localTableCount} tables`); } - // Step 6: Create session - logger.step(6, 6, "Creating session..."); + // Final step: Create session + logger.step(totalSteps, totalSteps, "Creating session..."); if (!options.dryRun) { const session = await createSession( targetRemoteUrl, - config.localDbUrl, + localDbUrl, targetRemoteName, + containerID, ); logger.success(`Session created (cloned at: ${session.clonedAt})`); } else { diff --git a/cli/src/modules/db/services/container.ts b/cli/src/modules/db/services/container.ts new file mode 100644 index 0000000..a55a61a --- /dev/null +++ b/cli/src/modules/db/services/container.ts @@ -0,0 +1,87 @@ +import net from "net"; +import {runCommand, runSpawnCommand, commandExists} from "../../../common/shell"; +import {testConnection} from "./database"; +import {PostkitError} from "../../../common/errors"; + +const POSTGRES_IMAGE = "postgres:16-alpine"; +const CONTAINER_PREFIX = "postkit-session"; +const DB_NAME = "postkit_local"; +const DB_USER = "postgres"; +const DB_PASSWORD = "postkit_local"; + +export interface ContainerInfo { + containerID: string; + localDbUrl: string; + port: number; +} + +export async function checkDockerAvailable(): Promise { + const installed = await commandExists("docker"); + if (!installed) { + throw new PostkitError( + "Docker not found.", + "Install Docker Desktop from https://docker.com or set localDbUrl in postkit.secrets.json to use an existing database.", + ); + } + const result = await runCommand("docker info"); + if (result.exitCode !== 0) { + throw new PostkitError( + "Docker is not running.", + "Start Docker Desktop and retry. Or set localDbUrl in postkit.secrets.json to use an existing database.", + ); + } +} + +export async function startSessionContainer(): Promise { + const port = await findFreePort(15432, 15532); + const containerName = `${CONTAINER_PREFIX}-${Date.now()}`; + + const result = await runSpawnCommand([ + "docker", "run", "-d", + "--name", containerName, + "-p", `${port}:5432`, + "-e", `POSTGRES_PASSWORD=${DB_PASSWORD}`, + "-e", `POSTGRES_DB=${DB_NAME}`, + "-e", `POSTGRES_USER=${DB_USER}`, + POSTGRES_IMAGE, + ]); + + if (result.exitCode !== 0) { + throw new Error(`Failed to start Postgres container: ${result.stderr}`); + } + + const containerID = result.stdout.trim(); + const localDbUrl = `postgres://${DB_USER}:${DB_PASSWORD}@localhost:${port}/${DB_NAME}`; + + await waitForPostgres(localDbUrl); + return {containerID, localDbUrl, port}; +} + +export async function stopSessionContainer(containerID: string): Promise { + await runCommand(`docker stop ${containerID}`); + await runCommand(`docker rm ${containerID}`); +} + +async function waitForPostgres(url: string, maxAttempts = 30): Promise { + for (let i = 0; i < maxAttempts; i++) { + if (await testConnection(url)) return; + await new Promise((r) => setTimeout(r, 1000)); + } + throw new Error("Postgres container did not become ready within 30 seconds."); +} + +async function findFreePort(start: number, end: number): Promise { + for (let port = start; port <= end; port++) { + if (await isPortFree(port)) return port; + } + throw new Error(`No free port found between ${start} and ${end}.`); +} + +function isPortFree(port: number): Promise { + return new Promise((resolve) => { + const server = net.createServer(); + server.once("error", () => resolve(false)); + server.once("listening", () => server.close(() => resolve(true))); + server.listen(port); + }); +} diff --git a/cli/src/modules/db/types/session.ts b/cli/src/modules/db/types/session.ts index 95fe098..a8e4bf1 100644 --- a/cli/src/modules/db/types/session.ts +++ b/cli/src/modules/db/types/session.ts @@ -9,6 +9,7 @@ export interface SessionState { remoteName?: string; localDbUrl: string; remoteDbUrl: string; + containerID?: string; pendingChanges: { planned: boolean; applied: boolean; diff --git a/cli/src/modules/db/utils/db-config.ts b/cli/src/modules/db/utils/db-config.ts index b73c9ee..79ea02c 100644 --- a/cli/src/modules/db/utils/db-config.ts +++ b/cli/src/modules/db/utils/db-config.ts @@ -35,7 +35,7 @@ const RemoteConfigInputSchema = z.object({ }); const DbConfigInputSchema = z.object({ - localDbUrl: z.string().min(1, "Local database URL is required"), + localDbUrl: z.string().default(""), schemaPath: z.string().optional(), schema: z.string().optional(), remotes: z.record(z.string(), RemoteConfigInputSchema).optional(), diff --git a/cli/src/modules/db/utils/session.ts b/cli/src/modules/db/utils/session.ts index 244b5b3..d977250 100644 --- a/cli/src/modules/db/utils/session.ts +++ b/cli/src/modules/db/utils/session.ts @@ -26,6 +26,7 @@ export async function createSession( remoteDbUrl: string, localDbUrl: string, remoteName?: string, + containerID?: string, ): Promise { const now = new Date(); const session: SessionState = { @@ -35,6 +36,7 @@ export async function createSession( remoteName, localDbUrl, remoteDbUrl, + containerID, pendingChanges: { planned: false, applied: false, From a6ac308652d14a2b9c701beb9b82d84da10aa7b4 Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Thu, 7 May 2026 19:20:11 +0530 Subject: [PATCH 03/18] feat: align local container Postgres version with remote and perform database cloning via container execution --- cli/src/modules/db/commands/deploy.ts | 21 +++++--- cli/src/modules/db/commands/start.ts | 19 +++++-- cli/src/modules/db/services/container.ts | 68 +++++++++++++++++++++--- cli/src/modules/db/services/database.ts | 16 ++++++ 4 files changed, 106 insertions(+), 18 deletions(-) diff --git a/cli/src/modules/db/commands/deploy.ts b/cli/src/modules/db/commands/deploy.ts index f95d08c..5c042aa 100644 --- a/cli/src/modules/db/commands/deploy.ts +++ b/cli/src/modules/db/commands/deploy.ts @@ -11,11 +11,12 @@ import { cloneDatabase, dropDatabase, getTableCount, + getRemotePgMajorVersion, } from "../services/database"; import {runCommittedMigrate, runDbmateStatus} from "../services/dbmate"; import {loadInfra, applyInfra} from "../services/infra-generator"; import {loadSeeds, applySeeds} from "../services/seed-generator"; -import {checkDockerAvailable, startSessionContainer, stopSessionContainer} from "../services/container"; +import {checkDockerAvailable, startSessionContainer, stopSessionContainer, cloneDatabaseViaContainer} from "../services/container"; import {getPendingCommittedMigrations} from "../utils/committed"; import {resolveRemote, maskRemoteUrl, normalizeUrl} from "../utils/remotes"; import type {CommandOptions} from "../../../common/types"; @@ -44,15 +45,16 @@ function resolveTargetUrl(options: DeployOptions): {url: string; label: string} async function resolveLocalDbUrl( config: DbConfig, spinner: ReturnType, + remotePgVersion: number, ): Promise<{url: string; tempContainerID?: string}> { if (config.localDbUrl) { return {url: config.localDbUrl}; } spinner.start("Checking Docker availability..."); await checkDockerAvailable(); - spinner.text = "Starting temporary Postgres container for dry-run..."; - const container = await startSessionContainer(); - spinner.succeed(`Temporary container started on port ${container.port}`); + spinner.text = `Starting temporary postgres:${remotePgVersion}-alpine container for dry-run...`; + const container = await startSessionContainer(remotePgVersion); + spinner.succeed(`Temporary Postgres ${remotePgVersion} container started on port ${container.port}`); return {url: container.localDbUrl, tempContainerID: container.containerID}; } @@ -200,7 +202,8 @@ export async function deployCommand(options: DeployOptions): Promise { } const targetTableCount = await getTableCount(targetUrl); - spinner.succeed(`Connected to target database (${targetTableCount} tables)`); + const remotePgVersion = await getRemotePgMajorVersion(targetUrl); + spinner.succeed(`Connected to target database (${targetTableCount} tables, PostgreSQL ${remotePgVersion})`); // Step 2: Check target migration status logger.step(2, totalSteps, "Checking target migration status..."); @@ -225,7 +228,7 @@ export async function deployCommand(options: DeployOptions): Promise { } // Step 3: Resolve local DB URL (spin up a container if localDbUrl is not configured) - const {url: localDbUrl, tempContainerID} = await resolveLocalDbUrl(config, spinner); + const {url: localDbUrl, tempContainerID} = await resolveLocalDbUrl(config, spinner, remotePgVersion); const cleanupLocal = async () => { try { await dropDatabase(localDbUrl); } catch { /* best effort */ } @@ -236,7 +239,11 @@ export async function deployCommand(options: DeployOptions): Promise { logger.step(3, totalSteps, "Cloning target database to local..."); spinner.start("Cloning target database to local for dry-run verification..."); - await cloneDatabase(targetUrl, localDbUrl); + if (tempContainerID) { + await cloneDatabaseViaContainer(tempContainerID, targetUrl, localDbUrl); + } else { + await cloneDatabase(targetUrl, localDbUrl); + } const localTableCount = await getTableCount(localDbUrl); spinner.succeed(`Target cloned to local (${localTableCount} tables)`); diff --git a/cli/src/modules/db/commands/start.ts b/cli/src/modules/db/commands/start.ts index d7b844e..bb44794 100644 --- a/cli/src/modules/db/commands/start.ts +++ b/cli/src/modules/db/commands/start.ts @@ -11,10 +11,11 @@ import { testConnection, cloneDatabase, getTableCount, + getRemotePgMajorVersion, } from "../services/database"; import {checkPgschemaInstalled} from "../services/pgschema"; import {checkDbmateInstalled, runDbmateStatus} from "../services/dbmate"; -import {checkDockerAvailable, startSessionContainer} from "../services/container"; +import {checkDockerAvailable, startSessionContainer, cloneDatabaseViaContainer} from "../services/container"; import {getPendingCommittedMigrations} from "../utils/committed"; import type {CommandOptions} from "../../../common/types"; import {PostkitError} from "../../../common/errors"; @@ -129,6 +130,9 @@ export async function startCommand(options: StartOptions): Promise { const remoteTableCount = await getTableCount(targetRemoteUrl); logger.info(`Remote database has ${remoteTableCount} tables`); + const remotePgVersion = await getRemotePgMajorVersion(targetRemoteUrl); + logger.debug(`Remote PostgreSQL version: ${remotePgVersion}`, options.verbose); + // Step 4: Verify database state logger.step(4, totalSteps, "Verifying database state..."); @@ -202,11 +206,11 @@ export async function startCommand(options: StartOptions): Promise { logger.step(5, totalSteps, "Starting local Postgres container..."); spinner.start("Checking Docker availability..."); await checkDockerAvailable(); - spinner.text = "Starting postgres:16-alpine container..."; - const container = await startSessionContainer(); + spinner.text = `Starting postgres:${remotePgVersion}-alpine container...`; + const container = await startSessionContainer(remotePgVersion); containerID = container.containerID; localDbUrl = container.localDbUrl; - spinner.succeed(`Postgres container started on port ${container.port}`); + spinner.succeed(`Postgres ${remotePgVersion} container started on port ${container.port}`); logger.debug(`Local DB (container): ${maskConnectionUrl(localDbUrl)}`, options.verbose); } @@ -218,7 +222,12 @@ export async function startCommand(options: StartOptions): Promise { if (options.dryRun) { spinner.info("Dry run - skipping database clone"); } else { - await cloneDatabase(targetRemoteUrl, localDbUrl); + if (containerID) { + // Run pg_dump/psql inside the container — version-matched with remote + await cloneDatabaseViaContainer(containerID, targetRemoteUrl, localDbUrl); + } else { + await cloneDatabase(targetRemoteUrl, localDbUrl); + } spinner.succeed("Database cloned successfully"); const localTableCount = await getTableCount(localDbUrl); diff --git a/cli/src/modules/db/services/container.ts b/cli/src/modules/db/services/container.ts index a55a61a..a888a3b 100644 --- a/cli/src/modules/db/services/container.ts +++ b/cli/src/modules/db/services/container.ts @@ -1,9 +1,9 @@ import net from "net"; import {runCommand, runSpawnCommand, commandExists} from "../../../common/shell"; -import {testConnection} from "./database"; +import {testConnection, parseConnectionUrl} from "./database"; +import {runPipedCommands} from "../../../common/shell"; import {PostkitError} from "../../../common/errors"; -const POSTGRES_IMAGE = "postgres:16-alpine"; const CONTAINER_PREFIX = "postkit-session"; const DB_NAME = "postkit_local"; const DB_USER = "postgres"; @@ -13,6 +13,7 @@ export interface ContainerInfo { containerID: string; localDbUrl: string; port: number; + pgVersion: number; } export async function checkDockerAvailable(): Promise { @@ -32,9 +33,14 @@ export async function checkDockerAvailable(): Promise { } } -export async function startSessionContainer(): Promise { +/** + * Start a Postgres container whose version matches the remote database. + * This ensures pg_dump (run inside the container) is always version-compatible. + */ +export async function startSessionContainer(pgVersion: number): Promise { const port = await findFreePort(15432, 15532); const containerName = `${CONTAINER_PREFIX}-${Date.now()}`; + const image = `postgres:${pgVersion}-alpine`; const result = await runSpawnCommand([ "docker", "run", "-d", @@ -43,18 +49,18 @@ export async function startSessionContainer(): Promise { "-e", `POSTGRES_PASSWORD=${DB_PASSWORD}`, "-e", `POSTGRES_DB=${DB_NAME}`, "-e", `POSTGRES_USER=${DB_USER}`, - POSTGRES_IMAGE, + image, ]); if (result.exitCode !== 0) { - throw new Error(`Failed to start Postgres container: ${result.stderr}`); + throw new Error(`Failed to start postgres:${pgVersion}-alpine container: ${result.stderr}`); } const containerID = result.stdout.trim(); const localDbUrl = `postgres://${DB_USER}:${DB_PASSWORD}@localhost:${port}/${DB_NAME}`; await waitForPostgres(localDbUrl); - return {containerID, localDbUrl, port}; + return {containerID, localDbUrl, port, pgVersion}; } export async function stopSessionContainer(containerID: string): Promise { @@ -62,6 +68,56 @@ export async function stopSessionContainer(containerID: string): Promise { await runCommand(`docker rm ${containerID}`); } +/** + * Clone sourceUrl into the container's local database by running pg_dump and psql + * *inside* the container. This guarantees the dump tools always match the remote's + * PostgreSQL version — no host binary version mismatch possible. + * + * pg_dump → runs inside container, connects to remote externally (version = remote) + * psql → runs inside container, connects to localhost:5432 (version = remote) + */ +export async function cloneDatabaseViaContainer( + containerID: string, + sourceUrl: string, + targetUrl: string, +): Promise { + const src = parseConnectionUrl(sourceUrl); + const dst = parseConnectionUrl(targetUrl); + + const result = await runPipedCommands( + { + args: [ + "docker", "exec", + "-e", `PGPASSWORD=${src.password}`, + "-i", containerID, + "pg_dump", + "-h", src.host, + "-p", String(src.port), + "-U", src.user, + "-d", src.database, + "--no-owner", + "--no-acl", + ], + }, + { + args: [ + "docker", "exec", + "-e", `PGPASSWORD=${dst.password}`, + "-i", containerID, + "psql", + "-h", "localhost", + "-p", "5432", // internal port — always 5432 inside the container + "-U", dst.user, + "-d", dst.database, + ], + }, + ); + + if (result.exitCode !== 0) { + throw new Error(`Failed to clone database via container: ${result.stderr}`); + } +} + async function waitForPostgres(url: string, maxAttempts = 30): Promise { for (let i = 0; i < maxAttempts; i++) { if (await testConnection(url)) return; diff --git a/cli/src/modules/db/services/database.ts b/cli/src/modules/db/services/database.ts index e951f12..a66982e 100644 --- a/cli/src/modules/db/services/database.ts +++ b/cli/src/modules/db/services/database.ts @@ -156,3 +156,19 @@ export async function getTableCount(url: string): Promise { await client.end(); } } + +/** + * Returns the major PostgreSQL version of a server (e.g. 14, 15, 16). + * Uses SHOW server_version_num which returns a zero-padded integer like "160003". + */ +export async function getRemotePgMajorVersion(url: string): Promise { + const client = new Client({connectionString: url}); + try { + await client.connect(); + const result = await client.query("SHOW server_version_num"); + const num = parseInt(result.rows[0].server_version_num as string, 10); + return Math.floor(num / 10000); + } finally { + await client.end(); + } +} From 3bda5c8451a5b9b96081fa507f9a63f4d1f5cadd Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Thu, 7 May 2026 19:31:10 +0530 Subject: [PATCH 04/18] feat: add container management for database sessions and implement remote PostgreSQL version detection --- .../modules/db/services/container.test.ts | 242 ++++++++++++++++++ cli/test/modules/db/services/database.test.ts | 34 ++- cli/test/modules/db/utils/session.test.ts | 19 ++ 3 files changed, 293 insertions(+), 2 deletions(-) create mode 100644 cli/test/modules/db/services/container.test.ts diff --git a/cli/test/modules/db/services/container.test.ts b/cli/test/modules/db/services/container.test.ts new file mode 100644 index 0000000..f21b62f --- /dev/null +++ b/cli/test/modules/db/services/container.test.ts @@ -0,0 +1,242 @@ +import {describe, it, expect, vi, beforeEach} from "vitest"; + +vi.mock("../../../../src/common/shell", () => ({ + runCommand: vi.fn(), + runSpawnCommand: vi.fn(), + commandExists: vi.fn(), + runPipedCommands: vi.fn(), +})); + +vi.mock("../../../../src/modules/db/services/database", () => ({ + testConnection: vi.fn(), + parseConnectionUrl: vi.fn((url: string) => { + const parsed = new URL(url); + return { + host: parsed.hostname, + port: parseInt(parsed.port || "5432", 10), + database: parsed.pathname.slice(1), + user: parsed.username, + password: decodeURIComponent(parsed.password), + }; + }), +})); + +vi.mock("../../../../src/common/errors", () => ({ + PostkitError: class PostkitError extends Error { + hint?: string; + constructor(message: string, hint?: string) { + super(message); + this.hint = hint; + } + }, +})); + +// Mock net to control port-free checks +vi.mock("net", () => { + const listeners: Record void> = {}; + const mockServer = { + once: vi.fn((event: string, cb: (...args: any[]) => void) => { + listeners[event] = cb; + return mockServer; + }), + listen: vi.fn(() => { + // Default: port is free — trigger "listening" event + listeners["listening"]?.(); + return mockServer; + }), + close: vi.fn((cb?: () => void) => { cb?.(); }), + }; + return { + default: {createServer: vi.fn(() => mockServer)}, + createServer: vi.fn(() => mockServer), + }; +}); + +import {runCommand, runSpawnCommand, commandExists, runPipedCommands} from "../../../../src/common/shell"; +import {testConnection} from "../../../../src/modules/db/services/database"; +import { + checkDockerAvailable, + startSessionContainer, + stopSessionContainer, + cloneDatabaseViaContainer, +} from "../../../../src/modules/db/services/container"; + +describe("container", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + // ─── checkDockerAvailable ────────────────────────────────────────────────── + + describe("checkDockerAvailable()", () => { + it("passes when docker is installed and running", async () => { + vi.mocked(commandExists).mockResolvedValue(true); + vi.mocked(runCommand).mockResolvedValue({stdout: "", stderr: "", exitCode: 0}); + await expect(checkDockerAvailable()).resolves.toBeUndefined(); + }); + + it("throws PostkitError when docker binary is not found", async () => { + vi.mocked(commandExists).mockResolvedValue(false); + await expect(checkDockerAvailable()).rejects.toThrow("Docker not found"); + }); + + it("throws PostkitError when docker daemon is not running", async () => { + vi.mocked(commandExists).mockResolvedValue(true); + vi.mocked(runCommand).mockResolvedValue({stdout: "", stderr: "Cannot connect", exitCode: 1}); + await expect(checkDockerAvailable()).rejects.toThrow("Docker is not running"); + }); + }); + + // ─── startSessionContainer ──────────────────────────────────────────────── + + describe("startSessionContainer()", () => { + it("starts a container with the correct versioned image", async () => { + vi.mocked(runSpawnCommand).mockResolvedValue({ + stdout: "abc123containerid\n", + stderr: "", + exitCode: 0, + }); + vi.mocked(testConnection).mockResolvedValue(true); + + const info = await startSessionContainer(16); + + const spawnArgs = vi.mocked(runSpawnCommand).mock.calls[0]![0]; + expect(spawnArgs).toContain("postgres:16-alpine"); + expect(spawnArgs).toContain("docker"); + expect(spawnArgs).toContain("run"); + }); + + it("uses the provided pg version in the image tag", async () => { + vi.mocked(runSpawnCommand).mockResolvedValue({stdout: "cid\n", stderr: "", exitCode: 0}); + vi.mocked(testConnection).mockResolvedValue(true); + + await startSessionContainer(14); + const spawnArgs = vi.mocked(runSpawnCommand).mock.calls[0]![0]; + expect(spawnArgs).toContain("postgres:14-alpine"); + }); + + it("returns containerID, localDbUrl, port and pgVersion", async () => { + vi.mocked(runSpawnCommand).mockResolvedValue({ + stdout: "mycontainerid\n", + stderr: "", + exitCode: 0, + }); + vi.mocked(testConnection).mockResolvedValue(true); + + const info = await startSessionContainer(15); + + expect(info.containerID).toBe("mycontainerid"); + expect(info.localDbUrl).toMatch(/^postgres:\/\//); + expect(info.localDbUrl).toContain("localhost"); + expect(info.port).toBeGreaterThanOrEqual(15432); + expect(info.port).toBeLessThanOrEqual(15532); + expect(info.pgVersion).toBe(15); + }); + + it("throws when docker run fails", async () => { + vi.mocked(runSpawnCommand).mockResolvedValue({ + stdout: "", + stderr: "image not found", + exitCode: 1, + }); + await expect(startSessionContainer(16)).rejects.toThrow("Failed to start"); + }); + + it("waits for postgres to become ready", async () => { + vi.mocked(runSpawnCommand).mockResolvedValue({stdout: "cid\n", stderr: "", exitCode: 0}); + // Fail twice then succeed + vi.mocked(testConnection) + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(true); + + const info = await startSessionContainer(16); + expect(testConnection).toHaveBeenCalledTimes(3); + expect(info.containerID).toBe("cid"); + }); + }); + + // ─── stopSessionContainer ───────────────────────────────────────────────── + + describe("stopSessionContainer()", () => { + it("runs docker stop then docker rm", async () => { + vi.mocked(runCommand) + .mockResolvedValueOnce({stdout: "", stderr: "", exitCode: 0}) // stop + .mockResolvedValueOnce({stdout: "", stderr: "", exitCode: 0}); // rm + + await stopSessionContainer("abc123"); + + const calls = vi.mocked(runCommand).mock.calls; + expect(calls[0]![0]).toContain("docker stop"); + expect(calls[0]![0]).toContain("abc123"); + expect(calls[1]![0]).toContain("docker rm"); + expect(calls[1]![0]).toContain("abc123"); + }); + }); + + // ─── cloneDatabaseViaContainer ──────────────────────────────────────────── + + describe("cloneDatabaseViaContainer()", () => { + const containerID = "testcontainer123"; + const sourceUrl = "postgres://srcuser:srcpass@remote-host:5432/sourcedb"; + const targetUrl = "postgres://postgres:postkit_local@localhost:15432/postkit_local"; + + it("runs pg_dump and psql inside the container via docker exec", async () => { + vi.mocked(runPipedCommands).mockResolvedValue({stdout: "", stderr: "", exitCode: 0}); + + await cloneDatabaseViaContainer(containerID, sourceUrl, targetUrl); + + expect(runPipedCommands).toHaveBeenCalledTimes(1); + const [producer, consumer] = vi.mocked(runPipedCommands).mock.calls[0]!; + + // Producer: docker exec ... pg_dump + expect(producer.args[0]).toBe("docker"); + expect(producer.args).toContain("exec"); + expect(producer.args).toContain(containerID); + expect(producer.args).toContain("pg_dump"); + expect(producer.args).toContain("remote-host"); + expect(producer.args).toContain("sourcedb"); + + // Consumer: docker exec ... psql + expect(consumer.args[0]).toBe("docker"); + expect(consumer.args).toContain("exec"); + expect(consumer.args).toContain(containerID); + expect(consumer.args).toContain("psql"); + }); + + it("psql connects to container-internal localhost:5432, not the mapped port", async () => { + vi.mocked(runPipedCommands).mockResolvedValue({stdout: "", stderr: "", exitCode: 0}); + + await cloneDatabaseViaContainer(containerID, sourceUrl, targetUrl); + + const [, consumer] = vi.mocked(runPipedCommands).mock.calls[0]!; + expect(consumer.args).toContain("localhost"); + expect(consumer.args).toContain("5432"); + // Must NOT use the external mapped port (15432) + expect(consumer.args).not.toContain("15432"); + }); + + it("passes PGPASSWORD for source via -e flag in docker exec args", async () => { + vi.mocked(runPipedCommands).mockResolvedValue({stdout: "", stderr: "", exitCode: 0}); + + await cloneDatabaseViaContainer(containerID, sourceUrl, targetUrl); + + const [producer] = vi.mocked(runPipedCommands).mock.calls[0]!; + const envFlag = producer.args.findIndex((a) => a === "-e"); + expect(envFlag).not.toBe(-1); + expect(producer.args[envFlag + 1]).toContain("PGPASSWORD=srcpass"); + }); + + it("throws on non-zero exit code", async () => { + vi.mocked(runPipedCommands).mockResolvedValue({ + stdout: "", + stderr: "dump error", + exitCode: 1, + }); + + await expect( + cloneDatabaseViaContainer(containerID, sourceUrl, targetUrl), + ).rejects.toThrow("Failed to clone database via container"); + }); + }); +}); diff --git a/cli/test/modules/db/services/database.test.ts b/cli/test/modules/db/services/database.test.ts index 02a20d5..c50a8d4 100644 --- a/cli/test/modules/db/services/database.test.ts +++ b/cli/test/modules/db/services/database.test.ts @@ -22,7 +22,7 @@ vi.mock("../../../../src/common/shell", () => ({ // Get references to the mocked functions via pg import import pg from "pg"; import {runPipedCommands} from "../../../../src/common/shell"; -import {parseConnectionUrl, testConnection, createDatabase, dropDatabase, cloneDatabase} from "../../../../src/modules/db/services/database"; +import {parseConnectionUrl, testConnection, createDatabase, dropDatabase, cloneDatabase, getRemotePgMajorVersion} from "../../../../src/modules/db/services/database"; // Access mock methods from the Client prototype const getMockClient = () => { @@ -100,6 +100,36 @@ describe("database", () => { }); }); + describe("getRemotePgMajorVersion()", () => { + it("parses version_num and returns major version", async () => { + mockClient.query.mockResolvedValue({rows: [{server_version_num: "160003"}]}); + const version = await getRemotePgMajorVersion("postgres://user:pass@host/db"); + expect(version).toBe(16); + }); + + it("returns correct major for PG 14", async () => { + mockClient.query.mockResolvedValue({rows: [{server_version_num: "140012"}]}); + expect(await getRemotePgMajorVersion("postgres://user:pass@host/db")).toBe(14); + }); + + it("returns correct major for PG 15", async () => { + mockClient.query.mockResolvedValue({rows: [{server_version_num: "150007"}]}); + expect(await getRemotePgMajorVersion("postgres://user:pass@host/db")).toBe(15); + }); + + it("always closes the connection", async () => { + mockClient.query.mockResolvedValue({rows: [{server_version_num: "160003"}]}); + await getRemotePgMajorVersion("postgres://user:pass@host/db"); + expect(mockClient.end).toHaveBeenCalledTimes(1); + }); + + it("closes connection even on query failure", async () => { + mockClient.query.mockRejectedValue(new Error("query failed")); + await expect(getRemotePgMajorVersion("postgres://user:pass@host/db")).rejects.toThrow(); + expect(mockClient.end).toHaveBeenCalledTimes(1); + }); + }); + describe("cloneDatabase()", () => { it("calls runPipedCommands with pg_dump and psql", async () => { vi.mocked(runPipedCommands).mockResolvedValue({stdout: "", stderr: "", exitCode: 0}); @@ -107,7 +137,7 @@ describe("database", () => { expect(runPipedCommands).toHaveBeenCalledTimes(1); const [producer, consumer] = vi.mocked(runPipedCommands).mock.calls[0]!; expect(producer.args[0]).toBe("pg_dump"); - expect(producer.env.PGPASSWORD).toBe("pass"); + expect(producer.env?.PGPASSWORD).toBe("pass"); expect(consumer.args[0]).toBe("psql"); }); diff --git a/cli/test/modules/db/utils/session.test.ts b/cli/test/modules/db/utils/session.test.ts index 7f2c888..ade9f17 100644 --- a/cli/test/modules/db/utils/session.test.ts +++ b/cli/test/modules/db/utils/session.test.ts @@ -102,6 +102,25 @@ describe("session", () => { const session = await createSession("postgres://remote/db", "postgres://local/db"); expect(session.clonedAt).toMatch(/^\d{14}$/); }); + + it("stores containerID when provided", async () => { + vi.mocked(fs.writeFile).mockResolvedValue(); + const session = await createSession( + "postgres://remote:5432/db", + "postgres://localhost:5432/local", + "dev", + "abc123containerID", + ); + expect(session.containerID).toBe("abc123containerID"); + const written = JSON.parse(vi.mocked(fs.writeFile).mock.calls[0]![1] as string); + expect(written.containerID).toBe("abc123containerID"); + }); + + it("containerID is undefined when not provided", async () => { + vi.mocked(fs.writeFile).mockResolvedValue(); + const session = await createSession("postgres://remote/db", "postgres://local/db", "dev"); + expect(session.containerID).toBeUndefined(); + }); }); describe("updateSession()", () => { From ffc54551f31686d9a63d272d383cd660b58def5a Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Thu, 7 May 2026 19:45:31 +0530 Subject: [PATCH 05/18] feat: implement auto-container management for local databases and refactor configuration into committed and ignored secrets files --- CLAUDE.md | 54 ++++++--- cli/README.md | 51 ++++---- cli/docs/architecture.md | 39 +++++-- cli/docs/db.md | 129 ++++++++++++++++----- docs/docs/getting-started/configuration.md | 83 ++++++------- docs/docs/getting-started/quick-start.md | 13 ++- docs/docs/modules/db/commands/abort.md | 6 +- docs/docs/modules/db/commands/deploy.md | 20 ++-- docs/docs/modules/db/commands/remote.md | 2 +- docs/docs/modules/db/commands/start.md | 16 ++- docs/docs/modules/db/overview.md | 2 +- docs/docs/modules/db/troubleshooting.md | 27 +++++ docs/docs/reference/session-state.md | 4 +- 13 files changed, 294 insertions(+), 152 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a8954da..7b566b5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -77,16 +77,23 @@ Then import and call the registration function in `cli/src/index.ts`. The `db` module implements a **session-based migration workflow**: -1. **Session state**: Tracked in `.postkit/db/session.json`. Includes `remoteName` to track which remote was used. +1. **Session state**: Tracked in `.postkit/db/session.json`. Includes `remoteName` to track which remote was used, and optional `containerID` for auto Docker containers. 2. **Named remotes**: Users can configure multiple named remote databases via `db.remotes` in config: - At least one remote must be configured - One remote can be marked as `default: true` - Managed via `postkit db remote` commands + - Metadata (name, default, addedAt) written to `postkit.config.json`; URL written to `postkit.secrets.json` 3. **Binary resolution**: Both `pgschema` and `dbmate` binaries are auto-resolved: - `pgschema`: Bundled in `vendor/pgschema/` for all platforms (darwin-{arm64,amd64}, linux-{arm64,amd64}, windows-{arm64,amd64}) - `dbmate`: npm-installed via the `dbmate` package -4. **Migration steps execution**: The `deploy` command uses `runSteps()` to execute multi-step operations with resume capability - if a step fails, re-running resumes from where it left off. -5. **Schema directory structure** (`db/schema/`): +4. **Auto Docker container** (`modules/db/services/container.ts`): When `localDbUrl` is empty, PostKit: + - Checks Docker availability (`checkDockerAvailable()`) + - Queries remote PG version via `getRemotePgMajorVersion()` (uses `SHOW server_version_num`) + - Starts `postgres:{version}-alpine` on a free port in range 15432–15532 + - Runs `pg_dump`/`psql` inside the container via `docker exec` (`cloneDatabaseViaContainer()`) + - Stores `containerID` in session; cleaned up on `db abort` or `db deploy` completion +5. **Migration steps execution**: The `deploy` command uses `runSteps()` to execute multi-step operations with resume capability - if a step fails, re-running resumes from where it left off. +6. **Schema directory structure** (`db/schema/`): - `infra/` - Pre-migration (roles, schemas, extensions) - excluded from pgschema - `extensions/`, `types/`, `enums/`, `tables/`, etc. - pgschema-managed - `seeds/` - Post-migration seed data - excluded from pgschema @@ -120,35 +127,48 @@ All PostKit runtime files are stored in `.postkit/` (gitignored): ### Configuration System -Config is loaded from `postkit.config.json` in the project root. Use `loadPostkitConfig()` from `common/config.ts`. +Config is loaded by `loadPostkitConfig()` from `common/config.ts`, which deep-merges two files: -**Database config structure:** +| File | Committed | Purpose | +|------|-----------|---------| +| `postkit.config.json` | Yes | Non-sensitive settings (schema paths, remote metadata, flags) | +| `postkit.secrets.json` | No (gitignored) | Credentials (database URLs, passwords) | + +**`postkit.config.json` (committed):** ```json { "db": { - "localDbUrl": "postgres://...", - "schemaPath": "schema", + "localDbUrl": "", + "schemaPath": "db/schema", "schema": "public", - "pgSchemaBin": "pgschema", - "dbmateBin": "dbmate", "remotes": { - "dev": { - "url": "postgres://...", - "default": true, - "addedAt": "2024-12-31T10:00:00.000Z" - }, - "staging": { - "url": "postgres://..." - } + "dev": { "default": true, "addedAt": "2024-12-31T10:00:00.000Z" }, + "staging": {} } } } ``` +**`postkit.secrets.json` (gitignored):** +```json +{ + "db": { + "localDbUrl": "postgres://...", + "remotes": { + "dev": { "url": "postgres://..." }, + "staging": { "url": "postgres://..." } + } + } +} +``` + +**`localDbUrl`**: Leave empty to have PostKit automatically start a `postgres:{version}-alpine` Docker container. The version is queried from the remote DB via `SHOW server_version_num`. The container is started on `db start` and stopped on `db abort`. + **Auto-migration:** When loading config, if `remotes` is missing but `remoteDbUrl` exists, it's auto-migrated to create a `default` remote. Key config paths: - `POSTKIT_CONFIG_FILE` = "postkit.config.json" +- `POSTKIT_SECRETS_FILE` = "postkit.secrets.json" - `POSTKIT_DIR` = ".postkit" (session state, staged files) - `vendor/` = Bundled binaries (resolved relative to CLI root, not project root) diff --git a/cli/README.md b/cli/README.md index 7a97042..e044399 100644 --- a/cli/README.md +++ b/cli/README.md @@ -21,11 +21,10 @@ npm install -g @appritech/postkit ### Requirements -| Requirement | Version | Download | -|-------------|---------|----------| -| **Node.js** | >= 18.0.0 | [nodejs.org](https://nodejs.org/) | -| **Docker** | Latest | [docker.com](https://www.docker.com/products/docker-desktop/) | -| **PostgreSQL CLI** | `psql`, `pg_dump` | [postgresql.org/download](https://www.postgresql.org/download/) | +| Requirement | Version | Notes | +|-------------|---------|-------| +| **Node.js** | >= 18.0.0 | Required | +| **Docker** | Latest | Required only when `localDbUrl` is empty (auto-container mode) | ### Basic Usage @@ -110,41 +109,37 @@ Full documentation available at: **[https://docs.postkitstack.com/](https://docs ## 🔧 Configuration -Create a `postkit.config.json` in your project root: +PostKit uses two config files — run `postkit init` to generate both. +**`postkit.config.json`** (commit this): ```json { "db": { - "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", + "localDbUrl": "", "schemaPath": "db/schema", "schema": "public", "remotes": { - "dev": { - "url": "postgres://user:pass@dev-host:5432/myapp", - "default": true - }, - "staging": { - "url": "postgres://user:pass@staging-host:5432/myapp" - } + "dev": { "default": true } } - }, - "auth": { - "source": { - "url": "https://keycloak-dev.example.com", - "adminUser": "admin", - "adminPass": "dev-password", - "realm": "myapp-realm" - }, - "target": { - "url": "https://keycloak-staging.example.com", - "adminUser": "admin", - "adminPass": "staging-password" - }, - "configCliImage": "adorsys/keycloak-config-cli:6.4.0-24" } } ``` +**`postkit.secrets.json`** (gitignored — contains credentials): +```json +{ + "db": { + "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", + "remotes": { + "dev": { "url": "postgres://user:pass@dev-host:5432/myapp" } + } + } +} +``` + +> Leave `localDbUrl` empty to have PostKit automatically spin up a version-matched Docker container for your local database. No PostgreSQL client tools required on your host. +``` + Run `postkit init` to create the `.postkit/` directory structure: ``` diff --git a/cli/docs/architecture.md b/cli/docs/architecture.md index 6b73170..d99cb7f 100644 --- a/cli/docs/architecture.md +++ b/cli/docs/architecture.md @@ -71,8 +71,9 @@ Session-based migration workflow: `start → plan → apply → commit → deplo **Key components:** - **pgschema** — Bundled binary for schema diffing (`vendor/pgschema/`) - **dbmate** — npm-installed migration runner (`--migrations-table postkit.schema_migrations`) -- **Session state** — Tracked in `.postkit/db/session.json` -- **Named remotes** — Multiple remote DBs via `db.remotes` in config +- **Session state** — Tracked in `.postkit/db/session.json`; includes optional `containerID` when auto Docker container is active +- **Named remotes** — Multiple remote DBs via `db.remotes` in config; URLs stored in `postkit.secrets.json`, metadata in `postkit.config.json` +- **Auto Docker container** — When `localDbUrl` is empty, `container.ts` starts a `postgres:{version}-alpine` container. Version is queried from remote via `SHOW server_version_num`. `pg_dump`/`psql` run inside the container via `docker exec` for version-matched tools. - **Schema directory** — User-maintained SQL files (`db/schema/`) with sections: `infra/`, `extensions/`, `types/`, `enums/`, `tables/`, `views/`, `functions/`, `triggers/`, `grants/`, `seeds/` **Import sub-workflow** (`postkit db import`): @@ -100,7 +101,7 @@ Shared utilities used by all modules, located in `cli/src/common/`: | File | Purpose | |------|---------| -| `config.ts` | Config loader (`.env`, `postkit.config.json`), path resolution | +| `config.ts` | Config loader — merges `postkit.config.json` + `postkit.secrets.json`, path resolution | | `logger.ts` | Chalk-based console output (respects `--verbose`) | | `shell.ts` | Shell command execution wrapper | | `types.ts` | Shared TypeScript types (`CommandOptions`) | @@ -110,26 +111,42 @@ Shared utilities used by all modules, located in `cli/src/common/`: ## Configuration -Loaded from `postkit.config.json` via `loadPostkitConfig()`: +Loaded via `loadPostkitConfig()`, which deep-merges two files: + +| File | Committed | Contains | +|------|-----------|---------| +| `postkit.config.json` | Yes | Non-sensitive settings (schema paths, remote metadata, flags) | +| `postkit.secrets.json` | No (gitignored) | Credentials (database URLs, passwords) | ```json +// postkit.config.json (committed) { "db": { - "localDbUrl": "postgres://...", + "localDbUrl": "", "schemaPath": "db/schema", "schema": "public", "remotes": { - "dev": { "url": "postgres://...", "default": true }, - "staging": { "url": "postgres://..." } + "dev": { "default": true, "addedAt": "2024-12-31T10:00:00.000Z" }, + "staging": {} + } + } +} + +// postkit.secrets.json (gitignored) +{ + "db": { + "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", + "remotes": { + "dev": { "url": "postgres://user:pass@dev-host:5432/myapp" }, + "staging": { "url": "postgres://user:pass@staging-host:5432/myapp" } } - }, - "auth": { - "sourceKeycloak": { "baseUrl": "...", "realm": "..." }, - "targetKeycloak": { "baseUrl": "...", "realm": "..." } } } ``` +`localDbUrl` can be empty — PostKit will automatically start a Docker container (`postgres:{version}-alpine`) for the session. The container image version is detected from the remote database at runtime via `SHOW server_version_num`. +``` + --- ## Binary Resolution diff --git a/cli/docs/db.md b/cli/docs/db.md index d41bc18..912b480 100644 --- a/cli/docs/db.md +++ b/cli/docs/db.md @@ -62,54 +62,109 @@ A session-based database migration workflow for safe schema changes. Clone your ## 🧰 Prerequisites -- **PostgreSQL** client tools (`pg_dump`, `psql`) - **pgschema** — Bundled with PostKit. Platform-specific binaries are shipped in `vendor/pgschema/` and resolved automatically. No separate installation needed. - **dbmate** — Installed automatically as an npm dependency. No separate installation needed. +- **Docker** _(optional)_ — Required only if `db.localDbUrl` is empty. PostKit will automatically spin up a version-matched `postgres:{version}-alpine` container for the session and tear it down when done. --- ## ⚙️ Configuration -### Config File (`postkit.config.json`) +### Split Configuration Files -| Property | Description | Required | -|----------|-------------|----------| -| `db.localDbUrl` | PostgreSQL connection URL for local clone database | Yes | -| `db.schemaPath` | Path to schema files (relative to project root) | No | -| `db.schema` | Database schema name | No | -| `db.pgSchemaBin` | Path to pgschema binary | No | -| `db.dbmateBin` | Path to dbmate binary | No | -| `db.remotes` | Named remote database configurations | Yes (at least one) | +PostKit uses two configuration files to separate non-sensitive settings from credentials: -### Remote Configuration +| File | Committed to Git | Purpose | +|------|-----------------|---------| +| `postkit.config.json` | **Yes** | Schema paths, remote metadata (names, flags), non-sensitive settings | +| `postkit.secrets.json` | **No** (gitignored) | Database URLs, passwords, credentials | + +Both files are deep-merged at load time. Use `postkit.secrets.example.json` (auto-generated by `postkit init`) as a template for team members to create their own `postkit.secrets.json`. -Configure named remotes in `postkit.config.json`: +### `postkit.config.json` (committed) ```json { "db": { - "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", - "schemaPath": "schema", + "localDbUrl": "", + "schemaPath": "db/schema", "schema": "public", "remotes": { "dev": { - "url": "postgres://user:pass@dev-host:5432/myapp", "default": true, "addedAt": "2024-12-31T10:00:00.000Z" }, "staging": { - "url": "postgres://user:pass@staging-host:5432/myapp" + "addedAt": "2024-12-31T10:00:00.000Z" + } + } + } +} +``` + +### `postkit.secrets.json` (gitignored) + +```json +{ + "db": { + "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", + "remotes": { + "dev": { + "url": "postgres://user:pass@dev-host:5432/myapp" }, - "production": { - "url": "postgres://user:pass@prod-host:5432/myapp" + "staging": { + "url": "postgres://user:pass@staging-host:5432/myapp" } } } } ``` -**Properties:** -- `url` - PostgreSQL connection URL (required) +> **Tip:** Leave `localDbUrl` empty (or omit it) to have PostKit automatically start a Docker container for your local database. The container image version is matched to your remote PostgreSQL version automatically. + +### Config Properties + +| Property | File | Description | Required | +|----------|------|-------------|----------| +| `db.localDbUrl` | secrets | PostgreSQL URL for local clone database. Leave empty to use auto Docker container. | No | +| `db.schemaPath` | config | Path to schema files (relative to project root) | No | +| `db.schema` | config | Database schema name | No | +| `db.pgSchemaBin` | config | Path to pgschema binary | No | +| `db.dbmateBin` | config | Path to dbmate binary | No | +| `db.remotes` | both | Named remote database configurations | Yes (at least one) | + +### Remote Configuration + +Remote metadata (name, default flag, addedAt) goes in `postkit.config.json`; the URL goes in `postkit.secrets.json`. When you run `postkit db remote add`, this split happens automatically. + +```json +// postkit.config.json +{ + "db": { + "remotes": { + "dev": { "default": true, "addedAt": "2024-12-31T10:00:00.000Z" }, + "staging": { "addedAt": "2024-12-31T10:00:00.000Z" }, + "production": { "addedAt": "2024-12-31T10:00:00.000Z" } + } + } +} + +// postkit.secrets.json +{ + "db": { + "remotes": { + "dev": { "url": "postgres://user:pass@dev-host:5432/myapp" }, + "staging": { "url": "postgres://user:pass@staging-host:5432/myapp" }, + "production": { "url": "postgres://user:pass@prod-host:5432/myapp" } + } + } +} +``` + +**Properties (in `postkit.secrets.json`):** +- `url` - PostgreSQL connection URL (required — sensitive, never committed) + +**Properties (in `postkit.config.json`):** - `default` - Mark as default remote (optional, one must be default) - `addedAt` - ISO timestamp when remote was added (auto-set) @@ -205,10 +260,13 @@ postkit db start --remote staging # Use specific remote **What it does:** 1. Checks prerequisites (pgschema, dbmate installed) 2. Resolves target remote (default or specified) -3. Checks for pending committed migrations by querying the remote's `postkit.schema_migrations` table -4. Tests connection to remote database -5. Clones remote database to local using `pg_dump` and `psql` -6. Creates a session file (`.postkit/db/session.json`) to track state +3. Tests connection to remote database and detects its PostgreSQL major version +4. Checks for pending committed migrations by querying the remote's `postkit.schema_migrations` table +5. **If `localDbUrl` is empty**: Checks Docker availability and starts a `postgres:{version}-alpine` container on a free port (15432–15532), where `{version}` matches the remote database's PostgreSQL major version +6. Clones remote database to local. When using an auto-container, `pg_dump` and `psql` run inside the container via `docker exec` (version-matched tools, no host binary required) +7. Creates a session file (`.postkit/db/session.json`) to track state, including the container ID if a container was started + +**Auto-container:** When `localDbUrl` is not configured, PostKit manages the full container lifecycle — start on `db start`, stop on `db abort`. The container image always matches the remote PostgreSQL version. --- @@ -284,13 +342,14 @@ postkit db deploy --dry-run # Verify only, don't touch target 1. Resolves the target database URL (from remote config or `--url` flag) 2. Checks for pending committed migrations by querying the remote's `postkit.schema_migrations` table 3. If an active session exists, removes it (with confirmation unless `-f`) -4. Tests the target database connection -5. Clones the target database to local (using `LOCAL_DATABASE_URL`) -6. Runs a full dry-run on the local clone: infra, dbmate migrate, seeds -7. If `--dry-run` is set, stops here and reports results without touching the target -8. Reports dry-run results and confirms deployment (unless `-f`) -9. Applies to target: infra, dbmate migrate, seeds -10. Drops the local clone database +4. Tests the target database connection and detects its PostgreSQL major version +5. **If `localDbUrl` is empty**: Starts a temporary `postgres:{version}-alpine` container (version-matched to the target) for the dry-run +6. Clones the target database to the local URL. When using a temp container, cloning runs via `docker exec` inside the container +7. Runs a full dry-run on the local clone: infra, dbmate migrate, seeds +8. If `--dry-run` is set, stops here and reports results without touching the target +9. Reports dry-run results and confirms deployment (unless `-f`) +10. Applies to target: infra, dbmate migrate, seeds +11. Drops the local clone database; stops and removes the temp container if one was used If the dry run fails, deployment is aborted and no changes are made to the target database. @@ -456,8 +515,9 @@ Session state is stored in `.postkit/db/session.json`: "startedAt": "2026-02-11T12:00:00Z", "clonedAt": "20260211120000", "remoteName": "staging", - "localDbUrl": "postgres://...", + "localDbUrl": "postgres://postgres:postkit_local@localhost:15432/postkit_local", "remoteDbUrl": "postgres://...", + "containerID": "abc123def456", "pendingChanges": { "planned": false, "applied": false, @@ -471,6 +531,8 @@ Session state is stored in `.postkit/db/session.json`: } ``` +> `containerID` is present only when PostKit started an auto Docker container. It is used by `postkit db abort` to stop and remove the container. + ### Committed Migrations (`committed.json`) Committed migrations are tracked in `.postkit/db/committed.json`. Deployment status is determined by querying the remote database's `postkit.schema_migrations` table — not stored locally. @@ -511,6 +573,9 @@ Session migrations are staged in `.postkit/db/session/` and committed migrations | `Schema files have changed since the plan was generated` | Schema files were modified after running `plan`. Run `postkit db plan` again | | `Seeds failed during apply` | Re-run `postkit db apply` — it resumes from where it left off | | `Deploy failed during dry run` | No changes were made to the target. Fix the issue and retry. | +| `Docker not found` | Install Docker Desktop and ensure the `docker` binary is on your PATH. Docker is only required when `localDbUrl` is empty. | +| `Docker is not running` | Start Docker Desktop before running `postkit db start` or `postkit db deploy`. | +| `Failed to start container` | Check that the `postgres:{version}-alpine` image can be pulled. Ensure you have internet access or the image is already cached locally. | | `Import: pgschema plan produced no output` | Schema directory may be empty after normalization. Check that the source DB has objects in the target schema. | | `Import: Could not insert migration tracking record` | Non-fatal. The local DB migration succeeded but the source DB tracking record failed. Manually insert the version into `postkit.schema_migrations` on the source DB. | | `Import: column does not exist during local apply` | Infrastructure SQL (roles, schemas) may not have been applied to the local database before dbmate. Ensure `schema/infra/` files exist and are valid. | diff --git a/docs/docs/getting-started/configuration.md b/docs/docs/getting-started/configuration.md index fa26d0c..29389c3 100644 --- a/docs/docs/getting-started/configuration.md +++ b/docs/docs/getting-started/configuration.md @@ -4,20 +4,47 @@ sidebar_position: 2 # Configuration -PostKit uses a `postkit.config.json` file in your project root for configuration. +PostKit separates non-sensitive settings from credentials using two config files. -## Basic Configuration +## Config Files + +| File | Commit to Git | Purpose | +|------|--------------|---------| +| `postkit.config.json` | **Yes** | Schema paths, remote names, non-sensitive settings | +| `postkit.secrets.json` | **No** (gitignored) | Database URLs, passwords, credentials | + +Both files are deep-merged at load time. `postkit init` creates all three files: the config, the secrets file, and a `postkit.secrets.example.json` template your team can use as a reference. + +## `postkit.config.json` (committed) ```json { "db": { - "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", + "localDbUrl": "", "schemaPath": "db/schema", "schema": "public", "remotes": { "dev": { - "url": "postgres://user:pass@dev-host:5432/myapp", - "default": true + "default": true, + "addedAt": "2024-12-31T10:00:00.000Z" + }, + "staging": { + "addedAt": "2024-12-31T10:00:00.000Z" + } + } + } +} +``` + +## `postkit.secrets.json` (gitignored) + +```json +{ + "db": { + "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", + "remotes": { + "dev": { + "url": "postgres://user:pass@dev-host:5432/myapp" }, "staging": { "url": "postgres://user:pass@staging-host:5432/myapp" @@ -29,9 +56,9 @@ PostKit uses a `postkit.config.json` file in your project root for configuration ## Configuration Options -### `db.localDbUrl` (required) +### `db.localDbUrl` (optional) -PostgreSQL connection URL for your local clone database. +PostgreSQL connection URL for your local clone database. **Leave empty** to have PostKit automatically start a Docker container (`postgres:{version}-alpine`) for you. The container image version is matched to your remote PostgreSQL version automatically and the container is cleaned up when you abort the session. ### `db.schemaPath` (optional) @@ -43,47 +70,22 @@ Database schema name. Default: `"public"`. ### `db.remotes` (required) -Named remote database configurations. At least one remote must be configured. +Named remote database configurations. At least one remote must be configured. Remote metadata (name, default flag, addedAt) goes in `postkit.config.json`; the URL goes in `postkit.secrets.json`. The `postkit db remote add` command handles this split automatically. #### Remote Properties -| Property | Type | Required | Description | +| Property | File | Required | Description | |----------|------|----------|-------------| -| `url` | string | Yes | PostgreSQL connection URL | -| `default` | boolean | No | Mark as default remote (one must be default) | -| `addedAt` | string | No | ISO timestamp when remote was added (auto-set) | - -## Environment Variables - -For sensitive data like database passwords, use environment variables: - -```bash -# .env file -DEV_DB_URL="postgres://user:pass@dev-host:5432/myapp" -STAGING_DB_URL="postgres://user:pass@staging-host:5432/myapp" -``` - -Then reference them in your config: - -```json -{ - "db": { - "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", - "remotes": { - "dev": { - "url": "${DEV_DB_URL}", - "default": true - } - } - } -} -``` +| `url` | secrets | Yes | PostgreSQL connection URL | +| `default` | config | No | Mark as default remote (one must be default) | +| `addedAt` | config | No | ISO timestamp when remote was added (auto-set) | ## Auth Module Configuration -The auth module is configured in `postkit.config.json`: +The auth module is configured in `postkit.config.json` (non-sensitive settings) and `postkit.secrets.json` (credentials): ```json +// postkit.secrets.json { "auth": { "source": { @@ -96,8 +98,7 @@ The auth module is configured in `postkit.config.json`: "url": "https://keycloak-staging.example.com", "adminUser": "admin", "adminPass": "staging-password" - }, - "configCliImage": "adorsys/keycloak-config-cli:6.4.0-24" + } } } ``` diff --git a/docs/docs/getting-started/quick-start.md b/docs/docs/getting-started/quick-start.md index 40f2a67..cb748ae 100644 --- a/docs/docs/getting-started/quick-start.md +++ b/docs/docs/getting-started/quick-start.md @@ -17,13 +17,15 @@ postkit init ``` This creates: -- `postkit.config.json` - Your configuration file +- `postkit.config.json` - Non-sensitive configuration (committed to git) +- `postkit.secrets.json` - Your credentials (gitignored) +- `postkit.secrets.example.json` - Credentials template for teammates (committed) - `db/schema/` - Your schema files directory - `.postkit/` - Runtime files (gitignored) ## 2. Configure Remotes -Add your remote databases: +Add your remote databases (URLs are stored in `postkit.secrets.json`, metadata in `postkit.config.json`): ```bash # Add development remote (set as default) @@ -42,9 +44,10 @@ postkit db start ``` This: -1. Clones the remote database to local -2. Creates a session to track your changes -3. Prepares for schema modifications +1. Detects the remote PostgreSQL version +2. Clones the remote database to local (auto-starts a Docker container if `localDbUrl` is empty) +3. Creates a session to track your changes +4. Prepares for schema modifications ## 4. Make Schema Changes diff --git a/docs/docs/modules/db/commands/abort.md b/docs/docs/modules/db/commands/abort.md index 646f669..560bb3f 100644 --- a/docs/docs/modules/db/commands/abort.md +++ b/docs/docs/modules/db/commands/abort.md @@ -23,8 +23,10 @@ postkit db abort [-f] ## What It Does 1. Prompts for confirmation (unless `-f`) -2. Removes the session file (`.postkit/db/session.json`) -3. Cleans up session-specific files +2. Drops the local clone database +3. If the session used an auto-started Docker container (`containerID` is set in the session), stops and removes it +4. Removes the session file (`.postkit/db/session.json`) +5. Cleans up session-specific files (plan file, generated schema) **Warning:** This will discard any uncommitted changes made during the session. diff --git a/docs/docs/modules/db/commands/deploy.md b/docs/docs/modules/db/commands/deploy.md index 3087b8c..0bbfcae 100644 --- a/docs/docs/modules/db/commands/deploy.md +++ b/docs/docs/modules/db/commands/deploy.md @@ -43,22 +43,22 @@ postkit db deploy --remote staging -f 1. Resolves the target database URL (from remote config or `--url` flag) 2. If an active session exists, removes it (with confirmation unless `-f`) -3. Tests the target database connection -4. Clones the target database to local (using `LOCAL_DATABASE_URL`) -5. Runs a full dry-run on the local clone: infra, dbmate migrate, seeds -6. If `--dry-run` is set, stops here and reports results -7. Reports dry-run results and confirms deployment (unless `-f`) -8. Applies to target: infra, dbmate migrate, seeds -9. Drops the local clone database -10. Marks migrations as deployed in `.postkit/db/committed.json` +3. Tests the target database connection and detects its PostgreSQL major version +4. **If `localDbUrl` is empty**: Starts a temporary `postgres:{version}-alpine` container for the dry-run, version-matched to the target +5. Clones the target database to local for dry-run verification. When using a temp container, cloning runs via `docker exec` inside the container +6. Runs a full dry-run on the local clone: infra, dbmate migrate, seeds +7. If `--dry-run` is set, stops here and reports results without touching the target +8. Reports dry-run results and confirms deployment (unless `-f`) +9. Applies to target: infra, dbmate migrate, seeds +10. Drops the local clone database; stops and removes the temp container if one was used If the dry run fails, deployment is aborted and no changes are made to the target database. ## Requirements - Committed migrations must exist (run `db commit` first) -- PostgreSQL client tools must be installed -- `localDbUrl` must be different from the target remote URL +- `localDbUrl` must be different from the target remote URL (or leave it empty to use an auto-container) +- Docker must be running if `localDbUrl` is empty ## Related diff --git a/docs/docs/modules/db/commands/remote.md b/docs/docs/modules/db/commands/remote.md index 1cd39c2..679a2dc 100644 --- a/docs/docs/modules/db/commands/remote.md +++ b/docs/docs/modules/db/commands/remote.md @@ -25,7 +25,7 @@ postkit db remote list --json ### add -Add a new remote. +Add a new remote. The URL is written to `postkit.secrets.json` (gitignored); the remote name and metadata are written to `postkit.config.json` (committed). ```bash postkit db remote add [--default] diff --git a/docs/docs/modules/db/commands/start.md b/docs/docs/modules/db/commands/start.md index 1a508ee..ee39ec6 100644 --- a/docs/docs/modules/db/commands/start.md +++ b/docs/docs/modules/db/commands/start.md @@ -35,9 +35,19 @@ postkit db start --remote staging 1. Checks prerequisites (pgschema, dbmate installed) 2. Resolves target remote (default or specified) -3. Tests connection to remote database -4. Clones remote database to local using `pg_dump` and `psql` -5. Creates a session file (`.postkit/db/session.json`) to track state +3. Tests connection to remote database and detects its PostgreSQL major version +4. Checks for pending committed migrations +5. **If `localDbUrl` is empty**: Checks Docker availability and starts a `postgres:{version}-alpine` container on a free port (15432–15532), version-matched to the remote +6. Clones remote database to local. When using an auto-container, cloning runs via `docker exec` inside the container (no host `pg_dump`/`psql` required) +7. Creates a session file (`.postkit/db/session.json`) to track state, including the `containerID` if a container was started + +## Auto Docker Container + +When `db.localDbUrl` is not set in your secrets file, PostKit automatically: +- Pulls and starts `postgres:{remote-version}-alpine` +- Uses `docker exec` to run `pg_dump`/`psql` inside the container (version-matched tools) +- Stores the container ID in the session +- Stops and removes the container when you run `postkit db abort` ## Related diff --git a/docs/docs/modules/db/overview.md b/docs/docs/modules/db/overview.md index 1f4e912..2ea854d 100644 --- a/docs/docs/modules/db/overview.md +++ b/docs/docs/modules/db/overview.md @@ -136,9 +136,9 @@ db/schema/ ## Prerequisites -- **PostgreSQL** client tools (`psql`, `pg_dump`) - **pgschema** - Bundled with PostKit (no separate installation needed) - **dbmate** - Auto-installed via npm (no separate installation needed) +- **Docker** _(optional)_ - Required only when `db.localDbUrl` is empty. PostKit starts a `postgres:{version}-alpine` container automatically, version-matched to your remote DB. ## Troubleshooting diff --git a/docs/docs/modules/db/troubleshooting.md b/docs/docs/modules/db/troubleshooting.md index 1f22271..0f561a1 100644 --- a/docs/docs/modules/db/troubleshooting.md +++ b/docs/docs/modules/db/troubleshooting.md @@ -42,6 +42,33 @@ sidebar_position: 100 **Solution:** No changes were made to the target. Fix the issue and retry. +## Docker / Auto-Container Issues + +### `Docker not found` + +**Solution:** Install [Docker Desktop](https://www.docker.com/products/docker-desktop/). Docker is only needed when `db.localDbUrl` is empty in `postkit.secrets.json`. Alternatively, set `localDbUrl` to use an existing local PostgreSQL instance. + +### `Docker is not running` + +**Solution:** Start Docker Desktop before running `postkit db start` or `postkit db deploy`. + +### `Failed to start container` + +**Solution:** The `postgres:{version}-alpine` image could not be started. Ensure you have internet access to pull the image, or pre-pull it: +```bash +docker pull postgres:16-alpine +``` +Check that Docker has enough memory allocated (at least 512MB recommended). + +### Container not cleaned up after abort + +**Solution:** If a container was left running after an interrupted session, stop it manually: +```bash +docker stop +docker rm +``` +The container ID is stored in `.postkit/db/session.json` under `containerID` if the session file still exists. + ## Import Issues ### `Import: pgschema plan produced no output` diff --git a/docs/docs/reference/session-state.md b/docs/docs/reference/session-state.md index e7f175b..c8da74c 100644 --- a/docs/docs/reference/session-state.md +++ b/docs/docs/reference/session-state.md @@ -14,8 +14,9 @@ The database module uses a session-based workflow. Session state is tracked in ` "startedAt": "2026-02-11T12:00:00Z", "clonedAt": "20260211120000", "remoteName": "staging", - "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", + "localDbUrl": "postgres://postgres:postkit_local@localhost:15432/postkit_local", "remoteDbUrl": "postgres://user:pass@staging-host:5432/myapp", + "containerID": "abc123def456", "pendingChanges": { "planned": false, "applied": false, @@ -39,6 +40,7 @@ The database module uses a session-based workflow. Session state is tracked in ` | `remoteName` | Name of the remote that was cloned | | `localDbUrl` | Local database connection URL | | `remoteDbUrl` | Remote database connection URL | +| `containerID` | Docker container ID (present only when PostKit auto-started a container) | | `pendingChanges` | Object tracking changes in the session | ### pendingChanges From 389defe23fb3e34dc3657058f1561b9144a1fb2e Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Thu, 7 May 2026 19:54:10 +0530 Subject: [PATCH 06/18] refactor: move all database remote data from public config to gitignored secrets file --- cli/src/commands/init.ts | 7 +- cli/src/common/config.ts | 14 +-- cli/src/modules/db/utils/remotes.ts | 136 ++++++++-------------- cli/test/common/config.test.ts | 22 ++-- cli/test/modules/db/utils/remotes.test.ts | 46 ++++++-- 5 files changed, 107 insertions(+), 118 deletions(-) diff --git a/cli/src/commands/init.ts b/cli/src/commands/init.ts index 057a839..8238e2e 100644 --- a/cli/src/commands/init.ts +++ b/cli/src/commands/init.ts @@ -24,12 +24,11 @@ const GITIGNORE_ENTRIES = [ "postkit.secrets.json", ]; -// Non-sensitive settings committed to git +// Non-sensitive settings committed to git — no remotes (user/env-specific, lives in secrets) const SCAFFOLD_PUBLIC_CONFIG: PostkitPublicConfig = { db: { schemaPath: "schema", schema: "public", - remotes: {}, }, auth: { configCliImage: "adorsys/keycloak-config-cli:6.4.0-24", @@ -209,8 +208,8 @@ export async function initCommand(options: CommandOptions): Promise { logger.success("Postkit project initialized!"); logger.blank(); logger.info("Config split:"); - logger.info(` ${POSTKIT_CONFIG_FILE} — committed to git (schema paths, remote metadata)`); - logger.info(` ${POSTKIT_SECRETS_FILE} — gitignored (DB URLs, passwords)`); + logger.info(` ${POSTKIT_CONFIG_FILE} — committed to git (schema paths, project settings)`); + logger.info(` ${POSTKIT_SECRETS_FILE} — gitignored (DB URLs, remotes, passwords)`); logger.info(` postkit.secrets.example.json — committed template for teammates`); logger.blank(); logger.info("Next steps:"); diff --git a/cli/src/common/config.ts b/cli/src/common/config.ts index f0a1526..fcc631c 100644 --- a/cli/src/common/config.ts +++ b/cli/src/common/config.ts @@ -64,17 +64,12 @@ export interface AuthInputConfig { } // ─── Public config (committed to git) ─────────────────────────────────────── -// Non-sensitive settings: schema paths, remote metadata, docker images, etc. - -export interface RemotePublicConfig { - default?: boolean; - addedAt?: string; -} +// Non-sensitive project settings: schema paths, docker images, etc. +// Remotes are user/environment-specific and live entirely in secrets. export interface DbPublicConfig { schemaPath?: string; schema?: string; - remotes?: Record; } export interface AuthPublicConfig { @@ -88,9 +83,12 @@ export interface PostkitPublicConfig { // ─── Secrets (gitignored) ───────────────────────────────────────────────────── // Sensitive credentials: DB URLs, passwords, auth tokens. +// Remotes are fully stored here (url + metadata). export interface RemoteSecretConfig { url: string; + default?: boolean; + addedAt?: string; } export interface DbSecretsConfig { @@ -219,6 +217,6 @@ export function loadPostkitConfig(): PostkitConfig { parsed = deepMerge(parsed as object, secrets as object) as Record; } - cachedConfig = parsed as PostkitConfig; + cachedConfig = parsed as unknown as PostkitConfig; return cachedConfig; } diff --git a/cli/src/modules/db/utils/remotes.ts b/cli/src/modules/db/utils/remotes.ts index 22e2174..0ff471a 100644 --- a/cli/src/modules/db/utils/remotes.ts +++ b/cli/src/modules/db/utils/remotes.ts @@ -1,7 +1,6 @@ import fs from "fs/promises"; -import {existsSync} from "fs"; import {logger} from "../../../common/logger"; -import {loadPostkitConfig, getConfigFilePath, getSecretsFilePath, invalidateConfig} from "../../../common/config"; +import {loadPostkitConfig, getSecretsFilePath, POSTKIT_SECRETS_FILE, invalidateConfig} from "../../../common/config"; import type {RemoteConfig} from "../../../common/config"; export interface RemoteInfo { @@ -83,25 +82,14 @@ async function writeJsonFile(filePath: string, data: Record): P await fs.writeFile(filePath, JSON.stringify(data, null, 2) + "\n", "utf-8"); } -/** - * Returns true when the project was initialized with the split-config layout - * (i.e. postkit.secrets.json exists next to postkit.config.json). - */ -function hasSplitConfig(): boolean { - return existsSync(getSecretsFilePath()); -} - // ─── Remote management ─────────────────────────────────────────────────────── /** * Add a new remote configuration. - * - * Split-config projects: metadata (default, addedAt) → postkit.config.json - * URL → postkit.secrets.json - * Legacy projects (no secrets file): everything → postkit.config.json + * All remote data (url, default flag, addedAt) is written to postkit.secrets.json. + * Nothing remote-related is written to postkit.config.json. */ export async function addRemote(name: string, url: string, setAsDefault: boolean = false): Promise { - // Validate name — only letters, numbers, hyphens, underscores if (!name || name.trim().length === 0) { throw new Error("Remote name cannot be empty"); } @@ -118,12 +106,21 @@ export async function addRemote(name: string, url: string, setAsDefault: boolean ); } - const configPath = getConfigFilePath(); - const config = await readJsonFile(configPath); - const db = (config.db ?? {}) as Record; - const existingRemotes = (db.remotes ?? {}) as Record>; + const secretsPath = getSecretsFilePath(); + let secrets: Record; + try { + secrets = await readJsonFile(secretsPath); + } catch { + throw new Error( + `Secrets file not found: ${POSTKIT_SECRETS_FILE}\n` + + 'Run "postkit init" to initialize your project first.', + ); + } + + const secretsDb = (secrets.db ?? {}) as Record; + const secretsRemotes = (secretsDb.remotes ?? {}) as Record>; - if (existingRemotes[name]) { + if (secretsRemotes[name]) { throw new Error(`Remote "${name}" already exists`); } @@ -144,52 +141,31 @@ export async function addRemote(name: string, url: string, setAsDefault: boolean ); } - const remoteCount = Object.keys(existingRemotes).length; + const remoteCount = Object.keys(secretsRemotes).length; const makeDefault = setAsDefault || remoteCount === 0; if (makeDefault) { - for (const key of Object.keys(existingRemotes)) { - delete existingRemotes[key].default; + for (const key of Object.keys(secretsRemotes)) { + delete secretsRemotes[key]!.default; } } const addedAt = new Date().toISOString(); + secretsRemotes[name] = {url, addedAt}; + if (makeDefault) secretsRemotes[name]!.default = true; - if (hasSplitConfig()) { - // Write metadata to public config - existingRemotes[name] = {addedAt}; - if (makeDefault) existingRemotes[name].default = true; - db.remotes = existingRemotes; - config.db = db; - await writeJsonFile(configPath, config); - - // Write URL to secrets - const secretsPath = getSecretsFilePath(); - const secrets = await readJsonFile(secretsPath); - const secretsDb = (secrets.db ?? {}) as Record; - const secretsRemotes = (secretsDb.remotes ?? {}) as Record; - secretsRemotes[name] = {url}; - secretsDb.remotes = secretsRemotes; - secrets.db = secretsDb; - await writeJsonFile(secretsPath, secrets); - } else { - // Legacy: write everything to postkit.config.json - existingRemotes[name] = {url, addedAt}; - if (makeDefault) existingRemotes[name].default = true; - db.remotes = existingRemotes; - config.db = db; - await writeJsonFile(configPath, config); - } + secretsDb.remotes = secretsRemotes; + secrets.db = secretsDb; + await writeJsonFile(secretsPath, secrets); invalidateConfig(); logger.success(`Remote "${name}" added successfully`); } /** - * Remove a remote configuration from both config and secrets files. + * Remove a remote configuration from postkit.secrets.json. */ export async function removeRemote(name: string, force: boolean = false): Promise { - // Use merged config to validate existence and count const merged = loadPostkitConfig(); const remotes = merged.db.remotes ?? {}; @@ -215,33 +191,20 @@ export async function removeRemote(name: string, force: boolean = false): Promis ); } - // Remove from public config - const configPath = getConfigFilePath(); - const config = await readJsonFile(configPath); - const db = (config.db ?? {}) as Record; - const configRemotes = (db.remotes ?? {}) as Record>; - delete configRemotes[name]; + const secretsPath = getSecretsFilePath(); + const secrets = await readJsonFile(secretsPath); + const secretsDb = (secrets.db ?? {}) as Record; + const secretsRemotes = (secretsDb.remotes ?? {}) as Record>; + delete secretsRemotes[name]; if (isDefault) { - const firstKey = Object.keys(configRemotes)[0]; - if (firstKey) configRemotes[firstKey].default = true; + const firstKey = Object.keys(secretsRemotes)[0]; + if (firstKey) secretsRemotes[firstKey]!.default = true; } - db.remotes = configRemotes; - config.db = db; - await writeJsonFile(configPath, config); - - // Remove from secrets if split-config layout is in use - if (hasSplitConfig()) { - const secretsPath = getSecretsFilePath(); - const secrets = await readJsonFile(secretsPath); - const secretsDb = (secrets.db ?? {}) as Record; - const secretsRemotes = (secretsDb.remotes ?? {}) as Record; - delete secretsRemotes[name]; - secretsDb.remotes = secretsRemotes; - secrets.db = secretsDb; - await writeJsonFile(secretsPath, secrets); - } + secretsDb.remotes = secretsRemotes; + secrets.db = secretsDb; + await writeJsonFile(secretsPath, secrets); invalidateConfig(); logger.success(`Remote "${name}" removed successfully`); @@ -249,7 +212,7 @@ export async function removeRemote(name: string, force: boolean = false): Promis /** * Set a remote as the default. - * Only updates postkit.config.json — the `default` flag is not sensitive. + * Updates postkit.secrets.json — all remote data lives there. */ export async function setDefaultRemote(name: string): Promise { const merged = loadPostkitConfig(); @@ -257,24 +220,23 @@ export async function setDefaultRemote(name: string): Promise { throw new Error(`Remote "${name}" not found`); } - const configPath = getConfigFilePath(); - const config = await readJsonFile(configPath); - const db = (config.db ?? {}) as Record; - const configRemotes = (db.remotes ?? {}) as Record>; + const secretsPath = getSecretsFilePath(); + const secrets = await readJsonFile(secretsPath); + const secretsDb = (secrets.db ?? {}) as Record; + const secretsRemotes = (secretsDb.remotes ?? {}) as Record>; - for (const key of Object.keys(configRemotes)) { - delete configRemotes[key].default; + for (const key of Object.keys(secretsRemotes)) { + delete secretsRemotes[key]!.default; } - // Ensure the entry exists in config (it may only exist in secrets for legacy remotes) - if (!configRemotes[name]) { - configRemotes[name] = {}; + if (!secretsRemotes[name]) { + secretsRemotes[name] = {}; } - configRemotes[name].default = true; + secretsRemotes[name]!.default = true; - db.remotes = configRemotes; - config.db = db; - await writeJsonFile(configPath, config); + secretsDb.remotes = secretsRemotes; + secrets.db = secretsDb; + await writeJsonFile(secretsPath, secrets); invalidateConfig(); logger.success(`Remote "${name}" set as default`); diff --git a/cli/test/common/config.test.ts b/cli/test/common/config.test.ts index 308f968..2e82123 100644 --- a/cli/test/common/config.test.ts +++ b/cli/test/common/config.test.ts @@ -86,8 +86,8 @@ describe("config", () => { db: {localDbUrl: "postgres://localhost:5432/test", remoteDbUrl: "postgres://remote:5432/test"}, })); const config = loadPostkitConfig(); - expect(config.db.remotes.default).toBeDefined(); - expect(config.db.remotes.default.url).toBe("postgres://remote:5432/test"); + expect(config.db.remotes!["default"]).toBeDefined(); + expect(config.db.remotes!["default"]!.url).toBe("postgres://remote:5432/test"); expect(fs.writeFileSync).toHaveBeenCalled(); }); @@ -97,8 +97,8 @@ describe("config", () => { db: {localDbUrl: "postgres://localhost:5432/test", environments: {staging: "postgres://staging:5432/test"}}, })); const config = loadPostkitConfig(); - expect(config.db.remotes.staging).toBeDefined(); - expect(config.db.environments).toBeUndefined(); + expect(config.db.remotes!["staging"]).toBeDefined(); + expect((config.db as Record)["environments"]).toBeUndefined(); }); it("does not re-migrate if remotes already exist", () => { @@ -109,12 +109,16 @@ describe("config", () => { }); it("merges secrets file when both files exist", () => { + // Public config has no remotes — all remote data lives in secrets const publicConfig = { - db: {schemaPath: "schema", schema: "public", remotes: {dev: {default: true, addedAt: "2024-01-01"}}}, + db: {schemaPath: "schema", schema: "public"}, auth: {configCliImage: "keycloak:latest"}, }; const secrets = { - db: {localDbUrl: "postgres://localhost:5432/test", remotes: {dev: {url: "postgres://dev:5432/test"}}}, + db: { + localDbUrl: "postgres://localhost:5432/test", + remotes: {dev: {url: "postgres://dev:5432/test", default: true, addedAt: "2024-01-01"}}, + }, auth: {source: {url: "http://kc:8080", adminUser: "admin", adminPass: "pass", realm: "r"}}, }; vi.mocked(fs.existsSync).mockReturnValue(true); // both files exist @@ -122,11 +126,11 @@ describe("config", () => { .mockReturnValueOnce(JSON.stringify(publicConfig)) // config file .mockReturnValueOnce(JSON.stringify(secrets)); // secrets file const config = loadPostkitConfig(); - // Secrets values are merged in + // All remote data comes from secrets expect(config.db.localDbUrl).toBe("postgres://localhost:5432/test"); - expect(config.db.remotes.dev.url).toBe("postgres://dev:5432/test"); + expect(config.db.remotes!["dev"]!.url).toBe("postgres://dev:5432/test"); + expect(config.db.remotes!["dev"]!.default).toBe(true); // Public config values are preserved - expect(config.db.remotes.dev.default).toBe(true); expect((config.auth as any).configCliImage).toBe("keycloak:latest"); }); }); diff --git a/cli/test/modules/db/utils/remotes.test.ts b/cli/test/modules/db/utils/remotes.test.ts index bad6861..eee0d37 100644 --- a/cli/test/modules/db/utils/remotes.test.ts +++ b/cli/test/modules/db/utils/remotes.test.ts @@ -127,14 +127,37 @@ describe("remotes", () => { }); describe("addRemote()", () => { - it("writes new remote to config", async () => { - vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(defaultMockConfig)); + // Secrets file content (all remote data lives here) + const secretsWithRemotes = { + db: { + localDbUrl: "postgres://localhost:5432/local", + remotes: { + dev: {url: "postgres://user:pass@dev-host:5432/db", default: true, addedAt: "2024-01-01T00:00:00.000Z"}, + staging: {url: "postgres://user:pass@staging-host:5432/db", addedAt: "2024-01-01T00:00:00.000Z"}, + }, + }, + }; + + it("writes new remote to secrets only", async () => { + vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(secretsWithRemotes)); vi.mocked(fs.writeFile).mockResolvedValue(); await addRemote("prod", "postgres://user:pass@prod-host:5432/db"); - expect(fs.writeFile).toHaveBeenCalled(); + expect(fs.writeFile).toHaveBeenCalledTimes(1); + const [writtenPath, writtenContent] = vi.mocked(fs.writeFile).mock.calls[0]!; + expect(writtenPath).toContain("postkit.secrets.json"); + const written = JSON.parse(writtenContent as string); + expect(written.db.remotes.prod).toBeDefined(); + expect(written.db.remotes.prod.url).toBe("postgres://user:pass@prod-host:5432/db"); expect(invalidateConfig).toHaveBeenCalled(); }); + it("throws when secrets file is missing", async () => { + vi.mocked(fs.readFile).mockRejectedValue(new Error("ENOENT")); + await expect(addRemote("prod", "postgres://user:pass@prod-host:5432/db")).rejects.toThrow( + "Secrets file not found", + ); + }); + it("throws for empty name", async () => { await expect(addRemote("", "postgres://host/db")).rejects.toThrow("cannot be empty"); }); @@ -144,7 +167,7 @@ describe("remotes", () => { }); it("throws for duplicate name", async () => { - vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(defaultMockConfig)); + vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(secretsWithRemotes)); await expect(addRemote("dev", "postgres://new/db")).rejects.toThrow("already exists"); }); @@ -153,23 +176,24 @@ describe("remotes", () => { }); it("throws when URL matches local DB", async () => { - vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(defaultMockConfig)); + vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(secretsWithRemotes)); await expect(addRemote("new", "postgres://localhost:5432/local")).rejects.toThrow("matches local database"); }); }); describe("removeRemote()", () => { - it("removes remote and reassigns default", async () => { + it("removes remote from secrets and reassigns default", async () => { vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(defaultMockConfig)); vi.mocked(fs.writeFile).mockResolvedValue(); await removeRemote("dev", true); - const written = JSON.parse(vi.mocked(fs.writeFile).mock.calls[0]![1] as string); + const [writtenPath, writtenContent] = vi.mocked(fs.writeFile).mock.calls[0]!; + expect(writtenPath).toContain("postkit.secrets.json"); + const written = JSON.parse(writtenContent as string); expect(written.db.remotes.dev).toBeUndefined(); expect(written.db.remotes.staging.default).toBe(true); }); it("throws when removing the only remote", async () => { - // Validation now uses the merged config, so mock loadPostkitConfig with a single remote vi.mocked(loadPostkitConfig).mockReturnValueOnce({ db: {localDbUrl: "postgres://localhost/db", remotes: {dev: {url: "postgres://dev/db", default: true}}}, } as any); @@ -178,11 +202,13 @@ describe("remotes", () => { }); describe("setDefaultRemote()", () => { - it("sets remote as default", async () => { + it("sets remote as default in secrets", async () => { vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(defaultMockConfig)); vi.mocked(fs.writeFile).mockResolvedValue(); await setDefaultRemote("staging"); - const written = JSON.parse(vi.mocked(fs.writeFile).mock.calls[0]![1] as string); + const [writtenPath, writtenContent] = vi.mocked(fs.writeFile).mock.calls[0]!; + expect(writtenPath).toContain("postkit.secrets.json"); + const written = JSON.parse(writtenContent as string); expect(written.db.remotes.staging.default).toBe(true); expect(written.db.remotes.dev.default).toBeUndefined(); }); From f21e8ecfbbc510f41c4401e972d02e3da4dbf91e Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Thu, 7 May 2026 19:57:57 +0530 Subject: [PATCH 07/18] refactor: move all database remote configurations from committed config to gitignored secrets file --- CLAUDE.md | 17 +++----- cli/README.md | 10 ++--- cli/docs/architecture.md | 20 ++++------ cli/docs/db.md | 45 +++++++--------------- docs/docs/getting-started/configuration.md | 37 ++++++++---------- docs/docs/getting-started/quick-start.md | 2 +- docs/docs/modules/db/commands/remote.md | 2 +- 7 files changed, 48 insertions(+), 85 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7b566b5..c88c921 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,11 +78,11 @@ Then import and call the registration function in `cli/src/index.ts`. The `db` module implements a **session-based migration workflow**: 1. **Session state**: Tracked in `.postkit/db/session.json`. Includes `remoteName` to track which remote was used, and optional `containerID` for auto Docker containers. -2. **Named remotes**: Users can configure multiple named remote databases via `db.remotes` in config: +2. **Named remotes**: Users can configure multiple named remote databases via `db.remotes` in secrets: - At least one remote must be configured - One remote can be marked as `default: true` - Managed via `postkit db remote` commands - - Metadata (name, default, addedAt) written to `postkit.config.json`; URL written to `postkit.secrets.json` + - All remote data (url, default, addedAt) stored entirely in `postkit.secrets.json` — nothing remote-related in `postkit.config.json` 3. **Binary resolution**: Both `pgschema` and `dbmate` binaries are auto-resolved: - `pgschema`: Bundled in `vendor/pgschema/` for all platforms (darwin-{arm64,amd64}, linux-{arm64,amd64}, windows-{arm64,amd64}) - `dbmate`: npm-installed via the `dbmate` package @@ -131,20 +131,15 @@ Config is loaded by `loadPostkitConfig()` from `common/config.ts`, which deep-me | File | Committed | Purpose | |------|-----------|---------| -| `postkit.config.json` | Yes | Non-sensitive settings (schema paths, remote metadata, flags) | -| `postkit.secrets.json` | No (gitignored) | Credentials (database URLs, passwords) | +| `postkit.config.json` | Yes | Non-sensitive project settings (schema paths, flags) | +| `postkit.secrets.json` | No (gitignored) | Credentials + all remote config (URLs, names, defaults) | **`postkit.config.json` (committed):** ```json { "db": { - "localDbUrl": "", "schemaPath": "db/schema", - "schema": "public", - "remotes": { - "dev": { "default": true, "addedAt": "2024-12-31T10:00:00.000Z" }, - "staging": {} - } + "schema": "public" } } ``` @@ -155,7 +150,7 @@ Config is loaded by `loadPostkitConfig()` from `common/config.ts`, which deep-me "db": { "localDbUrl": "postgres://...", "remotes": { - "dev": { "url": "postgres://..." }, + "dev": { "url": "postgres://...", "default": true, "addedAt": "2024-12-31T10:00:00.000Z" }, "staging": { "url": "postgres://..." } } } diff --git a/cli/README.md b/cli/README.md index e044399..1f5cc8a 100644 --- a/cli/README.md +++ b/cli/README.md @@ -115,23 +115,19 @@ PostKit uses two config files — run `postkit init` to generate both. ```json { "db": { - "localDbUrl": "", "schemaPath": "db/schema", - "schema": "public", - "remotes": { - "dev": { "default": true } - } + "schema": "public" } } ``` -**`postkit.secrets.json`** (gitignored — contains credentials): +**`postkit.secrets.json`** (gitignored — contains credentials and all remote config): ```json { "db": { "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", "remotes": { - "dev": { "url": "postgres://user:pass@dev-host:5432/myapp" } + "dev": { "url": "postgres://user:pass@dev-host:5432/myapp", "default": true } } } } diff --git a/cli/docs/architecture.md b/cli/docs/architecture.md index d99cb7f..79a46b8 100644 --- a/cli/docs/architecture.md +++ b/cli/docs/architecture.md @@ -72,7 +72,7 @@ Session-based migration workflow: `start → plan → apply → commit → deplo - **pgschema** — Bundled binary for schema diffing (`vendor/pgschema/`) - **dbmate** — npm-installed migration runner (`--migrations-table postkit.schema_migrations`) - **Session state** — Tracked in `.postkit/db/session.json`; includes optional `containerID` when auto Docker container is active -- **Named remotes** — Multiple remote DBs via `db.remotes` in config; URLs stored in `postkit.secrets.json`, metadata in `postkit.config.json` +- **Named remotes** — Multiple remote DBs via `db.remotes`; all remote data (url, default, addedAt) stored entirely in `postkit.secrets.json` - **Auto Docker container** — When `localDbUrl` is empty, `container.ts` starts a `postgres:{version}-alpine` container. Version is queried from remote via `SHOW server_version_num`. `pg_dump`/`psql` run inside the container via `docker exec` for version-matched tools. - **Schema directory** — User-maintained SQL files (`db/schema/`) with sections: `infra/`, `extensions/`, `types/`, `enums/`, `tables/`, `views/`, `functions/`, `triggers/`, `grants/`, `seeds/` @@ -115,29 +115,24 @@ Loaded via `loadPostkitConfig()`, which deep-merges two files: | File | Committed | Contains | |------|-----------|---------| -| `postkit.config.json` | Yes | Non-sensitive settings (schema paths, remote metadata, flags) | -| `postkit.secrets.json` | No (gitignored) | Credentials (database URLs, passwords) | +| `postkit.config.json` | Yes | Non-sensitive project settings (schema paths, flags) | +| `postkit.secrets.json` | No (gitignored) | Credentials + all remote config (URLs, names, defaults) | ```json -// postkit.config.json (committed) +// postkit.config.json (committed — no remotes) { "db": { - "localDbUrl": "", "schemaPath": "db/schema", - "schema": "public", - "remotes": { - "dev": { "default": true, "addedAt": "2024-12-31T10:00:00.000Z" }, - "staging": {} - } + "schema": "public" } } -// postkit.secrets.json (gitignored) +// postkit.secrets.json (gitignored — all remote data lives here) { "db": { "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", "remotes": { - "dev": { "url": "postgres://user:pass@dev-host:5432/myapp" }, + "dev": { "url": "postgres://user:pass@dev-host:5432/myapp", "default": true, "addedAt": "2024-12-31T10:00:00.000Z" }, "staging": { "url": "postgres://user:pass@staging-host:5432/myapp" } } } @@ -145,7 +140,6 @@ Loaded via `loadPostkitConfig()`, which deep-merges two files: ``` `localDbUrl` can be empty — PostKit will automatically start a Docker container (`postgres:{version}-alpine`) for the session. The container image version is detected from the remote database at runtime via `SHOW server_version_num`. -``` --- diff --git a/cli/docs/db.md b/cli/docs/db.md index 912b480..04b060c 100644 --- a/cli/docs/db.md +++ b/cli/docs/db.md @@ -76,41 +76,37 @@ PostKit uses two configuration files to separate non-sensitive settings from cre | File | Committed to Git | Purpose | |------|-----------------|---------| -| `postkit.config.json` | **Yes** | Schema paths, remote metadata (names, flags), non-sensitive settings | +| `postkit.config.json` | **Yes** | Schema paths, non-sensitive project settings (no remotes) | | `postkit.secrets.json` | **No** (gitignored) | Database URLs, passwords, credentials | Both files are deep-merged at load time. Use `postkit.secrets.example.json` (auto-generated by `postkit init`) as a template for team members to create their own `postkit.secrets.json`. ### `postkit.config.json` (committed) +Contains only non-sensitive project settings. Remotes are user/environment-specific and live entirely in secrets. + ```json { "db": { - "localDbUrl": "", "schemaPath": "db/schema", - "schema": "public", - "remotes": { - "dev": { - "default": true, - "addedAt": "2024-12-31T10:00:00.000Z" - }, - "staging": { - "addedAt": "2024-12-31T10:00:00.000Z" - } - } + "schema": "public" } } ``` ### `postkit.secrets.json` (gitignored) +Contains all credentials and remote configurations. + ```json { "db": { "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", "remotes": { "dev": { - "url": "postgres://user:pass@dev-host:5432/myapp" + "url": "postgres://user:pass@dev-host:5432/myapp", + "default": true, + "addedAt": "2024-12-31T10:00:00.000Z" }, "staging": { "url": "postgres://user:pass@staging-host:5432/myapp" @@ -131,29 +127,18 @@ Both files are deep-merged at load time. Use `postkit.secrets.example.json` (aut | `db.schema` | config | Database schema name | No | | `db.pgSchemaBin` | config | Path to pgschema binary | No | | `db.dbmateBin` | config | Path to dbmate binary | No | -| `db.remotes` | both | Named remote database configurations | Yes (at least one) | +| `db.remotes` | secrets | Named remote database configurations | Yes (at least one) | ### Remote Configuration -Remote metadata (name, default flag, addedAt) goes in `postkit.config.json`; the URL goes in `postkit.secrets.json`. When you run `postkit db remote add`, this split happens automatically. +All remote data lives entirely in `postkit.secrets.json` — nothing remote-related is written to `postkit.config.json`. Remotes are user/environment-specific and should not be committed. ```json -// postkit.config.json -{ - "db": { - "remotes": { - "dev": { "default": true, "addedAt": "2024-12-31T10:00:00.000Z" }, - "staging": { "addedAt": "2024-12-31T10:00:00.000Z" }, - "production": { "addedAt": "2024-12-31T10:00:00.000Z" } - } - } -} - // postkit.secrets.json { "db": { "remotes": { - "dev": { "url": "postgres://user:pass@dev-host:5432/myapp" }, + "dev": { "url": "postgres://user:pass@dev-host:5432/myapp", "default": true, "addedAt": "2024-12-31T10:00:00.000Z" }, "staging": { "url": "postgres://user:pass@staging-host:5432/myapp" }, "production": { "url": "postgres://user:pass@prod-host:5432/myapp" } } @@ -161,10 +146,8 @@ Remote metadata (name, default flag, addedAt) goes in `postkit.config.json`; the } ``` -**Properties (in `postkit.secrets.json`):** -- `url` - PostgreSQL connection URL (required — sensitive, never committed) - -**Properties (in `postkit.config.json`):** +**Remote properties (all in `postkit.secrets.json`):** +- `url` - PostgreSQL connection URL (required) - `default` - Mark as default remote (optional, one must be default) - `addedAt` - ISO timestamp when remote was added (auto-set) diff --git a/docs/docs/getting-started/configuration.md b/docs/docs/getting-started/configuration.md index 29389c3..f3e3d67 100644 --- a/docs/docs/getting-started/configuration.md +++ b/docs/docs/getting-started/configuration.md @@ -10,41 +10,36 @@ PostKit separates non-sensitive settings from credentials using two config files | File | Commit to Git | Purpose | |------|--------------|---------| -| `postkit.config.json` | **Yes** | Schema paths, remote names, non-sensitive settings | -| `postkit.secrets.json` | **No** (gitignored) | Database URLs, passwords, credentials | +| `postkit.config.json` | **Yes** | Schema paths, non-sensitive project settings | +| `postkit.secrets.json` | **No** (gitignored) | Database URLs, remotes, passwords, credentials | Both files are deep-merged at load time. `postkit init` creates all three files: the config, the secrets file, and a `postkit.secrets.example.json` template your team can use as a reference. ## `postkit.config.json` (committed) +Contains only non-sensitive project settings. Remotes are user/environment-specific and are not stored here. + ```json { "db": { - "localDbUrl": "", "schemaPath": "db/schema", - "schema": "public", - "remotes": { - "dev": { - "default": true, - "addedAt": "2024-12-31T10:00:00.000Z" - }, - "staging": { - "addedAt": "2024-12-31T10:00:00.000Z" - } - } + "schema": "public" } } ``` ## `postkit.secrets.json` (gitignored) +Contains all credentials and remote configurations. Each team member has their own copy. + ```json { "db": { "localDbUrl": "postgres://user:pass@localhost:5432/myapp_local", "remotes": { "dev": { - "url": "postgres://user:pass@dev-host:5432/myapp" + "url": "postgres://user:pass@dev-host:5432/myapp", + "default": true }, "staging": { "url": "postgres://user:pass@staging-host:5432/myapp" @@ -70,15 +65,15 @@ Database schema name. Default: `"public"`. ### `db.remotes` (required) -Named remote database configurations. At least one remote must be configured. Remote metadata (name, default flag, addedAt) goes in `postkit.config.json`; the URL goes in `postkit.secrets.json`. The `postkit db remote add` command handles this split automatically. +Named remote database configurations. At least one remote must be configured. All remote data (URL, default flag, addedAt timestamp) lives entirely in `postkit.secrets.json` — remotes are user/environment-specific and should never be committed. Use `postkit db remote add` to add remotes. -#### Remote Properties +#### Remote Properties (all in `postkit.secrets.json`) -| Property | File | Required | Description | -|----------|------|----------|-------------| -| `url` | secrets | Yes | PostgreSQL connection URL | -| `default` | config | No | Mark as default remote (one must be default) | -| `addedAt` | config | No | ISO timestamp when remote was added (auto-set) | +| Property | Required | Description | +|----------|----------|-------------| +| `url` | Yes | PostgreSQL connection URL | +| `default` | No | Mark as default remote (one must be default) | +| `addedAt` | No | ISO timestamp when remote was added (auto-set) | ## Auth Module Configuration diff --git a/docs/docs/getting-started/quick-start.md b/docs/docs/getting-started/quick-start.md index cb748ae..f4879a9 100644 --- a/docs/docs/getting-started/quick-start.md +++ b/docs/docs/getting-started/quick-start.md @@ -25,7 +25,7 @@ This creates: ## 2. Configure Remotes -Add your remote databases (URLs are stored in `postkit.secrets.json`, metadata in `postkit.config.json`): +Add your remote databases (all remote data is stored in `postkit.secrets.json` — remotes are user-specific and never committed): ```bash # Add development remote (set as default) diff --git a/docs/docs/modules/db/commands/remote.md b/docs/docs/modules/db/commands/remote.md index 679a2dc..da53670 100644 --- a/docs/docs/modules/db/commands/remote.md +++ b/docs/docs/modules/db/commands/remote.md @@ -25,7 +25,7 @@ postkit db remote list --json ### add -Add a new remote. The URL is written to `postkit.secrets.json` (gitignored); the remote name and metadata are written to `postkit.config.json` (committed). +Add a new remote. All remote data (URL, default flag, timestamp) is written to `postkit.secrets.json` (gitignored). Nothing is written to `postkit.config.json` — remotes are user-specific and should not be committed. ```bash postkit db remote add [--default] From 9ccb4fd4a11b0b25e08b07364979a42868e10a9c Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Fri, 8 May 2026 07:27:59 +0530 Subject: [PATCH 08/18] refactor: introduce resolveLocalDb to unify and simplify container startup logic across database commands --- cli/package-lock.json | 4 +- cli/src/modules/db/commands/deploy.ts | 25 ++------ cli/src/modules/db/commands/import.ts | 21 +++++- cli/src/modules/db/commands/start.ts | 12 ++-- cli/src/modules/db/services/container.ts | 33 +++++++++- .../modules/db/services/container.test.ts | 64 ++++++++++++++++++- 6 files changed, 126 insertions(+), 33 deletions(-) diff --git a/cli/package-lock.json b/cli/package-lock.json index 30df7a8..5998c46 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -1,12 +1,12 @@ { "name": "@appritech/postkit", - "version": "1.1.0", + "version": "1.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@appritech/postkit", - "version": "1.1.0", + "version": "1.2.0", "license": "Apache-2.0", "dependencies": { "chalk": "^5.3.0", diff --git a/cli/src/modules/db/commands/deploy.ts b/cli/src/modules/db/commands/deploy.ts index 5c042aa..6e2cf0b 100644 --- a/cli/src/modules/db/commands/deploy.ts +++ b/cli/src/modules/db/commands/deploy.ts @@ -2,7 +2,6 @@ import ora from "ora"; import {logger} from "../../../common/logger"; import {promptConfirm} from "../../../common/prompt"; import {getDbConfig} from "../utils/db-config"; -import type {DbConfig} from "../utils/db-config"; import {hasActiveSession, deleteSession} from "../utils/session"; import {deletePlanFile} from "../services/pgschema"; import {deleteGeneratedSchema} from "../services/schema-generator"; @@ -16,7 +15,7 @@ import { import {runCommittedMigrate, runDbmateStatus} from "../services/dbmate"; import {loadInfra, applyInfra} from "../services/infra-generator"; import {loadSeeds, applySeeds} from "../services/seed-generator"; -import {checkDockerAvailable, startSessionContainer, stopSessionContainer, cloneDatabaseViaContainer} from "../services/container"; +import {resolveLocalDb, stopSessionContainer, cloneDatabaseViaContainer} from "../services/container"; import {getPendingCommittedMigrations} from "../utils/committed"; import {resolveRemote, maskRemoteUrl, normalizeUrl} from "../utils/remotes"; import type {CommandOptions} from "../../../common/types"; @@ -42,21 +41,6 @@ function resolveTargetUrl(options: DeployOptions): {url: string; label: string} return {url: resolved.url, label: `${resolved.name} (default)`}; } -async function resolveLocalDbUrl( - config: DbConfig, - spinner: ReturnType, - remotePgVersion: number, -): Promise<{url: string; tempContainerID?: string}> { - if (config.localDbUrl) { - return {url: config.localDbUrl}; - } - spinner.start("Checking Docker availability..."); - await checkDockerAvailable(); - spinner.text = `Starting temporary postgres:${remotePgVersion}-alpine container for dry-run...`; - const container = await startSessionContainer(remotePgVersion); - spinner.succeed(`Temporary Postgres ${remotePgVersion} container started on port ${container.port}`); - return {url: container.localDbUrl, tempContainerID: container.containerID}; -} async function confirmAndRemoveSession( spinner: ReturnType, @@ -228,7 +212,12 @@ export async function deployCommand(options: DeployOptions): Promise { } // Step 3: Resolve local DB URL (spin up a container if localDbUrl is not configured) - const {url: localDbUrl, tempContainerID} = await resolveLocalDbUrl(config, spinner, remotePgVersion); + const {url: localDbUrl, containerID: tempContainerID} = await resolveLocalDb( + config.localDbUrl, + targetUrl, + spinner, + "Starting temporary container for dry-run...", + ); const cleanupLocal = async () => { try { await dropDatabase(localDbUrl); } catch { /* best effort */ } diff --git a/cli/src/modules/db/commands/import.ts b/cli/src/modules/db/commands/import.ts index f75442e..ad5eecf 100644 --- a/cli/src/modules/db/commands/import.ts +++ b/cli/src/modules/db/commands/import.ts @@ -9,6 +9,7 @@ import {getDbConfig, getTmpImportDir, getCommittedMigrationsPath, toRelativePath import {hasActiveSession} from "../utils/session"; import {addCommittedMigration, saveCommittedState} from "../utils/committed"; import {testConnection, getTableCount, createDatabase} from "../services/database"; +import {resolveLocalDb, stopSessionContainer} from "../services/container"; import {checkPgschemaInstalled, deletePlanFile} from "../services/pgschema"; import {checkDbmateInstalled, createMigrationFile, runCommittedMigrate} from "../services/dbmate"; import {deleteGeneratedSchema} from "../services/schema-generator"; @@ -41,6 +42,13 @@ export async function importCommand(options: ImportOptions): Promise { const spinner = ora(); const migrationName = options.name || "imported_baseline"; const schemaName = options.schema || "public"; + let tempContainerID: string | undefined; + + async function cleanupContainer(): Promise { + if (tempContainerID) { + try { await stopSessionContainer(tempContainerID); } catch { /* best effort */ } + } + } try { // Step 0: Check prerequisites @@ -257,9 +265,14 @@ export async function importCommand(options: ImportOptions): Promise { // Step 7: Set up local database logger.step(7, 8, "Setting up local database..."); + // Resolve local DB URL — start a Docker container if localDbUrl is not configured + const resolved = await resolveLocalDb(config.localDbUrl, targetUrl, spinner); + const localDbUrl = resolved.url; + tempContainerID = resolved.containerID; + spinner.start("Creating local database..."); try { - await createDatabase(config.localDbUrl); + await createDatabase(localDbUrl); spinner.succeed("Local database created"); } catch { spinner.warn("Local database may already exist — continuing"); @@ -268,14 +281,14 @@ export async function importCommand(options: ImportOptions): Promise { // Apply infra SQL (roles, schemas) before running migration spinner.start("Applying infrastructure SQL to local database..."); try { - await applyInfraToDatabase(config.localDbUrl, config.schemaPath); + await applyInfraToDatabase(localDbUrl, config.schemaPath); spinner.succeed("Infrastructure SQL applied"); } catch { spinner.warn("Could not apply infrastructure SQL — continuing"); } spinner.start("Applying baseline migration to local database..."); - const migrateResult = await runCommittedMigrate(config.localDbUrl); + const migrateResult = await runCommittedMigrate(localDbUrl); if (migrateResult.success) { spinner.succeed("Baseline migration applied to local database"); } else { @@ -313,6 +326,7 @@ export async function importCommand(options: ImportOptions): Promise { } await deletePlanFile(); await deleteGeneratedSchema(); + await cleanupContainer(); } // Summary @@ -330,6 +344,7 @@ export async function importCommand(options: ImportOptions): Promise { logger.info(' 3. Start working: modify schema files, then "postkit db plan" to see changes'); } catch (error) { spinner.fail("Import failed"); + await cleanupContainer(); throw error; } } diff --git a/cli/src/modules/db/commands/start.ts b/cli/src/modules/db/commands/start.ts index bb44794..a8a4058 100644 --- a/cli/src/modules/db/commands/start.ts +++ b/cli/src/modules/db/commands/start.ts @@ -15,7 +15,7 @@ import { } from "../services/database"; import {checkPgschemaInstalled} from "../services/pgschema"; import {checkDbmateInstalled, runDbmateStatus} from "../services/dbmate"; -import {checkDockerAvailable, startSessionContainer, cloneDatabaseViaContainer} from "../services/container"; +import {resolveLocalDb, cloneDatabaseViaContainer} from "../services/container"; import {getPendingCommittedMigrations} from "../utils/committed"; import type {CommandOptions} from "../../../common/types"; import {PostkitError} from "../../../common/errors"; @@ -204,13 +204,9 @@ export async function startCommand(options: StartOptions): Promise { // Step 5 (only when no localDbUrl): Start local Postgres container if (needsContainer) { logger.step(5, totalSteps, "Starting local Postgres container..."); - spinner.start("Checking Docker availability..."); - await checkDockerAvailable(); - spinner.text = `Starting postgres:${remotePgVersion}-alpine container...`; - const container = await startSessionContainer(remotePgVersion); - containerID = container.containerID; - localDbUrl = container.localDbUrl; - spinner.succeed(`Postgres ${remotePgVersion} container started on port ${container.port}`); + const resolved = await resolveLocalDb(localDbUrl, targetRemoteUrl, spinner); + containerID = resolved.containerID; + localDbUrl = resolved.url; logger.debug(`Local DB (container): ${maskConnectionUrl(localDbUrl)}`, options.verbose); } diff --git a/cli/src/modules/db/services/container.ts b/cli/src/modules/db/services/container.ts index a888a3b..722d615 100644 --- a/cli/src/modules/db/services/container.ts +++ b/cli/src/modules/db/services/container.ts @@ -1,6 +1,7 @@ import net from "net"; +import type {Ora} from "ora"; import {runCommand, runSpawnCommand, commandExists} from "../../../common/shell"; -import {testConnection, parseConnectionUrl} from "./database"; +import {testConnection, parseConnectionUrl, getRemotePgMajorVersion} from "./database"; import {runPipedCommands} from "../../../common/shell"; import {PostkitError} from "../../../common/errors"; @@ -68,6 +69,36 @@ export async function stopSessionContainer(containerID: string): Promise { await runCommand(`docker rm ${containerID}`); } +export interface ResolvedLocalDb { + url: string; + containerID?: string; +} + +/** + * Resolve the local database URL. + * If localDbUrl is already set, return it directly without touching Docker. + * If empty, detect the remote PG version, check Docker availability, and start + * a version-matched container. The caller is responsible for stopping it via containerID. + */ +export async function resolveLocalDb( + localDbUrl: string, + remoteUrl: string, + spinner: Ora, + spinnerText?: string, +): Promise { + if (localDbUrl) { + return {url: localDbUrl}; + } + spinner.start("Checking Docker availability..."); + await checkDockerAvailable(); + spinner.text = "Detecting remote PostgreSQL version..."; + const pgVersion = await getRemotePgMajorVersion(remoteUrl); + spinner.text = spinnerText ?? `Starting postgres:${pgVersion}-alpine container...`; + const container = await startSessionContainer(pgVersion); + spinner.succeed(`Postgres ${pgVersion} container started on port ${container.port}`); + return {url: container.localDbUrl, containerID: container.containerID}; +} + /** * Clone sourceUrl into the container's local database by running pg_dump and psql * *inside* the container. This guarantees the dump tools always match the remote's diff --git a/cli/test/modules/db/services/container.test.ts b/cli/test/modules/db/services/container.test.ts index f21b62f..d939bfd 100644 --- a/cli/test/modules/db/services/container.test.ts +++ b/cli/test/modules/db/services/container.test.ts @@ -9,6 +9,7 @@ vi.mock("../../../../src/common/shell", () => ({ vi.mock("../../../../src/modules/db/services/database", () => ({ testConnection: vi.fn(), + getRemotePgMajorVersion: vi.fn().mockResolvedValue(16), parseConnectionUrl: vi.fn((url: string) => { const parsed = new URL(url); return { @@ -53,12 +54,13 @@ vi.mock("net", () => { }); import {runCommand, runSpawnCommand, commandExists, runPipedCommands} from "../../../../src/common/shell"; -import {testConnection} from "../../../../src/modules/db/services/database"; +import {testConnection, getRemotePgMajorVersion} from "../../../../src/modules/db/services/database"; import { checkDockerAvailable, startSessionContainer, stopSessionContainer, cloneDatabaseViaContainer, + resolveLocalDb, } from "../../../../src/modules/db/services/container"; describe("container", () => { @@ -239,4 +241,64 @@ describe("container", () => { ).rejects.toThrow("Failed to clone database via container"); }); }); + + // ─── resolveLocalDb ─────────────────────────────────────────────────────── + + describe("resolveLocalDb()", () => { + const remoteUrl = "postgres://user:pass@remote-host:5432/mydb"; + const mockSpinner = { + start: vi.fn(), + succeed: vi.fn(), + fail: vi.fn(), + text: "", + } as any; + + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getRemotePgMajorVersion).mockResolvedValue(16); + }); + + it("returns existing URL directly without touching Docker", async () => { + const result = await resolveLocalDb( + "postgres://localhost:5432/mydb", + remoteUrl, + mockSpinner, + ); + expect(result.url).toBe("postgres://localhost:5432/mydb"); + expect(result.containerID).toBeUndefined(); + expect(commandExists).not.toHaveBeenCalled(); + expect(getRemotePgMajorVersion).not.toHaveBeenCalled(); + }); + + it("fetches PG version from remoteUrl and starts a container when localDbUrl is empty", async () => { + vi.mocked(commandExists).mockResolvedValue(true); + vi.mocked(runCommand).mockResolvedValue({stdout: "", stderr: "", exitCode: 0}); + vi.mocked(runSpawnCommand).mockResolvedValue({stdout: "newcontainer\n", stderr: "", exitCode: 0}); + vi.mocked(testConnection).mockResolvedValue(true); + + const result = await resolveLocalDb("", remoteUrl, mockSpinner); + + expect(getRemotePgMajorVersion).toHaveBeenCalledWith(remoteUrl); + expect(result.containerID).toBe("newcontainer"); + expect(result.url).toMatch(/^postgres:\/\//); + expect(result.url).toContain("localhost"); + }); + + it("propagates PostkitError when Docker is not available", async () => { + vi.mocked(commandExists).mockResolvedValue(false); + + await expect(resolveLocalDb("", remoteUrl, mockSpinner)).rejects.toThrow("Docker not found"); + }); + + it("uses custom spinnerText when provided", async () => { + vi.mocked(commandExists).mockResolvedValue(true); + vi.mocked(runCommand).mockResolvedValue({stdout: "", stderr: "", exitCode: 0}); + vi.mocked(runSpawnCommand).mockResolvedValue({stdout: "cid\n", stderr: "", exitCode: 0}); + vi.mocked(testConnection).mockResolvedValue(true); + + await resolveLocalDb("", remoteUrl, mockSpinner, "Custom spinner text"); + + expect(mockSpinner.text).toBe("Custom spinner text"); + }); + }); }); From 7031717f2d7b645b484f0e1d85266033da6f1c72 Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Fri, 8 May 2026 07:38:40 +0530 Subject: [PATCH 09/18] refactor: migrate sensitive credentials and remote configurations from postkit.config.json to postkit.secrets.json in E2E tests --- cli/test/e2e/smoke/basic-commands.test.ts | 24 ++++++++++++------- .../e2e/workflows/remote-management.test.ts | 22 ++++++++--------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/cli/test/e2e/smoke/basic-commands.test.ts b/cli/test/e2e/smoke/basic-commands.test.ts index c75ac79..219e295 100644 --- a/cli/test/e2e/smoke/basic-commands.test.ts +++ b/cli/test/e2e/smoke/basic-commands.test.ts @@ -119,17 +119,24 @@ describe("init command — detailed tests (no Docker)", () => { const config = JSON.parse( fs.readFileSync(path.join(tmpDir, "postkit.config.json"), "utf-8"), ); + const secrets = JSON.parse( + fs.readFileSync(path.join(tmpDir, "postkit.secrets.json"), "utf-8"), + ); - // DB section - expect(config.db.localDbUrl).toBe(""); + // postkit.config.json — non-sensitive project settings only (no remotes, no localDbUrl) expect(config.db.schemaPath).toBe("schema"); expect(config.db.schema).toBe("public"); - expect(config.db.remotes).toEqual({}); + expect(config.db.localDbUrl).toBeUndefined(); + expect(config.db.remotes).toBeUndefined(); - // Auth section - expect(config.auth).toBeDefined(); - expect(config.auth.source).toBeDefined(); - expect(config.auth.target).toBeDefined(); + // postkit.secrets.json — credentials and remotes + expect(secrets.db.localDbUrl).toBe(""); + expect(secrets.db.remotes).toEqual({}); + + // Auth section in secrets + expect(secrets.auth).toBeDefined(); + expect(secrets.auth.source).toBeDefined(); + expect(secrets.auth.target).toBeDefined(); } finally { await cleanupDir(tmpDir); } @@ -156,7 +163,8 @@ describe("init command — detailed tests (no Docker)", () => { const gitignore = fs.readFileSync(path.join(tmpDir, ".gitignore"), "utf-8"); expect(gitignore).toContain(".postkit/"); - expect(gitignore).toContain("postkit.config.json"); + expect(gitignore).toContain("postkit.secrets.json"); + expect(gitignore).not.toContain("postkit.config.json"); } finally { await cleanupDir(tmpDir); } diff --git a/cli/test/e2e/workflows/remote-management.test.ts b/cli/test/e2e/workflows/remote-management.test.ts index 3552eb8..0e591bb 100644 --- a/cli/test/e2e/workflows/remote-management.test.ts +++ b/cli/test/e2e/workflows/remote-management.test.ts @@ -60,11 +60,11 @@ describe("Remote management", () => { ); expect(result.exitCode).toBe(0); - const config = await readJson<{ + const secrets = await readJson<{ db: {remotes: Record}; - }>(project, "postkit.config.json"); - expect(config.db.remotes.prod).toBeDefined(); - expect(config.db.remotes.prod?.default).toBe(true); + }>(project, "postkit.secrets.json"); + expect(secrets.db.remotes.prod).toBeDefined(); + expect(secrets.db.remotes.prod?.default).toBe(true); }); it("sets default remote with 'use'", async () => { @@ -87,11 +87,11 @@ describe("Remote management", () => { }); expect(result.exitCode).toBe(0); - const config = await readJson<{ + const secrets = await readJson<{ db: {remotes: Record}; - }>(project, "postkit.config.json"); - expect(config.db.remotes.staging).toBeDefined(); - expect(config.db.remotes.staging?.default).toBe(true); + }>(project, "postkit.secrets.json"); + expect(secrets.db.remotes.staging).toBeDefined(); + expect(secrets.db.remotes.staging?.default).toBe(true); }); it("removes a remote with --force", async () => { @@ -114,9 +114,9 @@ describe("Remote management", () => { }); expect(result.exitCode).toBe(0); - const config = await readJson<{ + const secrets = await readJson<{ db: {remotes: Record}; - }>(project, "postkit.config.json"); - expect(config.db.remotes.staging).toBeUndefined(); + }>(project, "postkit.secrets.json"); + expect(secrets.db.remotes.staging).toBeUndefined(); }); }); From 026e2c6900b6f702c6c54a50e0701af844e81562 Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Fri, 8 May 2026 07:48:30 +0530 Subject: [PATCH 10/18] refactor: update gitignore strategy to track migrations and auth state while ignoring ephemeral .postkit files --- CLAUDE.md | 23 ++++++++------- cli/src/commands/init.ts | 24 ++++++++++++---- cli/src/modules/db/commands/import.ts | 28 +++++++++++-------- .../modules/db/services/schema-importer.ts | 7 ++--- cli/test/e2e/smoke/basic-commands.test.ts | 11 +++++++- 5 files changed, 61 insertions(+), 32 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c88c921..bbba0ac 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -100,17 +100,20 @@ The `db` module implements a **session-based migration workflow**: ### PostKit Directory Structure -All PostKit runtime files are stored in `.postkit/` (gitignored): +PostKit files are split between committed (shared with team) and gitignored (user-specific/ephemeral): ``` .postkit/ -└── db/ - ├── session.json # Current session state - ├── committed.json # Committed migrations tracking - ├── plan.sql # Generated migration plan - ├── schema.sql # Generated schema from files - ├── session/ # Session migrations (temporary) - └── migrations/ # Committed migrations (for deploy) +├── db/ +│ ├── session.json # GITIGNORED — active session state, local DB URL, container ID +│ ├── plan.sql # GITIGNORED — generated migration diff (ephemeral) +│ ├── schema.sql # GITIGNORED — generated schema artifact (ephemeral) +│ ├── session/ # GITIGNORED — temporary in-progress migrations +│ ├── committed.json # COMMITTED — migration tracking index (shared) +│ └── migrations/ # COMMITTED — committed SQL migrations for deploy (shared) +└── auth/ + ├── raw/ # COMMITTED — auth raw config (shared) + └── realm/ # COMMITTED — auth realm config (shared) ``` **Key paths** (from `modules/db/utils/db-config.ts`): @@ -251,8 +254,8 @@ logger.debug(`Remote URL: ${maskRemoteUrl(url)}`, options.verbose); - All paths in `common/config.ts` are resolved relative to either `cliRoot` (the CLI installation) or `projectRoot` (where the user runs commands). - Session files in `.postkit/db/` track migration state and enable resume capability. - The `vendor/` directory contains platform-specific binaries that are bundled with the CLI - no separate installation required. -- The `.gitignore` should include `.postkit/` to ignore all runtime files. -- All migration-related files are in `.postkit/db/` - the only user-maintained DB files should be in `db/schema/`. +- The `.gitignore` includes specific ephemeral paths (session.json, plan.sql, schema.sql, session/) — NOT the whole `.postkit/` directory. Committed migrations and auth state ARE tracked by git. +- All migration-related files are in `.postkit/db/` — the only user-maintained DB files should be in `db/schema/`. ## Claude Code Skills & Agents diff --git a/cli/src/commands/init.ts b/cli/src/commands/init.ts index 8238e2e..9045262 100644 --- a/cli/src/commands/init.ts +++ b/cli/src/commands/init.ts @@ -16,11 +16,14 @@ import { import type {CommandOptions} from "../common/types"; import type {PostkitPublicConfig, PostkitSecrets} from "../common/config"; -// Only .postkit/ and the secrets file are gitignored. +// Ephemeral/user-specific files are gitignored; committed migrations and auth state are tracked. // postkit.config.json is safe to commit. const GITIGNORE_ENTRIES = [ "# Postkit", - ".postkit/", + ".postkit/db/session.json", + ".postkit/db/plan.sql", + ".postkit/db/schema.sql", + ".postkit/db/session/", "postkit.secrets.json", ]; @@ -207,10 +210,19 @@ export async function initCommand(options: CommandOptions): Promise { logger.blank(); logger.success("Postkit project initialized!"); logger.blank(); - logger.info("Config split:"); - logger.info(` ${POSTKIT_CONFIG_FILE} — committed to git (schema paths, project settings)`); - logger.info(` ${POSTKIT_SECRETS_FILE} — gitignored (DB URLs, remotes, passwords)`); - logger.info(` postkit.secrets.example.json — committed template for teammates`); + logger.info("What gets committed to git:"); + logger.info(` ${POSTKIT_CONFIG_FILE} — schema paths, project settings`); + logger.info(` postkit.secrets.example.json — secrets template for teammates`); + logger.info(` .postkit/db/migrations/ — committed migration SQL files`); + logger.info(` .postkit/db/committed.json — migration tracking index`); + logger.info(` .postkit/auth/ — auth realm and raw config`); + logger.blank(); + logger.info("What is gitignored:"); + logger.info(` ${POSTKIT_SECRETS_FILE} — DB URLs, remotes, passwords`); + logger.info(` .postkit/db/session.json — active session state`); + logger.info(` .postkit/db/plan.sql — generated diff (ephemeral)`); + logger.info(` .postkit/db/schema.sql — generated schema (ephemeral)`); + logger.info(` .postkit/db/session/ — temporary session migrations`); logger.blank(); logger.info("Next steps:"); logger.info(` 1. Fill in ${POSTKIT_SECRETS_FILE} with your database credentials`); diff --git a/cli/src/modules/db/commands/import.ts b/cli/src/modules/db/commands/import.ts index ad5eecf..28ae88b 100644 --- a/cli/src/modules/db/commands/import.ts +++ b/cli/src/modules/db/commands/import.ts @@ -217,14 +217,25 @@ export async function importCommand(options: ImportOptions): Promise { } } - // Step 5: Generate baseline migration using pgschema plan - logger.step(6, 8, "Generating baseline migration..."); + // Step 5: Resolve local DB URL — start a Docker container if localDbUrl is not configured + // (must happen before generateBaselineDDL, which needs a live Postgres to create a temp DB) + logger.step(6, 8, "Setting up local database..."); + + let localDbUrl = config.localDbUrl; + if (!options.dryRun) { + const resolved = await resolveLocalDb(config.localDbUrl, targetUrl, spinner); + localDbUrl = resolved.url; + tempContainerID = resolved.containerID; + } + + // Step 6: Generate baseline migration using pgschema plan + logger.step(7, 8, "Generating baseline migration..."); if (options.dryRun) { spinner.info("Dry run — skipping baseline generation"); } else { spinner.start("Generating baseline DDL via pgschema plan..."); - const baselineDDL = await generateBaselineDDL(config.schemaPath, schemaName); + const baselineDDL = await generateBaselineDDL(config.schemaPath, schemaName, localDbUrl); spinner.succeed("Baseline DDL generated"); // Clear migrations directory and reset committed state before creating baseline migration @@ -262,13 +273,8 @@ export async function importCommand(options: ImportOptions): Promise { committedAt: new Date().toISOString(), }); - // Step 7: Set up local database - logger.step(7, 8, "Setting up local database..."); - - // Resolve local DB URL — start a Docker container if localDbUrl is not configured - const resolved = await resolveLocalDb(config.localDbUrl, targetUrl, spinner); - const localDbUrl = resolved.url; - tempContainerID = resolved.containerID; + // Step 7: Apply to local database + logger.step(8, 8, "Applying to local database..."); spinner.start("Creating local database..."); try { @@ -296,7 +302,7 @@ export async function importCommand(options: ImportOptions): Promise { logger.warn(` ${migrateResult.output}`); } - // Step 8: Sync migration state with source database (only after successful local apply) + // Sync migration state with source database (only after successful local apply) logger.step(8, 8, "Syncing migration state..."); spinner.start("Inserting migration tracking record..."); diff --git a/cli/src/modules/db/services/schema-importer.ts b/cli/src/modules/db/services/schema-importer.ts index c8c08e2..90ef2ae 100644 --- a/cli/src/modules/db/services/schema-importer.ts +++ b/cli/src/modules/db/services/schema-importer.ts @@ -446,11 +446,10 @@ export async function applyInfraToDatabase(databaseUrl: string, schemaPath: stri export async function generateBaselineDDL( schemaPath: string, schemaName: string, + localDbUrl: string, ): Promise { - const config = getDbConfig(); - - // Construct a temp database URL based on localDbUrl - const localInfo = parseConnectionUrl(config.localDbUrl); + // Construct a temp database URL based on the resolved localDbUrl + const localInfo = parseConnectionUrl(localDbUrl); const tmpDbName = `postkit_import_${Date.now()}`; const tmpDbUrl = `postgres://${localInfo.user}:${encodeURIComponent(localInfo.password)}@${localInfo.host}:${localInfo.port}/${tmpDbName}`; diff --git a/cli/test/e2e/smoke/basic-commands.test.ts b/cli/test/e2e/smoke/basic-commands.test.ts index 219e295..57f8d30 100644 --- a/cli/test/e2e/smoke/basic-commands.test.ts +++ b/cli/test/e2e/smoke/basic-commands.test.ts @@ -106,6 +106,7 @@ describe("init command — detailed tests (no Docker)", () => { expect(fs.existsSync(path.join(tmpDir, ".postkit", "db", "committed.json"))).toBe(true); expect(fs.existsSync(path.join(tmpDir, ".postkit", "db", "plan.sql"))).toBe(true); expect(fs.existsSync(path.join(tmpDir, ".postkit", "db", "schema.sql"))).toBe(true); + } finally { await cleanupDir(tmpDir); } @@ -162,9 +163,17 @@ describe("init command — detailed tests (no Docker)", () => { await runCli(["init", "--force"], {cwd: tmpDir}); const gitignore = fs.readFileSync(path.join(tmpDir, ".gitignore"), "utf-8"); - expect(gitignore).toContain(".postkit/"); + // Ephemeral/session-specific paths are gitignored + expect(gitignore).toContain(".postkit/db/session.json"); + expect(gitignore).toContain(".postkit/db/plan.sql"); + expect(gitignore).toContain(".postkit/db/schema.sql"); + expect(gitignore).toContain(".postkit/db/session/"); expect(gitignore).toContain("postkit.secrets.json"); + // Committed files must NOT be gitignored expect(gitignore).not.toContain("postkit.config.json"); + expect(gitignore).not.toContain(".postkit/db/migrations"); + expect(gitignore).not.toContain(".postkit/db/committed.json"); + expect(gitignore).not.toContain(".postkit/auth"); } finally { await cleanupDir(tmpDir); } From dcc817d50194e362ae640ee75ed2b2d4c93a2933 Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Fri, 8 May 2026 08:17:33 +0530 Subject: [PATCH 11/18] feat: implement auto-container mode for local database provisioning in CLI tests --- cli/test/e2e/helpers/test-project.ts | 17 +- cli/test/e2e/workflows/auto-container.test.ts | 185 ++++++++++++++++++ 2 files changed, 189 insertions(+), 13 deletions(-) create mode 100644 cli/test/e2e/workflows/auto-container.test.ts diff --git a/cli/test/e2e/helpers/test-project.ts b/cli/test/e2e/helpers/test-project.ts index 54755dd..02104fe 100644 --- a/cli/test/e2e/helpers/test-project.ts +++ b/cli/test/e2e/helpers/test-project.ts @@ -13,7 +13,7 @@ export interface TestProject { } export interface CreateTestProjectOptions { - localDbUrl: string; + localDbUrl?: string; // omit or pass "" for auto-Docker mode remoteDbUrl?: string; remoteName?: string; schemaPath?: string; @@ -46,24 +46,15 @@ export async function createTestProject( // Ensure schema directory exists (init doesn't create it) await fs.mkdir(schemaPath, {recursive: true}); - // Patch the public config: add remote name/metadata (no URLs) + // Patch secrets: all credentials and remote data live exclusively in postkit.secrets.json const secretsPath = path.join(rootDir, "postkit.secrets.json"); const remoteName = config.remoteName ?? "test-remote"; - if (config.remoteDbUrl) { - const existingConfig = JSON.parse(await fs.readFile(configPath, "utf-8")); - existingConfig.db.remotes = { - [remoteName]: {default: true, addedAt: new Date().toISOString()}, - }; - await fs.writeFile(configPath, JSON.stringify(existingConfig, null, 2)); - } - - // Patch secrets: write localDbUrl and remote URLs const existingSecrets = JSON.parse(await fs.readFile(secretsPath, "utf-8")); - existingSecrets.db.localDbUrl = config.localDbUrl; + existingSecrets.db.localDbUrl = config.localDbUrl ?? ""; if (config.remoteDbUrl) { existingSecrets.db.remotes = { - [remoteName]: {url: config.remoteDbUrl}, + [remoteName]: {url: config.remoteDbUrl, default: true, addedAt: new Date().toISOString()}, }; } await fs.writeFile(secretsPath, JSON.stringify(existingSecrets, null, 2)); diff --git a/cli/test/e2e/workflows/auto-container.test.ts b/cli/test/e2e/workflows/auto-container.test.ts new file mode 100644 index 0000000..845d006 --- /dev/null +++ b/cli/test/e2e/workflows/auto-container.test.ts @@ -0,0 +1,185 @@ +import {execSync} from "child_process"; +import fs from "fs"; +import path from "path"; +import {describe, it, expect, beforeAll, afterAll} from "vitest"; +import {runCli} from "../helpers/cli-runner"; +import {createTestProject, cleanupTestProject, type TestProject} from "../helpers/test-project"; +import {startPostgres, stopPostgres, type TestDatabase} from "../helpers/test-database"; +import {executeSql} from "../helpers/db-query"; + +// Check Docker availability once at module load time so tests are skipped +// cleanly when Docker is not installed or not running. +function isDockerAvailable(): boolean { + try { + execSync("docker info", {stdio: "ignore"}); + return true; + } catch { + return false; + } +} + +const dockerAvailable = isDockerAvailable(); + +// Minimal schema to seed the "remote" database with +const SEED_SQL = ` + CREATE TABLE public.item ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name VARCHAR(100) NOT NULL + ); + INSERT INTO public.item (name) VALUES ('alpha'), ('beta'); +`; + +/** + * Auto-container: db import with empty localDbUrl + * + * When localDbUrl is empty in postkit.secrets.json, resolveLocalDb() should + * automatically start a postgres:{version}-alpine Docker container, use it + * for the import, and clean it up when the command finishes. + * + * Network note: import runs pg_dump on the HOST (not inside Docker), so + * testcontainer remote DBs are accessible without Docker-in-Docker issues. + */ +describe.skipIf(!dockerAvailable)( + "auto-container — db import with empty localDbUrl", + () => { + let remoteDb: TestDatabase; + let project: TestProject; + + beforeAll(async () => { + remoteDb = await startPostgres(); + await executeSql(remoteDb.url, SEED_SQL); + + // No localDbUrl — PostKit must auto-start a Docker container + project = await createTestProject({ + remoteDbUrl: remoteDb.url, + remoteName: "dev", + // localDbUrl intentionally omitted + }); + }, 120_000); + + afterAll(async () => { + if (project) await cleanupTestProject(project); + if (remoteDb) await stopPostgres(remoteDb); + }); + + it("import exits 0 and reports completion", async () => { + const result = await runCli( + ["db", "import", "--force", "--name", "auto_container_baseline", "--url", remoteDb.url], + {cwd: project.rootDir, timeout: 180_000}, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("import complete"); + }, 180_000); + + it("committed.json has exactly one baseline migration", () => { + const committed = JSON.parse( + fs.readFileSync(path.join(project.dbDir, "committed.json"), "utf-8"), + ); + expect(committed.migrations).toHaveLength(1); + expect(committed.migrations[0].description).toContain("Baseline import"); + }); + + it("baseline migration SQL file exists and contains CREATE TABLE", () => { + const migrationsDir = path.join(project.dbDir, "migrations"); + const files = fs.readdirSync(migrationsDir).filter((f) => f.endsWith(".sql")); + expect(files.length).toBeGreaterThan(0); + const content = fs.readFileSync(path.join(migrationsDir, files[0]!), "utf-8"); + expect(content).toContain("CREATE TABLE"); + expect(content).toContain("item"); + }); + + it("schema files were created for the imported table", () => { + const tablesDir = path.join(project.schemaPath, "tables"); + expect(fs.existsSync(tablesDir)).toBe(true); + const files = fs.readdirSync(tablesDir).filter((f) => f.endsWith(".sql")); + expect(files.length).toBeGreaterThan(0); + }); + + it("import cleaned up its temporary Docker container", () => { + // PostKit names its session containers with a predictable pattern. + // After import completes, no postkit_local containers should be running. + const output = execSync( + 'docker ps --filter "name=postkit_local" --format "{{.Names}}"', + {encoding: "utf-8"}, + ).trim(); + expect(output).toBe(""); + }); + + it("ephemeral artifacts are cleaned up (plan.sql, schema.sql)", () => { + const dbDir = project.dbDir; + const planSql = path.join(dbDir, "plan.sql"); + const schemaSql = path.join(dbDir, "schema.sql"); + // Either absent or empty — both mean cleaned up + const planContent = fs.existsSync(planSql) + ? fs.readFileSync(planSql, "utf-8").trim() + : ""; + const schemaContent = fs.existsSync(schemaSql) + ? fs.readFileSync(schemaSql, "utf-8").trim() + : ""; + expect(planContent).toBe(""); + expect(schemaContent).toBe(""); + }); + }, +); + +/** + * Auto-container: db start with empty localDbUrl + * + * Network limitation: `db start` clones the remote DB by running pg_dump + * *inside* the auto-started Docker container. When the remote is a + * testcontainer (bound to localhost), it is not reachable from inside another + * Docker container. Therefore we only verify that: + * 1. PostKit reaches the Docker step (output mentions container) + * 2. The failure is a clone/network error, NOT a "Docker not found" error + * + * Full happy-path coverage for `start` auto-container requires a remote DB + * accessible inside Docker (e.g. a service in the same Docker network). + */ +describe.skipIf(!dockerAvailable)( + "auto-container — db start with empty localDbUrl (partial: Docker step reached)", + () => { + let remoteDb: TestDatabase; + let project: TestProject; + + beforeAll(async () => { + remoteDb = await startPostgres(); + await executeSql( + remoteDb.url, + `CREATE TABLE public.item (id UUID PRIMARY KEY DEFAULT gen_random_uuid(), name VARCHAR(100) NOT NULL);`, + ); + + project = await createTestProject({ + remoteDbUrl: remoteDb.url, + remoteName: "dev", + // localDbUrl intentionally omitted + }); + }, 120_000); + + afterAll(async () => { + // Abort any lingering session so cleanup is clean + await runCli(["db", "abort", "--force"], {cwd: project.rootDir}).catch(() => {}); + if (project) await cleanupTestProject(project); + if (remoteDb) await stopPostgres(remoteDb); + }); + + it("start reaches the Docker/container step (not a Docker-unavailable error)", async () => { + const result = await runCli(["db", "start", "--force"], { + cwd: project.rootDir, + timeout: 120_000, + }); + + // Should NOT fail with "Docker not found" — PostKit found Docker fine + expect(result.stdout + result.stderr).not.toContain("Docker not found"); + expect(result.stdout + result.stderr).not.toContain("Docker is not running"); + + // Either fully succeeded (unlikely with localhost remote inside Docker) + // or failed at the clone step (expected with testcontainer network isolation) + if (result.exitCode === 0) { + expect(result.stdout).toContain("Migration session started"); + } else { + // Acceptable failure: clone failed due to network, not Docker setup + expect(result.stdout + result.stderr).toMatch(/clone|pg_dump|connect/i); + } + }, 120_000); + }, +); From 37882c63c94a2ed83ae924aeef33bdbfbb8199a4 Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Fri, 8 May 2026 08:36:45 +0530 Subject: [PATCH 12/18] chore: remove local Claude settings file --- cli/.claude/settings.local.json | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 cli/.claude/settings.local.json diff --git a/cli/.claude/settings.local.json b/cli/.claude/settings.local.json deleted file mode 100644 index f56755b..0000000 --- a/cli/.claude/settings.local.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(npm install:*)", - "Bash(npx vitest:*)", - "Bash(npm run:*)", - "Bash(node:*)", - "Read(//Users/supunnilakshana/development/appri/postkit/Postkit/test-proj/schema/**)", - "Read(//Users/supunnilakshana/development/appri/postkit/Postkit/**)", - "Bash(for dir:*)", - "Bash(do echo:*)", - "Read(//Users/supunnilakshana/development/appri/postkit/Postkit/cli/**)", - "Bash(done)", - "Bash(docker run:*)" - ] - } -} From 12046bde2f74390a50dd4d08414aeb0afbb8a354 Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Fri, 8 May 2026 09:31:15 +0530 Subject: [PATCH 13/18] refactor: simplify CLI session handling and error reporting by centralizing utility functions and standardizing deployment steps. --- cli/src/modules/auth/commands/export.ts | 3 +- cli/src/modules/auth/commands/import.ts | 3 +- cli/src/modules/auth/commands/sync.ts | 3 +- cli/src/modules/db/commands/apply.ts | 104 ++---------------- cli/src/modules/db/commands/commit.ts | 12 +- cli/src/modules/db/commands/deploy.ts | 25 +---- cli/src/modules/db/commands/import.ts | 41 ++----- cli/src/modules/db/commands/infra.ts | 44 ++------ cli/src/modules/db/commands/migration.ts | 34 +----- cli/src/modules/db/commands/plan.ts | 29 +---- cli/src/modules/db/commands/remote.ts | 12 +- cli/src/modules/db/commands/seed.ts | 44 ++------ cli/src/modules/db/commands/start.ts | 37 +------ cli/src/modules/db/services/database.ts | 82 +++++--------- .../modules/db/services/infra-generator.ts | 12 ++ cli/src/modules/db/services/prerequisites.ts | 25 +++++ .../modules/db/services/schema-generator.ts | 63 ----------- .../modules/db/services/schema-importer.ts | 47 ++------ cli/src/modules/db/services/seed-generator.ts | 21 ++++ cli/src/modules/db/utils/apply-target.ts | 35 ++++++ cli/src/modules/db/utils/committed.ts | 37 +++---- cli/src/modules/db/utils/json-file.ts | 10 ++ cli/src/modules/db/utils/remotes.ts | 53 ++++----- cli/src/modules/db/utils/session.ts | 37 ++++++- .../e2e/workflows/remote-management.test.ts | 95 +++++++--------- .../db/services/schema-generator.test.ts | 12 +- cli/test/modules/db/utils/session.test.ts | 1 - 27 files changed, 322 insertions(+), 599 deletions(-) create mode 100644 cli/src/modules/db/services/prerequisites.ts create mode 100644 cli/src/modules/db/utils/apply-target.ts create mode 100644 cli/src/modules/db/utils/json-file.ts diff --git a/cli/src/modules/auth/commands/export.ts b/cli/src/modules/auth/commands/export.ts index ed37978..d35c1f3 100644 --- a/cli/src/modules/auth/commands/export.ts +++ b/cli/src/modules/auth/commands/export.ts @@ -79,7 +79,6 @@ export async function exportCommand(options: CommandOptions): Promise { logger.info(`Clean → ${config.cleanFilePath}`); } catch (error) { spinner.fail("Export failed"); - logger.error(error instanceof Error ? error.message : String(error)); - process.exit(1); + throw error; } } diff --git a/cli/src/modules/auth/commands/import.ts b/cli/src/modules/auth/commands/import.ts index 156e433..1e9f8af 100644 --- a/cli/src/modules/auth/commands/import.ts +++ b/cli/src/modules/auth/commands/import.ts @@ -45,7 +45,6 @@ export async function importCommand(options: CommandOptions): Promise { logger.success("Import complete!"); } catch (error) { spinner.fail("Import failed"); - logger.error(error instanceof Error ? error.message : String(error)); - process.exit(1); + throw error; } } diff --git a/cli/src/modules/auth/commands/sync.ts b/cli/src/modules/auth/commands/sync.ts index e5cea39..b8b3e2f 100644 --- a/cli/src/modules/auth/commands/sync.ts +++ b/cli/src/modules/auth/commands/sync.ts @@ -19,7 +19,6 @@ export async function syncCommand(options: CommandOptions): Promise { logger.blank(); logger.success("Sync complete! Realm exported and imported successfully."); } catch (error) { - logger.error(error instanceof Error ? error.message : String(error)); - process.exit(1); + throw error; } } diff --git a/cli/src/modules/db/commands/apply.ts b/cli/src/modules/db/commands/apply.ts index e585fb7..f089b8f 100644 --- a/cli/src/modules/db/commands/apply.ts +++ b/cli/src/modules/db/commands/apply.ts @@ -4,72 +4,26 @@ import fs from "fs/promises"; import {promptConfirm, promptInput} from "../../../common/prompt"; import {existsSync} from "fs"; import {logger} from "../../../common/logger"; -import {getSession, updatePendingChanges} from "../utils/session"; +import {requireActiveSession, assertLocalConnection, updatePendingChanges} from "../utils/session"; import {getSessionMigrationsPath, toRelativePath, resolveProjectPath} from "../utils/db-config"; import {wrapPlanSQL, getPlanFileContent} from "../services/pgschema"; -import {testConnection} from "../services/database"; import { createMigrationFile, runSessionMigrate, deleteMigrationFile, } from "../services/dbmate"; -import {generateSchemaFingerprint} from "../services/schema-generator"; -import {applyInfra, loadInfra} from "../services/infra-generator"; -import {applySeeds, loadSeeds} from "../services/seed-generator"; +import {generateSchemaSQLAndFingerprint} from "../services/schema-generator"; +import {applyInfraStep} from "../services/infra-generator"; +import {applySeedsStep} from "../services/seed-generator"; import type {CommandOptions} from "../../../common/types"; import type {SessionState} from "../types/index"; import {PostkitError} from "../../../common/errors"; -async function applyInfraStep( - spinner: ReturnType, - dbUrl: string, -): Promise { - const infra = await loadInfra(); - if (infra.length === 0) { - spinner.info("No infra files found - skipping"); - return; - } - spinner.start("Applying infra..."); - await applyInfra(dbUrl); - spinner.succeed(`Infra applied (${infra.length} file(s))`); -} - -async function applySeedsStep( - spinner: ReturnType, - dbUrl: string, - retryHint: string, -): Promise { - const seeds = await loadSeeds(); - if (seeds.length === 0) { - spinner.info("No seed files found - skipping"); - return; - } - try { - spinner.start("Applying seed data..."); - await applySeeds(dbUrl); - spinner.succeed(`Seeds applied (${seeds.length} file(s))`); - } catch (error) { - spinner.fail("Failed to apply seeds"); - throw new PostkitError( - `Seeds failed: ${error instanceof Error ? error.message : String(error)}`, - retryHint, - ); - } -} - export async function applyCommand(options: CommandOptions): Promise { const spinner = ora(); try { - // Check for active session - const session = await getSession(); - - if (!session || !session.active) { - throw new PostkitError( - "No active migration session.", - 'Run "postkit db start" to begin a new session.', - ); - } + const session = await requireActiveSession(); // Confirm apply operation (unless force flag) const confirmed = await promptConfirm( @@ -206,11 +160,7 @@ async function handleResume( // Seeds if (!pc.seedsApplied) { logger.step(step, totalSteps, "Applying seeds..."); - await applySeedsStep( - spinner, - session.localDbUrl, - 'Run "postkit db apply" again to retry from seeds.', - ); + await applySeedsStep(spinner, session.localDbUrl); await updatePendingChanges({seedsApplied: true}); } else { logger.step(step, totalSteps, "Seeds already applied - skipping"); @@ -281,7 +231,7 @@ async function handlePlanApply( // Plan-based migration flow (original logic) // Validate schema fingerprint if (session.pendingChanges.schemaFingerprint) { - const currentFingerprint = await generateSchemaFingerprint(); + const {fingerprint: currentFingerprint} = await generateSchemaSQLAndFingerprint(); if (currentFingerprint !== session.pendingChanges.schemaFingerprint) { throw new PostkitError( @@ -312,19 +262,7 @@ async function handlePlanApply( // Step 2: Test local connection logger.step(2, 7, "Testing local database connection..."); - spinner.start("Connecting to local database..."); - - const localConnected = await testConnection(session.localDbUrl); - - if (!localConnected) { - spinner.fail("Failed to connect to local database"); - throw new PostkitError( - "Could not connect to the local database.", - 'The local clone may have been removed. Run "postkit db start" again.', - ); - } - - spinner.succeed("Connected to local database"); + await assertLocalConnection(session, spinner); // Step 3: Apply infra (roles, schemas, extensions) logger.step(3, 7, "Applying infrastructure..."); @@ -395,11 +333,7 @@ async function handlePlanApply( // Step 6: Apply seeds logger.step(6, 7, "Applying seeds..."); - await applySeedsStep( - spinner, - session.localDbUrl, - 'Migration is already applied. Run "postkit db apply" again to retry from seeds.', - ); + await applySeedsStep(spinner, session.localDbUrl); await updatePendingChanges({seedsApplied: true}); // Step 7: Mark fully applied and clean up plan file @@ -467,19 +401,7 @@ async function handleManualApply( // Step 1: Test local connection logger.step(1, 4, "Testing local database connection..."); - spinner.start("Connecting to local database..."); - - const localConnected = await testConnection(session.localDbUrl); - - if (!localConnected) { - spinner.fail("Failed to connect to local database"); - throw new PostkitError( - "Could not connect to the local database.", - 'The local clone may have been removed. Run "postkit db start" again.', - ); - } - - spinner.succeed("Connected to local database"); + await assertLocalConnection(session, spinner); // Step 2: Apply infra logger.step(2, 4, "Applying infrastructure..."); @@ -518,11 +440,7 @@ async function handleManualApply( // Step 4: Apply seeds logger.step(4, 4, "Applying seeds..."); - await applySeedsStep( - spinner, - session.localDbUrl, - 'Migration(s) are already applied. Run "postkit db apply" again to retry from seeds.', - ); + await applySeedsStep(spinner, session.localDbUrl); await updatePendingChanges({seedsApplied: true, applied: true}); logger.blank(); diff --git a/cli/src/modules/db/commands/commit.ts b/cli/src/modules/db/commands/commit.ts index ee49ed9..c068dce 100644 --- a/cli/src/modules/db/commands/commit.ts +++ b/cli/src/modules/db/commands/commit.ts @@ -1,7 +1,7 @@ import ora from "ora"; import {logger} from "../../../common/logger"; import {promptInput} from "../../../common/prompt"; -import {getSession, deleteSession} from "../utils/session"; +import {requireActiveSession, deleteSession} from "../utils/session"; import {getSessionMigrationsPath, toRelativePath} from "../utils/db-config"; import {mergeSessionMigrations, deleteSessionMigrations} from "../services/dbmate"; import {deletePlanFile} from "../services/pgschema"; @@ -19,15 +19,7 @@ export async function commitCommand(options: CommitOptions): Promise { const spinner = ora(); try { - // Check for active session - const session = await getSession(); - - if (!session || !session.active) { - throw new PostkitError( - "No active migration session.", - 'Run "postkit db start" to begin a new session.', - ); - } + const session = await requireActiveSession(); if (!session.pendingChanges.applied) { throw new PostkitError( diff --git a/cli/src/modules/db/commands/deploy.ts b/cli/src/modules/db/commands/deploy.ts index 6e2cf0b..8d4f286 100644 --- a/cli/src/modules/db/commands/deploy.ts +++ b/cli/src/modules/db/commands/deploy.ts @@ -13,8 +13,8 @@ import { getRemotePgMajorVersion, } from "../services/database"; import {runCommittedMigrate, runDbmateStatus} from "../services/dbmate"; -import {loadInfra, applyInfra} from "../services/infra-generator"; -import {loadSeeds, applySeeds} from "../services/seed-generator"; +import {applyInfraStep} from "../services/infra-generator"; +import {applySeedsStep} from "../services/seed-generator"; import {resolveLocalDb, stopSessionContainer, cloneDatabaseViaContainer} from "../services/container"; import {getPendingCommittedMigrations} from "../utils/committed"; import {resolveRemote, maskRemoteUrl, normalizeUrl} from "../utils/remotes"; @@ -74,16 +74,7 @@ async function runSteps( // Infra logger.step(step, totalSteps, `Applying infra to ${label}...`); - const infra = await loadInfra(); - - if (infra.length === 0) { - spinner.info("No infra files found - skipping"); - } else { - spinner.start(`Applying infra to ${label}...`); - await applyInfra(dbUrl); - spinner.succeed(`Infra applied to ${label} (${infra.length} file(s))`); - } - + await applyInfraStep(spinner, dbUrl, label); step++; // Dbmate migrate @@ -101,15 +92,7 @@ async function runSteps( // Seeds logger.step(step, totalSteps, `Applying seeds to ${label}...`); - const seeds = await loadSeeds(); - - if (seeds.length === 0) { - spinner.info("No seed files found - skipping"); - } else { - spinner.start(`Applying seeds to ${label}...`); - await applySeeds(dbUrl); - spinner.succeed(`Seeds applied to ${label} (${seeds.length} file(s))`); - } + await applySeedsStep(spinner, dbUrl, label); } export async function deployCommand(options: DeployOptions): Promise { diff --git a/cli/src/modules/db/commands/import.ts b/cli/src/modules/db/commands/import.ts index 28ae88b..78db262 100644 --- a/cli/src/modules/db/commands/import.ts +++ b/cli/src/modules/db/commands/import.ts @@ -7,11 +7,13 @@ import {promptConfirm} from "../../../common/prompt"; import {PostkitError} from "../../../common/errors"; import {getDbConfig, getTmpImportDir, getCommittedMigrationsPath, toRelativePath} from "../utils/db-config"; import {hasActiveSession} from "../utils/session"; +import {maskRemoteUrl} from "../utils/remotes"; import {addCommittedMigration, saveCommittedState} from "../utils/committed"; import {testConnection, getTableCount, createDatabase} from "../services/database"; import {resolveLocalDb, stopSessionContainer} from "../services/container"; -import {checkPgschemaInstalled, deletePlanFile} from "../services/pgschema"; -import {checkDbmateInstalled, createMigrationFile, runCommittedMigrate} from "../services/dbmate"; +import {deletePlanFile} from "../services/pgschema"; +import {createMigrationFile, runCommittedMigrate} from "../services/dbmate"; +import {checkDbPrerequisites} from "../services/prerequisites"; import {deleteGeneratedSchema} from "../services/schema-generator"; import { runPgschemaDump, @@ -28,16 +30,6 @@ interface ImportOptions extends CommandOptions { name?: string; } -function maskConnectionUrl(url: string): string { - try { - const parsed = new URL(url); - parsed.password = "****"; - return parsed.toString(); - } catch { - return url.replace(/:([^@]+)@/, ":****@"); - } -} - export async function importCommand(options: ImportOptions): Promise { const spinner = ora(); const migrationName = options.name || "imported_baseline"; @@ -63,24 +55,7 @@ export async function importCommand(options: ImportOptions): Promise { logger.step(1, 8, "Checking prerequisites..."); - const pgschemaInstalled = await checkPgschemaInstalled(); - const dbmateInstalled = await checkDbmateInstalled(); - - if (!pgschemaInstalled) { - throw new PostkitError( - "pgschema binary not found.", - "Visit: https://github.com/pgschema/pgschema", - ); - } - - if (!dbmateInstalled) { - throw new PostkitError( - "dbmate binary not found.", - "Install with: brew install dbmate or go install github.com/amacneil/dbmate@latest", - ); - } - - logger.debug("Prerequisites check passed", options.verbose); + await checkDbPrerequisites(options.verbose ?? false); // Step 1: Resolve target database and test connection logger.step(2, 8, "Validating database connection..."); @@ -95,7 +70,7 @@ export async function importCommand(options: ImportOptions): Promise { ); } - logger.debug(`Target database: ${maskConnectionUrl(targetUrl)}`, options.verbose); + logger.debug(`Target database: ${maskRemoteUrl(targetUrl)}`, options.verbose); spinner.start("Connecting to database..."); const connected = await testConnection(targetUrl); @@ -103,7 +78,7 @@ export async function importCommand(options: ImportOptions): Promise { if (!connected) { spinner.fail("Failed to connect to database"); throw new PostkitError( - `Could not connect to database: ${maskConnectionUrl(targetUrl)}`, + `Could not connect to database: ${maskRemoteUrl(targetUrl)}`, "Check the database URL and ensure the database is running.", ); } @@ -155,7 +130,7 @@ export async function importCommand(options: ImportOptions): Promise { } logger.info("This command will:"); - logger.info(` 1. Dump schema from ${maskConnectionUrl(targetUrl)} (schema: ${schemaName})`); + logger.info(` 1. Dump schema from ${maskRemoteUrl(targetUrl)} (schema: ${schemaName})`); logger.info(" 2. Normalize the dump into PostKit schema directory structure"); logger.info(` 3. Generate baseline migration: "${migrationName}"`); logger.info(" 4. Insert migration tracking record in the source database"); diff --git a/cli/src/modules/db/commands/infra.ts b/cli/src/modules/db/commands/infra.ts index 65a8349..11703fd 100644 --- a/cli/src/modules/db/commands/infra.ts +++ b/cli/src/modules/db/commands/infra.ts @@ -5,13 +5,13 @@ import { getInfraSQL, applyInfra, } from "../services/infra-generator"; -import {getSession} from "../utils/session"; import {testConnection} from "../services/database"; +import {resolveApplyTarget} from "../utils/apply-target"; import type {CommandOptions} from "../../../common/types"; interface InfraOptions extends CommandOptions { apply?: boolean; - target?: "local" | "remote"; + target?: string; } export async function infraCommand(options: InfraOptions): Promise { @@ -51,48 +51,25 @@ export async function infraCommand(options: InfraOptions): Promise { // Apply if requested if (options.apply) { - const session = await getSession(); - let targetUrl: string | null = null; - let targetName: string; - - if (options.target === "remote") { - if (session) { - targetUrl = session.remoteDbUrl; - } else { - const {resolveRemote} = await import("../utils/remotes"); - const {url} = resolveRemote(); - targetUrl = url; - } - targetName = "remote"; - } else { - if (!session || !session.active) { - logger.error( - "No active session. Cannot apply infra to local database.", - ); - logger.info('Run "postkit db start" first or use --target=remote.'); - process.exit(1); - } - targetUrl = session.localDbUrl; - targetName = "local"; - } + const target = await resolveApplyTarget(options.target); - logger.info(`Applying infra to ${targetName} database...`); + logger.info(`Applying infra to ${target.label} database...`); spinner.start("Testing connection..."); - const connected = await testConnection(targetUrl); + const connected = await testConnection(target.url); if (!connected) { - spinner.fail(`Failed to connect to ${targetName} database`); - process.exit(1); + spinner.fail(`Failed to connect to ${target.label} database`); + throw new Error(`Could not connect to ${target.label} database`); } - spinner.succeed(`Connected to ${targetName} database`); + spinner.succeed(`Connected to ${target.label} database`); if (options.dryRun) { spinner.info("Dry run - skipping infra application"); } else { spinner.start("Applying infra..."); - await applyInfra(targetUrl); + await applyInfra(target.url); spinner.succeed("Infra applied successfully"); } } @@ -105,7 +82,6 @@ export async function infraCommand(options: InfraOptions): Promise { ); } catch (error) { spinner.fail("Failed to generate infra statements"); - logger.error(error instanceof Error ? error.message : String(error)); - process.exit(1); + throw error; } } diff --git a/cli/src/modules/db/commands/migration.ts b/cli/src/modules/db/commands/migration.ts index 9b408c4..c65552a 100644 --- a/cli/src/modules/db/commands/migration.ts +++ b/cli/src/modules/db/commands/migration.ts @@ -1,12 +1,12 @@ import ora from "ora"; import {logger} from "../../../common/logger"; import {promptInput} from "../../../common/prompt"; -import {getSession, updatePendingChanges} from "../utils/session"; +import {requireActiveSession, assertLocalConnection, updatePendingChanges} from "../utils/session"; import {getSessionMigrationsPath} from "../utils/db-config"; import {createMigrationFile} from "../services/dbmate"; -import {testConnection} from "../services/database"; import {getDbConfig} from "../utils/db-config"; import type {CommandOptions} from "../../../common/types"; +import {PostkitError} from "../../../common/errors"; interface MigrateOptions extends CommandOptions { name?: string; @@ -40,14 +40,7 @@ export async function migrationCommand(options: MigrateOptions, name?: string): const spinner = ora(); try { - // Check for active session - const session = await getSession(); - - if (!session || !session.active) { - logger.error("No active migration session."); - logger.info('Run "postkit db start" to begin a new session.'); - process.exit(1); - } + const session = await requireActiveSession(); // Get migration name let migrationName = name || options.name; @@ -61,8 +54,7 @@ export async function migrationCommand(options: MigrateOptions, name?: string): // Ensure migrationName is defined (TypeScript safety) if (!migrationName) { - logger.error("Migration name is required."); - process.exit(1); + throw new PostkitError("Migration name is required."); } logger.heading("Create Manual Migration"); @@ -72,20 +64,7 @@ export async function migrationCommand(options: MigrateOptions, name?: string): // Test local connection logger.step(1, 3, "Testing local database connection..."); - spinner.start("Connecting to local database..."); - - const localConnected = await testConnection(session.localDbUrl); - - if (!localConnected) { - spinner.fail("Failed to connect to local database"); - logger.error("Could not connect to the local database."); - logger.info( - 'The local clone may have been removed. Run "postkit db start" again.', - ); - process.exit(1); - } - - spinner.succeed("Connected to local database"); + await assertLocalConnection(session, spinner); // Create migration file logger.step(2, 3, "Creating migration file..."); @@ -150,7 +129,6 @@ export async function migrationCommand(options: MigrateOptions, name?: string): logger.info(' - Run "postkit db migration " to create more migrations in this session'); } catch (error) { spinner.fail("Failed to create migration"); - logger.error(error instanceof Error ? error.message : String(error)); - process.exit(1); + throw error; } } diff --git a/cli/src/modules/db/commands/plan.ts b/cli/src/modules/db/commands/plan.ts index b7ce790..23d1899 100644 --- a/cli/src/modules/db/commands/plan.ts +++ b/cli/src/modules/db/commands/plan.ts @@ -1,44 +1,21 @@ import ora from "ora"; import {logger} from "../../../common/logger"; -import {getSession, updatePendingChanges} from "../utils/session"; +import {requireActiveSession, assertLocalConnection, updatePendingChanges} from "../utils/session"; import {toRelativePath} from "../utils/db-config"; import {generateSchemaSQLAndFingerprint} from "../services/schema-generator"; import {runPgschemaplan} from "../services/pgschema"; -import {testConnection} from "../services/database"; import type {CommandOptions} from "../../../common/types"; -import {PostkitError} from "../../../common/errors"; - export async function planCommand(options: CommandOptions): Promise { const spinner = ora(); try { - // Check for active session - const session = await getSession(); - - if (!session || !session.active) { - throw new PostkitError( - "No active migration session.", - 'Run "postkit db start" to begin a new session.', - ); - } + const session = await requireActiveSession(); logger.heading("Generating Migration Plan"); // Step 1: Test local connection logger.step(1, 3, "Testing local database connection..."); - spinner.start("Connecting to local database..."); - - const localConnected = await testConnection(session.localDbUrl); - - if (!localConnected) { - spinner.fail("Failed to connect to local database"); - throw new PostkitError( - "Could not connect to the local database.", - 'The local clone may have been removed. Run "postkit db start" again.', - ); - } - - spinner.succeed("Connected to local database"); + await assertLocalConnection(session, spinner); // Step 2: Generate combined schema logger.step(2, 3, "Generating schema SQL..."); diff --git a/cli/src/modules/db/commands/remote.ts b/cli/src/modules/db/commands/remote.ts index 04fbeae..91f84f3 100644 --- a/cli/src/modules/db/commands/remote.ts +++ b/cli/src/modules/db/commands/remote.ts @@ -66,8 +66,7 @@ export async function remoteListCommand(options: CommandOptions = {}): Promise { @@ -52,48 +52,25 @@ export async function seedCommand(options: SeedOptions): Promise { // Apply if requested if (options.apply) { - const session = await getSession(); - let targetUrl: string | null = null; - let targetName: string; - - if (options.target === "remote") { - if (session) { - targetUrl = session.remoteDbUrl; - } else { - const {resolveRemote} = await import("../utils/remotes"); - const {url} = resolveRemote(); - targetUrl = url; - } - targetName = "remote"; - } else { - if (!session || !session.active) { - logger.error( - "No active session. Cannot apply seeds to local database.", - ); - logger.info('Run "postkit db start" first or use --target=remote.'); - process.exit(1); - } - targetUrl = session.localDbUrl; - targetName = "local"; - } + const target = await resolveApplyTarget(options.target); - logger.info(`Applying seeds to ${targetName} database...`); + logger.info(`Applying seeds to ${target.label} database...`); spinner.start("Testing connection..."); - const connected = await testConnection(targetUrl); + const connected = await testConnection(target.url); if (!connected) { - spinner.fail(`Failed to connect to ${targetName} database`); - process.exit(1); + spinner.fail(`Failed to connect to ${target.label} database`); + throw new Error(`Could not connect to ${target.label} database`); } - spinner.succeed(`Connected to ${targetName} database`); + spinner.succeed(`Connected to ${target.label} database`); if (options.dryRun) { spinner.info("Dry run - skipping seed application"); } else { spinner.start("Applying seeds..."); - await applySeeds(targetUrl); + await applySeeds(target.url); spinner.succeed("Seeds applied successfully"); } } @@ -106,7 +83,6 @@ export async function seedCommand(options: SeedOptions): Promise { ); } catch (error) { spinner.fail("Failed to generate seeds"); - logger.error(error instanceof Error ? error.message : String(error)); - process.exit(1); + throw error; } } diff --git a/cli/src/modules/db/commands/start.ts b/cli/src/modules/db/commands/start.ts index a8a4058..7d2ba08 100644 --- a/cli/src/modules/db/commands/start.ts +++ b/cli/src/modules/db/commands/start.ts @@ -13,8 +13,8 @@ import { getTableCount, getRemotePgMajorVersion, } from "../services/database"; -import {checkPgschemaInstalled} from "../services/pgschema"; -import {checkDbmateInstalled, runDbmateStatus} from "../services/dbmate"; +import {runDbmateStatus} from "../services/dbmate"; +import {checkDbPrerequisites} from "../services/prerequisites"; import {resolveLocalDb, cloneDatabaseViaContainer} from "../services/container"; import {getPendingCommittedMigrations} from "../utils/committed"; import type {CommandOptions} from "../../../common/types"; @@ -41,24 +41,7 @@ export async function startCommand(options: StartOptions): Promise { // Step 1: Check prerequisites logger.step(1, 5, "Checking prerequisites..."); - const pgschemaInstalled = await checkPgschemaInstalled(); - const dbmateInstalled = await checkDbmateInstalled(); - - if (!pgschemaInstalled) { - throw new PostkitError( - "pgschema binary not found.", - "Visit: https://github.com/pgschema/pgschema", - ); - } - - if (!dbmateInstalled) { - throw new PostkitError( - "dbmate binary not found.", - "Install with: brew install dbmate or go install github.com/amacneil/dbmate@latest", - ); - } - - logger.debug("Prerequisites check passed", options.verbose); + await checkDbPrerequisites(options.verbose ?? false); // Step 2: Load configuration logger.step(2, 5, "Loading configuration..."); @@ -103,7 +86,7 @@ export async function startCommand(options: StartOptions): Promise { ); if (localDbUrl) { logger.debug( - `Local DB: ${maskConnectionUrl(localDbUrl)}`, + `Local DB: ${maskRemoteUrl(localDbUrl)}`, options.verbose, ); } @@ -207,7 +190,7 @@ export async function startCommand(options: StartOptions): Promise { const resolved = await resolveLocalDb(localDbUrl, targetRemoteUrl, spinner); containerID = resolved.containerID; localDbUrl = resolved.url; - logger.debug(`Local DB (container): ${maskConnectionUrl(localDbUrl)}`, options.verbose); + logger.debug(`Local DB (container): ${maskRemoteUrl(localDbUrl)}`, options.verbose); } // Step 5/6: Clone database @@ -274,13 +257,3 @@ async function ensurePgschemaIgnore(schemaPath: string): Promise { await fs.writeFile(ignorePath, content, "utf-8"); } - -function maskConnectionUrl(url: string): string { - try { - const parsed = new URL(url); - parsed.password = "****"; - return parsed.toString(); - } catch { - return url.replace(/:([^@]+)@/, ":****@"); - } -} diff --git a/cli/src/modules/db/services/database.ts b/cli/src/modules/db/services/database.ts index a66982e..1632870 100644 --- a/cli/src/modules/db/services/database.ts +++ b/cli/src/modules/db/services/database.ts @@ -10,6 +10,19 @@ import { const {Client} = pg; +export async function withPgClient( + url: string, + fn: (client: InstanceType) => Promise, +): Promise { + const client = new Client({connectionString: url}); + await client.connect(); + try { + return await fn(client); + } finally { + await client.end(); + } +} + export function parseConnectionUrl(url: string): DatabaseConnectionInfo { const parsed = new URL(url); @@ -28,16 +41,13 @@ function buildConnectionUrl(info: DatabaseConnectionInfo): string { } export async function testConnection(url: string): Promise { - const client = new Client({connectionString: url}); - try { - await client.connect(); - await client.query(TEST_CONNECTION); - return true; + return await withPgClient(url, async (client) => { + await client.query(TEST_CONNECTION); + return true; + }); } catch { return false; - } finally { - await client.end(); } } @@ -47,23 +57,13 @@ export async function createDatabase(url: string): Promise { // Connect to postgres database to create the new one const adminUrl = buildConnectionUrl({...info, database: "postgres"}); - const client = new Client({connectionString: adminUrl}); - - try { - await client.connect(); - + await withPgClient(adminUrl, async (client) => { // Check if database exists - const result = await client.query( - CHECK_DB_EXISTS, - [targetDb], - ); - + const result = await client.query(CHECK_DB_EXISTS, [targetDb]); if (result.rows.length === 0) { await client.query(`CREATE DATABASE "${targetDb}"`); } - } finally { - await client.end(); - } + }); } export async function dropDatabase(url: string): Promise { @@ -72,22 +72,12 @@ export async function dropDatabase(url: string): Promise { // Connect to postgres database to drop the target const adminUrl = buildConnectionUrl({...info, database: "postgres"}); - const client = new Client({connectionString: adminUrl}); - - try { - await client.connect(); - + await withPgClient(adminUrl, async (client) => { // Terminate existing connections - await client.query( - TERMINATE_CONNECTIONS, - [targetDb], - ); - + await client.query(TERMINATE_CONNECTIONS, [targetDb]); // Drop database if exists await client.query(`DROP DATABASE IF EXISTS "${targetDb}"`); - } finally { - await client.end(); - } + }); } export async function cloneDatabase( @@ -134,27 +124,17 @@ export async function cloneDatabase( } export async function executeSQL(url: string, sql: string): Promise { - const client = new Client({connectionString: url}); - - try { - await client.connect(); + return withPgClient(url, async (client) => { const result = await client.query(sql); return JSON.stringify(result.rows, null, 2); - } finally { - await client.end(); - } + }); } export async function getTableCount(url: string): Promise { - const client = new Client({connectionString: url}); - - try { - await client.connect(); + return withPgClient(url, async (client) => { const result = await client.query(COUNT_TABLES); return parseInt(result.rows[0].count, 10); - } finally { - await client.end(); - } + }); } /** @@ -162,13 +142,9 @@ export async function getTableCount(url: string): Promise { * Uses SHOW server_version_num which returns a zero-padded integer like "160003". */ export async function getRemotePgMajorVersion(url: string): Promise { - const client = new Client({connectionString: url}); - try { - await client.connect(); + return withPgClient(url, async (client) => { const result = await client.query("SHOW server_version_num"); const num = parseInt(result.rows[0].server_version_num as string, 10); return Math.floor(num / 10000); - } finally { - await client.end(); - } + }); } diff --git a/cli/src/modules/db/services/infra-generator.ts b/cli/src/modules/db/services/infra-generator.ts index bfa3464..68c159c 100644 --- a/cli/src/modules/db/services/infra-generator.ts +++ b/cli/src/modules/db/services/infra-generator.ts @@ -1,6 +1,7 @@ import fs from "fs/promises"; import path from "path"; import {existsSync} from "fs"; +import type {Ora} from "ora"; import {getDbConfig} from "../utils/db-config"; import {parseConnectionUrl} from "./database"; import {runSpawnCommand} from "../../../common/shell"; @@ -91,3 +92,14 @@ export async function applyInfra(databaseUrl: string): Promise { throw new Error(`Failed to apply infra: ${result.stderr || result.stdout}`); } } + +export async function applyInfraStep(spinner: Ora, dbUrl: string, label = "local"): Promise { + const infra = await loadInfra(); + if (infra.length === 0) { + spinner.info("No infra files found - skipping"); + return; + } + spinner.start(`Applying infra to ${label}...`); + await applyInfra(dbUrl); + spinner.succeed(`Infra applied to ${label} (${infra.length} file(s))`); +} diff --git a/cli/src/modules/db/services/prerequisites.ts b/cli/src/modules/db/services/prerequisites.ts new file mode 100644 index 0000000..9301b56 --- /dev/null +++ b/cli/src/modules/db/services/prerequisites.ts @@ -0,0 +1,25 @@ +import {logger} from "../../../common/logger"; +import {PostkitError} from "../../../common/errors"; +import {checkPgschemaInstalled} from "./pgschema"; +import {checkDbmateInstalled} from "./dbmate"; + +export async function checkDbPrerequisites(verbose: boolean): Promise { + const pgschemaInstalled = await checkPgschemaInstalled(); + const dbmateInstalled = await checkDbmateInstalled(); + + if (!pgschemaInstalled) { + throw new PostkitError( + "pgschema binary not found.", + "Visit: https://github.com/pgschema/pgschema", + ); + } + + if (!dbmateInstalled) { + throw new PostkitError( + "dbmate binary not found.", + "Install with: brew install dbmate or go install github.com/amacneil/dbmate@latest", + ); + } + + logger.debug("Prerequisites check passed", verbose); +} diff --git a/cli/src/modules/db/services/schema-generator.ts b/cli/src/modules/db/services/schema-generator.ts index 3ceb984..57237c9 100644 --- a/cli/src/modules/db/services/schema-generator.ts +++ b/cli/src/modules/db/services/schema-generator.ts @@ -44,8 +44,6 @@ export async function generateSchemaSQL(): Promise { /** * Generate schema SQL and compute fingerprint in a single filesystem pass. - * Use this instead of calling generateSchemaSQL() and generateSchemaFingerprint() - * separately to avoid reading schema files twice. */ export async function generateSchemaSQLAndFingerprint(): Promise<{ schemaFile: string; @@ -158,67 +156,6 @@ async function loadSectionFiles(sectionPath: string): Promise { return ""; } -async function getSchemaFiles(): Promise { - const config = getDbConfig(); - const schemaPath = config.schemaPath; - - if (!existsSync(schemaPath)) { - return []; - } - - return collectSqlFiles(schemaPath); -} - -const SKIP_DIRECTORIES = new Set(["seed", "seeds"]); - -async function collectSqlFiles( - dirPath: string, - isRoot = true, -): Promise { - const files: string[] = []; - const entries = await fs.readdir(dirPath, {withFileTypes: true}); - - for (const entry of entries) { - const fullPath = path.join(dirPath, entry.name); - - if (entry.isDirectory()) { - // Skip seed and grant directories at the schema root level - if (isRoot && SKIP_DIRECTORIES.has(entry.name.toLowerCase())) { - continue; - } - const subFiles = await collectSqlFiles(fullPath, false); - files.push(...subFiles); - } else if (entry.isFile() && entry.name.endsWith(".sql")) { - files.push(fullPath); - } - } - - return files.sort(); -} - -export async function generateSchemaFingerprint(): Promise { - const config = getDbConfig(); - const schemaPath = config.schemaPath; - - if (!existsSync(schemaPath)) { - return createHash("sha256").digest("hex"); - } - - const sections = await discoverSchemaSections(schemaPath); - const sortedSections = sections.sort((a, b) => a.order - b.order); - const hash = createHash("sha256"); - - for (const section of sortedSections) { - const sectionContent = await loadSectionFiles(section.path); - if (sectionContent) { - hash.update(section.path); - hash.update(sectionContent); - } - } - - return hash.digest("hex"); -} - export async function deleteGeneratedSchema(): Promise { const outputPath = getGeneratedSchemaPath(); diff --git a/cli/src/modules/db/services/schema-importer.ts b/cli/src/modules/db/services/schema-importer.ts index 90ef2ae..6ff86df 100644 --- a/cli/src/modules/db/services/schema-importer.ts +++ b/cli/src/modules/db/services/schema-importer.ts @@ -1,9 +1,8 @@ -import pg from "pg"; import fs from "fs/promises"; import path from "path"; import {existsSync} from "fs"; import {runCommand} from "../../../common/shell"; -import {getDbConfig, getTmpImportDir, MIGRATIONS_TABLE} from "../utils/db-config"; +import {getDbConfig} from "../utils/db-config"; import { CREATE_POSTKIT_SCHEMA, CREATE_MIGRATIONS_TABLE, @@ -11,12 +10,10 @@ import { FETCH_SCHEMAS, FETCH_ROLES, } from "../config/queries"; -import {parseConnectionUrl, createDatabase, dropDatabase} from "./database"; +import {parseConnectionUrl, createDatabase, dropDatabase, withPgClient} from "./database"; import {runPgschemaplan} from "./pgschema"; import {generateSchemaSQLAndFingerprint} from "./schema-generator"; -const {Client} = pg; - /** * Parse schema.sql \i include directives to determine file ordering per directory. * Returns a Map where key = directory name (e.g. "tables") and value = ordered filenames. @@ -231,17 +228,12 @@ export async function fetchInfraFromDatabase( databaseUrl: string, schemaName: string, ): Promise<{roles: string[]; schemas: string[]}> { - const client = new Client({connectionString: databaseUrl}); - const roles: string[] = []; - const schemas: string[] = []; - - try { - await client.connect(); + return withPgClient(databaseUrl, async (client) => { + const roles: string[] = []; + const schemas: string[] = []; // Fetch non-system schemas - const schemaRows = await client.query<{nspname: string; owner: string}>( - FETCH_SCHEMAS, - ); + const schemaRows = await client.query<{nspname: string; owner: string}>(FETCH_SCHEMAS); for (const row of schemaRows.rows) { schemas.push(`CREATE SCHEMA IF NOT EXISTS ${row.nspname} AUTHORIZATION ${row.owner};`); @@ -274,11 +266,9 @@ export async function fetchInfraFromDatabase( `DO $$\nBEGIN\n IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${row.rolname}') THEN\n CREATE ROLE ${row.rolname} ${attrs};\n END IF;\nEND\n$$;`, ); } - } finally { - await client.end(); - } - return {roles, schemas}; + return {roles, schemas}; + }); } /** @@ -416,10 +406,7 @@ export async function applyInfraToDatabase(databaseUrl: string, schemaPath: stri const infraDir = path.join(schemaPath, "infra"); if (!existsSync(infraDir)) return; - const client = new Client({connectionString: databaseUrl}); - try { - await client.connect(); - + await withPgClient(databaseUrl, async (client) => { const entries = (await fs.readdir(infraDir)).sort(); for (const entry of entries) { if (!entry.endsWith(".sql")) continue; @@ -428,9 +415,7 @@ export async function applyInfraToDatabase(databaseUrl: string, schemaPath: stri await client.query(sql); } } - } finally { - await client.end(); - } + }); } /** @@ -498,22 +483,14 @@ export async function syncMigrationState( databaseUrl: string, version: string, ): Promise { - const client = new Client({connectionString: databaseUrl}); - - try { - await client.connect(); - + await withPgClient(databaseUrl, async (client) => { // Ensure postkit schema exists await client.query(CREATE_POSTKIT_SCHEMA); - // Create schema_migrations table in postkit schema await client.query(CREATE_MIGRATIONS_TABLE); - // Insert the baseline version await client.query(INSERT_MIGRATION_VERSION, [version]); - } finally { - await client.end(); - } + }); } /** diff --git a/cli/src/modules/db/services/seed-generator.ts b/cli/src/modules/db/services/seed-generator.ts index 4dea2f3..907bb54 100644 --- a/cli/src/modules/db/services/seed-generator.ts +++ b/cli/src/modules/db/services/seed-generator.ts @@ -1,9 +1,11 @@ import fs from "fs/promises"; import path from "path"; import {existsSync} from "fs"; +import type {Ora} from "ora"; import {getDbConfig} from "../utils/db-config"; import {loadSqlGroup} from "../utils/sql-loader"; import type {SeedStatement} from "../types/index"; +import {PostkitError} from "../../../common/errors"; export async function loadSeeds(): Promise { const config = getDbConfig(); @@ -87,3 +89,22 @@ export async function applySeeds(databaseUrl: string): Promise { } } } + +export async function applySeedsStep(spinner: Ora, dbUrl: string, label = "local"): Promise { + const seeds = await loadSeeds(); + if (seeds.length === 0) { + spinner.info("No seed files found - skipping"); + return; + } + try { + spinner.start(`Applying seeds to ${label}...`); + await applySeeds(dbUrl); + spinner.succeed(`Seeds applied to ${label} (${seeds.length} file(s))`); + } catch (error) { + spinner.fail("Failed to apply seeds"); + throw new PostkitError( + `Seeds failed: ${error instanceof Error ? error.message : String(error)}`, + 'Run "postkit db apply" again to retry from seeds.', + ); + } +} diff --git a/cli/src/modules/db/utils/apply-target.ts b/cli/src/modules/db/utils/apply-target.ts new file mode 100644 index 0000000..7398c56 --- /dev/null +++ b/cli/src/modules/db/utils/apply-target.ts @@ -0,0 +1,35 @@ +import {PostkitError} from "../../../common/errors"; +import {getSession} from "./session"; +import {resolveRemote} from "./remotes"; + +export interface ApplyTarget { + url: string; + label: string; +} + +export async function resolveApplyTarget( + targetOption: string | undefined, +): Promise { + if (!targetOption || targetOption === "local") { + const session = await getSession(); + if (!session?.active) { + throw new PostkitError( + "No active session — cannot resolve local target.", + 'Run "postkit db start" first.', + ); + } + return {url: session.localDbUrl, label: "local"}; + } + + if (targetOption === "remote") { + const session = await getSession(); + if (session?.active) { + return {url: session.remoteDbUrl, label: `remote (${session.remoteName ?? "unknown"})`}; + } + // No session — fall back to default remote + const {name, url} = resolveRemote(); + return {url, label: `remote (${name})`}; + } + + throw new PostkitError(`Unknown target: ${targetOption}`, 'Use "local" or "remote".'); +} diff --git a/cli/src/modules/db/utils/committed.ts b/cli/src/modules/db/utils/committed.ts index 25775f6..962094f 100644 --- a/cli/src/modules/db/utils/committed.ts +++ b/cli/src/modules/db/utils/committed.ts @@ -1,16 +1,14 @@ -import fs from "fs/promises"; import {existsSync} from "fs"; -import pg from "pg"; import type {CommittedState, CommittedMigration} from "../types/index"; -import {getCommittedFilePath, MIGRATIONS_TABLE} from "./db-config"; +import {getCommittedFilePath} from "./db-config"; +import {readJsonFile, writeJsonFile} from "./json-file"; +import {withPgClient} from "../services/database"; import { CHECK_MIGRATIONS_TABLE_EXISTS, GET_APPLIED_VERSIONS, } from "../config/queries"; import {logger} from "../../../common/logger"; -const {Client} = pg; - /** * Reads the committed state from .postkit/committed.json */ @@ -22,8 +20,7 @@ export async function getCommittedState(): Promise { } try { - const content = await fs.readFile(committedFilePath, "utf-8"); - const state = JSON.parse(content) as CommittedState; + const state = await readJsonFile(committedFilePath); return {migrations: state.migrations ?? []}; } catch (error) { logger.warn( @@ -40,7 +37,7 @@ export async function getCommittedState(): Promise { */ export async function saveCommittedState(state: CommittedState): Promise { const committedFilePath = getCommittedFilePath(); - await fs.writeFile(committedFilePath, JSON.stringify(state, null, 2), "utf-8"); + await writeJsonFile(committedFilePath, state); } /** @@ -65,24 +62,18 @@ export async function getAllCommittedMigrations(): Promise * and returns the set of applied migration version timestamps. */ async function getAppliedMigrationVersions(remoteUrl: string): Promise> { - const client = new Client({connectionString: remoteUrl}); - try { - await client.connect(); - - // Check if migrations table exists in postkit schema - const tableCheck = await client.query(CHECK_MIGRATIONS_TABLE_EXISTS); - - if (tableCheck.rows.length === 0) { - return new Set(); - } - - const result = await client.query(GET_APPLIED_VERSIONS); - return new Set(result.rows.map((row: {version: string}) => row.version)); + return await withPgClient(remoteUrl, async (client) => { + // Check if migrations table exists in postkit schema + const tableCheck = await client.query(CHECK_MIGRATIONS_TABLE_EXISTS); + if (tableCheck.rows.length === 0) { + return new Set(); + } + const result = await client.query(GET_APPLIED_VERSIONS); + return new Set(result.rows.map((row: {version: string}) => row.version)); + }); } catch { return new Set(); - } finally { - await client.end(); } } diff --git a/cli/src/modules/db/utils/json-file.ts b/cli/src/modules/db/utils/json-file.ts new file mode 100644 index 0000000..59c6250 --- /dev/null +++ b/cli/src/modules/db/utils/json-file.ts @@ -0,0 +1,10 @@ +import fs from "fs/promises"; + +export async function readJsonFile(filePath: string): Promise { + const raw = await fs.readFile(filePath, "utf-8"); + return JSON.parse(raw) as T; +} + +export async function writeJsonFile(filePath: string, data: unknown): Promise { + await fs.writeFile(filePath, JSON.stringify(data, null, 2) + "\n", "utf-8"); +} diff --git a/cli/src/modules/db/utils/remotes.ts b/cli/src/modules/db/utils/remotes.ts index 0ff471a..8fc875c 100644 --- a/cli/src/modules/db/utils/remotes.ts +++ b/cli/src/modules/db/utils/remotes.ts @@ -1,6 +1,6 @@ -import fs from "fs/promises"; import {logger} from "../../../common/logger"; import {loadPostkitConfig, getSecretsFilePath, POSTKIT_SECRETS_FILE, invalidateConfig} from "../../../common/config"; +import {readJsonFile, writeJsonFile} from "./json-file"; import type {RemoteConfig} from "../../../common/config"; export interface RemoteInfo { @@ -71,17 +71,6 @@ export function getDefaultRemote(): string | null { return defaultName; } -// ─── File read helpers ─────────────────────────────────────────────────────── - -async function readJsonFile(filePath: string): Promise> { - const raw = await fs.readFile(filePath, "utf-8"); - return JSON.parse(raw) as Record; -} - -async function writeJsonFile(filePath: string, data: Record): Promise { - await fs.writeFile(filePath, JSON.stringify(data, null, 2) + "\n", "utf-8"); -} - // ─── Remote management ─────────────────────────────────────────────────────── /** @@ -109,7 +98,7 @@ export async function addRemote(name: string, url: string, setAsDefault: boolean const secretsPath = getSecretsFilePath(); let secrets: Record; try { - secrets = await readJsonFile(secretsPath); + secrets = await readJsonFile>(secretsPath); } catch { throw new Error( `Secrets file not found: ${POSTKIT_SECRETS_FILE}\n` + @@ -117,8 +106,8 @@ export async function addRemote(name: string, url: string, setAsDefault: boolean ); } - const secretsDb = (secrets.db ?? {}) as Record; - const secretsRemotes = (secretsDb.remotes ?? {}) as Record>; + const secretsDb = (secrets["db"] ?? {}) as Record; + const secretsRemotes = (secretsDb["remotes"] ?? {}) as Record>; if (secretsRemotes[name]) { throw new Error(`Remote "${name}" already exists`); @@ -146,16 +135,16 @@ export async function addRemote(name: string, url: string, setAsDefault: boolean if (makeDefault) { for (const key of Object.keys(secretsRemotes)) { - delete secretsRemotes[key]!.default; + delete secretsRemotes[key]!["default"]; } } const addedAt = new Date().toISOString(); secretsRemotes[name] = {url, addedAt}; - if (makeDefault) secretsRemotes[name]!.default = true; + if (makeDefault) secretsRemotes[name]!["default"] = true; - secretsDb.remotes = secretsRemotes; - secrets.db = secretsDb; + secretsDb["remotes"] = secretsRemotes; + secrets["db"] = secretsDb; await writeJsonFile(secretsPath, secrets); invalidateConfig(); @@ -192,18 +181,18 @@ export async function removeRemote(name: string, force: boolean = false): Promis } const secretsPath = getSecretsFilePath(); - const secrets = await readJsonFile(secretsPath); - const secretsDb = (secrets.db ?? {}) as Record; - const secretsRemotes = (secretsDb.remotes ?? {}) as Record>; + const secrets = await readJsonFile>(secretsPath); + const secretsDb = (secrets["db"] ?? {}) as Record; + const secretsRemotes = (secretsDb["remotes"] ?? {}) as Record>; delete secretsRemotes[name]; if (isDefault) { const firstKey = Object.keys(secretsRemotes)[0]; - if (firstKey) secretsRemotes[firstKey]!.default = true; + if (firstKey) secretsRemotes[firstKey]!["default"] = true; } - secretsDb.remotes = secretsRemotes; - secrets.db = secretsDb; + secretsDb["remotes"] = secretsRemotes; + secrets["db"] = secretsDb; await writeJsonFile(secretsPath, secrets); invalidateConfig(); @@ -221,21 +210,21 @@ export async function setDefaultRemote(name: string): Promise { } const secretsPath = getSecretsFilePath(); - const secrets = await readJsonFile(secretsPath); - const secretsDb = (secrets.db ?? {}) as Record; - const secretsRemotes = (secretsDb.remotes ?? {}) as Record>; + const secrets = await readJsonFile>(secretsPath); + const secretsDb = (secrets["db"] ?? {}) as Record; + const secretsRemotes = (secretsDb["remotes"] ?? {}) as Record>; for (const key of Object.keys(secretsRemotes)) { - delete secretsRemotes[key]!.default; + delete secretsRemotes[key]!["default"]; } if (!secretsRemotes[name]) { secretsRemotes[name] = {}; } - secretsRemotes[name]!.default = true; + secretsRemotes[name]!["default"] = true; - secretsDb.remotes = secretsRemotes; - secrets.db = secretsDb; + secretsDb["remotes"] = secretsRemotes; + secrets["db"] = secretsDb; await writeJsonFile(secretsPath, secrets); invalidateConfig(); diff --git a/cli/src/modules/db/utils/session.ts b/cli/src/modules/db/utils/session.ts index d977250..f4e5b1c 100644 --- a/cli/src/modules/db/utils/session.ts +++ b/cli/src/modules/db/utils/session.ts @@ -1,7 +1,9 @@ -import fs from "fs/promises"; import {existsSync} from "fs"; +import type {Ora} from "ora"; import type {SessionState} from "../types/index"; import {getSessionFilePath} from "./db-config"; +import {readJsonFile, writeJsonFile} from "./json-file"; +import {PostkitError} from "../../../common/errors"; export async function getSession(): Promise { const sessionPath = getSessionFilePath(); @@ -11,8 +13,7 @@ export async function getSession(): Promise { } try { - const content = await fs.readFile(sessionPath, "utf-8"); - const state = JSON.parse(content) as SessionState; + const state = await readJsonFile(sessionPath); if (!state || typeof state.active !== "boolean" || !state.pendingChanges) { return null; } @@ -88,7 +89,8 @@ export async function deleteSession(): Promise { const sessionPath = getSessionFilePath(); if (existsSync(sessionPath)) { - await fs.unlink(sessionPath); + const {unlink} = await import("fs/promises"); + await unlink(sessionPath); } } @@ -97,9 +99,34 @@ export async function hasActiveSession(): Promise { return session !== null && session.active; } +export async function requireActiveSession(): Promise { + const session = await getSession(); + if (!session || !session.active) { + throw new PostkitError( + "No active migration session.", + 'Run "postkit db start" to begin a new session.', + ); + } + return session; +} + +export async function assertLocalConnection(session: SessionState, spinner: Ora): Promise { + const {testConnection} = await import("../services/database"); + spinner.start("Connecting to local database..."); + const connected = await testConnection(session.localDbUrl); + if (!connected) { + spinner.fail("Failed to connect to local database"); + throw new PostkitError( + "Could not connect to the local database.", + 'The local clone may have been removed. Run "postkit db start" again.', + ); + } + spinner.succeed("Connected to local database"); +} + async function saveSession(session: SessionState): Promise { const sessionPath = getSessionFilePath(); - await fs.writeFile(sessionPath, JSON.stringify(session, null, 2), "utf-8"); + await writeJsonFile(sessionPath, session); } export function formatTimestamp(date: Date): string { diff --git a/cli/test/e2e/workflows/remote-management.test.ts b/cli/test/e2e/workflows/remote-management.test.ts index 0e591bb..8d9271a 100644 --- a/cli/test/e2e/workflows/remote-management.test.ts +++ b/cli/test/e2e/workflows/remote-management.test.ts @@ -1,61 +1,48 @@ -import {describe, it, expect, afterAll} from "vitest"; +import {describe, it, expect, beforeAll, afterAll} from "vitest"; import {runCli} from "../helpers/cli-runner"; import {createTestProject, cleanupTestProject, type TestProject, readJson} from "../helpers/test-project"; describe("Remote management", () => { - const projects: TestProject[] = []; + let project: TestProject; - afterAll(async () => { - for (const p of projects) { - await cleanupTestProject(p); - } - }); - - it("lists remotes", async () => { - const project = await createTestProject({ + beforeAll(async () => { + project = await createTestProject({ localDbUrl: "postgres://localhost:5432/test", remoteDbUrl: "postgres://localhost:5432/remote", remoteName: "dev", }); - projects.push(project); + }); + + afterAll(async () => { + await cleanupTestProject(project); + }); + it("lists remotes", async () => { const result = await runCli(["db", "remote", "list"], {cwd: project.rootDir}); expect(result.exitCode).toBe(0); expect(result.stdout).toContain("dev"); }); it("adds a new remote", async () => { - const project = await createTestProject({ - localDbUrl: "postgres://localhost:5432/test", - remoteDbUrl: "postgres://localhost:5432/remote", - remoteName: "dev", - }); - projects.push(project); - const result = await runCli( - ["db", "remote", "add", "staging", "postgres://localhost:5432/staging"], + ["db", "remote", "add", "staging-add", "postgres://localhost:5432/staging-add"], {cwd: project.rootDir}, ); expect(result.exitCode).toBe(0); - // URL is in secrets file; public config only has metadata const secrets = await readJson<{ db: {remotes: Record}; }>(project, "postkit.secrets.json"); - expect(secrets.db.remotes.staging).toBeDefined(); - expect(secrets.db.remotes.staging?.url).toBe("postgres://localhost:5432/staging"); + expect(secrets.db.remotes["staging-add"]).toBeDefined(); + expect(secrets.db.remotes["staging-add"]?.url).toBe("postgres://localhost:5432/staging-add"); + + // Clean up + await runCli(["db", "remote", "remove", "staging-add", "--force"], {cwd: project.rootDir}); }); it("adds a remote with --default flag", async () => { - const project = await createTestProject({ - localDbUrl: "postgres://localhost:5432/test", - remoteDbUrl: "postgres://localhost:5432/remote", - remoteName: "dev", - }); - projects.push(project); - const result = await runCli( - ["db", "remote", "add", "prod", "postgres://localhost:5432/prod", "--default"], + ["db", "remote", "add", "prod-default", "postgres://localhost:5432/prod-default", "--default"], {cwd: project.rootDir}, ); expect(result.exitCode).toBe(0); @@ -63,26 +50,23 @@ describe("Remote management", () => { const secrets = await readJson<{ db: {remotes: Record}; }>(project, "postkit.secrets.json"); - expect(secrets.db.remotes.prod).toBeDefined(); - expect(secrets.db.remotes.prod?.default).toBe(true); + expect(secrets.db.remotes["prod-default"]).toBeDefined(); + expect(secrets.db.remotes["prod-default"]?.default).toBe(true); + + // Restore dev as default and clean up + await runCli(["db", "remote", "use", "dev"], {cwd: project.rootDir}); + await runCli(["db", "remote", "remove", "prod-default", "--force"], {cwd: project.rootDir}); }); it("sets default remote with 'use'", async () => { - const project = await createTestProject({ - localDbUrl: "postgres://localhost:5432/test", - remoteDbUrl: "postgres://localhost:5432/remote", - remoteName: "dev", - }); - projects.push(project); - - // Add another remote + // Add a staging-use remote await runCli( - ["db", "remote", "add", "staging", "postgres://localhost:5432/staging"], + ["db", "remote", "add", "staging-use", "postgres://localhost:5432/staging-use"], {cwd: project.rootDir}, ); - // Set staging as default - const result = await runCli(["db", "remote", "use", "staging"], { + // Set staging-use as default + const result = await runCli(["db", "remote", "use", "staging-use"], { cwd: project.rootDir, }); expect(result.exitCode).toBe(0); @@ -90,26 +74,23 @@ describe("Remote management", () => { const secrets = await readJson<{ db: {remotes: Record}; }>(project, "postkit.secrets.json"); - expect(secrets.db.remotes.staging).toBeDefined(); - expect(secrets.db.remotes.staging?.default).toBe(true); + expect(secrets.db.remotes["staging-use"]).toBeDefined(); + expect(secrets.db.remotes["staging-use"]?.default).toBe(true); + + // Restore and clean up + await runCli(["db", "remote", "use", "dev"], {cwd: project.rootDir}); + await runCli(["db", "remote", "remove", "staging-use", "--force"], {cwd: project.rootDir}); }); it("removes a remote with --force", async () => { - const project = await createTestProject({ - localDbUrl: "postgres://localhost:5432/test", - remoteDbUrl: "postgres://localhost:5432/remote", - remoteName: "dev", - }); - projects.push(project); - - // Add a second remote + // Add a second remote to remove await runCli( - ["db", "remote", "add", "staging", "postgres://localhost:5432/staging"], + ["db", "remote", "add", "staging-remove", "postgres://localhost:5432/staging-remove"], {cwd: project.rootDir}, ); - // Remove staging - const result = await runCli(["db", "remote", "remove", "staging", "--force"], { + // Remove it + const result = await runCli(["db", "remote", "remove", "staging-remove", "--force"], { cwd: project.rootDir, }); expect(result.exitCode).toBe(0); @@ -117,6 +98,6 @@ describe("Remote management", () => { const secrets = await readJson<{ db: {remotes: Record}; }>(project, "postkit.secrets.json"); - expect(secrets.db.remotes.staging).toBeUndefined(); + expect(secrets.db.remotes["staging-remove"]).toBeUndefined(); }); }); diff --git a/cli/test/modules/db/services/schema-generator.test.ts b/cli/test/modules/db/services/schema-generator.test.ts index 1c17854..056aa28 100644 --- a/cli/test/modules/db/services/schema-generator.test.ts +++ b/cli/test/modules/db/services/schema-generator.test.ts @@ -32,7 +32,7 @@ vi.mock("fs", async () => { import fs from "fs/promises"; import {existsSync} from "fs"; -import {generateSchemaSQLAndFingerprint, generateSchemaFingerprint, deleteGeneratedSchema} from "../../../../src/modules/db/services/schema-generator"; +import {generateSchemaSQLAndFingerprint, deleteGeneratedSchema} from "../../../../src/modules/db/services/schema-generator"; describe("schema-generator", () => { beforeEach(() => { @@ -98,10 +98,12 @@ describe("schema-generator", () => { }); }); - describe("generateSchemaFingerprint()", () => { - it("returns valid hex when no schema dir", async () => { - vi.mocked(existsSync).mockReturnValue(false); - const fingerprint = await generateSchemaFingerprint(); + describe("generateSchemaSQLAndFingerprint() fingerprint edge case", () => { + it("returns valid hex fingerprint when schema dir is empty", async () => { + vi.mocked(existsSync).mockReturnValue(true); + vi.mocked(fs.readdir).mockResolvedValue([]); + vi.mocked(fs.writeFile).mockResolvedValue(); + const {fingerprint} = await generateSchemaSQLAndFingerprint(); expect(fingerprint).toMatch(/^[a-f0-9]{64}$/); }); }); diff --git a/cli/test/modules/db/utils/session.test.ts b/cli/test/modules/db/utils/session.test.ts index ade9f17..d30d76f 100644 --- a/cli/test/modules/db/utils/session.test.ts +++ b/cli/test/modules/db/utils/session.test.ts @@ -44,7 +44,6 @@ const validSession = { description: null, schemaFingerprint: null, migrationApplied: false, - grantsApplied: false, seedsApplied: false, }, }; From 3d96b7b06ff04d95367266e47541baacd2299d94 Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Fri, 8 May 2026 10:03:29 +0530 Subject: [PATCH 14/18] refactor: move connection logic into try block and modernize session file deletion using fs/promises --- cli/src/modules/db/services/database.ts | 2 +- cli/src/modules/db/utils/session.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/src/modules/db/services/database.ts b/cli/src/modules/db/services/database.ts index 1632870..a3ba7cc 100644 --- a/cli/src/modules/db/services/database.ts +++ b/cli/src/modules/db/services/database.ts @@ -15,8 +15,8 @@ export async function withPgClient( fn: (client: InstanceType) => Promise, ): Promise { const client = new Client({connectionString: url}); - await client.connect(); try { + await client.connect(); return await fn(client); } finally { await client.end(); diff --git a/cli/src/modules/db/utils/session.ts b/cli/src/modules/db/utils/session.ts index f4e5b1c..67ac895 100644 --- a/cli/src/modules/db/utils/session.ts +++ b/cli/src/modules/db/utils/session.ts @@ -1,4 +1,5 @@ import {existsSync} from "fs"; +import fsp from "fs/promises"; import type {Ora} from "ora"; import type {SessionState} from "../types/index"; import {getSessionFilePath} from "./db-config"; @@ -89,8 +90,7 @@ export async function deleteSession(): Promise { const sessionPath = getSessionFilePath(); if (existsSync(sessionPath)) { - const {unlink} = await import("fs/promises"); - await unlink(sessionPath); + await fsp.unlink(sessionPath); } } From a4d5e0f3100b483ca5f3c8e6cd0ad2221653de65 Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Fri, 8 May 2026 10:12:20 +0530 Subject: [PATCH 15/18] docs: update documentation regarding db file persistence, remote configuration storage, and internal service functions --- CLAUDE.md | 12 ++++++-- cli/docs/architecture.md | 37 +++++++++++++++++++----- cli/docs/db.md | 16 +++++----- cli/docs/e2e-testing.md | 8 ++--- docs/docs/getting-started/quick-start.md | 2 +- 5 files changed, 53 insertions(+), 22 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index bbba0ac..0ca0d92 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,12 +86,13 @@ The `db` module implements a **session-based migration workflow**: 3. **Binary resolution**: Both `pgschema` and `dbmate` binaries are auto-resolved: - `pgschema`: Bundled in `vendor/pgschema/` for all platforms (darwin-{arm64,amd64}, linux-{arm64,amd64}, windows-{arm64,amd64}) - `dbmate`: npm-installed via the `dbmate` package -4. **Auto Docker container** (`modules/db/services/container.ts`): When `localDbUrl` is empty, PostKit: +4. **Auto Docker container** (`modules/db/services/container.ts`): When `localDbUrl` is empty, PostKit uses `resolveLocalDb(localDbUrl, remoteUrl, spinner)` which: - Checks Docker availability (`checkDockerAvailable()`) - - Queries remote PG version via `getRemotePgMajorVersion()` (uses `SHOW server_version_num`) + - Queries remote PG version via `getRemotePgMajorVersion()` (uses `SHOW server_version_num`) — callers do not pass the version - Starts `postgres:{version}-alpine` on a free port in range 15432–15532 - Runs `pg_dump`/`psql` inside the container via `docker exec` (`cloneDatabaseViaContainer()`) - Stores `containerID` in session; cleaned up on `db abort` or `db deploy` completion + - Used by `start`, `deploy`, and `import` commands 5. **Migration steps execution**: The `deploy` command uses `runSteps()` to execute multi-step operations with resume capability - if a step fails, re-running resumes from where it left off. 6. **Schema directory structure** (`db/schema/`): - `infra/` - Pre-migration (roles, schemas, extensions) - excluded from pgschema @@ -127,6 +128,13 @@ PostKit files are split between committed (shared with team) and gitignored (use **Key functions** (from `modules/db/services/`): - `generateSchemaSQLAndFingerprint()` - Reads all schema files once and returns both the output path (`.postkit/db/schema.sql`) and a SHA-256 fingerprint of the source files +- `resolveLocalDb(localDbUrl, remoteUrl, spinner, spinnerText?)` (`container.ts`) - When `localDbUrl` is empty, fetches PG version from `remoteUrl` and starts an auto Docker container. Used by `start`, `deploy`, and `import` commands. +- `withPgClient(url, fn)` (`database.ts`) - Scoped pg client wrapper; opens a connection, runs `fn`, closes on completion or error +- `checkDbPrerequisites(verbose)` (`prerequisites.ts`) - Shared pgschema + dbmate availability check used by all commands that need them +- `requireActiveSession()` (`utils/session.ts`) - Returns active session or throws a descriptive error +- `assertLocalConnection(session, spinner)` (`utils/session.ts`) - Tests local DB connection from session; throws if unreachable +- `resolveApplyTarget(target?)` (`utils/apply-target.ts`) - Resolves `"local"` or `"remote"` apply target; used by infra and seed commands +- `readJsonFile(path)` / `writeJsonFile(path, data)` (`utils/json-file.ts`) - Typed JSON helpers used by remotes and committed migration tracking ### Configuration System diff --git a/cli/docs/architecture.md b/cli/docs/architecture.md index 79a46b8..60e72c0 100644 --- a/cli/docs/architecture.md +++ b/cli/docs/architecture.md @@ -107,6 +107,19 @@ Shared utilities used by all modules, located in `cli/src/common/`: | `types.ts` | Shared TypeScript types (`CommandOptions`) | | `init-check.ts` | Project initialization validation | +### DB Module Shared Utilities + +Key shared utilities within the `db` module (used by multiple commands): + +| File | Purpose | +|------|---------| +| `utils/json-file.ts` | `readJsonFile()` / `writeJsonFile()` — typed JSON read/write | +| `utils/apply-target.ts` | `resolveApplyTarget(target?)` — resolves `local` or `remote` for infra/seed commands | +| `utils/session.ts` | `requireActiveSession()`, `assertLocalConnection(session, spinner)` | +| `services/prerequisites.ts` | `checkDbPrerequisites(verbose)` — verifies pgschema + dbmate are available | +| `services/database.ts` | `withPgClient(url, fn)` — scoped pg client wrapper | +| `services/container.ts` | `resolveLocalDb(localDbUrl, remoteUrl, spinner, spinnerText?)` — starts auto Docker container when `localDbUrl` is empty; fetches PG version from `remoteUrl` internally | + --- ## Configuration @@ -192,17 +205,25 @@ cli/test/ ## Runtime Directory Structure -All PostKit runtime files in `.postkit/` (gitignored): +PostKit files in `.postkit/` are split between gitignored (ephemeral/user-specific) and committed (shared with team): ``` .postkit/ ├── db/ -│ ├── session.json # Current session state -│ ├── committed.json # Committed migration tracking -│ ├── plan.sql # Generated migration plan -│ ├── schema.sql # Generated schema from files -│ ├── session/ # Session migrations (temporary) -│ └── migrations/ # Committed migrations (for deploy) +│ ├── session.json # GITIGNORED — active session state, local DB URL, container ID +│ ├── plan.sql # GITIGNORED — generated migration diff (ephemeral) +│ ├── schema.sql # GITIGNORED — generated schema artifact (ephemeral) +│ ├── session/ # GITIGNORED — temporary in-progress migrations +│ ├── committed.json # COMMITTED — migration tracking index (shared) +│ └── migrations/ # COMMITTED — committed SQL migrations for deploy (shared) └── auth/ - └── raw/ # Exported realm config (pre-clean) + ├── raw/ # COMMITTED — auth raw config (shared) + └── realm/ # COMMITTED — auth realm config (shared) ``` + +`.gitignore` (written by `postkit init`) covers only the ephemeral paths: +- `.postkit/db/session.json` +- `.postkit/db/plan.sql` +- `.postkit/db/schema.sql` +- `.postkit/db/session/` +- `postkit.secrets.json` diff --git a/cli/docs/db.md b/cli/docs/db.md index 04b060c..5de41c0 100644 --- a/cli/docs/db.md +++ b/cli/docs/db.md @@ -211,22 +211,24 @@ db/schema/ ### PostKit Directory Structure -All PostKit runtime files are stored in `.postkit/` (gitignored): +PostKit files in `.postkit/db/` are split between gitignored (ephemeral) and committed (shared with team): ``` .postkit/ └── db/ - ├── session.json # Current session state - ├── committed.json # Committed migrations tracking - ├── plan.sql # Generated migration plan - ├── schema.sql # Generated schema from files - ├── session/ # Session migrations (temporary) + ├── session.json # GITIGNORED — current session state + ├── plan.sql # GITIGNORED — generated migration plan + ├── schema.sql # GITIGNORED — generated schema from files + ├── session/ # GITIGNORED — session migrations (temporary) │ └── 20250131_*.sql - └── migrations/ # Committed migrations (for deploy) + ├── committed.json # COMMITTED — migrations tracking index (shared) + └── migrations/ # COMMITTED — committed migrations (for deploy) ├── 20250130_add_users.sql └── 20250131_add_posts.sql ``` +`postkit init` adds only the ephemeral paths to `.gitignore` (`.postkit/db/session.json`, `.postkit/db/plan.sql`, `.postkit/db/schema.sql`, `.postkit/db/session/`). The `migrations/` directory and `committed.json` are committed to git and shared across the team. + --- ## 🚀 Commands diff --git a/cli/docs/e2e-testing.md b/cli/docs/e2e-testing.md index 6eaf0a0..7284eb4 100644 --- a/cli/docs/e2e-testing.md +++ b/cli/docs/e2e-testing.md @@ -254,10 +254,10 @@ Tests infrastructure SQL (roles) and seed data management. Grant permissions are | Test | What It Tests | |------|---------------| | `db remote list` | Shows configured remote name | -| `db remote add` | Persists to `postkit.config.json` | -| `db remote add --default` | Sets `default: true` flag | -| `db remote use` | Switches default remote | -| `db remote remove --force` | Deletes from config | +| `db remote add` | Persists to `postkit.secrets.json` | +| `db remote add --default` | Sets `default: true` flag in `postkit.secrets.json` | +| `db remote use` | Switches default remote in `postkit.secrets.json` | +| `db remote remove --force` | Deletes from `postkit.secrets.json` | ### Error Handling (4 files) diff --git a/docs/docs/getting-started/quick-start.md b/docs/docs/getting-started/quick-start.md index f4879a9..f625851 100644 --- a/docs/docs/getting-started/quick-start.md +++ b/docs/docs/getting-started/quick-start.md @@ -21,7 +21,7 @@ This creates: - `postkit.secrets.json` - Your credentials (gitignored) - `postkit.secrets.example.json` - Credentials template for teammates (committed) - `db/schema/` - Your schema files directory -- `.postkit/` - Runtime files (gitignored) +- `.postkit/` - Runtime state (ephemeral files gitignored; committed migrations and auth config are tracked by git) ## 2. Configure Remotes From e1cd6aa82b156a766554aff16e152f13b104c06b Mon Sep 17 00:00:00 2001 From: supunappri99 Date: Fri, 8 May 2026 10:24:37 +0530 Subject: [PATCH 16/18] docs: update PR template and refine create-pr skill to support configurable base branches --- .claude/skills/create-pr/SKILL.md | 58 ++++++++++++++++++++++++------- .github/pull_request_template.md | 2 ++ 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/.claude/skills/create-pr/SKILL.md b/.claude/skills/create-pr/SKILL.md index db8ba1e..876f1d1 100644 --- a/.claude/skills/create-pr/SKILL.md +++ b/.claude/skills/create-pr/SKILL.md @@ -7,6 +7,11 @@ description: Generate a PR description and save to temp/pr-description.md. Generate a standardized PR description for the PostKit project and save it to `temp/pr-description.md`. +## Arguments + +Optional: base branch to compare against (e.g. `/create-pr dev` or `/create-pr main`). +Default to `main` if no argument is provided. + ## Project Context Read `CLAUDE.md` at the project root for project conventions. @@ -15,16 +20,16 @@ Use the PR template at `.github/pull_request_template.md`. ## Workflow ### Step 1: Analyze Changes -Gather branch information: + +Determine the base branch from the argument (default: `main`). Then gather branch information: ```bash -git log origin/main...HEAD --oneline -git diff origin/main...HEAD --stat -git diff origin/main...HEAD +git log origin/...HEAD --oneline +git diff origin/...HEAD --stat +git diff origin/...HEAD ``` Categorize changes into: features, fixes, refactors, tests, docs, chore. ### Step 2: Generate PR Description -Using the PR template structure, generate: **Title** — under 70 characters with conventional commit prefix: - `feat: ` for new features @@ -34,15 +39,44 @@ Using the PR template structure, generate: - `docs: ` for documentation changes - `chore: ` for build/tooling changes -**Body** — using the template sections: -- Summary (1-3 bullet points) -- Changes (specific list) -- Type of Change (check one) -- Test Plan (checklist) +**Body** — follow the exact section order from `.github/pull_request_template.md`: +1. **Summary** — 1-3 bullet points describing what this PR does +2. **Changes** — specific list of changes made +3. **Type of Change** — check one box (`[x]`) matching the primary change type +4. **Test Plan** — check completed items; fill in "Manually tested" description +5. **Breaking Changes** — check "No breaking changes" if none; otherwise describe them ### Step 3: Save to File + Create `temp/` directory if needed and save to `temp/pr-description.md`. -Use the exact format from `.github/pull_request_template.md` — read that file and follow its structure. + +The file must follow this exact structure: +``` +# + +**Branch:** `<current-branch>` → `<base-branch>` + +## Summary +... + +## Changes +... + +## Type of Change +... + +## Test Plan +... + +## Breaking Changes +... +``` + +Get the current branch name with: +```bash +git rev-parse --abbrev-ref HEAD +``` ### Step 4: Show to User -Display the generated PR description and ask for confirmation or edits before saving. + +Display the full generated PR description from the saved file and invite the user to request edits. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 621d163..4aef0d0 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,5 @@ +**Branch:** `<from-branch>` → `<to-branch>` + ## Summary <!-- 1-3 bullet points describing what this PR does --> From ea64fddda54eed52adafa7f9b56943c25300737d Mon Sep 17 00:00:00 2001 From: supunappri99 <supun@appritechnologies.com> Date: Fri, 8 May 2026 10:36:07 +0530 Subject: [PATCH 17/18] docs: update create-pr skill instructions to use remote tracking branches for git comparisons --- .claude/skills/create-pr/SKILL.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.claude/skills/create-pr/SKILL.md b/.claude/skills/create-pr/SKILL.md index 876f1d1..42c2d64 100644 --- a/.claude/skills/create-pr/SKILL.md +++ b/.claude/skills/create-pr/SKILL.md @@ -21,12 +21,17 @@ Use the PR template at `.github/pull_request_template.md`. ### Step 1: Analyze Changes -Determine the base branch from the argument (default: `main`). Then gather branch information: +Determine the base branch from the argument (default: `main`). Always compare against the **remote** tracking branch, not a local branch. First fetch to ensure the remote ref is up to date, then gather branch information: + ```bash +git fetch origin <base> git log origin/<base>...HEAD --oneline git diff origin/<base>...HEAD --stat git diff origin/<base>...HEAD ``` + +Always use `origin/<base>` (not `<base>`) in all git commands so the comparison is against the remote state, not a potentially stale local branch. + Categorize changes into: features, fixes, refactors, tests, docs, chore. ### Step 2: Generate PR Description From 6ce6256c7579deffae6284bea5a2579ff628393d Mon Sep 17 00:00:00 2001 From: supunappri99 <supun@appritechnologies.com> Date: Fri, 8 May 2026 19:50:24 +0530 Subject: [PATCH 18/18] chore: bump package version to 1.2.1 --- cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/package.json b/cli/package.json index b53f504..47d25ae 100644 --- a/cli/package.json +++ b/cli/package.json @@ -1,6 +1,6 @@ { "name": "@appritech/postkit", - "version": "1.2.0", + "version": "1.2.1", "description": "PostKit - Developer toolkit for database management and more", "type": "module", "main": "dist/index.js",