feat: schema-drift detection + dev auto-reconcile + os migrate (#2186)

.changeset/issue-2186-schema-migrate.md

@@ -0,0 +1,31 @@
+---
+"@objectstack/driver-sql": minor
+"@objectstack/cli": minor
+"@objectstack/rest": patch
+---
+
+Schema drift detection + `os migrate` for non-additive metadata changes (#2186).
+
+The metadata→DB schema sync was additive-only: it created tables and added
+columns but never altered/dropped existing ones, so relaxing `required`,
+changing a type/length, or dropping a field silently diverged from an existing
+database. The physical column won at write time, surfacing a misleading
+`organization_id is required` 400 even though `/meta` reported the field
+optional.
+
+- **driver-sql** — the SQL driver now detects managed-schema drift (metadata is
+  the source of truth) and categorises each divergence `safe` / `needs_confirm`
+  / `destructive`. `initObjects` warns once per divergence with an actionable
+  hint. A new opt-in `SqlDriverConfig.autoMigrate: 'safe'` auto-applies the
+  *loosening* subset (relax `NOT NULL`, widen varchar) so an existing dev DB
+  self-heals on restart — never destructive, force-disabled under
+  `NODE_ENV=production`. New public methods `detectManagedDrift()` /
+  `applyMigrationEntries()`. SQLite reconciles via the official table-rebuild
+  (copy → swap), preserving data; Postgres/MySQL alter in place.
+- **cli** — new `os migrate plan` (dry-run, categorised diff) and
+  `os migrate apply` (`--allow-destructive` for drops/tightenings, confirm gate,
+  `--json`). `os dev`/`serve` now pass `autoMigrate: 'safe'` in dev only.
+- **rest** — a `NOT NULL` violation that reaches the driver (metadata validation
+  already passed) now carries a drift-aware `hint` pointing at `os migrate`,
+  instead of only the misleading "field is required" message. The
+  `VALIDATION_FAILED` / `fields` envelope is unchanged for back-compat.

packages/cli/src/commands/migrate/apply.ts

@@ -0,0 +1,160 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { Command, Flags } from '@oclif/core';
+import chalk from 'chalk';
+import { createInterface } from 'node:readline';
+import {
+  printHeader,
+  printSuccess,
+  printWarning,
+  printError,
+  printInfo,
+  printStep,
+  createTimer,
+} from '../../utils/format.js';
+import {
+  bootSchemaStack,
+  renderPlan,
+  summarize,
+  groupByCategory,
+} from '../../utils/schema-migrate.js';
+
+async function confirm(question: string): Promise<boolean> {
+  if (!process.stdin.isTTY) return false; // non-interactive → require --yes
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+  try {
+    const answer: string = await new Promise((resolve) => rl.question(question, resolve));
+    return /^y(es)?$/i.test(answer.trim());
+  } finally {
+    rl.close();
+  }
+}
+
+/**
+ * `os migrate apply` — reconcile the physical database to metadata (#2186).
+ * Applies safe (loosening) + needs-confirm changes by default; destructive
+ * changes (drop column, tighten NOT NULL, narrow type) require
+ * `--allow-destructive`.
+ */
+export default class MigrateApply extends Command {
+  static override description =
+    'Reconcile the physical database to metadata (safe by default; destructive changes need --allow-destructive)';
+
+  static override examples = [
+    '$ os migrate apply',
+    '$ os migrate apply --yes',
+    '$ os migrate apply --allow-destructive --yes',
+    '$ os migrate apply --json',
+  ];
+
+  static override flags = {
+    'database-url': Flags.string({
+      description: 'Database URL to reconcile (defaults to $OS_DATABASE_URL / the project DB)',
+      env: 'OS_DATABASE_URL',
+    }),
+    'allow-destructive': Flags.boolean({
+      description: 'Also apply destructive changes (drop column, tighten NOT NULL, narrow type)',
+      default: false,
+    }),
+    yes: Flags.boolean({ char: 'y', description: 'Skip the confirmation prompt', default: false }),
+    json: Flags.boolean({ description: 'Output as JSON (implies non-interactive; requires --yes to mutate)' }),
+  };
+
+  async run(): Promise<void> {
+    const { flags } = await this.parse(MigrateApply);
+    const timer = createTimer();
+    const allowDestructive = flags['allow-destructive'];
+
+    if (!flags.json) {
+      printHeader('Migrate · apply');
+      printStep('Booting schema stack…');
+    }
+
+    let stack;
+    try {
+      stack = await bootSchemaStack({ databaseUrl: flags['database-url'] });
+    } catch (error: any) {
+      if (flags.json) { console.log(JSON.stringify({ error: error.message })); this.exit(1); }
+      printError(error.message || String(error));
+      this.exit(1);
+      return;
+    }
+
+    try {
+      if (!stack.driver) {
+        if (flags.json) { console.log(JSON.stringify({ error: 'no_sql_driver' })); return; }
+        printWarning('Schema migration is only supported on SQL drivers (SQLite / Postgres). No SQL driver is active.');
+        return;
+      }
+
+      const drift = await stack.driver.detectManagedDrift();
+      const grouped = groupByCategory(drift);
+
+      if (drift.length === 0) {
+        if (flags.json) { console.log(JSON.stringify({ applied: [], skipped: [], message: 'in_sync' })); return; }
+        printSuccess('Physical schema is already in sync with metadata — nothing to apply.');
+        return;
+      }
+
+      // Entries we intend to apply this run.
+      const intended = drift.filter((d) => d.category !== 'destructive' || allowDestructive);
+      const deferred = drift.filter((d) => d.category === 'destructive' && !allowDestructive);
+
+      if (!flags.json) {
+        printInfo(`Database: ${chalk.white(stack.dbLabel)}`);
+        console.log('');
+        renderPlan(drift);
+        printInfo(summarize(drift));
+        if (deferred.length > 0) {
+          printWarning(`${deferred.length} destructive change(s) will be SKIPPED (re-run with --allow-destructive to include them).`);
+        }
+        if (allowDestructive && grouped.destructive.length > 0) {
+          printWarning('Destructive changes assume your full app/plugin set is loaded. A column that looks "orphaned" here may belong to a plugin that is not part of this build.');
+        }
+      }
+
+      if (intended.length === 0) {
+        if (flags.json) { console.log(JSON.stringify({ applied: [], skipped: deferred, message: 'nothing_safe_to_apply' })); return; }
+        printWarning('No changes to apply without --allow-destructive.');
+        return;
+      }
+
+      // Confirmation gate.
+      if (!flags.yes) {
+        if (flags.json || !process.stdin.isTTY) {
+          if (flags.json) { console.log(JSON.stringify({ applied: [], skipped: drift, message: 'confirmation_required', hint: 'pass --yes' })); return; }
+          printWarning('Confirmation required. Re-run with --yes to apply, or use "os migrate plan" to preview.');
+          return;
+        }
+        const ok = await confirm(chalk.bold(`\nApply ${intended.length} change(s) to ${stack.dbLabel}? [y/N] `));
+        if (!ok) { printInfo('Aborted — no changes made.'); return; }
+      }
+
+      const { applied, skipped } = await stack.driver.applyMigrationEntries(drift, { allowDestructive });
+
+      if (flags.json) {
+        console.log(JSON.stringify({
+          database: stack.dbLabel,
+          applied,
+          skipped,
+          duration: timer.elapsed(),
+        }, null, 2));
+        return;
+      }
+
+      console.log('');
+      printSuccess(`Applied ${applied.length} change(s).`);
+      if (skipped.length > 0) {
+        printWarning(`Skipped ${skipped.length} change(s) (destructive without --allow-destructive, or unsupported on this dialect).`);
+      }
+      console.log(chalk.dim(`  ${timer.display()}`));
+      console.log('');
+    } catch (error: any) {
+      if (flags.json) { console.log(JSON.stringify({ error: error.message })); this.exit(1); }
+      printError(error.message || String(error));
+      this.exit(1);
+    } finally {
+      await stack.shutdown();
+    }
+  }
+}

packages/cli/src/commands/migrate/index.ts

@@ -0,0 +1,12 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import MigratePlan from './plan.js';
+
+/**
+ * `os migrate` with no subcommand defaults to the (read-only) plan, so the
+ * bare command can never mutate the schema by surprise (issue #2186).
+ */
+export default class Migrate extends MigratePlan {
+  static override description =
+    'Inspect / reconcile physical-database drift from metadata. Defaults to a dry-run plan; use "os migrate apply" to reconcile.';
+}

packages/cli/src/commands/migrate/plan.ts

@@ -0,0 +1,102 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { Command, Flags } from '@oclif/core';
+import chalk from 'chalk';
+import {
+  printHeader,
+  printSuccess,
+  printWarning,
+  printError,
+  printInfo,
+  printStep,
+  createTimer,
+} from '../../utils/format.js';
+import { bootSchemaStack, renderPlan, summarize } from '../../utils/schema-migrate.js';
+
+/**
+ * `os migrate plan` — dry-run diff of metadata vs the physical database,
+ * categorised safe / needs-confirm / destructive (issue #2186). Never mutates
+ * the schema.
+ */
+export default class MigratePlan extends Command {
+  static override description =
+    'Show how the physical database has drifted from metadata (dry run; no changes applied)';
+
+  static override examples = [
+    '$ os migrate plan',
+    '$ os migrate plan --json',
+    '$ os migrate plan --database-url postgres://localhost/app',
+  ];
+
+  static override flags = {
+    'database-url': Flags.string({
+      description: 'Database URL to inspect (defaults to $OS_DATABASE_URL / the project DB)',
+      env: 'OS_DATABASE_URL',
+    }),
+    json: Flags.boolean({ description: 'Output as JSON' }),
+  };
+
+  async run(): Promise<void> {
+    const { flags } = await this.parse(MigratePlan);
+    const timer = createTimer();
+
+    if (!flags.json) {
+      printHeader('Migrate · plan');
+      printStep('Booting schema stack…');
+    }
+
+    let stack;
+    try {
+      stack = await bootSchemaStack({ databaseUrl: flags['database-url'] });
+    } catch (error: any) {
+      if (flags.json) { console.log(JSON.stringify({ error: error.message })); this.exit(1); }
+      printError(error.message || String(error));
+      this.exit(1);
+      return;
+    }
+
+    try {
+      if (!stack.driver) {
+        if (flags.json) { console.log(JSON.stringify({ error: 'no_sql_driver', changes: [] })); return; }
+        printWarning('Schema migration is only supported on SQL drivers (SQLite / Postgres). No SQL driver is active.');
+        return;
+      }
+
+      const drift = await stack.driver.detectManagedDrift();
+
+      if (flags.json) {
+        console.log(JSON.stringify({
+          database: stack.dbLabel,
+          managedTables: stack.managedTableCount,
+          total: drift.length,
+          changes: drift,
+          duration: timer.elapsed(),
+        }, null, 2));
+        return;
+      }
+
+      printInfo(`Database: ${chalk.white(stack.dbLabel)}`);
+      printInfo(`Examined ${chalk.white(String(stack.managedTableCount))} managed table(s).`);
+      console.log('');
+
+      if (drift.length === 0) {
+        printSuccess('Physical schema is in sync with metadata — nothing to migrate.');
+        console.log('');
+        return;
+      }
+
+      renderPlan(drift);
+      printInfo(summarize(drift));
+      console.log(chalk.dim('  Apply with: ') + chalk.white('os migrate apply') +
+        chalk.dim(' (add --allow-destructive for drops / tightenings)'));
+      console.log(chalk.dim(`  ${timer.display()}`));
+      console.log('');
+    } catch (error: any) {
+      if (flags.json) { console.log(JSON.stringify({ error: error.message })); this.exit(1); }
+      printError(error.message || String(error));
+      this.exit(1);
+    } finally {
+      await stack.shutdown();
+    }
+  }
+}

packages/cli/src/commands/serve.ts

@@ -606,6 +606,9 @@ export default class Serve extends Command {
                client: 'better-sqlite3',
                connection: { filename: filePath },
                useNullAsDefault: true,
+               // #2186: in dev, self-heal a persisted DB when a metadata change
+               // relaxes a constraint (loosen-only; never destructive / never in prod).
+               autoMigrate: isDev ? 'safe' : undefined,
              }) as any));
              trackPlugin('SqlDriver');
              resolvedDriverLabel = 'SqlDriver(sqlite)';
@@ -626,6 +629,7 @@ export default class Serve extends Command {
                client: 'pg',
                connection: databaseUrl,
                pool: { min: 0, max: 5 },
+               autoMigrate: isDev ? 'safe' : undefined, // #2186 dev loosen-only self-heal
              }) as any));
              trackPlugin('PostgresDriver');
              resolvedDriverLabel = 'SqlDriver(pg)';
@@ -636,6 +640,7 @@ export default class Serve extends Command {
                client: 'mysql2',
                connection: databaseUrl,
                pool: { min: 0, max: 5 },
+               autoMigrate: isDev ? 'safe' : undefined, // #2186 dev loosen-only self-heal
              }) as any));
              trackPlugin('MySQLDriver');
              resolvedDriverLabel = 'SqlDriver(mysql2)';
@@ -666,6 +671,7 @@ export default class Serve extends Command {
                  client: 'better-sqlite3',
                  connection: { filename: ':memory:' },
                  useNullAsDefault: true,
+                 autoMigrate: 'safe', // #2186 dev loosen-only self-heal
                });
                await sqliteDriver.connect();
                sqliteOk = true;

packages/cli/src/index.ts

@@ -17,6 +17,11 @@ export { default as StartCommand } from './commands/start.js';
 export { default as TestCommand } from './commands/test.js';
 export { default as DoctorCommand } from './commands/doctor.js';
 
+// ─── Migrate topic subcommands (#2186) ──────────────────────────────
+export { default as MigrateCommand } from './commands/migrate/index.js';
+export { default as MigratePlanCommand } from './commands/migrate/plan.js';
+export { default as MigrateApplyCommand } from './commands/migrate/apply.js';
+
 // ─── Environments topic subcommands ─────────────────────────────────
 export { default as EnvironmentsListCommand } from './commands/environments/list.js';
 export { default as EnvironmentsShowCommand } from './commands/environments/show.js';