Add v1.1 migration CLI for legacy data

docs/migration_v1_1.md

@@ -0,0 +1,85 @@
+# Firebase v1.1 migration (Rails/Postgres ➜ Firestore)
+
+This guide describes how to migrate the legacy Rails/Postgres data into the Firebase v1 architecture without changing the legacy app.
+
+## What moves (and what does not)
+
+- **Migrated**: instances, rooms, messages, memberships, and deterministic anonymous user profiles (nymTag + glyphBits) created from legacy user IDs.
+- **Not migrated**: Firebase Auth identities or passwords. Legacy users become Firestore users with IDs like `legacy:{legacyUserId}`.
+- **Attachments/avatars**: not yet copied. The CLI can export references, but files are left for a later v1.2 pass.
+
+## Prerequisites
+
+- Legacy stack running (Postgres reachable). Example: `docker-compose up -d postgres redis website`.
+- Firebase Emulator Suite running for Firestore (preferred for dry runs). Example: `firebase emulators:start`.
+- Node 18+ available locally.
+- No production credentials required for emulator runs.
+
+Environment variables for the migration CLI live in `scripts/migrate_legacy_to_firestore/.env` (see `.env.example`). Key values:
+
+- `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE` — Postgres connection.
+- `MIGRATION_NYM_SALT` — must match the Cloud Functions salt so nym generation stays deterministic.
+- `GOOGLE_APPLICATION_CREDENTIALS` — only required for `--target=prod` runs.
+
+## Commands
+
+All commands are executed from `scripts/migrate_legacy_to_firestore`.
+
+```bash
+cd scripts/migrate_legacy_to_firestore
+npm install
+```
+
+### Export (Postgres ➜ JSONL)
+
+```bash
+npm run export -- --output=./out
+```
+
+- Reads legacy tables and writes JSONL files (`instances.jsonl`, `rooms.jsonl`, `users.jsonl`, `memberships.jsonl`, `messages.jsonl`) that already match the Firestore v1 model.
+- Document IDs are deterministic (`legacy:{id}`) to make the export idempotent.
+
+### Import (JSONL ➜ Firestore)
+
+```bash
+npm run import -- --target=emulator --output=./out
+# or with a dry run
+npm run import -- --target=emulator --output=./out -- --dry-run
+```
+
+- Defaults to the emulator; pass `--target=prod` only when authenticated with production credentials.
+- Uses batch writes with `{ merge: true }` so re-running the import overwrites safely.
+- `--dry-run` logs intended writes without touching Firestore.
+
+### Validate (counts)
+
+```bash
+npm run validate -- --target=emulator
+```
+
+- Compares Postgres counts (rooms, memberships, messages) to Firestore collection group counts.
+- Prints a diff summary to highlight mismatches.
+
+## Data mapping
+
+- **Instances**: `instances/{instanceId}` where `instanceId` is the legacy numeric ID as a string. `settings.cloakMode` is forced to `true`.
+- **Users**: `instances/{instanceId}/users/legacy:{legacyUserId}` with deterministic `nymTag` and `glyphBits` using the shared salt. Legacy username/email are stored only for admin reference.
+- **Rooms**: `instances/{instanceId}/rooms/legacy:{roomId}` with lock fields, owner UID, message counters, and last message preview/time derived from messages.
+- **Memberships**: `instances/{instanceId}/rooms/{roomId}/members/legacy:{userId}`. Role resolution:
+  - `admin` if the user is the instance owner or has the legacy `admin` role.
+  - `mod` if the user appears in `moderatorships` for the instance.
+  - `member` otherwise.
+  - `mutedUntil` is set to a far-future timestamp when present in `muted_room_users`.
+  - Nicknames from `room_user_nicknames` are preserved.
+- **Messages**: `instances/{instanceId}/rooms/{roomId}/messages/legacy:{messageId}` with cloak-safe fields only (`authorUid`, `nymTag`, `glyphBits`, `text`, `createdAt`).
+
+## Production run guidance
+
+1. **Backup first**: snapshot the production database and Firestore.
+2. **Dry-run**: run `export` and `import --dry-run` against an emulator or staging project.
+3. **Test**: open the Next.js client against the emulator to verify migrated rooms/messages render correctly.
+4. **Prod import**: set `--target=prod` with `GOOGLE_APPLICATION_CREDENTIALS` pointing at a service account that can write to Firestore. Re-run `import` until validation diffs are zero.
+
+## Attachment follow-up
+
+Legacy uploads (CarrierWave) are not copied in v1.1. If needed later, extend the CLI to copy files into `instances/{instanceId}/rooms/{roomId}/attachments/{uid}/{file}` in Cloud Storage and update message docs with references.

scripts/migrate_legacy_to_firestore/.env.example

@@ -0,0 +1,12 @@
+# Postgres connection (legacy Rails database)
+PGHOST=localhost
+PGPORT=5432
+PGUSER=postgres
+PGPASSWORD=postgres
+PGDATABASE=threads_development
+
+# Firestore target (emulator by default)
+# GOOGLE_APPLICATION_CREDENTIALS=/absolute/path/service-account.json
+
+# Salt used for deterministic nym generation; match functions config
+MIGRATION_NYM_SALT=dev-salt

scripts/migrate_legacy_to_firestore/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "migrate-legacy-to-firestore",
+  "version": "1.0.0",
+  "private": true,
+  "type": "commonjs",
+  "scripts": {
+    "build": "tsc",
+    "export": "ts-node src/index.ts export",
+    "import": "ts-node src/index.ts import",
+    "validate": "ts-node src/index.ts validate"
+  },
+  "dependencies": {
+    "dotenv": "^16.4.5",
+    "firebase-admin": "^12.1.1",
+    "pg": "^8.11.5",
+    "yargs": "^17.7.2"
+  },
+  "devDependencies": {
+    "@types/node": "^20.12.8",
+    "@types/pg": "^8.11.6",
+    "@types/yargs": "^17.0.32",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.5"
+  }
+}

scripts/migrate_legacy_to_firestore/src/config.ts

@@ -0,0 +1,40 @@
+import path from 'path';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+export type Target = 'emulator' | 'prod';
+
+export interface Config {
+  pg: {
+    host: string;
+    port: number;
+    user: string;
+    password: string;
+    database: string;
+  };
+  outputDir: string;
+  nymSalt: string;
+  target: Target;
+  dryRun: boolean;
+}
+
+export function loadConfig(overrides?: Partial<Pick<Config, 'target' | 'dryRun' | 'outputDir'>>): Config {
+  const target = overrides?.target ?? 'emulator';
+  const dryRun = overrides?.dryRun ?? false;
+  const outputDir = overrides?.outputDir ?? path.resolve(process.cwd(), 'out');
+
+  return {
+    pg: {
+      host: process.env.PGHOST || 'localhost',
+      port: Number(process.env.PGPORT || 5432),
+      user: process.env.PGUSER || 'postgres',
+      password: process.env.PGPASSWORD || 'postgres',
+      database: process.env.PGDATABASE || 'threads_development',
+    },
+    outputDir,
+    nymSalt: process.env.MIGRATION_NYM_SALT || 'dev-salt',
+    target,
+    dryRun,
+  };
+}

scripts/migrate_legacy_to_firestore/src/firestore.ts

@@ -0,0 +1,100 @@
+import admin from 'firebase-admin';
+import fs from 'fs';
+import readline from 'readline';
+import { FirestoreDoc } from './transform';
+import { Config, Target } from './config';
+
+function reviveTimestamps(value: unknown): unknown {
+  if (value === null || value === undefined) return value;
+  if (typeof value === 'string') {
+    if (/\d{4}-\d{2}-\d{2}T/.test(value)) {
+      const parsed = Date.parse(value);
+      if (!Number.isNaN(parsed)) {
+        return admin.firestore.Timestamp.fromDate(new Date(parsed));
+      }
+    }
+  }
+  if (Array.isArray(value)) {
+    return value.map((v) => reviveTimestamps(v));
+  }
+  if (typeof value === 'object') {
+    const result: Record<string, unknown> = {};
+    Object.entries(value as Record<string, unknown>).forEach(([k, v]) => {
+      result[k] = reviveTimestamps(v);
+    });
+    return result;
+  }
+  return value;
+}
+
+export function initFirestore(target: Target): admin.firestore.Firestore {
+  if (admin.apps.length === 0) {
+    const options: admin.AppOptions = {};
+    if (target === 'emulator') {
+      process.env.FIRESTORE_EMULATOR_HOST = process.env.FIRESTORE_EMULATOR_HOST || '127.0.0.1:8080';
+    } else {
+      if (!process.env.GOOGLE_APPLICATION_CREDENTIALS) {
+        throw new Error('GOOGLE_APPLICATION_CREDENTIALS is required for production runs');
+      }
+    }
+    admin.initializeApp(options);
+  }
+  return admin.firestore();
+}
+
+export async function importDocs(docs: FirestoreDoc[], db: admin.firestore.Firestore, cfg: Config): Promise<void> {
+  if (cfg.dryRun) {
+    console.log(`[dry-run] Would import ${docs.length} documents`);
+    return;
+  }
+
+  const batchSize = 450;
+  let batch = db.batch();
+  let counter = 0;
+
+  for (const doc of docs) {
+    const ref = db.doc(doc.path);
+    const data = reviveTimestamps(doc.data);
+    batch.set(ref, data, { merge: true });
+    counter++;
+    if (counter % batchSize === 0) {
+      await batch.commit();
+      console.log(`Committed ${counter} docs`);
+      batch = db.batch();
+      await new Promise((resolve) => setTimeout(resolve, 50));
+    }
+  }
+  if (counter % batchSize !== 0) {
+    await batch.commit();
+    console.log(`Committed ${counter} docs total`);
+  }
+}
+
+export async function readJsonl(filePath: string): Promise<FirestoreDoc[]> {
+  const file = fs.createReadStream(filePath, 'utf8');
+  const rl = readline.createInterface({ input: file, crlfDelay: Infinity });
+  const docs: FirestoreDoc[] = [];
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+    docs.push(JSON.parse(line) as FirestoreDoc);
+  }
+  return docs;
+}
+
+export async function countFirestore(db: admin.firestore.Firestore): Promise<{
+  rooms: number;
+  messages: number;
+  memberships: number;
+}> {
+  const [roomsSnap, messageSnap, memberSnap] = await Promise.all([
+    db.collectionGroup('rooms').get(),
+    db.collectionGroup('messages').get(),
+    db.collectionGroup('members').get(),
+  ]);
+
+  return {
+    rooms: roomsSnap.size,
+    messages: messageSnap.size,
+    memberships: memberSnap.size,
+  };
+}