From 88bdf5765533fd09643d5c8728d35b01a0dd3687 Mon Sep 17 00:00:00 2001 From: BenAppleDev Date: Mon, 15 Dec 2025 00:07:08 -0500 Subject: [PATCH] Add v1.1 migration CLI for legacy data --- .gitignore | 13 + README.md | 29 ++ apps/web/.env.example | 13 + apps/web/.eslintrc.json | 3 + apps/web/app/globals.css | 81 +++++ .../[instanceId]/rooms/[roomId]/page.tsx | 176 ++++++++++ .../app/instances/[instanceId]/rooms/page.tsx | 115 +++++++ apps/web/app/layout.tsx | 23 ++ apps/web/app/page.tsx | 30 ++ apps/web/components/Glyph.tsx | 49 +++ apps/web/lib/firebaseClient.ts | 66 ++++ apps/web/lib/nym.ts | 15 + apps/web/next-env.d.ts | 5 + apps/web/next.config.js | 9 + apps/web/package.json | 25 ++ apps/web/tsconfig.json | 22 ++ docs/migration_v1_1.md | 85 +++++ docs/modernization_v1.md | 87 +++++ firebase/.firebaserc | 5 + firebase/firebase.json | 27 ++ firebase/firestore.indexes.json | 19 ++ firebase/firestore.rules | 72 +++++ firebase/storage.rules | 13 + functions/.env.example | 2 + functions/package.json | 24 ++ functions/src/index.ts | 93 ++++++ functions/tsconfig.json | 16 + .../migrate_legacy_to_firestore/.env.example | 12 + .../migrate_legacy_to_firestore/package.json | 25 ++ .../migrate_legacy_to_firestore/src/config.ts | 40 +++ .../src/firestore.ts | 100 ++++++ .../migrate_legacy_to_firestore/src/index.ts | 143 ++++++++ .../src/postgres.ts | 133 ++++++++ .../src/transform.ts | 305 ++++++++++++++++++ .../src/validate.ts | 17 + .../migrate_legacy_to_firestore/tsconfig.json | 12 + scripts/package.json | 10 + scripts/seed_demo.ts | 41 +++ 38 files changed, 1955 insertions(+) create mode 100644 apps/web/.env.example create mode 100644 apps/web/.eslintrc.json create mode 100644 apps/web/app/globals.css create mode 100644 apps/web/app/instances/[instanceId]/rooms/[roomId]/page.tsx create mode 100644 apps/web/app/instances/[instanceId]/rooms/page.tsx create mode 100644 apps/web/app/layout.tsx create mode 100644 apps/web/app/page.tsx create mode 100644 apps/web/components/Glyph.tsx create mode 100644 apps/web/lib/firebaseClient.ts create mode 100644 apps/web/lib/nym.ts create mode 100644 apps/web/next-env.d.ts create mode 100644 apps/web/next.config.js create mode 100644 apps/web/package.json create mode 100644 apps/web/tsconfig.json create mode 100644 docs/migration_v1_1.md create mode 100644 docs/modernization_v1.md create mode 100644 firebase/.firebaserc create mode 100644 firebase/firebase.json create mode 100644 firebase/firestore.indexes.json create mode 100644 firebase/firestore.rules create mode 100644 firebase/storage.rules create mode 100644 functions/.env.example create mode 100644 functions/package.json create mode 100644 functions/src/index.ts create mode 100644 functions/tsconfig.json create mode 100644 scripts/migrate_legacy_to_firestore/.env.example create mode 100644 scripts/migrate_legacy_to_firestore/package.json create mode 100644 scripts/migrate_legacy_to_firestore/src/config.ts create mode 100644 scripts/migrate_legacy_to_firestore/src/firestore.ts create mode 100644 scripts/migrate_legacy_to_firestore/src/index.ts create mode 100644 scripts/migrate_legacy_to_firestore/src/postgres.ts create mode 100644 scripts/migrate_legacy_to_firestore/src/transform.ts create mode 100644 scripts/migrate_legacy_to_firestore/src/validate.ts create mode 100644 scripts/migrate_legacy_to_firestore/tsconfig.json create mode 100644 scripts/package.json create mode 100644 scripts/seed_demo.ts diff --git a/.gitignore b/.gitignore index 81f8b90..12ad554 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,16 @@ coverage .env config/database.yml + +# Node/Firebase artifacts +node_modules +apps/web/.next +apps/web/.env +apps/web/.turbo +functions/lib +functions/.env +firebase-debug.log +storage-debug.log +firestore-debug.log +scripts/migrate_legacy_to_firestore/out +scripts/migrate_legacy_to_firestore/.env diff --git a/README.md b/README.md index 7a23f3f..be92e50 100644 --- a/README.md +++ b/README.md @@ -32,3 +32,32 @@ To discuss current or potential uses, see our new [Threads forum](https://github ## License MIT + +## Modern Firebase v1 + +A parallel Firebase/GCS implementation lives in the `apps/web`, `firebase`, and `functions` directories. Rails remains unchanged. + +## Legacy ➜ Firebase migration (v1.1) + +Use the TypeScript CLI in `scripts/migrate_legacy_to_firestore` to export legacy Postgres data, import it into the Firestore emulator (or production), and validate counts. See `docs/migration_v1_1.md` for prerequisites, commands, and data mapping notes. + +### Local quickstart + +1. Install dependencies: + ```bash + cd functions && npm install + cd ../apps/web && npm install + ``` +2. In `firebase/`, start the Emulator Suite (auth, firestore, functions): + ```bash + firebase emulators:start + ``` +3. In another terminal run the Next.js client: + ```bash + cd apps/web + cp .env.example .env + npm run dev + ``` +4. Visit `http://localhost:3000`, sign in anonymously, create/join rooms, and chat in real time. The callable `ensureNymProfile` sets your deterministic nym identity; Firestore rules enforce cloak mode and membership. + +See [`docs/modernization_v1.md`](docs/modernization_v1.md) for architecture details, rules, and seeding helpers. diff --git a/apps/web/.env.example b/apps/web/.env.example new file mode 100644 index 0000000..4da47ea --- /dev/null +++ b/apps/web/.env.example @@ -0,0 +1,13 @@ +# Firebase web config +NEXT_PUBLIC_FIREBASE_API_KEY=your-api-key +NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN=localhost +NEXT_PUBLIC_FIREBASE_PROJECT_ID=threads-modern +NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET=threads-modern.appspot.com +NEXT_PUBLIC_FIREBASE_APP_ID=your-app-id +NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID= + +# When running against the emulator suite +NEXT_PUBLIC_USE_EMULATORS=true +NEXT_PUBLIC_FIRESTORE_EMULATOR_HOST=localhost:8080 +NEXT_PUBLIC_AUTH_EMULATOR_HOST=localhost:9099 +NEXT_PUBLIC_FUNCTIONS_EMULATOR_HOST=localhost:5001 diff --git a/apps/web/.eslintrc.json b/apps/web/.eslintrc.json new file mode 100644 index 0000000..957cd15 --- /dev/null +++ b/apps/web/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": ["next/core-web-vitals"] +} diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css new file mode 100644 index 0000000..003d636 --- /dev/null +++ b/apps/web/app/globals.css @@ -0,0 +1,81 @@ +:root { + color-scheme: light dark; + --bg: #0f0c29; + --bg-alt: #302b63; + --accent: #b18cff; + --text: #e9e9ff; + --muted: #c7c7e8; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + padding: 0; + font-family: 'Inter', system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; + background: radial-gradient(circle at 20% 20%, rgba(177, 140, 255, 0.1), transparent 25%), + radial-gradient(circle at 80% 0%, rgba(106, 255, 204, 0.08), transparent 25%), + linear-gradient(135deg, var(--bg), var(--bg-alt)); + color: var(--text); + min-height: 100vh; +} + +a { + color: var(--accent); + text-decoration: none; +} + +button { + background: linear-gradient(120deg, #8a5cff, #00e5ff); + color: #0b0b1c; + border: none; + padding: 10px 16px; + border-radius: 12px; + font-weight: 700; + cursor: pointer; + transition: transform 0.1s ease, box-shadow 0.2s ease; + box-shadow: 0 8px 32px rgba(0, 229, 255, 0.2); +} + +button:hover { + transform: translateY(-1px); + box-shadow: 0 12px 48px rgba(177, 140, 255, 0.35); +} + +.container { + max-width: 900px; + margin: 0 auto; + padding: 24px 16px 64px; +} + +.card { + background: rgba(255, 255, 255, 0.04); + border: 1px solid rgba(255, 255, 255, 0.06); + border-radius: 16px; + padding: 16px; + box-shadow: 0 12px 32px rgba(0, 0, 0, 0.35); +} + +.input { + width: 100%; + padding: 10px 12px; + border-radius: 10px; + border: 1px solid rgba(255, 255, 255, 0.2); + background: rgba(0, 0, 0, 0.2); + color: var(--text); +} + +.chip { + display: inline-flex; + align-items: center; + padding: 6px 10px; + border-radius: 9999px; + background: rgba(177, 140, 255, 0.15); + border: 1px solid rgba(177, 140, 255, 0.35); + color: var(--text); + font-size: 12px; + letter-spacing: 0.04em; + text-transform: uppercase; +} diff --git a/apps/web/app/instances/[instanceId]/rooms/[roomId]/page.tsx b/apps/web/app/instances/[instanceId]/rooms/[roomId]/page.tsx new file mode 100644 index 0000000..7f9ea80 --- /dev/null +++ b/apps/web/app/instances/[instanceId]/rooms/[roomId]/page.tsx @@ -0,0 +1,176 @@ +'use client'; + +import { useEffect, useMemo, useState } from 'react'; +import { useParams } from 'next/navigation'; +import { + addDoc, + collection, + doc, + getDoc, + onSnapshot, + orderBy, + query, + serverTimestamp, + setDoc, +} from 'firebase/firestore'; +import { Glyph } from '../../../../../components/Glyph'; +import { ensureNymProfile, NymProfile } from '../../../../../lib/nym'; +import { getFirebaseClients, requireAnonymousUser } from '../../../../../lib/firebaseClient'; + +interface Message { + id: string; + text: string; + nymTag: string; + glyphBits: string; + createdAt?: { seconds: number; nanoseconds: number }; +} + +export default function RoomPage() { + const params = useParams<{ instanceId: string; roomId: string }>(); + const instanceId = params.instanceId; + const roomId = params.roomId; + const { firestore } = useMemo(() => getFirebaseClients(), []); + + const [messages, setMessages] = useState([]); + const [text, setText] = useState(''); + const [profile, setProfile] = useState(null); + const [status, setStatus] = useState('Joining room...'); + const [isMember, setIsMember] = useState(false); + + useEffect(() => { + let unsub: (() => void) | undefined; + requireAnonymousUser() + .then((user) => ensureNymProfile(instanceId).then((p) => ({ user, profile: p }))) + .then(async ({ user, profile: p }) => { + setProfile(p); + const membershipRef = doc(firestore, 'instances', instanceId, 'rooms', roomId, 'members', user.uid); + const membership = await getDoc(membershipRef); + if (membership.exists()) { + setIsMember(true); + } + const messagesQuery = query( + collection(firestore, 'instances', instanceId, 'rooms', roomId, 'messages'), + orderBy('createdAt', 'asc') + ); + unsub = onSnapshot( + messagesQuery, + (snap) => { + setMessages( + snap.docs.map((d) => ({ + id: d.id, + text: d.get('text') || '', + nymTag: d.get('nymTag'), + glyphBits: d.get('glyphBits'), + createdAt: d.get('createdAt'), + })) + ); + setStatus(''); + }, + (err) => setStatus(`Error loading messages: ${err.message}`) + ); + }) + .catch((err) => setStatus(`Auth error: ${err.message}`)); + + return () => { + if (unsub) unsub(); + }; + }, [firestore, instanceId, roomId]); + + const joinRoom = async () => { + const user = await requireAnonymousUser(); + await setDoc(doc(firestore, 'instances', instanceId, 'rooms', roomId, 'members', user.uid), { + role: 'member', + lastReadAt: serverTimestamp(), + }, { merge: true }); + setIsMember(true); + }; + + const sendMessage = async () => { + if (!profile) return; + const user = await requireAnonymousUser(); + await addDoc(collection(firestore, 'instances', instanceId, 'rooms', roomId, 'messages'), { + authorUid: user.uid, + nymTag: profile.nymTag, + glyphBits: profile.glyphBits, + text, + createdAt: serverTimestamp(), + }); + setText(''); + }; + + return ( +
+
+
+
+

Room {roomId}

+
Instance: {instanceId}
+
+ {profile && ( +
+
+
You are
+ {profile.nymTag} +
+ +
+ )} +
+ + {!isMember && ( +
+

Join to read and send messages.

+ +
+ )} + +
+
+ {status &&

{status}

} + {messages.map((msg) => ( +
+ +
+
{msg.nymTag}
+
{msg.text}
+
+
+ ))} +
+ +
+ setText(e.target.value)} + disabled={!isMember} + /> + +
+
+
+
+ ); +} diff --git a/apps/web/app/instances/[instanceId]/rooms/page.tsx b/apps/web/app/instances/[instanceId]/rooms/page.tsx new file mode 100644 index 0000000..57b7067 --- /dev/null +++ b/apps/web/app/instances/[instanceId]/rooms/page.tsx @@ -0,0 +1,115 @@ +'use client'; + +import { useEffect, useMemo, useState } from 'react'; +import { useParams, useRouter } from 'next/navigation'; +import { + addDoc, + collection, + doc, + onSnapshot, + orderBy, + query, + serverTimestamp, + setDoc, +} from 'firebase/firestore'; +import { ensureNymProfile } from '../../../../lib/nym'; +import { getFirebaseClients, requireAnonymousUser } from '../../../../lib/firebaseClient'; + +export default function RoomsPage() { + const params = useParams<{ instanceId: string }>(); + const instanceId = params.instanceId; + const router = useRouter(); + const [rooms, setRooms] = useState>([]); + const [title, setTitle] = useState(''); + const [status, setStatus] = useState('Loading rooms...'); + + const { firestore } = useMemo(() => getFirebaseClients(), []); + + useEffect(() => { + let unsub: (() => void) | undefined; + requireAnonymousUser() + .then((user) => ensureNymProfile(instanceId).then(() => user)) + .then((user) => { + const roomsQuery = query( + collection(firestore, 'instances', instanceId, 'rooms'), + orderBy('createdAt', 'asc') + ); + unsub = onSnapshot( + roomsQuery, + (snap) => { + const data = snap.docs.map((d) => ({ id: d.id, title: d.get('title') || 'Untitled', messagesCount: d.get('messagesCount') })); + setRooms(data); + setStatus(data.length ? '' : 'No rooms yet. Create one!'); + }, + (err) => setStatus(`Error loading rooms: ${err.message}`) + ); + }) + .catch((err) => setStatus(`Auth error: ${err.message}`)); + + return () => { + if (unsub) unsub(); + }; + }, [firestore, instanceId]); + + const createRoom = async () => { + const user = await requireAnonymousUser(); + const roomRef = await addDoc(collection(firestore, 'instances', instanceId, 'rooms'), { + title: title || 'New room', + createdAt: serverTimestamp(), + locked: false, + ownerUid: user.uid, + messagesCount: 0, + lastMessageAt: null, + lastMessagePreview: '', + }); + await setDoc(doc(firestore, 'instances', instanceId, 'rooms', roomRef.id, 'members', user.uid), { + role: 'admin', + lastReadAt: serverTimestamp(), + }); + setTitle(''); + router.push(`/instances/${instanceId}/rooms/${roomRef.id}`); + }; + + const joinRoom = async (roomId: string) => { + const user = await requireAnonymousUser(); + const membershipRef = doc(firestore, 'instances', instanceId, 'rooms', roomId, 'members', user.uid); + await setDoc(membershipRef, { + role: 'member', + lastReadAt: serverTimestamp(), + }, { merge: true }); + router.push(`/instances/${instanceId}/rooms/${roomId}`); + }; + + return ( +
+
+

Rooms in instance {instanceId}

+
+ setTitle(e.target.value)} + /> + +
+ {status &&

{status}

} +
    + {rooms.map((room) => ( +
  • +
    +
    + {room.title} +
    + {room.messagesCount ?? 0} messages +
    +
    + +
    +
    • + ))} +
+
+
+ ); +} diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx new file mode 100644 index 0000000..e7defc5 --- /dev/null +++ b/apps/web/app/layout.tsx @@ -0,0 +1,23 @@ +import './globals.css'; +import type { ReactNode } from 'react'; + +export const metadata = { + title: 'Threads Modern Nymspace', + description: 'Firebase-backed anonymous rooms with cloak mode', +}; + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + +
+
+

Threads Nymspace

+ Cloak: ON +
+ {children} +
+ + + ); +} diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx new file mode 100644 index 0000000..67afc80 --- /dev/null +++ b/apps/web/app/page.tsx @@ -0,0 +1,30 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { useRouter } from 'next/navigation'; +import { requireAnonymousUser } from '../lib/firebaseClient'; + +export default function Home() { + const [status, setStatus] = useState(''); + const router = useRouter(); + const defaultInstance = 'demo-instance'; + + useEffect(() => { + requireAnonymousUser() + .then(() => setStatus('Signed in anonymously')) + .catch((err) => setStatus(`Auth error: ${err.message}`)); + }, []); + + const goToRooms = () => router.push(`/instances/${defaultInstance}/rooms`); + + return ( +
+
+

Welcome to the Nymspace

+

Anonymous-first rooms with cloak mode enforced via Firebase.

+

{status || 'Preparing anonymous identity...'}

+ +
+
+ ); +} diff --git a/apps/web/components/Glyph.tsx b/apps/web/components/Glyph.tsx new file mode 100644 index 0000000..d4912ad --- /dev/null +++ b/apps/web/components/Glyph.tsx @@ -0,0 +1,49 @@ +'use client'; + +import React from 'react'; + +function bitsToMatrix(bits: string): number[][] { + const normalized = bits.padEnd(64, '0').slice(0, 64); + const matrix: number[][] = []; + for (let i = 0; i < 8; i += 1) { + const row = normalized + .slice(i * 8, i * 8 + 8) + .split('') + .map((b) => (b === '1' ? 1 : 0)); + matrix.push(row); + } + return matrix; +} + +export function Glyph({ bits, size = 96 }: { bits: string; size?: number }) { + const matrix = bitsToMatrix(bits); + return ( +
+ {matrix.flatMap((row, rowIndex) => + row.map((cell, cellIndex) => ( +
+ )) + )} +
+ ); +} diff --git a/apps/web/lib/firebaseClient.ts b/apps/web/lib/firebaseClient.ts new file mode 100644 index 0000000..ff195fd --- /dev/null +++ b/apps/web/lib/firebaseClient.ts @@ -0,0 +1,66 @@ +import { initializeApp, getApps, getApp } from 'firebase/app'; +import { connectAuthEmulator, getAuth, onAuthStateChanged, signInAnonymously, User } from 'firebase/auth'; +import { connectFirestoreEmulator, getFirestore } from 'firebase/firestore'; +import { connectFunctionsEmulator, getFunctions } from 'firebase/functions'; + +type FirebaseClients = { + auth: ReturnType; + firestore: ReturnType; + functions: ReturnType; +}; + +const useEmulators = process.env.NEXT_PUBLIC_USE_EMULATORS === 'true'; + +function createApp() { + const config = { + apiKey: process.env.NEXT_PUBLIC_FIREBASE_API_KEY, + authDomain: process.env.NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN, + projectId: process.env.NEXT_PUBLIC_FIREBASE_PROJECT_ID, + storageBucket: process.env.NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET, + appId: process.env.NEXT_PUBLIC_FIREBASE_APP_ID, + measurementId: process.env.NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID, + }; + + return getApps().length ? getApp() : initializeApp(config); +} + +export function getFirebaseClients(): FirebaseClients { + const app = createApp(); + const firestore = getFirestore(app); + const auth = getAuth(app); + const functions = getFunctions(app); + + if (useEmulators) { + const firestoreHost = process.env.NEXT_PUBLIC_FIRESTORE_EMULATOR_HOST || 'localhost:8080'; + const [fsHost, fsPort] = firestoreHost.split(':'); + connectFirestoreEmulator(firestore, fsHost, Number(fsPort)); + + const authHost = process.env.NEXT_PUBLIC_AUTH_EMULATOR_HOST || 'localhost:9099'; + connectAuthEmulator(auth, `http://${authHost}`); + + const functionsHost = process.env.NEXT_PUBLIC_FUNCTIONS_EMULATOR_HOST || 'localhost:5001'; + const [fnHost, fnPort] = functionsHost.split(':'); + connectFunctionsEmulator(functions, fnHost, Number(fnPort)); + } + + return { auth, firestore, functions }; +} + +export async function requireAnonymousUser(): Promise { + const { auth } = getFirebaseClients(); + const currentUser = auth.currentUser; + if (currentUser) return currentUser; + await signInAnonymously(auth); + return new Promise((resolve, reject) => { + const unsub = onAuthStateChanged( + auth, + (user) => { + if (user) { + resolve(user); + unsub(); + } + }, + reject + ); + }); +} diff --git a/apps/web/lib/nym.ts b/apps/web/lib/nym.ts new file mode 100644 index 0000000..dfefabe --- /dev/null +++ b/apps/web/lib/nym.ts @@ -0,0 +1,15 @@ +import { httpsCallable } from 'firebase/functions'; +import { getFirebaseClients } from './firebaseClient'; + +export type NymProfile = { + nymTag: string; + glyphBits: string; + createdAt?: string; +}; + +export async function ensureNymProfile(instanceId: string): Promise { + const { functions } = getFirebaseClients(); + const callable = httpsCallable<{ instanceId: string }, NymProfile>(functions, 'ensureNymProfile'); + const result = await callable({ instanceId }); + return result.data; +} diff --git a/apps/web/next-env.d.ts b/apps/web/next-env.d.ts new file mode 100644 index 0000000..4f11a03 --- /dev/null +++ b/apps/web/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/basic-features/typescript for more information. diff --git a/apps/web/next.config.js b/apps/web/next.config.js new file mode 100644 index 0000000..b84e085 --- /dev/null +++ b/apps/web/next.config.js @@ -0,0 +1,9 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + reactStrictMode: true, + eslint: { + dirs: ['.'] + } +}; + +module.exports = nextConfig; diff --git a/apps/web/package.json b/apps/web/package.json new file mode 100644 index 0000000..ac216b0 --- /dev/null +++ b/apps/web/package.json @@ -0,0 +1,25 @@ +{ + "name": "threads-modern-web", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "next lint" + }, + "dependencies": { + "firebase": "^10.12.2", + "next": "^14.2.3", + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@types/node": "^20.12.12", + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "eslint": "^8.57.0", + "eslint-config-next": "^14.2.3", + "typescript": "^5.4.5" + } +} diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json new file mode 100644 index 0000000..3492aca --- /dev/null +++ b/apps/web/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "es5", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "types": ["node"], + "baseUrl": "." + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"] +} diff --git a/docs/migration_v1_1.md b/docs/migration_v1_1.md new file mode 100644 index 0000000..a5affec --- /dev/null +++ b/docs/migration_v1_1.md @@ -0,0 +1,85 @@ +# Firebase v1.1 migration (Rails/Postgres ➜ Firestore) + +This guide describes how to migrate the legacy Rails/Postgres data into the Firebase v1 architecture without changing the legacy app. + +## What moves (and what does not) + +- **Migrated**: instances, rooms, messages, memberships, and deterministic anonymous user profiles (nymTag + glyphBits) created from legacy user IDs. +- **Not migrated**: Firebase Auth identities or passwords. Legacy users become Firestore users with IDs like `legacy:{legacyUserId}`. +- **Attachments/avatars**: not yet copied. The CLI can export references, but files are left for a later v1.2 pass. + +## Prerequisites + +- Legacy stack running (Postgres reachable). Example: `docker-compose up -d postgres redis website`. +- Firebase Emulator Suite running for Firestore (preferred for dry runs). Example: `firebase emulators:start`. +- Node 18+ available locally. +- No production credentials required for emulator runs. + +Environment variables for the migration CLI live in `scripts/migrate_legacy_to_firestore/.env` (see `.env.example`). Key values: + +- `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE` — Postgres connection. +- `MIGRATION_NYM_SALT` — must match the Cloud Functions salt so nym generation stays deterministic. +- `GOOGLE_APPLICATION_CREDENTIALS` — only required for `--target=prod` runs. + +## Commands + +All commands are executed from `scripts/migrate_legacy_to_firestore`. + +```bash +cd scripts/migrate_legacy_to_firestore +npm install +``` + +### Export (Postgres ➜ JSONL) + +```bash +npm run export -- --output=./out +``` + +- Reads legacy tables and writes JSONL files (`instances.jsonl`, `rooms.jsonl`, `users.jsonl`, `memberships.jsonl`, `messages.jsonl`) that already match the Firestore v1 model. +- Document IDs are deterministic (`legacy:{id}`) to make the export idempotent. + +### Import (JSONL ➜ Firestore) + +```bash +npm run import -- --target=emulator --output=./out +# or with a dry run +npm run import -- --target=emulator --output=./out -- --dry-run +``` + +- Defaults to the emulator; pass `--target=prod` only when authenticated with production credentials. +- Uses batch writes with `{ merge: true }` so re-running the import overwrites safely. +- `--dry-run` logs intended writes without touching Firestore. + +### Validate (counts) + +```bash +npm run validate -- --target=emulator +``` + +- Compares Postgres counts (rooms, memberships, messages) to Firestore collection group counts. +- Prints a diff summary to highlight mismatches. + +## Data mapping + +- **Instances**: `instances/{instanceId}` where `instanceId` is the legacy numeric ID as a string. `settings.cloakMode` is forced to `true`. +- **Users**: `instances/{instanceId}/users/legacy:{legacyUserId}` with deterministic `nymTag` and `glyphBits` using the shared salt. Legacy username/email are stored only for admin reference. +- **Rooms**: `instances/{instanceId}/rooms/legacy:{roomId}` with lock fields, owner UID, message counters, and last message preview/time derived from messages. +- **Memberships**: `instances/{instanceId}/rooms/{roomId}/members/legacy:{userId}`. Role resolution: + - `admin` if the user is the instance owner or has the legacy `admin` role. + - `mod` if the user appears in `moderatorships` for the instance. + - `member` otherwise. + - `mutedUntil` is set to a far-future timestamp when present in `muted_room_users`. + - Nicknames from `room_user_nicknames` are preserved. +- **Messages**: `instances/{instanceId}/rooms/{roomId}/messages/legacy:{messageId}` with cloak-safe fields only (`authorUid`, `nymTag`, `glyphBits`, `text`, `createdAt`). + +## Production run guidance + +1. **Backup first**: snapshot the production database and Firestore. +2. **Dry-run**: run `export` and `import --dry-run` against an emulator or staging project. +3. **Test**: open the Next.js client against the emulator to verify migrated rooms/messages render correctly. +4. **Prod import**: set `--target=prod` with `GOOGLE_APPLICATION_CREDENTIALS` pointing at a service account that can write to Firestore. Re-run `import` until validation diffs are zero. + +## Attachment follow-up + +Legacy uploads (CarrierWave) are not copied in v1.1. If needed later, extend the CLI to copy files into `instances/{instanceId}/rooms/{roomId}/attachments/{uid}/{file}` in Cloud Storage and update message docs with references. diff --git a/docs/modernization_v1.md b/docs/modernization_v1.md new file mode 100644 index 0000000..e7d2725 --- /dev/null +++ b/docs/modernization_v1.md @@ -0,0 +1,87 @@ +# Modernization v1: Firebase-first architecture + +This document describes the parallel Firebase/GCS stack that lives alongside the legacy Rails app. + +## Data model (Cloud Firestore) + +``` +instances/{instanceId} + name, ownerUid, createdAt, roomsCount, settings { cloakMode: boolean }, theme { ... } + rooms/{roomId} + title, createdAt, locked, plannedLock, ownerUid, lastMessageAt, messagesCount, lastMessagePreview + messages/{messageId} + authorUid, nymTag, glyphBits, text, createdAt, deletedAt?, flags? + members/{uid} + role: "member"|"mod"|"admin", mutedUntil?, lastReadAt, nickname? + users/{uid} + nymTag, glyphBits, createdAt, roles, isBanned? +``` + +### Cloak mode +All messaging UI and writes must avoid real identity fields. Only `nymTag` and `glyphBits` identify a sender. Security rules reject forbidden keys such as `email`, `username`, or `realName` in message documents. + +## Security rules + +* Authentication is required for all reads and writes (anonymous auth allowed). +* Reading messages requires a membership document under `members/{uid}`. +* Writing a message requires membership, room unlocked (unless `mod/admin`), and the user must not be muted. +* Mods/admins can change other members or delete messages; users can create/update their own membership docs when joining. +* Storage uploads are limited to members at `instances/{instanceId}/rooms/{roomId}/attachments/{uid}/...` and reads are likewise restricted. + +See [`firebase/firestore.rules`](../firebase/firestore.rules) and [`firebase/storage.rules`](../firebase/storage.rules) for the exact logic. + +## Cloud Functions (TypeScript) + +* **ensureNymProfile**: HTTPS callable that deterministically generates a `nymTag` and `glyphBits` (8x8 pattern) from `instanceId + uid + salt`. The salt is provided via `functions:config:set nym.salt="..."` or `NYM_SALT` in the environment. Creates `instances/{instanceId}/users/{uid}` if missing. +* **onMessageCreated**: Firestore trigger that updates room counters (`messagesCount`, `lastMessageAt`, `lastMessagePreview`). +* **moderationStub**: Callable placeholder that flags messages containing simple banned keywords. + +Functions live in [`functions/src/index.ts`](../functions/src/index.ts) and compile to `functions/lib` via `npm run build`. + +## Web client (Next.js) + +Located at [`apps/web`](../apps/web): + +* Anonymous sign-in via Firebase Auth. +* `ensureNymProfile` invoked on instance entry. +* Room list with creation + join (`/instances/[instanceId]/rooms`). +* Realtime message view with cloak chip, glyph renderer, and composer (`/instances/[instanceId]/rooms/[roomId]`). +* Styling uses simple Vaporwave/space gradients with minimal components. + +## Local development + +1. Install dependencies: + ```bash + cd firebase && npm install -g firebase-tools # if not already + cd ../functions && npm install + cd ../apps/web && npm install + ``` +2. From `firebase/`, start emulators: + ```bash + firebase emulators:start + ``` +3. In another terminal, run the web app: + ```bash + cd apps/web + cp .env.example .env # update values if needed + npm run dev + ``` +4. Navigate to `http://localhost:3000`. The default flow signs you in anonymously, ensures a nym profile, allows creating/joining rooms, and sending messages in real time. + +### Seeding a demo instance + +The script [`scripts/seed_demo.ts`](../scripts/seed_demo.ts) seeds a demo instance/room against the emulators using the Admin SDK. + +Run with: +```bash +cd scripts +npm install +node seed_demo.ts +``` + +Ensure `FIREBASE_EMULATOR_HOST` variables are set (the script defaults to localhost ports used in `firebase.json`). + +## Deployment notes + +* No secrets are committed. Provide `NYM_SALT` via environment or `functions:config:set nym.salt="your-secret"` before deploying functions. +* Rails remains untouched; the Firebase stack lives in parallel directories. diff --git a/firebase/.firebaserc b/firebase/.firebaserc new file mode 100644 index 0000000..f9e9370 --- /dev/null +++ b/firebase/.firebaserc @@ -0,0 +1,5 @@ +{ + "projects": { + "default": "threads-modern" + } +} diff --git a/firebase/firebase.json b/firebase/firebase.json new file mode 100644 index 0000000..0cf6910 --- /dev/null +++ b/firebase/firebase.json @@ -0,0 +1,27 @@ +{ + "firestore": { + "rules": "firestore.rules", + "indexes": "firestore.indexes.json" + }, + "storage": { + "rules": "storage.rules" + }, + "functions": { + "source": "../functions", + "runtime": "nodejs18" + }, + "emulators": { + "firestore": { + "port": 8080 + }, + "auth": { + "port": 9099 + }, + "functions": { + "port": 5001 + }, + "ui": { + "enabled": true + } + } +} diff --git a/firebase/firestore.indexes.json b/firebase/firestore.indexes.json new file mode 100644 index 0000000..a18f511 --- /dev/null +++ b/firebase/firestore.indexes.json @@ -0,0 +1,19 @@ +{ + "indexes": [ + { + "collectionGroup": "rooms", + "queryScope": "COLLECTION", + "fields": [ + { "fieldPath": "createdAt", "order": "ASCENDING" } + ] + }, + { + "collectionGroup": "messages", + "queryScope": "COLLECTION", + "fields": [ + { "fieldPath": "createdAt", "order": "ASCENDING" } + ] + } + ], + "fieldOverrides": [] +} diff --git a/firebase/firestore.rules b/firebase/firestore.rules new file mode 100644 index 0000000..1af99fd --- /dev/null +++ b/firebase/firestore.rules @@ -0,0 +1,72 @@ +rules_version = '2'; +service cloud.firestore { + match /databases/{database}/documents { + function isAuth() { + return request.auth != null; + } + + function membershipDoc(instanceId, roomId) { + return get(/databases/$(database)/documents/instances/$(instanceId)/rooms/$(roomId)/members/$(request.auth.uid)); + } + + function isMember(instanceId, roomId) { + return membershipDoc(instanceId, roomId).exists(); + } + + function roleFor(instanceId, roomId) { + return membershipDoc(instanceId, roomId).data.role; + } + + function isModAdmin(instanceId, roomId) { + return roleFor(instanceId, roomId) in ['mod', 'admin']; + } + + function isMuted(instanceId, roomId) { + let doc = membershipDoc(instanceId, roomId); + return doc.exists() && doc.data.mutedUntil != null && request.time < doc.data.mutedUntil; + } + + function isLocked(instanceId, roomId) { + let room = get(/databases/$(database)/documents/instances/$(instanceId)/rooms/$(roomId)); + return room.exists() && room.data.locked == true; + } + + function hasForbiddenIdentityFields() { + let forbidden = ['email', 'username', 'realName', 'displayName']; + return request.resource.data.keys().hasAny(forbidden); + } + + match /instances/{instanceId} { + allow read, create: if isAuth(); + allow update, delete: if isAuth(); + + match /users/{uid} { + allow read, write: if isAuth() && request.auth.uid == uid; + } + + match /rooms/{roomId} { + allow read: if isAuth(); + allow create: if isAuth(); + allow update, delete: if isAuth(); + + match /members/{uid} { + allow read: if isAuth() && request.auth.uid == uid || isModAdmin(instanceId, roomId); + allow create: if isAuth() && request.auth.uid == uid; + allow update, delete: if isAuth() && (request.auth.uid == uid || isModAdmin(instanceId, roomId)); + } + + match /messages/{messageId} { + allow read: if isAuth() && isMember(instanceId, roomId); + + allow create: if isAuth() + && isMember(instanceId, roomId) + && !isMuted(instanceId, roomId) + && (!isLocked(instanceId, roomId) || isModAdmin(instanceId, roomId)) + && !hasForbiddenIdentityFields(); + + allow update, delete: if isAuth() && isModAdmin(instanceId, roomId); + } + } + } + } +} diff --git a/firebase/storage.rules b/firebase/storage.rules new file mode 100644 index 0000000..c18d4fe --- /dev/null +++ b/firebase/storage.rules @@ -0,0 +1,13 @@ +rules_version = '2'; +service firebase.storage { + match /b/{bucket}/o { + function isAuth() { return request.auth != null; } + function isMember(instanceId, roomId) { + return exists(/databases/(default)/documents/instances/$(instanceId)/rooms/$(roomId)/members/$(request.auth.uid)); + } + + match /instances/{instanceId}/rooms/{roomId}/attachments/{uid}/{fileName} { + allow read, write: if isAuth() && isMember(instanceId, roomId) && request.auth.uid == uid; + } + } +} diff --git a/functions/.env.example b/functions/.env.example new file mode 100644 index 0000000..646af15 --- /dev/null +++ b/functions/.env.example @@ -0,0 +1,2 @@ +# Secret salt used for deterministic nym generation +NYM_SALT=change-me diff --git a/functions/package.json b/functions/package.json new file mode 100644 index 0000000..4f02208 --- /dev/null +++ b/functions/package.json @@ -0,0 +1,24 @@ +{ + "name": "threads-functions", + "scripts": { + "build": "tsc", + "serve": "npm run build && firebase emulators:start --only functions", + "lint": "eslint .", + "deploy": "firebase deploy --only functions", + "watch": "tsc --watch" + }, + "main": "lib/index.js", + "engines": { + "node": "18" + }, + "dependencies": { + "firebase-admin": "^12.2.0", + "firebase-functions": "^5.0.1" + }, + "devDependencies": { + "@types/node": "^20.12.12", + "eslint": "^8.57.0", + "typescript": "^5.4.5" + }, + "private": true +} diff --git a/functions/src/index.ts b/functions/src/index.ts new file mode 100644 index 0000000..d294471 --- /dev/null +++ b/functions/src/index.ts @@ -0,0 +1,93 @@ +import * as crypto from 'crypto'; +import * as admin from 'firebase-admin'; +import * as functions from 'firebase-functions'; + +admin.initializeApp(); +const db = admin.firestore(); + +function getSalt(): string { + return process.env.NYM_SALT || functions.config().nym?.salt || 'dev-salt'; +} + +function hashString(input: string): string { + return crypto.createHash('sha256').update(input).digest('hex'); +} + +function glyphFromHash(hash: string): string { + // Convert first 64 bits of hash into bit string + const bytes = Buffer.from(hash.slice(0, 16), 'hex'); + return Array.from(bytes) + .map((b) => b.toString(2).padStart(8, '0')) + .join(''); +} + +function adjective(hash: string) { + const words = ['aurora', 'nebula', 'stardust', 'eclipse', 'quantum', 'plasma', 'luminous', 'orbit']; + const idx = parseInt(hash.slice(0, 2), 16) % words.length; + return words[idx]; +} + +function numberTag(hash: string) { + return (parseInt(hash.slice(2, 6), 16) % 900 + 100).toString(); +} + +export const ensureNymProfile = functions.https.onCall(async (data, context) => { + if (!context.auth?.uid) { + throw new functions.https.HttpsError('unauthenticated', 'Sign in required'); + } + + const instanceId = data.instanceId as string; + if (!instanceId) { + throw new functions.https.HttpsError('invalid-argument', 'instanceId is required'); + } + + const uid = context.auth.uid; + const userRef = db.doc(`instances/${instanceId}/users/${uid}`); + const snapshot = await userRef.get(); + if (snapshot.exists) { + return snapshot.data(); + } + + const salt = getSalt(); + const hash = hashString(`${instanceId}:${uid}:${salt}`); + const nymTag = `nym:${adjective(hash)}-${numberTag(hash)}`; + const glyphBits = glyphFromHash(hash); + const profile = { + nymTag, + glyphBits, + createdAt: admin.firestore.FieldValue.serverTimestamp(), + roles: [], + isBanned: false, + }; + + await userRef.set(profile); + return { nymTag, glyphBits }; +}); + +export const onMessageCreated = functions.firestore + .document('instances/{instanceId}/rooms/{roomId}/messages/{messageId}') + .onCreate(async (snap, context) => { + const data = snap.data(); + const { instanceId, roomId } = context.params; + + const preview = (data.text as string | undefined)?.slice(0, 140) ?? ''; + const roomRef = db.doc(`instances/${instanceId}/rooms/${roomId}`); + await roomRef.set( + { + messagesCount: admin.firestore.FieldValue.increment(1), + lastMessageAt: admin.firestore.FieldValue.serverTimestamp(), + lastMessagePreview: preview, + }, + { merge: true } + ); + }); + +export const moderationStub = functions.https.onCall(async (data, context) => { + if (!context.auth) { + throw new functions.https.HttpsError('unauthenticated', 'Auth required'); + } + const text = (data.text as string | undefined) || ''; + const blocked = ['spam', 'banword']; + const flagged = blocked.some((w) => text.toLowerCase().includes(w)); + return { flagged, reason: flagged ? 'Contains blocked keyword' : null }; +}); diff --git a/functions/tsconfig.json b/functions/tsconfig.json new file mode 100644 index 0000000..74509b8 --- /dev/null +++ b/functions/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "module": "commonjs", + "noImplicitReturns": true, + "noUnusedLocals": true, + "outDir": "lib", + "sourceMap": true, + "target": "es2017", + "lib": ["es2020"], + "moduleResolution": "node", + "esModuleInterop": true, + "resolveJsonModule": true + }, + "compileOnSave": true, + "include": ["src"] +} diff --git a/scripts/migrate_legacy_to_firestore/.env.example b/scripts/migrate_legacy_to_firestore/.env.example new file mode 100644 index 0000000..763d6cc --- /dev/null +++ b/scripts/migrate_legacy_to_firestore/.env.example @@ -0,0 +1,12 @@ +# Postgres connection (legacy Rails database) +PGHOST=localhost +PGPORT=5432 +PGUSER=postgres +PGPASSWORD=postgres +PGDATABASE=threads_development + +# Firestore target (emulator by default) +# GOOGLE_APPLICATION_CREDENTIALS=/absolute/path/service-account.json + +# Salt used for deterministic nym generation; match functions config +MIGRATION_NYM_SALT=dev-salt diff --git a/scripts/migrate_legacy_to_firestore/package.json b/scripts/migrate_legacy_to_firestore/package.json new file mode 100644 index 0000000..ffd59f5 --- /dev/null +++ b/scripts/migrate_legacy_to_firestore/package.json @@ -0,0 +1,25 @@ +{ + "name": "migrate-legacy-to-firestore", + "version": "1.0.0", + "private": true, + "type": "commonjs", + "scripts": { + "build": "tsc", + "export": "ts-node src/index.ts export", + "import": "ts-node src/index.ts import", + "validate": "ts-node src/index.ts validate" + }, + "dependencies": { + "dotenv": "^16.4.5", + "firebase-admin": "^12.1.1", + "pg": "^8.11.5", + "yargs": "^17.7.2" + }, + "devDependencies": { + "@types/node": "^20.12.8", + "@types/pg": "^8.11.6", + "@types/yargs": "^17.0.32", + "ts-node": "^10.9.2", + "typescript": "^5.4.5" + } +} diff --git a/scripts/migrate_legacy_to_firestore/src/config.ts b/scripts/migrate_legacy_to_firestore/src/config.ts new file mode 100644 index 0000000..5b47721 --- /dev/null +++ b/scripts/migrate_legacy_to_firestore/src/config.ts @@ -0,0 +1,40 @@ +import path from 'path'; +import dotenv from 'dotenv'; + +dotenv.config(); + +export type Target = 'emulator' | 'prod'; + +export interface Config { + pg: { + host: string; + port: number; + user: string; + password: string; + database: string; + }; + outputDir: string; + nymSalt: string; + target: Target; + dryRun: boolean; +} + +export function loadConfig(overrides?: Partial>): Config { + const target = overrides?.target ?? 'emulator'; + const dryRun = overrides?.dryRun ?? false; + const outputDir = overrides?.outputDir ?? path.resolve(process.cwd(), 'out'); + + return { + pg: { + host: process.env.PGHOST || 'localhost', + port: Number(process.env.PGPORT || 5432), + user: process.env.PGUSER || 'postgres', + password: process.env.PGPASSWORD || 'postgres', + database: process.env.PGDATABASE || 'threads_development', + }, + outputDir, + nymSalt: process.env.MIGRATION_NYM_SALT || 'dev-salt', + target, + dryRun, + }; +} diff --git a/scripts/migrate_legacy_to_firestore/src/firestore.ts b/scripts/migrate_legacy_to_firestore/src/firestore.ts new file mode 100644 index 0000000..577b961 --- /dev/null +++ b/scripts/migrate_legacy_to_firestore/src/firestore.ts @@ -0,0 +1,100 @@ +import admin from 'firebase-admin'; +import fs from 'fs'; +import readline from 'readline'; +import { FirestoreDoc } from './transform'; +import { Config, Target } from './config'; + +function reviveTimestamps(value: unknown): unknown { + if (value === null || value === undefined) return value; + if (typeof value === 'string') { + if (/\d{4}-\d{2}-\d{2}T/.test(value)) { + const parsed = Date.parse(value); + if (!Number.isNaN(parsed)) { + return admin.firestore.Timestamp.fromDate(new Date(parsed)); + } + } + } + if (Array.isArray(value)) { + return value.map((v) => reviveTimestamps(v)); + } + if (typeof value === 'object') { + const result: Record = {}; + Object.entries(value as Record).forEach(([k, v]) => { + result[k] = reviveTimestamps(v); + }); + return result; + } + return value; +} + +export function initFirestore(target: Target): admin.firestore.Firestore { + if (admin.apps.length === 0) { + const options: admin.AppOptions = {}; + if (target === 'emulator') { + process.env.FIRESTORE_EMULATOR_HOST = process.env.FIRESTORE_EMULATOR_HOST || '127.0.0.1:8080'; + } else { + if (!process.env.GOOGLE_APPLICATION_CREDENTIALS) { + throw new Error('GOOGLE_APPLICATION_CREDENTIALS is required for production runs'); + } + } + admin.initializeApp(options); + } + return admin.firestore(); +} + +export async function importDocs(docs: FirestoreDoc[], db: admin.firestore.Firestore, cfg: Config): Promise { + if (cfg.dryRun) { + console.log(`[dry-run] Would import ${docs.length} documents`); + return; + } + + const batchSize = 450; + let batch = db.batch(); + let counter = 0; + + for (const doc of docs) { + const ref = db.doc(doc.path); + const data = reviveTimestamps(doc.data); + batch.set(ref, data, { merge: true }); + counter++; + if (counter % batchSize === 0) { + await batch.commit(); + console.log(`Committed ${counter} docs`); + batch = db.batch(); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } + if (counter % batchSize !== 0) { + await batch.commit(); + console.log(`Committed ${counter} docs total`); + } +} + +export async function readJsonl(filePath: string): Promise { + const file = fs.createReadStream(filePath, 'utf8'); + const rl = readline.createInterface({ input: file, crlfDelay: Infinity }); + const docs: FirestoreDoc[] = []; + for await (const line of rl) { + if (!line.trim()) continue; + docs.push(JSON.parse(line) as FirestoreDoc); + } + return docs; +} + +export async function countFirestore(db: admin.firestore.Firestore): Promise<{ + rooms: number; + messages: number; + memberships: number; +}> { + const [roomsSnap, messageSnap, memberSnap] = await Promise.all([ + db.collectionGroup('rooms').get(), + db.collectionGroup('messages').get(), + db.collectionGroup('members').get(), + ]); + + return { + rooms: roomsSnap.size, + messages: messageSnap.size, + memberships: memberSnap.size, + }; +} diff --git a/scripts/migrate_legacy_to_firestore/src/index.ts b/scripts/migrate_legacy_to_firestore/src/index.ts new file mode 100644 index 0000000..7829f16 --- /dev/null +++ b/scripts/migrate_legacy_to_firestore/src/index.ts @@ -0,0 +1,143 @@ +import fs from 'fs'; +import path from 'path'; +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; +import { loadConfig, Target } from './config'; +import { connect, loadLegacyData } from './postgres'; +import { transformAll } from './transform'; +import { importDocs, initFirestore, readJsonl } from './firestore'; +import { validateCounts } from './validate'; + +async function ensureOutDir(dir: string) { + await fs.promises.mkdir(dir, { recursive: true }); +} + +async function writeJsonl(filePath: string, docs: unknown[]) { + const stream = fs.createWriteStream(filePath, { encoding: 'utf8' }); + docs.forEach((doc) => { + stream.write(JSON.stringify(doc)); + stream.write('\n'); + }); + stream.end(); + await new Promise((resolve) => stream.on('finish', resolve)); +} + +async function runExport(target: Target, outputDir: string) { + const cfg = loadConfig({ target, outputDir }); + const pool = await connect(cfg); + const data = await loadLegacyData(pool); + const docs = transformAll(data, cfg); + await ensureOutDir(cfg.outputDir); + + await writeJsonl(path.join(cfg.outputDir, 'instances.jsonl'), docs.filter((d) => d.path.split('/').length === 2)); + await writeJsonl( + path.join(cfg.outputDir, 'rooms.jsonl'), + docs.filter((d) => d.path.split('/').length === 4 && d.path.split('/')[2] === 'rooms') + ); + await writeJsonl( + path.join(cfg.outputDir, 'users.jsonl'), + docs.filter((d) => d.path.split('/').length === 4 && d.path.split('/')[2] === 'users') + ); + await writeJsonl( + path.join(cfg.outputDir, 'memberships.jsonl'), + docs.filter((d) => d.path.split('/').length === 6 && d.path.split('/')[4] === 'members') + ); + await writeJsonl( + path.join(cfg.outputDir, 'messages.jsonl'), + docs.filter((d) => d.path.split('/').length === 6 && d.path.split('/')[4] === 'messages') + ); + + await pool.end(); + console.log(`Exported ${docs.length} docs to ${cfg.outputDir}`); +} + +async function runImport(target: Target, outputDir: string, dryRun: boolean) { + const cfg = loadConfig({ target, dryRun, outputDir }); + const db = initFirestore(cfg.target); + + const files = ['instances.jsonl', 'rooms.jsonl', 'users.jsonl', 'memberships.jsonl', 'messages.jsonl']; + for (const file of files) { + const filePath = path.join(cfg.outputDir, file); + if (!fs.existsSync(filePath)) { + console.warn(`Skipping missing file ${filePath}`); + continue; + } + const docs = await readJsonl(filePath); + console.log(`Importing ${docs.length} docs from ${file}`); + await importDocs(docs, db, cfg); + } +} + +async function runValidate(target: Target, outputDir: string) { + const cfg = loadConfig({ target, outputDir }); + const pool = await connect(cfg); + const db = initFirestore(cfg.target); + await validateCounts(pool, db); + await pool.end(); +} + +yargs(hideBin(process.argv)) + .command( + 'export', + 'Export legacy data to JSONL files', + (y) => + y.option('output', { + alias: 'o', + type: 'string', + default: path.resolve(process.cwd(), 'out'), + describe: 'Output directory for JSONL files', + }), + async (argv) => { + await runExport((argv.target as Target) || 'emulator', argv.output as string); + } + ) + .command( + 'import', + 'Import JSONL files into Firestore', + (y) => + y + .option('output', { + alias: 'o', + type: 'string', + default: path.resolve(process.cwd(), 'out'), + describe: 'Directory containing JSONL files', + }) + .option('target', { + choices: ['emulator', 'prod'], + default: 'emulator', + }) + .option('dry-run', { + type: 'boolean', + default: false, + describe: 'Only log intended writes', + }), + async (argv) => { + await runImport((argv.target as Target) || 'emulator', argv.output as string, (argv['dry-run'] as boolean) || false); + } + ) + .command( + 'validate', + 'Validate counts between Postgres and Firestore', + (y) => + y + .option('target', { + choices: ['emulator', 'prod'], + default: 'emulator', + }) + .option('output', { + alias: 'o', + type: 'string', + default: path.resolve(process.cwd(), 'out'), + describe: 'Output directory (for parity with export/import)', + }), + async (argv) => { + await runValidate((argv.target as Target) || 'emulator', argv.output as string); + } + ) + .option('target', { + choices: ['emulator', 'prod'], + default: 'emulator', + describe: 'Firestore target', + }) + .demandCommand(1) + .help().argv; diff --git a/scripts/migrate_legacy_to_firestore/src/postgres.ts b/scripts/migrate_legacy_to_firestore/src/postgres.ts new file mode 100644 index 0000000..1e8d64f --- /dev/null +++ b/scripts/migrate_legacy_to_firestore/src/postgres.ts @@ -0,0 +1,133 @@ +import { Pool } from 'pg'; +import { Config } from './config'; + +export interface InstanceRow { + id: number; + title: string; + owner_id: number; + created_at: string; + rooms_count: number | null; +} + +export interface RoomRow { + id: number; + instance_id: number | null; + owner_id: number; + title: string; + created_at: string; + locked: boolean; + planned_lock: string | null; + messages_count: number | null; +} + +export interface UserRow { + id: number; + username: string; + email: string; + created_at: string; +} + +export interface MessageRow { + id: number; + room_id: number; + user_id: number; + content: string; + created_at: string; +} + +export interface RoomUserRow { + room_id: number; + user_id: number; + last_read_message_id: number | null; + created_at: string; + updated_at: string; +} + +export interface ModeratorRow { + instance_id: number | null; + user_id: number; +} + +export interface RoleUserRow { + role_id: number; + user_id: number; + role_name: string; +} + +export interface MutedRow { + room_id: number; + user_id: number; +} + +export interface NicknameRow { + room_id: number; + user_id: number; + nickname: string | null; +} + +export interface LegacyData { + instances: InstanceRow[]; + rooms: RoomRow[]; + users: UserRow[]; + messages: MessageRow[]; + roomUsers: RoomUserRow[]; + moderators: ModeratorRow[]; + roles: RoleUserRow[]; + muted: MutedRow[]; + nicknames: NicknameRow[]; +} + +export async function connect(config: Config): Promise { + const pool = new Pool({ + host: config.pg.host, + port: config.pg.port, + user: config.pg.user, + password: config.pg.password, + database: config.pg.database, + }); + return pool; +} + +async function fetchRows(pool: Pool, query: string): Promise { + const { rows } = await pool.query(query); + return rows; +} + +export async function loadLegacyData(pool: Pool): Promise { + const [instances, rooms, users, messages, roomUsers, moderators, roles, muted, nicknames] = await Promise.all([ + fetchRows(pool, 'select id, title, owner_id, created_at, rooms_count from instances'), + fetchRows( + pool, + 'select id, instance_id, owner_id, title, created_at, locked, planned_lock, messages_count from rooms' + ), + fetchRows(pool, 'select id, username, email, created_at from users'), + fetchRows(pool, 'select id, room_id, user_id, content, created_at from messages'), + fetchRows( + pool, + 'select room_id, user_id, last_read_message_id, created_at, updated_at from room_users' + ), + fetchRows(pool, 'select instance_id, user_id from moderatorships'), + fetchRows( + pool, + "select roles_users.role_id, roles_users.user_id, roles.name as role_name from roles_users join roles on roles.id = roles_users.role_id" + ), + fetchRows(pool, 'select room_id, user_id from muted_room_users'), + fetchRows(pool, 'select room_id, user_id, nickname from room_user_nicknames'), + ]); + + return { instances, rooms, users, messages, roomUsers, moderators, roles, muted, nicknames }; +} + +export async function loadCounts(pool: Pool): Promise<{ rooms: number; messages: number; memberships: number }> { + const [rooms, messages, memberships] = await Promise.all([ + pool.query<{ count: string }>('select count(*)::int as count from rooms'), + pool.query<{ count: string }>('select count(*)::int as count from messages'), + pool.query<{ count: string }>('select count(*)::int as count from room_users'), + ]); + + return { + rooms: Number(rooms.rows[0].count), + messages: Number(messages.rows[0].count), + memberships: Number(memberships.rows[0].count), + }; +} diff --git a/scripts/migrate_legacy_to_firestore/src/transform.ts b/scripts/migrate_legacy_to_firestore/src/transform.ts new file mode 100644 index 0000000..84f10f0 --- /dev/null +++ b/scripts/migrate_legacy_to_firestore/src/transform.ts @@ -0,0 +1,305 @@ +import crypto from 'crypto'; +import { + InstanceRow, + LegacyData, + MessageRow, + RoomRow, + RoomUserRow, + UserRow, +} from './postgres'; +import { Config } from './config'; + +export interface FirestoreDoc { + path: string; + data: Record; +} + +function hashString(input: string): string { + return crypto.createHash('sha256').update(input).digest('hex'); +} + +function glyphFromHash(hash: string): string { + const bytes = Buffer.from(hash.slice(0, 16), 'hex'); + return Array.from(bytes) + .map((b) => b.toString(2).padStart(8, '0')) + .join(''); +} + +function adjective(hash: string) { + const words = ['aurora', 'nebula', 'stardust', 'eclipse', 'quantum', 'plasma', 'luminous', 'orbit']; + const idx = parseInt(hash.slice(0, 2), 16) % words.length; + return words[idx]; +} + +function numberTag(hash: string) { + return (parseInt(hash.slice(2, 6), 16) % 900 + 100).toString(); +} + +function toDate(value: string | null): Date | null { + if (!value) return null; + return new Date(value); +} + +function buildLegacyUid(userId: number) { + return `legacy:${userId}`; +} + +function buildRoomId(roomId: number) { + return `legacy:${roomId}`; +} + +function buildMessageId(messageId: number) { + return `legacy:${messageId}`; +} + +function buildNym(instanceId: string, legacyUserId: number, salt: string) { + const uid = buildLegacyUid(legacyUserId); + const hash = hashString(`${instanceId}:${uid}:${salt}`); + const nymTag = `nym:${adjective(hash)}-${numberTag(hash)}`; + const glyphBits = glyphFromHash(hash); + return { uid, nymTag, glyphBits }; +} + +interface LookupMaps { + userById: Map; + roomById: Map; + instanceById: Map; + moderatorsByInstance: Map>; + adminUsers: Set; + mutedByRoom: Map>; + nicknameByRoomUser: Map; +} + +function buildLookups(data: LegacyData): LookupMaps { + const userById = new Map(); + data.users.forEach((u) => userById.set(u.id, u)); + + const roomById = new Map(); + data.rooms.forEach((r) => roomById.set(r.id, r)); + + const instanceById = new Map(); + data.instances.forEach((i) => instanceById.set(i.id, i)); + + const moderatorsByInstance = new Map>(); + data.moderators.forEach((m) => { + if (m.instance_id == null) return; + if (!moderatorsByInstance.has(m.instance_id)) { + moderatorsByInstance.set(m.instance_id, new Set()); + } + moderatorsByInstance.get(m.instance_id)!.add(m.user_id); + }); + + const adminUsers = new Set(); + data.roles.forEach((r) => { + if (r.role_name === 'admin') { + adminUsers.add(r.user_id); + } + }); + + const mutedByRoom = new Map>(); + data.muted.forEach((m) => { + if (!mutedByRoom.has(m.room_id)) { + mutedByRoom.set(m.room_id, new Set()); + } + mutedByRoom.get(m.room_id)!.add(m.user_id); + }); + + const nicknameByRoomUser = new Map(); + data.nicknames.forEach((n) => { + if (n.nickname) { + nicknameByRoomUser.set(`${n.room_id}:${n.user_id}`, n.nickname); + } + }); + + return { userById, roomById, instanceById, moderatorsByInstance, adminUsers, mutedByRoom, nicknameByRoomUser }; +} + +function transformInstances(data: LegacyData): FirestoreDoc[] { + return data.instances.map((instance) => { + const instanceId = String(instance.id); + return { + path: `instances/${instanceId}`, + data: { + name: instance.title, + ownerUid: buildLegacyUid(instance.owner_id), + createdAt: toDate(instance.created_at), + roomsCount: instance.rooms_count ?? 0, + settings: { cloakMode: true }, + }, + }; + }); +} + +function transformUsers(data: LegacyData, cfg: Config, lookups: LookupMaps): FirestoreDoc[] { + const docs: FirestoreDoc[] = []; + const instanceUserSet = new Map>(); + + const ensureSet = (instanceId: number) => { + if (!instanceUserSet.has(String(instanceId))) { + instanceUserSet.set(String(instanceId), new Set()); + } + return instanceUserSet.get(String(instanceId))!; + }; + + data.instances.forEach((i) => ensureSet(i.id).add(i.owner_id)); + data.rooms.forEach((r) => { + if (r.instance_id != null) { + ensureSet(r.instance_id).add(r.owner_id); + } + }); + data.roomUsers.forEach((ru) => { + const room = lookups.roomById.get(ru.room_id); + if (room?.instance_id != null) { + ensureSet(room.instance_id).add(ru.user_id); + } + }); + data.messages.forEach((m) => { + const room = lookups.roomById.get(m.room_id); + if (room?.instance_id != null) { + ensureSet(room.instance_id).add(m.user_id); + } + }); + + instanceUserSet.forEach((userIds, instanceId) => { + userIds.forEach((userId) => { + const profile = buildNym(instanceId, userId, cfg.nymSalt); + const user = lookups.userById.get(userId); + docs.push({ + path: `instances/${instanceId}/users/${profile.uid}`, + data: { + nymTag: profile.nymTag, + glyphBits: profile.glyphBits, + createdAt: user ? toDate(user.created_at) : null, + legacyUserId: userId, + legacyUsername: user?.username, + legacyEmail: user?.email, + }, + }); + }); + }); + + return docs; +} + +function deriveRole( + instanceId: number, + userId: number, + lookups: LookupMaps, + instanceOwnerId: number | null +): 'admin' | 'mod' | 'member' { + if (instanceOwnerId === userId || lookups.adminUsers.has(userId)) { + return 'admin'; + } + const mods = lookups.moderatorsByInstance.get(instanceId); + if (mods?.has(userId)) { + return 'mod'; + } + return 'member'; +} + +function transformRooms(data: LegacyData): FirestoreDoc[] { + const docs: FirestoreDoc[] = []; + data.rooms.forEach((room) => { + if (room.instance_id == null) return; + const instanceId = String(room.instance_id); + const roomId = buildRoomId(room.id); + docs.push({ + path: `instances/${instanceId}/rooms/${roomId}`, + data: { + title: room.title, + createdAt: toDate(room.created_at), + locked: room.locked, + plannedLock: toDate(room.planned_lock), + ownerUid: buildLegacyUid(room.owner_id), + messagesCount: room.messages_count ?? 0, + }, + }); + }); + return docs; +} + +function transformMemberships(data: LegacyData, lookups: LookupMaps): FirestoreDoc[] { + const docs: FirestoreDoc[] = []; + data.roomUsers.forEach((ru) => { + const room = lookups.roomById.get(ru.room_id); + if (!room || room.instance_id == null) return; + const instanceId = String(room.instance_id); + const role = deriveRole(room.instance_id, ru.user_id, lookups, lookups.instanceById.get(room.instance_id)?.owner_id ?? null); + const isMuted = lookups.mutedByRoom.get(ru.room_id)?.has(ru.user_id); + const nickname = lookups.nicknameByRoomUser.get(`${ru.room_id}:${ru.user_id}`); + docs.push({ + path: `instances/${instanceId}/rooms/legacy:${ru.room_id}/members/${buildLegacyUid(ru.user_id)}`, + data: { + role, + lastReadAt: toDate(ru.updated_at), + nickname: nickname || undefined, + mutedUntil: isMuted ? new Date('9999-12-31T00:00:00Z') : null, + }, + }); + }); + return docs; +} + +function transformMessages(data: LegacyData, cfg: Config, lookups: LookupMaps): FirestoreDoc[] { + const docs: FirestoreDoc[] = []; + data.messages.forEach((message) => { + const room = lookups.roomById.get(message.room_id); + if (!room || room.instance_id == null) return; + const instanceId = String(room.instance_id); + const { uid, nymTag, glyphBits } = buildNym(instanceId, message.user_id, cfg.nymSalt); + const messageId = buildMessageId(message.id); + docs.push({ + path: `instances/${instanceId}/rooms/${buildRoomId(room.id)}/messages/${messageId}`, + data: { + authorUid: uid, + nymTag, + glyphBits, + text: message.content, + createdAt: toDate(message.created_at), + }, + }); + }); + return docs; +} + +function computeRoomMessageStats(messages: MessageRow[]): Map +{ + const stats = new Map(); + messages.forEach((m) => { + const ts = toDate(m.created_at); + const preview = (m.content || '').slice(0, 140); + const current = stats.get(m.room_id); + if (!current || (ts && current.lastMessageAt && ts.getTime() > current.lastMessageAt.getTime())) { + stats.set(m.room_id, { lastMessageAt: ts, lastMessagePreview: preview }); + } else if (!current) { + stats.set(m.room_id, { lastMessageAt: ts, lastMessagePreview: preview }); + } + }); + return stats; +} + +export function transformAll(data: LegacyData, cfg: Config): FirestoreDoc[] { + const lookups = buildLookups(data); + const docs: FirestoreDoc[] = []; + const stats = computeRoomMessageStats(data.messages); + + docs.push(...transformInstances(data)); + docs.push(...transformRooms(data).map((doc) => { + const roomId = doc.path.split('/')[3]; + const roomNumericId = Number(roomId.replace('legacy:', '')); + const stat = stats.get(roomNumericId); + return { + ...doc, + data: { + ...doc.data, + lastMessageAt: stat?.lastMessageAt ?? null, + lastMessagePreview: stat?.lastMessagePreview ?? '', + }, + }; + })); + docs.push(...transformUsers(data, cfg, lookups)); + docs.push(...transformMemberships(data, lookups)); + docs.push(...transformMessages(data, cfg, lookups)); + + return docs; +} diff --git a/scripts/migrate_legacy_to_firestore/src/validate.ts b/scripts/migrate_legacy_to_firestore/src/validate.ts new file mode 100644 index 0000000..f93e3be --- /dev/null +++ b/scripts/migrate_legacy_to_firestore/src/validate.ts @@ -0,0 +1,17 @@ +import { Pool } from 'pg'; +import { countFirestore } from './firestore'; +import { loadCounts } from './postgres'; +import admin from 'firebase-admin'; + +export async function validateCounts(pool: Pool, db: admin.firestore.Firestore): Promise { + const [pgCounts, fsCounts] = await Promise.all([loadCounts(pool), countFirestore(db)]); + console.log('Postgres counts:', pgCounts); + console.log('Firestore counts:', fsCounts); + + const diff = { + rooms: fsCounts.rooms - pgCounts.rooms, + messages: fsCounts.messages - pgCounts.messages, + memberships: fsCounts.memberships - pgCounts.memberships, + }; + console.log('Diff (Firestore - Postgres):', diff); +} diff --git a/scripts/migrate_legacy_to_firestore/tsconfig.json b/scripts/migrate_legacy_to_firestore/tsconfig.json new file mode 100644 index 0000000..a1ef8e4 --- /dev/null +++ b/scripts/migrate_legacy_to_firestore/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2019", + "module": "CommonJS", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "outDir": "dist" + }, + "include": ["src/**/*"] +} diff --git a/scripts/package.json b/scripts/package.json new file mode 100644 index 0000000..d7c219a --- /dev/null +++ b/scripts/package.json @@ -0,0 +1,10 @@ +{ + "name": "threads-modern-scripts", + "private": true, + "scripts": { + "seed": "node seed_demo.ts" + }, + "dependencies": { + "firebase-admin": "^12.2.0" + } +} diff --git a/scripts/seed_demo.ts b/scripts/seed_demo.ts new file mode 100644 index 0000000..e880db7 --- /dev/null +++ b/scripts/seed_demo.ts @@ -0,0 +1,41 @@ +import admin from 'firebase-admin'; + +const projectId = process.env.FIREBASE_PROJECT_ID || 'threads-modern'; + +if (!admin.apps.length) { + admin.initializeApp({ projectId }); +} + +const db = admin.firestore(); + +db.settings({ host: process.env.FIRESTORE_EMULATOR_HOST || 'localhost:8080', ssl: false }); + +async function run() { + const instanceId = 'demo-instance'; + const roomId = 'welcome-room'; + const instanceRef = db.doc(`instances/${instanceId}`); + await instanceRef.set({ + name: 'Demo Instance', + createdAt: admin.firestore.FieldValue.serverTimestamp(), + settings: { cloakMode: true }, + roomsCount: 1, + }, { merge: true }); + + const roomRef = instanceRef.collection('rooms').doc(roomId); + await roomRef.set({ + title: 'Welcome Room', + createdAt: admin.firestore.FieldValue.serverTimestamp(), + locked: false, + messagesCount: 0, + ownerUid: 'seed-bot', + }, { merge: true }); + + console.log('Seeded demo instance and room'); +} + +run() + .catch((err) => { + console.error(err); + process.exit(1); + }) + .finally(() => process.exit());