diff --git a/apps/stack/.sst/types/index.ts b/apps/stack/.sst/types/index.ts index 8284fd5aa..4855f5be1 100644 --- a/apps/stack/.sst/types/index.ts +++ b/apps/stack/.sst/types/index.ts @@ -12,6 +12,20 @@ declare module "sst/node/config" { } } }import "sst/node/config"; +declare module "sst/node/config" { + export interface SecretResources { + "SUPER_DATABASE_URL": { + value: string; + } + } +}import "sst/node/config"; +declare module "sst/node/config" { + export interface SecretResources { + "SUPER_DATABASE_AUTH_TOKEN": { + value: string; + } + } +}import "sst/node/config"; declare module "sst/node/config" { export interface SecretResources { "CLERK_SECRET_KEY": { @@ -46,13 +60,6 @@ declare module "sst/node/config" { value: string; } } -}import "sst/node/config"; -declare module "sst/node/config" { - export interface SecretResources { - "TENANTS": { - value: string; - } - } }import "sst/node/event-bus"; declare module "sst/node/event-bus" { export interface EventBusResources { diff --git a/apps/stack/package.json b/apps/stack/package.json index c70df73e1..dc38ffdf7 100644 --- a/apps/stack/package.json +++ b/apps/stack/package.json @@ -22,6 +22,7 @@ "@acme/extract-functions": "*", "@acme/extract-schema": "*", "@acme/source-control": "*", + "@acme/super-schema": "*", "@acme/transform-functions": "*", "@acme/transform-schema": "*", "@aws-sdk/client-sqs": "^3.470.0", diff --git a/apps/stack/src/config/crawl.ts b/apps/stack/src/config/crawl.ts index 8c38477b4..59b95f3d9 100644 --- a/apps/stack/src/config/crawl.ts +++ b/apps/stack/src/config/crawl.ts @@ -3,7 +3,7 @@ import { events } from "@acme/crawl-schema"; import type { Context, InsertEventEntities } from "@acme/crawl-functions"; import type { EventNamespaceType } from "@acme/crawl-schema/src/events"; import { getTenantDb, type OmitDb } from "./get-tenant-db"; -import type { Tenant } from "./tenants"; +import type { Tenant } from "@acme/super-schema"; const context: OmitDb> = { entities: { diff --git a/apps/stack/src/config/create-event.ts b/apps/stack/src/config/create-event.ts index 03bb4f55e..eddc7b295 100644 --- a/apps/stack/src/config/create-event.ts +++ b/apps/stack/src/config/create-event.ts @@ -7,7 +7,7 @@ import { EventBus } from "sst/node/event-bus"; import { z } from "zod"; import { crawlComplete, crawlFailed } from "./crawl"; import type { EventNamespaceType } from "@acme/crawl-schema"; -import type { Tenant } from "./tenants"; +import type { Tenant } from "@acme/super-schema"; const client = new EventBridgeClient({}); type InferShapeOutput = z.infer< diff --git a/apps/stack/src/config/create-message.ts b/apps/stack/src/config/create-message.ts index 0829adfb0..4be9f9a0a 100644 --- a/apps/stack/src/config/create-message.ts +++ b/apps/stack/src/config/create-message.ts @@ -7,7 +7,7 @@ import type { SQSEvent } from "aws-lambda"; import { Queue } from 'sst/node/queue' import type { EventNamespaceType } from "@acme/crawl-schema"; import { crawlComplete, crawlFailed } from "./crawl"; -import type { Tenant } from "./tenants"; +import type { Tenant } from "@acme/super-schema"; const sqs = new SQSClient(); diff --git a/apps/stack/src/config/get-tenant-db.ts b/apps/stack/src/config/get-tenant-db.ts index 9b6a40e84..d236e0ff9 100644 --- a/apps/stack/src/config/get-tenant-db.ts +++ b/apps/stack/src/config/get-tenant-db.ts @@ -2,7 +2,7 @@ import { createClient } from "@libsql/client"; import { drizzle } from "drizzle-orm/libsql"; import { Config } from "sst/node/config"; import { getTenants } from "./tenants"; -import type { Tenant } from "./tenants"; +import type { Tenant } from "@acme/super-schema"; export type OmitDb = Omit; diff --git a/apps/stack/src/config/tenants.ts b/apps/stack/src/config/tenants.ts index 4919385e1..5036d3bcb 100644 --- a/apps/stack/src/config/tenants.ts +++ b/apps/stack/src/config/tenants.ts @@ -1,22 +1,10 @@ +import { createClient } from "@libsql/client"; +import { drizzle } from "drizzle-orm/libsql"; import { Config } from "sst/node/config"; -import { z } from "zod"; +import { getTenants as getTenantsFromDb } from "@acme/super-schema"; -const TenantSchema = z.object({ - id: z.number(), - tenant: z.string(), - dbUrl: z.string(), -}); +const superDb = drizzle(createClient({ url: Config.SUPER_DATABASE_URL, authToken: Config.SUPER_DATABASE_AUTH_TOKEN })) -const TenantArraySchema = z.array(TenantSchema); +const TENANTS = await getTenantsFromDb(superDb); -export type Tenant = z.infer; - -export const getTenants = () => { - const validTenants = TenantArraySchema.safeParse(JSON.parse(Config.TENANTS)); - if (!validTenants.success) { - console.error("Invalid Config 'TENANTS' value:", ...validTenants.error.issues); - throw new Error("Invalid Config 'TENANTS' value"); - } - - return validTenants.data; -} +export const getTenants = () => TENANTS; diff --git a/apps/stack/src/info/tenants.ts b/apps/stack/src/info/tenants.ts index 0fefde74f..09441485c 100644 --- a/apps/stack/src/info/tenants.ts +++ b/apps/stack/src/info/tenants.ts @@ -6,7 +6,7 @@ export const handler = ApiHandler(async (_evt) => { return { statusCode: 200, body: JSON.stringify({ - tenants: getTenants().map(({tenant}) => tenant), + tenants: getTenants().map(({name}) => ({tenant:name})), }), }; }); diff --git a/apps/stack/stacks/ExtractStack.ts b/apps/stack/stacks/ExtractStack.ts index b84da117a..f84d3bf6f 100644 --- a/apps/stack/stacks/ExtractStack.ts +++ b/apps/stack/stacks/ExtractStack.ts @@ -18,12 +18,13 @@ export function ExtractStack({ stack }: StackContext) { const ENV = ENVSchema.parse(process.env); const TENANT_DATABASE_AUTH_TOKEN = new Config.Secret(stack, "TENANT_DATABASE_AUTH_TOKEN"); + const SUPER_DATABASE_URL = new Config.Secret(stack, "SUPER_DATABASE_URL"); + const SUPER_DATABASE_AUTH_TOKEN = new Config.Secret(stack, "SUPER_DATABASE_AUTH_TOKEN"); const CLERK_SECRET_KEY = new Config.Secret(stack, "CLERK_SECRET_KEY"); const REDIS_URL = new Config.Secret(stack, "REDIS_URL"); const REDIS_TOKEN = new Config.Secret(stack, "REDIS_TOKEN"); const REDIS_USER_TOKEN_TTL = new Config.Parameter(stack, "REDIS_USER_TOKEN_TTL", { value: (20 * 60).toString() }); const PER_PAGE = new Config.Parameter(stack, "PER_PAGE", { value: (30).toString() }); - const TENANTS = new Config.Secret(stack, "TENANTS"); const bus = new EventBus(stack, "ExtractBus", { rules: { @@ -61,8 +62,9 @@ export function ExtractStack({ stack }: StackContext) { retries: 10, function: { bind: [ - TENANTS, TENANT_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_URL, CLERK_SECRET_KEY, REDIS_URL, REDIS_TOKEN, @@ -86,8 +88,9 @@ export function ExtractStack({ stack }: StackContext) { bind: [ bus, extractQueue, - TENANTS, TENANT_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_URL, CLERK_SECRET_KEY, REDIS_URL, REDIS_TOKEN, @@ -164,8 +167,9 @@ export function ExtractStack({ stack }: StackContext) { function: { bind: [ bus, - TENANTS, TENANT_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_URL, CLERK_SECRET_KEY, REDIS_URL, REDIS_TOKEN, @@ -200,7 +204,8 @@ export function ExtractStack({ stack }: StackContext) { }, bind: [ extractQueue, - TENANTS, + SUPER_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_URL, CLERK_SECRET_KEY, REDIS_URL, REDIS_TOKEN, @@ -218,7 +223,8 @@ export function ExtractStack({ stack }: StackContext) { return { ExtractBus: bus, - TENANTS, TENANT_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_URL, }; } diff --git a/apps/stack/stacks/InfoStack.ts b/apps/stack/stacks/InfoStack.ts index ff12d0474..52bebc5b2 100644 --- a/apps/stack/stacks/InfoStack.ts +++ b/apps/stack/stacks/InfoStack.ts @@ -8,14 +8,18 @@ import { ExtractStack } from "./ExtractStack"; export function InfoStack({ stack }: StackContext) { const { - TENANTS, + SUPER_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_URL, } = use(ExtractStack); const api = new Api(stack, "InfoApi", { defaults: { function: { - bind: [TENANTS] + bind: [ + SUPER_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_URL, + ] } }, routes: { diff --git a/apps/stack/stacks/TransformStack.ts b/apps/stack/stacks/TransformStack.ts index 15547983e..8cc936979 100644 --- a/apps/stack/stacks/TransformStack.ts +++ b/apps/stack/stacks/TransformStack.ts @@ -17,7 +17,8 @@ export function TransformStack({ stack }: StackContext) { const ENV = ENVSchema.parse(process.env); const { - TENANTS, + SUPER_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_URL, TENANT_DATABASE_AUTH_TOKEN, } = use(ExtractStack); @@ -32,7 +33,8 @@ export function TransformStack({ stack }: StackContext) { function: { bind: [ transformQueue, - TENANTS, + SUPER_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_URL, TENANT_DATABASE_AUTH_TOKEN, ], handler: "src/transform/queue.handler", @@ -46,7 +48,8 @@ export function TransformStack({ stack }: StackContext) { function: { bind: [ transformQueue, - TENANTS + SUPER_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_URL, ], runtime: "nodejs18.x", }, @@ -74,7 +77,8 @@ export function TransformStack({ stack }: StackContext) { handler: "src/transform/transform-tenant.cronHandler", bind: [ transformQueue, - TENANTS + SUPER_DATABASE_AUTH_TOKEN, + SUPER_DATABASE_URL, ], runtime: "nodejs18.x", } diff --git a/migrations/super/0000_smooth_enchantress.sql b/migrations/super/0000_smooth_enchantress.sql new file mode 100644 index 000000000..b7eb21002 --- /dev/null +++ b/migrations/super/0000_smooth_enchantress.sql @@ -0,0 +1,7 @@ +CREATE TABLE `tenants` ( + `id` integer PRIMARY KEY NOT NULL, + `name` text NOT NULL, + `db_url` text NOT NULL, + `__created_at` integer DEFAULT (strftime('%s', 'now')), + `__updated_at` integer DEFAULT (strftime('%s', 'now')) +); diff --git a/migrations/super/meta/0000_snapshot.json b/migrations/super/meta/0000_snapshot.json new file mode 100644 index 000000000..da748b436 --- /dev/null +++ b/migrations/super/meta/0000_snapshot.json @@ -0,0 +1,60 @@ +{ + "version": "5", + "dialect": "sqlite", + "id": "33f5c656-1746-4e5c-945b-d4767b78431c", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "tenants": { + "name": "tenants", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "db_url": { + "name": "db_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "__created_at": { + "name": "__created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(strftime('%s', 'now'))" + }, + "__updated_at": { + "name": "__updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(strftime('%s', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + } +} \ No newline at end of file diff --git a/migrations/super/meta/_journal.json b/migrations/super/meta/_journal.json new file mode 100644 index 000000000..78eaddc34 --- /dev/null +++ b/migrations/super/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "5", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "5", + "when": 1705062905089, + "tag": "0000_smooth_enchantress", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 8dec1629f..170785395 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,6 +38,7 @@ "version": "0.1.0", "dependencies": { "@acme/eslint-config": "*", + "@acme/super-schema": "*", "@acme/tailwind-config": "*", "@clerk/nextjs": "^4.23.2", "@radix-ui/react-avatar": "^1.0.3", @@ -73,6 +74,7 @@ "@acme/extract-functions": "*", "@acme/extract-schema": "*", "@acme/source-control": "*", + "@acme/super-schema": "*", "@acme/transform-functions": "*", "@acme/transform-schema": "*", "@aws-sdk/client-sqs": "^3.470.0", @@ -145,6 +147,10 @@ "resolved": "apps/stack", "link": true }, + "node_modules/@acme/super-schema": { + "resolved": "packages/schemas/super", + "link": true + }, "node_modules/@acme/tailwind-config": { "resolved": "packages/config/tailwind", "link": true @@ -21829,19 +21835,26 @@ "name": "@acme/crawl-schema", "version": "1.0.0", "devDependencies": { - "dotenv-cli": "^7.3.0", "drizzle-kit": "^0.20.6" } }, "packages/schemas/extract": { "name": "@acme/extract-schema", "version": "1.0.0", + "devDependencies": { + "drizzle-kit": "^0.20.6" + } + }, + "packages/schemas/super": { + "name": "@acme/super-schema", + "version": "1.0.0", "devDependencies": { "dotenv-cli": "^7.3.0", "drizzle-kit": "^0.20.6" } }, "packages/schemas/tenant-db": { + "name": "@acme/tenant-db-schema", "version": "1.0.0", "license": "ISC", "dependencies": { diff --git a/packages/schemas/super/drizzle.config.ts b/packages/schemas/super/drizzle.config.ts new file mode 100644 index 000000000..ef0918e83 --- /dev/null +++ b/packages/schemas/super/drizzle.config.ts @@ -0,0 +1,7 @@ +import type { Config } from 'drizzle-kit'; + +// deprecated since [TOOL-189] +export default { + schema: "./src/index.ts", + out: "../../../migrations/super", +} satisfies Config; diff --git a/packages/schemas/super/package.json b/packages/schemas/super/package.json new file mode 100644 index 000000000..d708f5739 --- /dev/null +++ b/packages/schemas/super/package.json @@ -0,0 +1,19 @@ +{ + "name": "@acme/super-schema", + "version": "1.0.0", + "main": "src/index.ts", + "type": "module", + "scripts": { + "db:generate": "drizzle-kit generate:sqlite", + "type-check": "tsc --noEmit && echo \"✔ No TypeScript warnings or errors\"", + "test": "echo \"Warning: no test specified\"", + "lint": "eslint . && echo \"✔ No ESLint warnings or errors\"", + "studio:turso": "npm run with-env drizzle-kit studio -- --config turso.config.ts", + "with-env": "dotenv -e ../../../.env --" + }, + "dependencies": {}, + "devDependencies": { + "dotenv-cli": "^7.3.0", + "drizzle-kit": "^0.20.6" + } +} diff --git a/packages/schemas/super/src/index.ts b/packages/schemas/super/src/index.ts new file mode 100644 index 000000000..3e97503e4 --- /dev/null +++ b/packages/schemas/super/src/index.ts @@ -0,0 +1,5 @@ +export { tenants } from "./tenants"; + +export type { NewTenant, Tenant } from "./tenants"; + +export { getTenants } from "./tenants"; \ No newline at end of file diff --git a/packages/schemas/super/src/tenants.ts b/packages/schemas/super/src/tenants.ts new file mode 100644 index 000000000..aef456a17 --- /dev/null +++ b/packages/schemas/super/src/tenants.ts @@ -0,0 +1,17 @@ +import type { InferSelectModel, InferInsertModel } from 'drizzle-orm'; +import { sql } from 'drizzle-orm'; +import { text, integer, sqliteTable } from 'drizzle-orm/sqlite-core'; +import type { LibSQLDatabase } from 'drizzle-orm/libsql'; + +export const tenants = sqliteTable('tenants', { + id: integer('id').primaryKey(), + name: text('name').notNull(), + dbUrl: text('db_url').notNull(), + _createdAt: integer('__created_at', { mode: 'timestamp' }).default(sql`(strftime('%s', 'now'))`), + _updatedAt: integer('__updated_at', { mode: 'timestamp' }).default(sql`(strftime('%s', 'now'))`), +}); + +export type Tenant = InferSelectModel; +export type NewTenant = InferInsertModel; + +export const getTenants = async (db: LibSQLDatabase) => db.select().from(tenants).all(); \ No newline at end of file diff --git a/packages/schemas/super/tsconfig.json b/packages/schemas/super/tsconfig.json new file mode 100644 index 000000000..deb4e9308 --- /dev/null +++ b/packages/schemas/super/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "baseUrl": ".", + }, + "exclude": [], + "include": [ + "drizzle.config.ts", + "turso.config.ts", + "src/**/*.ts", + "src/**/*.ts", + "src/**/*.cjs", + "src/**/*.mjs", + ] +} diff --git a/scripts/migrate.mjs b/scripts/migrate.mjs index f6968c775..2934a7ff4 100755 --- a/scripts/migrate.mjs +++ b/scripts/migrate.mjs @@ -67,20 +67,23 @@ const maybeOutDirAlreadyExistsMaybeNot = () => fs.existsSync('scripts/out') || f /** * @param {string} databaseId - * @param {string} [dbUrl] - * @param {string} [dbToken] + * @param {string} dbUrlEnvKey + * @param {string} dbTokenEnvKey */ -const tryMigrateDatabase = async (databaseId, dbUrl, dbToken) => { +const tryMigrateDatabase = async (databaseId, dbUrlEnvKey, dbTokenEnvKey) => { console.log(`Migrating db '${databaseId}' ...`); + const dbUrl = /**@type {string | undefined}*/(process.env[dbUrlEnvKey]); + const dbToken = /**@type {string | undefined}*/(process.env[dbTokenEnvKey]); + const localMigrationFiles = fs.readdirSync(`migrations/${databaseId}`).filter(file => file !== 'meta').sort(); if (localMigrationFiles.length === 0) return console.warn(`Skipping migration for db '${databaseId}'. Reason: No migration files found :o`); const upstreamMigrations = readUpstreamMigrationState(databaseId); if (!upstreamMigrations && noMigrationMode) return console.error(`Migration upstream state not found, need to specify migration mode.\n\nUsage: ./scripts/migrate.ts .\n fingers-crossed will try to run the migration without syncing with upstream.\n yolo will try to delete all tables in database before migrating.`); if (upstreamMigrations && isMigrationStateEqual(upstreamMigrations, localMigrationFiles)) return console.warn(`Skipping migration for db '${databaseId}'. Reason: Upstream migrations are in sync.`); - if (!dbUrl) return console.warn(`Skipping migration for db '${databaseId}'. Reason: Environment variable ${databaseId.toUpperCase()}_DB_URL is not defined :(`); - if (!dbToken) return console.warn(`Skipping migration for db '${databaseId}'. Reason: Environment variable ${databaseId.toUpperCase()}_DB_TOKEN is not defined :(`); + if (!dbUrl) return console.warn(`Skipping migration for db '${databaseId}'. Reason: Environment variable ${dbUrlEnvKey} is not defined :(`); + if (!dbToken) return console.warn(`Skipping migration for db '${databaseId}'. Reason: Environment variable ${dbTokenEnvKey} is not defined :(`); const client = createClient({ url: dbUrl, authToken: dbToken }); const db = drizzle(client); @@ -98,8 +101,9 @@ const tryMigrateDatabase = async (databaseId, dbUrl, dbToken) => { console.log('DONE'); } -console.log(process.env.TENANT_DATABASE_URL, process.env.TENANT_DATABASE_AUTH_TOKEN); -await tryMigrateDatabase('tenant-db', process.env.TENANT_DATABASE_URL, process.env.TENANT_DATABASE_AUTH_TOKEN); + +await tryMigrateDatabase('tenant-db', 'TENANT_DATABASE_URL', 'TENANT_DATABASE_AUTH_TOKEN'); +await tryMigrateDatabase('super', 'SUPER_DATABASE_URL', 'SUPER_DATABASE_AUTH_TOKEN'); // await tryMigrateDatabase('extract', process.env.EXTRACT_DATABASE_URL, process.env.EXTRACT_DATABASE_AUTH_TOKEN); // await tryMigrateDatabase('transform', process.env.TRANSFORM_DATABASE_URL, process.env.TRANSFORM_DATABASE_AUTH_TOKEN); // await tryMigrateDatabase('crawl', process.env.CRAWL_DATABASE_URL, process.env.CRAWL_DATABASE_AUTH_TOKEN); diff --git a/turbo.json b/turbo.json index 049327875..0984fdfd4 100644 --- a/turbo.json +++ b/turbo.json @@ -36,6 +36,8 @@ "NEXT_PUBLIC_CLERK_AFTER_SIGN_UP_URL", "TENANT_DATABASE_URL", "TENANT_DATABASE_AUTH_TOKEN", + "SUPER_DATABASE_URL", + "SUPER_DATABASE_AUTH_TOKEN", "NEXT_PUBLIC_EXTRACT_API_URL", "NEXT_PUBLIC_TRANSFORM_API_URL", "TENANTS",