feat: super db

apps/stack/.sst/types/index.ts

@@ -12,6 +12,20 @@ declare module "sst/node/config" {
     }
   }
 }import "sst/node/config";
+declare module "sst/node/config" {
+  export interface SecretResources {
+    "SUPER_DATABASE_URL": {
+      value: string;
+    }
+  }
+}import "sst/node/config";
+declare module "sst/node/config" {
+  export interface SecretResources {
+    "SUPER_DATABASE_AUTH_TOKEN": {
+      value: string;
+    }
+  }
+}import "sst/node/config";
 declare module "sst/node/config" {
   export interface SecretResources {
     "CLERK_SECRET_KEY": {
@@ -46,13 +60,6 @@ declare module "sst/node/config" {
       value: string;
     }
   }
-}import "sst/node/config";
-declare module "sst/node/config" {
-  export interface SecretResources {
-    "TENANTS": {
-      value: string;
-    }
-  }
 }import "sst/node/event-bus";
 declare module "sst/node/event-bus" {
   export interface EventBusResources {

apps/stack/package.json

@@ -22,6 +22,7 @@
     "@acme/extract-functions": "*",
     "@acme/extract-schema": "*",
     "@acme/source-control": "*",
+    "@acme/super-schema": "*",
     "@acme/transform-functions": "*",
     "@acme/transform-schema": "*",
     "@aws-sdk/client-sqs": "^3.470.0",

apps/stack/src/config/crawl.ts

@@ -3,7 +3,7 @@ import { events } from "@acme/crawl-schema";
 import type { Context, InsertEventEntities } from "@acme/crawl-functions";
 import type { EventNamespaceType } from "@acme/crawl-schema/src/events";
 import { getTenantDb, type OmitDb } from "./get-tenant-db";
-import type { Tenant } from "./tenants";
+import type { Tenant } from "@acme/super-schema";
 
 const context: OmitDb<Context<InsertEventEntities>> = {
   entities: {

apps/stack/src/config/create-event.ts

@@ -7,7 +7,7 @@ import { EventBus } from "sst/node/event-bus";
 import { z } from "zod";
 import { crawlComplete, crawlFailed } from "./crawl";
 import type { EventNamespaceType } from "@acme/crawl-schema";
-import type { Tenant } from "./tenants";
+import type { Tenant } from "@acme/super-schema";
 
 const client = new EventBridgeClient({});
 type InferShapeOutput<Shape extends z.ZodRawShape> = z.infer<

apps/stack/src/config/create-message.ts

@@ -7,7 +7,7 @@ import type { SQSEvent } from "aws-lambda";
 import { Queue } from 'sst/node/queue'
 import type { EventNamespaceType } from "@acme/crawl-schema";
 import { crawlComplete, crawlFailed } from "./crawl";
-import type { Tenant } from "./tenants";
+import type { Tenant } from "@acme/super-schema";
 
 const sqs = new SQSClient();
 

apps/stack/src/config/get-tenant-db.ts

@@ -2,7 +2,7 @@ import { createClient } from "@libsql/client";
 import { drizzle } from "drizzle-orm/libsql";
 import { Config } from "sst/node/config";
 import { getTenants } from "./tenants";
-import type { Tenant } from "./tenants";
+import type { Tenant } from "@acme/super-schema";
 
 export type OmitDb<T> = Omit<T, 'db'>;
 

apps/stack/src/config/tenants.ts

@@ -1,22 +1,10 @@
+import { createClient } from "@libsql/client";
+import { drizzle } from "drizzle-orm/libsql";
 import { Config } from "sst/node/config";
-import { z } from "zod";
+import { getTenants as getTenantsFromDb } from "@acme/super-schema";
 
-const TenantSchema = z.object({
-  id: z.number(),
-  tenant: z.string(),
-  dbUrl: z.string(),
-});
+const superDb = drizzle(createClient({ url: Config.SUPER_DATABASE_URL, authToken: Config.SUPER_DATABASE_AUTH_TOKEN }))
 
-const TenantArraySchema = z.array(TenantSchema);
+const TENANTS = await getTenantsFromDb(superDb);
 
-export type Tenant = z.infer<typeof TenantSchema>;
-
-export const getTenants = () => {
-  const validTenants = TenantArraySchema.safeParse(JSON.parse(Config.TENANTS));
-  if (!validTenants.success) {
-    console.error("Invalid Config 'TENANTS' value:", ...validTenants.error.issues);
-    throw new Error("Invalid Config 'TENANTS' value");
-  }
-
-  return validTenants.data;
-}
+export const getTenants = () => TENANTS;

apps/stack/src/info/tenants.ts

@@ -6,7 +6,7 @@ export const handler = ApiHandler(async (_evt) => {
   return {
     statusCode: 200,
     body: JSON.stringify({
-      tenants: getTenants().map(({tenant}) => tenant),
+      tenants: getTenants().map(({name}) => ({tenant:name})),
     }),
   };
 });

apps/stack/stacks/ExtractStack.ts

@@ -18,12 +18,13 @@ export function ExtractStack({ stack }: StackContext) {
   const ENV = ENVSchema.parse(process.env);
 
   const TENANT_DATABASE_AUTH_TOKEN = new Config.Secret(stack, "TENANT_DATABASE_AUTH_TOKEN");
+  const SUPER_DATABASE_URL = new Config.Secret(stack, "SUPER_DATABASE_URL");
+  const SUPER_DATABASE_AUTH_TOKEN = new Config.Secret(stack, "SUPER_DATABASE_AUTH_TOKEN");
   const CLERK_SECRET_KEY = new Config.Secret(stack, "CLERK_SECRET_KEY");
   const REDIS_URL = new Config.Secret(stack, "REDIS_URL");
   const REDIS_TOKEN = new Config.Secret(stack, "REDIS_TOKEN");
   const REDIS_USER_TOKEN_TTL = new Config.Parameter(stack, "REDIS_USER_TOKEN_TTL", { value: (20 * 60).toString() });
   const PER_PAGE = new Config.Parameter(stack, "PER_PAGE", { value: (30).toString() });
-  const TENANTS = new Config.Secret(stack, "TENANTS");
 
   const bus = new EventBus(stack, "ExtractBus", {
     rules: {
@@ -61,8 +62,9 @@ export function ExtractStack({ stack }: StackContext) {
       retries: 10,
       function: {
         bind: [
-          TENANTS,
           TENANT_DATABASE_AUTH_TOKEN,
+          SUPER_DATABASE_AUTH_TOKEN,
+          SUPER_DATABASE_URL,      
           CLERK_SECRET_KEY,
           REDIS_URL,
           REDIS_TOKEN,
@@ -86,8 +88,9 @@ export function ExtractStack({ stack }: StackContext) {
       bind: [
         bus,
         extractQueue,
-        TENANTS,
         TENANT_DATABASE_AUTH_TOKEN,
+        SUPER_DATABASE_AUTH_TOKEN,
+        SUPER_DATABASE_URL,    
         CLERK_SECRET_KEY,
         REDIS_URL,
         REDIS_TOKEN,
@@ -164,8 +167,9 @@ export function ExtractStack({ stack }: StackContext) {
       function: {
         bind: [
           bus,
-          TENANTS,
           TENANT_DATABASE_AUTH_TOKEN,
+          SUPER_DATABASE_AUTH_TOKEN,
+          SUPER_DATABASE_URL,      
           CLERK_SECRET_KEY,
           REDIS_URL,
           REDIS_TOKEN,
@@ -200,7 +204,8 @@ export function ExtractStack({ stack }: StackContext) {
           },
           bind: [
             extractQueue,
-            TENANTS,
+            SUPER_DATABASE_AUTH_TOKEN,
+            SUPER_DATABASE_URL,
             CLERK_SECRET_KEY,
             REDIS_URL,
             REDIS_TOKEN,
@@ -218,7 +223,8 @@ export function ExtractStack({ stack }: StackContext) {
 
   return {
     ExtractBus: bus,
-    TENANTS,
     TENANT_DATABASE_AUTH_TOKEN,
+    SUPER_DATABASE_AUTH_TOKEN,
+    SUPER_DATABASE_URL,
   };
 }

apps/stack/stacks/InfoStack.ts

@@ -8,14 +8,18 @@ import { ExtractStack } from "./ExtractStack";
 export function InfoStack({ stack }: StackContext) {
 
   const {
-    TENANTS,
+    SUPER_DATABASE_AUTH_TOKEN,
+    SUPER_DATABASE_URL,
   } = use(ExtractStack);
 
 
   const api = new Api(stack, "InfoApi", {
     defaults: {
       function: {
-        bind: [TENANTS]
+        bind: [
+          SUPER_DATABASE_AUTH_TOKEN,
+          SUPER_DATABASE_URL,
+        ]
       }
     },
     routes: {

apps/stack/stacks/TransformStack.ts

@@ -17,7 +17,8 @@ export function TransformStack({ stack }: StackContext) {
   const ENV = ENVSchema.parse(process.env);
 
   const {
-    TENANTS,
+    SUPER_DATABASE_AUTH_TOKEN,
+    SUPER_DATABASE_URL,
     TENANT_DATABASE_AUTH_TOKEN,
   } = use(ExtractStack);
 
@@ -32,7 +33,8 @@ export function TransformStack({ stack }: StackContext) {
     function: {
       bind: [
         transformQueue,
-        TENANTS,
+        SUPER_DATABASE_AUTH_TOKEN,
+        SUPER_DATABASE_URL,
         TENANT_DATABASE_AUTH_TOKEN,
       ],
       handler: "src/transform/queue.handler",
@@ -46,7 +48,8 @@ export function TransformStack({ stack }: StackContext) {
       function: {
         bind: [
           transformQueue,
-          TENANTS
+          SUPER_DATABASE_AUTH_TOKEN,
+          SUPER_DATABASE_URL,
         ],
         runtime: "nodejs18.x",
       },
@@ -74,7 +77,8 @@ export function TransformStack({ stack }: StackContext) {
           handler: "src/transform/transform-tenant.cronHandler",
           bind: [
             transformQueue,
-            TENANTS
+            SUPER_DATABASE_AUTH_TOKEN,
+            SUPER_DATABASE_URL,
           ],
           runtime: "nodejs18.x",
         }

migrations/super/0000_smooth_enchantress.sql

@@ -0,0 +1,7 @@
+CREATE TABLE `tenants` (
+	`id` integer PRIMARY KEY NOT NULL,
+	`name` text NOT NULL,
+	`db_url` text NOT NULL,
+	`__created_at` integer DEFAULT (strftime('%s', 'now')),
+	`__updated_at` integer DEFAULT (strftime('%s', 'now'))
+);

migrations/super/meta/0000_snapshot.json

@@ -0,0 +1,60 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "id": "33f5c656-1746-4e5c-945b-d4767b78431c",
+  "prevId": "00000000-0000-0000-0000-000000000000",
+  "tables": {
+    "tenants": {
+      "name": "tenants",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "integer",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "db_url": {
+          "name": "db_url",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "__created_at": {
+          "name": "__created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false,
+          "default": "(strftime('%s', 'now'))"
+        },
+        "__updated_at": {
+          "name": "__updated_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false,
+          "default": "(strftime('%s', 'now'))"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    }
+  },
+  "enums": {},
+  "_meta": {
+    "schemas": {},
+    "tables": {},
+    "columns": {}
+  }
+}

migrations/super/meta/_journal.json

@@ -0,0 +1,13 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "5",
+      "when": 1705062905089,
+      "tag": "0000_smooth_enchantress",
+      "breakpoints": true
+    }
+  ]
+}

packages/schemas/super/drizzle.config.ts

@@ -0,0 +1,7 @@
+import type { Config } from 'drizzle-kit';
+
+// deprecated since [TOOL-189]
+export default {
+  schema: "./src/index.ts",
+  out: "../../../migrations/super",
+} satisfies Config;