feat: htmx test app

apps/app-htmx/build.mjs

@@ -0,0 +1,29 @@
+import * as esbuild from "esbuild";
+
+await esbuild.build({
+    keepNames: true,
+    bundle: true,
+    platform: "node",
+    format: "esm",
+    target: "esnext",
+    metafile: true,
+    outdir: "out",
+    entryPoints: ["src/index.ts"],
+    mainFields: ["module", "main"],
+    external: [
+        '@clerk/fastify',
+        '@fastify/formbody',
+        '@fastify/view',
+        'fastify',
+        'nunjucks',
+    ],
+    banner: {
+        js: [
+            `import { createRequire as topLevelCreateRequire } from 'module';`,
+            `const require = topLevelCreateRequire(import.meta.url);`,
+            `import { fileURLToPath as topLevelFileUrlToPath, URL as topLevelURL } from "url"`,
+            `const __dirname = topLevelFileUrlToPath(new topLevelURL(".", import.meta.url))`,
+        ].join("\n"),
+    },
+
+});

apps/app-htmx/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@acme/app-htmx",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "start": "esbuild src/index.ts --bundle --platform=node | npm run with-env node",
+    "dev": "node build.mjs && npm run with-env node ./out/index.js",
+    "with-env": "dotenv -e ../../.env --"
+  },
+  "devDependencies": {
+    "@types/nunjucks": "3.2.6",
+    "dotenv-cli": "^7.3.0",
+    "esbuild": "^0.19.9"
+  },
+  "dependencies": {
+    "@acme/source-control": "*",
+    "@acme/super-schema": "*",
+    "@clerk/fastify": "0.6.30",
+    "@fastify/formbody": "7.4.0",
+    "@fastify/view": "8.2.0",
+    "date-fns": "3.3.1",
+    "fastify": "4.25.2",
+    "nunjucks": "3.2.4"
+  }
+}

apps/app-htmx/src/app-config.ts

@@ -0,0 +1,37 @@
+import { z } from "zod"
+
+const ENVSchema = z.object({
+  SUPER_DATABASE_URL: z.string(),
+  SUPER_DATABASE_AUTH_TOKEN: z.string().optional(),
+  TENANT_DATABASE_AUTH_TOKEN: z.string().optional(),
+
+  // TODO: remove after removing next
+  NEXT_PUBLIC_EXTRACT_API_URL: z.string(),
+  NEXT_PUBLIC_TRANSFORM_API_URL: z.string(),
+
+  CLERK_PUBLISHABLE_KEY: z.string(),
+  CLERK_SECRET_KEY: z.string(),
+  CLERK_DOMAIN: z.string(),
+  APP_HTMX_PORT: z.coerce.number().optional(),
+});
+
+const parsedEnv = ENVSchema.safeParse(process.env);
+if (!parsedEnv.success) throw new Error(`Invalid environment: ${parsedEnv.error}`);
+
+export const AppConfig = {
+  port: parsedEnv.data.APP_HTMX_PORT,
+  superDatabase: {
+    url: parsedEnv.data.SUPER_DATABASE_URL,
+    authToken: parsedEnv.data.SUPER_DATABASE_AUTH_TOKEN,
+  },
+  tenantDatabaseAuthToken: parsedEnv.data.TENANT_DATABASE_AUTH_TOKEN,
+  apis: {
+    extractStart: parsedEnv.data.NEXT_PUBLIC_EXTRACT_API_URL,
+    transformStart: parsedEnv.data.NEXT_PUBLIC_TRANSFORM_API_URL,
+  },
+  clerk: {
+    domain: parsedEnv.data.CLERK_DOMAIN,
+    publishableKey: parsedEnv.data.CLERK_PUBLISHABLE_KEY,
+    secretKey: "", // should be left in ENV
+  }
+}

apps/app-htmx/src/context/htmx.context.ts

@@ -0,0 +1,7 @@
+import type { IncomingHttpHeaders } from "http";
+
+export const htmxContext = (headers: IncomingHttpHeaders) => ({
+  htmx: {
+    boosted: 'hx-boosted' in headers,
+  }
+});

apps/app-htmx/src/context/page.context.ts

@@ -0,0 +1,5 @@
+export const pageContext = (title: string) => ({
+  page: {
+    title
+  },
+});

apps/app-htmx/src/context/tenant-list.context.ts

@@ -0,0 +1,17 @@
+import { getTenants } from "@acme/super-schema";
+import { createClient } from "@libsql/client";
+import { drizzle } from "drizzle-orm/libsql";
+import { AppConfig } from "src/app-config";
+
+const loadTenants = async ()=> {
+  const superDb = drizzle(createClient(AppConfig.superDatabase));
+
+  const tenants = await getTenants(superDb);
+
+  return tenants;
+}
+const TENANT_LIST = await loadTenants();
+
+export const tenantListContext = () => ({
+  tenantList: TENANT_LIST
+})

apps/app-htmx/src/functions/fetch-repository.ts

@@ -0,0 +1,42 @@
+import { GitHubSourceControl, GitlabSourceControl, type SourceControl } from "@acme/source-control";
+import { clerkClient } from "@clerk/fastify";
+
+const getUserForgeAccessToken = async (userId: string, forge: "github" | "gitlab") => {
+  const userTokens = await clerkClient.users.getUserOauthAccessToken(userId, `oauth_${forge}`);
+  if (userTokens[0] === undefined) throw new Error("no token");
+  return userTokens[0].token;
+}
+
+type Props = {
+  forge: "github" | "gitlab";
+  userId: string;
+  repositoryId: number;
+  repositoryName: string;
+  namespaceName: string;
+}
+export const tryFetchRepository = async ({
+  userId,
+  forge,
+  namespaceName,
+  repositoryId,
+  repositoryName,
+}: Props) => {
+  const token = await getUserForgeAccessToken(userId, forge);
+
+  let sc: SourceControl;
+  if (forge === 'github') sc = new GitHubSourceControl(token);
+  else sc = new GitlabSourceControl(token);
+
+  try {
+    const { repository, namespace } = await sc.fetchRepository(repositoryId, namespaceName, repositoryName);
+    return { repository, namespace };
+  } catch (error) {
+    console.log("FAILED TO FETCH REPOSITORY");
+    console.log(error);
+  }
+
+  return {
+    repository: undefined,
+    namespace: undefined
+  }
+}

apps/app-htmx/src/functions/get-crawl-stats.ts

@@ -0,0 +1,53 @@
+import type { Tenant } from "@acme/super-schema";
+import { type Client, createClient } from "@libsql/client";
+import { fromUnixTime } from "date-fns/fromUnixTime";
+import { formatDistanceToNow } from "date-fns/formatDistanceToNow";
+import { AppConfig } from "src/app-config";
+import { z } from "zod";
+
+const formatDuration = (s: number) => {
+  const seconds = s % 60;
+  if (s < 60) return `${seconds}s`
+  const m = (s - seconds) / 60;
+  const minutes = m % 60;
+  if (m < 60) return `${minutes}m ${seconds.toString().padStart(2,'0')}s`
+  const h = (m - minutes) / 60;
+  return `${h}h ${minutes.toString().padStart(2, '0')}m ${seconds.toString().padStart(2, '0')}s`
+}
+
+const rowSchema = z.object({
+  crawl_instance: z.number(),
+  event_count: z.number(),
+  crawl_duration_sec: z.number(),
+  crawl_started_at: z.number()
+});
+
+const readCrawlStats =async (client:Client) => {
+  const x = await client.execute(`
+  SELECT
+    ce.instance_id as crawl_instance,
+    COUNT(ce.instance_id) as event_count,
+    MAX(ce.timestamp) - ci.started_at AS crawl_duration_sec,
+    ci.started_at as crawl_started_at
+FROM crawl_events ce
+JOIN crawl_instances ci ON ce.instance_id = ci.id
+GROUP BY ce.instance_id;
+  `);
+  const rawStats = x.rows.map(row => rowSchema.parse(row));
+
+  return rawStats.map((rawStat)=>({
+    instanceId: rawStat.crawl_instance,
+    eventCount: rawStat.event_count,
+    duration: formatDuration(rawStat.crawl_duration_sec),
+    startedAt: formatDistanceToNow(fromUnixTime(rawStat.crawl_started_at))
+  }))
+}
+
+export const getCrawlStats = async (tenant: Tenant) => {
+  const client = createClient({
+    url: tenant.dbUrl,
+    authToken: AppConfig.tenantDatabaseAuthToken
+  });
+
+  return await readCrawlStats(client);
+}

apps/app-htmx/src/functions/get-repositories.ts

@@ -0,0 +1,14 @@
+import { namespaces, repositories } from "@acme/extract-schema";
+import { eq } from "drizzle-orm";
+import type { LibSQLDatabase } from "drizzle-orm/libsql";
+
+export const getRepositories = async (db: LibSQLDatabase) => {
+  const repos = await db.select({
+    forge: repositories.forgeType,
+    name: repositories.name,
+    org: namespaces.name,
+    projectId: repositories.externalId,
+  }).from(repositories).innerJoin(namespaces, eq(repositories.namespaceId, namespaces.id)).all()
+
+  return repos;
+}

apps/app-htmx/src/index.ts

@@ -0,0 +1,40 @@
+import Fastify from "fastify";
+import nunjucks from "nunjucks";
+import { fastifyView } from "@fastify/view";
+import path from "path";
+import { fileURLToPath } from "url";
+import { clerkPlugin } from "@clerk/fastify";
+import { Home } from "./pages/page.home.js";
+import fastifyFormbody from "@fastify/formbody";
+import { SignIn } from "./pages/page.sign-in.js";
+import { ExtractRepository } from "./pages/repository/extract.js";
+import { RegisterRepository } from "./pages/repository/register.js";
+import { AppConfig } from "./app-config.js";
+import { StartTransform } from "./pages/start-transform.js";
+
+AppConfig; // ensure loaded before starting server
+
+const TEMPLATE_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "../", "src", "views");
+
+const fastify = Fastify({ logger: true });
+
+
+await fastify.register(fastifyView, {
+  engine: {
+    nunjucks: nunjucks,
+  },
+  templates: TEMPLATE_DIR
+});
+await fastify.register(clerkPlugin);
+await fastify.register(fastifyFormbody);
+
+fastify.get('/', Home);
+fastify.get('/sign-in',SignIn);
+fastify.post('/repository/extract', ExtractRepository); 
+fastify.post('/repository/register', RegisterRepository);
+fastify.post('/transform', StartTransform);
+
+const PORT = AppConfig.port || 3001;
+
+await fastify.listen({ port: PORT, host: "127.0.0.1" })
+console.log(`Server started on http://127.0.0.1:${PORT}/`);

apps/app-htmx/src/pages/page.home.ts

@@ -0,0 +1,59 @@
+import { getAuth } from "@clerk/fastify";
+import type { RouteHandlerMethod } from "fastify";
+import { redirectToSignIn } from "./redirect-to-sign-in";
+import { pageContext } from "src/context/page.context";
+import { htmxContext } from "src/context/htmx.context";
+import { tenantListContext } from "src/context/tenant-list.context";
+import { drizzle } from "drizzle-orm/libsql";
+import { createClient } from "@libsql/client";
+import { getRepositories } from "src/functions/get-repositories";
+import { z } from "zod";
+import { AppConfig } from "src/app-config";
+import { getCrawlStats } from "src/functions/get-crawl-stats";
+
+const QuerySchema = z.object({
+  tenant: z.string().optional()
+});
+
+export const Home: RouteHandlerMethod = async (request, reply) => {
+  const auth = getAuth(request);
+  if (!auth.sessionId) return redirectToSignIn(reply);
+
+  const parsedQuery = QuerySchema.safeParse(request.query);
+  const query = parsedQuery.success ? parsedQuery.data : {};
+
+  const page = pageContext("Home");
+  const htmx = htmxContext(request.headers);
+  const tenantList = tenantListContext();
+
+
+  const tenantQuery = query.tenant || tenantList.tenantList[0]!.name;
+  const tenant = tenantList.tenantList.find(t => t.name === tenantQuery);
+  if (!tenant) return reply.redirect(303, "/");
+  const targetTenantId = tenant.id;
+
+  const db = drizzle(createClient({
+    url: tenant.dbUrl,
+    authToken: AppConfig.tenantDatabaseAuthToken
+  }));
+
+  const repos = await getRepositories(db);
+
+  const tenantCrawlStats = await getCrawlStats(tenant);
+
+  return reply.view("page.home.html", {
+    auth,
+    ...page,
+    ...htmx,
+    ...tenantList,
+    targetTenant: tenant,
+    targetTenantId,
+    repos,
+    dates: {
+      yesterday: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString().slice(0, 10),
+      today: new Date().toISOString().slice(0, 10)
+    },
+    AppConfig,
+    tenantCrawlStats
+  });
+}

apps/app-htmx/src/pages/page.sign-in.ts

@@ -0,0 +1,17 @@
+import { getAuth } from "@clerk/fastify";
+import type { RouteHandlerMethod } from "fastify";
+import { pageContext } from "../context/page.context.js";
+import { htmxContext } from "../context/htmx.context.js";
+import { AppConfig } from "src/app-config.js";
+
+export const SignIn: RouteHandlerMethod = async (request, reply) => {
+  const _auth = getAuth(request);
+
+  const page = pageContext("Sign In");
+  const htmx = htmxContext(request.headers);
+  return reply.view("page.sign-in.html", {
+    ...page,
+    ...htmx,
+    AppConfig
+  });
+}

apps/app-htmx/src/pages/redirect-to-sign-in.ts

@@ -0,0 +1,3 @@
+import type { FastifyReply } from "fastify";
+
+export const redirectToSignIn = (reply: FastifyReply) => reply.redirect(303, '/sign-in');