From f615c46053eebeba33e6e38b5c40564275b9f806 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Thu, 8 Feb 2024 15:47:09 +0100 Subject: [PATCH 1/7] refactor app --- apps/app-htmx/package.json | 4 +- .../app-htmx/src/functions/extract-context.ts | 45 ++++++++++++ .../src/functions/fetch-repository.ts | 42 ----------- .../src/functions/get-repositories.ts | 6 +- apps/app-htmx/src/functions/tenant-db.ts | 12 ++++ apps/app-htmx/src/index.ts | 4 +- apps/app-htmx/src/pages/page.home.ts | 4 +- apps/app-htmx/src/pages/repository/extract.ts | 72 ------------------- .../app-htmx/src/pages/repository/register.ts | 71 +++++++++++------- apps/app-htmx/src/pages/start-extract.ts | 64 +++++++++++++++++ apps/app-htmx/src/pages/start-transform.ts | 37 ++++++---- .../src/views/component.repository.html | 32 +-------- apps/app-htmx/src/views/page.home.html | 65 +++++++++-------- 13 files changed, 233 insertions(+), 225 deletions(-) create mode 100644 apps/app-htmx/src/functions/extract-context.ts delete mode 100644 apps/app-htmx/src/functions/fetch-repository.ts create mode 100644 apps/app-htmx/src/functions/tenant-db.ts delete mode 100644 apps/app-htmx/src/pages/repository/extract.ts create mode 100644 apps/app-htmx/src/pages/start-extract.ts diff --git a/apps/app-htmx/package.json b/apps/app-htmx/package.json index d023b3272..90ba7b92e 100644 --- a/apps/app-htmx/package.json +++ b/apps/app-htmx/package.json @@ -16,6 +16,8 @@ "dependencies": { "@acme/source-control": "*", "@acme/super-schema": "*", + "@acme/extract-schema": "*", + "@acme/extract-functions": "*", "@clerk/fastify": "0.6.30", "@fastify/formbody": "7.4.0", "@fastify/view": "8.2.0", @@ -23,4 +25,4 @@ "fastify": "4.25.2", "nunjucks": "3.2.4" } -} +} \ No newline at end of file diff --git a/apps/app-htmx/src/functions/extract-context.ts b/apps/app-htmx/src/functions/extract-context.ts new file mode 100644 index 000000000..51d054f1f --- /dev/null +++ b/apps/app-htmx/src/functions/extract-context.ts @@ -0,0 +1,45 @@ +import { GitHubSourceControl, GitlabSourceControl, type SourceControl } from "@acme/source-control" +import type { Tenant } from "@acme/super-schema" +import type { LibSQLDatabase } from "drizzle-orm/libsql" +import { clerkClient } from "@clerk/fastify"; +import { tenantDb } from "./tenant-db"; + +const getUserForgeAccessToken = async (userId: string, forge: "github" | "gitlab") => { + const userTokens = await clerkClient.users.getUserOauthAccessToken(userId, `oauth_${forge}`); + if (userTokens[0] === undefined) throw new Error("no token"); + return userTokens[0].token; +} + +type BaseExtractContext = { + db: LibSQLDatabase, + integrations: { + sourceControl: SourceControl | null + } +} + +type ExtractContextOptions = { + tenant: Tenant; + userId: string; + forge: "github" | "gitlab"; +} + +export const extractContext: { + (opts: ExtractContextOptions, ctx: TExtendedContext): Promise; +} = async ({ tenant, userId, forge }, ctx) => { + const token = await getUserForgeAccessToken(userId, forge); + + let sourceControl = null; + if (forge === "github") sourceControl = new GitHubSourceControl({ auth: token }); + else if (forge === "gitlab") sourceControl = new GitlabSourceControl(token); + + const ctxIntegrations = ('integrations' in ctx) ? ctx.integrations : {}; + + return { + ...ctx, + db: tenantDb(tenant), + integrations: { + ...ctxIntegrations, + sourceControl + } + } +} diff --git a/apps/app-htmx/src/functions/fetch-repository.ts b/apps/app-htmx/src/functions/fetch-repository.ts deleted file mode 100644 index ccad357a0..000000000 --- a/apps/app-htmx/src/functions/fetch-repository.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { GitHubSourceControl, GitlabSourceControl, type SourceControl } from "@acme/source-control"; -import { clerkClient } from "@clerk/fastify"; - -const getUserForgeAccessToken = async (userId: string, forge: "github" | "gitlab") => { - const userTokens = await clerkClient.users.getUserOauthAccessToken(userId, `oauth_${forge}`); - if (userTokens[0] === undefined) throw new Error("no token"); - return userTokens[0].token; -} - -type Props = { - forge: "github" | "gitlab"; - userId: string; - repositoryId: number; - repositoryName: string; - namespaceName: string; -} -export const tryFetchRepository = async ({ - userId, - forge, - namespaceName, - repositoryId, - repositoryName, -}: Props) => { - const token = await getUserForgeAccessToken(userId, forge); - - let sc: SourceControl; - if (forge === 'github') sc = new GitHubSourceControl({ auth: token }); - else sc = new GitlabSourceControl(token); - - try { - const { repository, namespace } = await sc.fetchRepository(repositoryId, namespaceName, repositoryName); - return { repository, namespace }; - } catch (error) { - console.log("FAILED TO FETCH REPOSITORY"); - console.log(error); - } - - return { - repository: undefined, - namespace: undefined - } -} \ No newline at end of file diff --git a/apps/app-htmx/src/functions/get-repositories.ts b/apps/app-htmx/src/functions/get-repositories.ts index 88279aaa1..ad7ddf5da 100644 --- a/apps/app-htmx/src/functions/get-repositories.ts +++ b/apps/app-htmx/src/functions/get-repositories.ts @@ -4,11 +4,15 @@ import type { LibSQLDatabase } from "drizzle-orm/libsql"; export const getRepositories = async (db: LibSQLDatabase) => { const repos = await db.select({ + id: repositories.id, forge: repositories.forgeType, name: repositories.name, org: namespaces.name, projectId: repositories.externalId, }).from(repositories).innerJoin(namespaces, eq(repositories.namespaceId, namespaces.id)).all() - return repos; + return repos.map(repo=>({ + ...repo, + key: `${repo.forge}-${repo.projectId}`, + })); } \ No newline at end of file diff --git a/apps/app-htmx/src/functions/tenant-db.ts b/apps/app-htmx/src/functions/tenant-db.ts new file mode 100644 index 000000000..58a892304 --- /dev/null +++ b/apps/app-htmx/src/functions/tenant-db.ts @@ -0,0 +1,12 @@ +import type { Tenant } from "@acme/super-schema"; +import { createClient } from "@libsql/client"; +import { drizzle } from "drizzle-orm/libsql"; +import { AppConfig } from "src/app-config"; + +export const tenantDb = (tenant:Tenant)=> { + const client = createClient({ + url: tenant.dbUrl, + authToken: AppConfig.tenantDatabaseAuthToken + }); + return drizzle(client); +} \ No newline at end of file diff --git a/apps/app-htmx/src/index.ts b/apps/app-htmx/src/index.ts index 77f72bce7..5c68abfff 100644 --- a/apps/app-htmx/src/index.ts +++ b/apps/app-htmx/src/index.ts @@ -7,10 +7,10 @@ import { clerkPlugin } from "@clerk/fastify"; import { Home } from "./pages/page.home.js"; import fastifyFormbody from "@fastify/formbody"; import { SignIn } from "./pages/page.sign-in.js"; -import { ExtractRepository } from "./pages/repository/extract.js"; import { RegisterRepository } from "./pages/repository/register.js"; import { AppConfig } from "./app-config.js"; import { StartTransform } from "./pages/start-transform.js"; +import { StartExtract } from "./pages/start-extract.js"; AppConfig; // ensure loaded before starting server @@ -30,7 +30,7 @@ await fastify.register(fastifyFormbody); fastify.get('/', Home); fastify.get('/sign-in',SignIn); -fastify.post('/repository/extract', ExtractRepository); +fastify.post('/extract', StartExtract); fastify.post('/repository/register', RegisterRepository); fastify.post('/transform', StartTransform); diff --git a/apps/app-htmx/src/pages/page.home.ts b/apps/app-htmx/src/pages/page.home.ts index a57015bc6..f887af852 100644 --- a/apps/app-htmx/src/pages/page.home.ts +++ b/apps/app-htmx/src/pages/page.home.ts @@ -30,7 +30,6 @@ export const Home: RouteHandlerMethod = async (request, reply) => { const tenantQuery = query.tenant || tenantList.tenantList[0]!.name; const tenant = tenantList.tenantList.find(t => t.name === tenantQuery); if (!tenant) return reply.redirect(303, "/"); - const targetTenantId = tenant.id; const db = drizzle(createClient({ url: tenant.dbUrl, @@ -46,8 +45,7 @@ export const Home: RouteHandlerMethod = async (request, reply) => { ...page, ...htmx, ...tenantList, - targetTenant: tenant, - targetTenantId, + tenant, repos, dates: { yesterday: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString().slice(0, 10), diff --git a/apps/app-htmx/src/pages/repository/extract.ts b/apps/app-htmx/src/pages/repository/extract.ts deleted file mode 100644 index e69d32615..000000000 --- a/apps/app-htmx/src/pages/repository/extract.ts +++ /dev/null @@ -1,72 +0,0 @@ -import type { RouteHandlerMethod } from "fastify"; -import { clerkClient, getAuth } from "@clerk/fastify" -import { z } from "zod"; -import { tenantListContext } from "src/context/tenant-list.context"; -import { AppConfig } from "src/app-config"; - -const ExtractInputSchema = z.object({ - target_tenant_id: z.coerce.number(), - forge: z.literal("github").or(z.literal("gitlab")), - owner: z.string().optional(), - repo: z.string().optional(), - project_id: z.coerce.number().optional() -}).and(z.discriminatedUnion("qtype", [ - z.object({ - qtype: z.literal('last'), - last: z.coerce.number(), - }), - z.object({ - qtype: z.literal('between'), - from: z.coerce.date(), - to: z.coerce.date(), - }) -])); - -export const ExtractRepository: RouteHandlerMethod = async (request, reply) => { - const auth = getAuth(request); - const {tenantList} = tenantListContext(); - if (!auth.userId) return reply.status(404).send(); - const parsedExtractInput = ExtractInputSchema.safeParse(request.body); - - if (!parsedExtractInput.success) return reply.view("component.log.html", { error: parsedExtractInput.error }); - - const extractInput = parsedExtractInput.data; - const tenant = tenantList.find(tenant=>tenant.id === extractInput.target_tenant_id); - if (!tenant) return reply.view("component.log.html", { error: `Invalid target_tenant_id: ${extractInput.target_tenant_id}` }); - - let to = new Date(); - let from = new Date(to); - if (extractInput.qtype === 'last') from.setDate(to.getDate() - extractInput.last); - if (extractInput.qtype === 'between') { - from = extractInput.from; - to = extractInput.to; - // from.setUTCHours(0, 0, 0, 0); - // to.setUTCHours(23, 59, 59, 999); - } - - const requestBody = JSON.stringify({ - repositoryId: extractInput.project_id || 0, - repositoryName: extractInput.repo || "", - namespaceName: extractInput.owner || "", - sourceControl: extractInput.forge, - from, - to, - tenantId: tenant.id, - }); - - const apiToken = await clerkClient.sessions.getToken(auth.sessionId, "dashboard") as string; - - const res = await fetch(AppConfig.apis.extractStart, { - method: 'post', - body: requestBody, - headers: { - 'Authorization': 'Bearer ' + apiToken - } - }); - - const body = await res.text(); - - if (res.status !== 200) return reply.view('component.log.html', { error: body }); - - return reply.view('component.log.html', { log: body }); -} \ No newline at end of file diff --git a/apps/app-htmx/src/pages/repository/register.ts b/apps/app-htmx/src/pages/repository/register.ts index 8d6e4bf52..8b576a5ee 100644 --- a/apps/app-htmx/src/pages/repository/register.ts +++ b/apps/app-htmx/src/pages/repository/register.ts @@ -1,8 +1,10 @@ import type { RouteHandlerMethod } from "fastify"; import { getAuth } from "@clerk/fastify" import { z } from "zod"; -import { tryFetchRepository } from "src/functions/fetch-repository"; import { tenantListContext } from "src/context/tenant-list.context"; +import { type GetRepositoryFunction, getRepository } from "@acme/extract-functions"; +import { namespaces, repositories } from "@acme/extract-schema"; +import { extractContext } from "src/functions/extract-context"; const RegisterInput = z.object({ target_tenant_id: z.coerce.number(), @@ -12,6 +14,13 @@ const RegisterInput = z.object({ project_id: z.string().optional() }) +const getRepositoryEntitiesContext = { + entities: { + namespaces: namespaces, + repositories: repositories, + }, +} satisfies Pick[1], 'entities'>; + export const RegisterRepository: RouteHandlerMethod = async (request, reply) => { const auth = getAuth(request); if (!auth.userId) return reply.status(404).send(); // hide actions from unauthenticated users @@ -21,37 +30,49 @@ export const RegisterRepository: RouteHandlerMethod = async (request, reply) => const safeInput = RegisterInput.safeParse(request.body); if (!safeInput.success) return reply.status(400).send(); + const targetTenantId = safeInput.data.target_tenant_id; + const tenant = tenantList.find(tenant => tenant.id === targetTenantId); + if (!tenant) return reply.view("component.log.html", { error: `Invalid target_tenant_id: ${targetTenantId}` }); + + const { userId } = auth; + const { forge } = safeInput.data; + const input = { - userId: auth.userId, - forge: safeInput.data.forge, + externalRepositoryId: Number(safeInput.data.project_id) || 0, namespaceName: safeInput.data.owner || "", repositoryName: safeInput.data.repo || "", - repositoryId: Number(safeInput.data.project_id) || 0 } - const { repository, namespace } = await tryFetchRepository(input); + const getRepositoryContext = await extractContext({ tenant, userId, forge }, getRepositoryEntitiesContext); - if (!repository || !namespace) return reply.view("component.log.html", - safeInput.data.forge === "github" ? { error: `Repository ${input.namespaceName}/${input.repositoryName} not found` } - : { error: `Project ${input.repositoryId} not found` }); + try { + const { repository, namespace } = await getRepository(input, getRepositoryContext); - const targetTenantId = safeInput.data.target_tenant_id; - const tenant = tenantList.find(tenant=>tenant.id === targetTenantId); - if (!tenant) return reply.view("component.log.html", { error: `Invalid target_tenant_id: ${targetTenantId}` }); + if (repository._createdAt?.getTime() !== repository._updatedAt?.getTime()) { + return reply.view("component.log.html", { log: `Repository : ${namespace.name}/${repository.name} is already registered.` }); + } - const repo = { - name: repository.name, - org: namespace.name, - forge: repository.forgeType, - projectId: repository.externalId - } - - return reply.view("component.repository.html", { - repo, - targetTenantId, - dates: { - yesterday: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString().slice(0, 10), - today: new Date().toISOString().slice(0, 10) + const repo = { + key: `${forge}-${repository.externalId}`, + name: repository.name, + org: namespace.name, + forge, + projectId: repository.externalId, } - }); + + return reply.view("component.repository.html", { + repo, + tenant, + dates: { + yesterday: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString().slice(0, 10), + today: new Date().toISOString().slice(0, 10) + } + }); + + } catch (error) { + console.log(error); + const errorMessage = error instanceof Error? error.message : error; + return reply.view("component.log.html", { error: errorMessage }); + } + } \ No newline at end of file diff --git a/apps/app-htmx/src/pages/start-extract.ts b/apps/app-htmx/src/pages/start-extract.ts new file mode 100644 index 000000000..b6587c5ea --- /dev/null +++ b/apps/app-htmx/src/pages/start-extract.ts @@ -0,0 +1,64 @@ +import { clerkClient, getAuth } from "@clerk/fastify"; +import type { RouteHandlerMethod } from "fastify"; +import { AppConfig } from "src/app-config"; +import { tenantListContext } from "src/context/tenant-list.context"; +import { z } from "zod"; + +const QuerySchema = z.object({ + tenant: z.string(), +}); + +const InputSchema = z.discriminatedUnion("qtype", [ + z.object({ + qtype: z.literal('last'), + last: z.coerce.number(), + }), + z.object({ + qtype: z.literal('between'), + from: z.coerce.date(), + to: z.coerce.date(), + }) +]); + +export const StartExtract: RouteHandlerMethod = async (request, reply) => { + const auth = getAuth(request); + if (!auth.userId) return reply.status(404).send(); + + const parsedQuery = QuerySchema.safeParse(request.query); + if (!parsedQuery.success) return reply.status(400).send(); + + const { tenant: tenantKey } = parsedQuery.data; + const { tenantList } = tenantListContext(); + const tenant = tenantList.find(tenant => tenant.name === tenantKey); + if (!tenant) return reply.status(400).send(); + + const apiToken = await clerkClient.sessions.getToken(auth.sessionId, "dashboard") as string; + + const parsedInput = InputSchema.safeParse(request.body); + if (!parsedInput.success) return reply.status(400).send(); + const input = parsedInput.data; + + let to = new Date(); + let from = new Date(to); + if (input.qtype === 'last') from.setDate(to.getDate() - input.last); + if (input.qtype === 'between') { + from = input.from; + to = input.to; + } + + const requestBody = JSON.stringify({ tenant: tenant.id, from, to }); + + const res = await fetch(AppConfig.apis.extractStart, { + method: 'post', + body: requestBody, + headers: { + 'Authorization': 'Bearer ' + apiToken + } + }); + + const body = await res.text(); + + if (res.status !== 200) return reply.view('component.log.html', { error: body }); + + return reply.view('component.log.html', { log: body }); +} \ No newline at end of file diff --git a/apps/app-htmx/src/pages/start-transform.ts b/apps/app-htmx/src/pages/start-transform.ts index 95db7d6ff..4d5e95677 100644 --- a/apps/app-htmx/src/pages/start-transform.ts +++ b/apps/app-htmx/src/pages/start-transform.ts @@ -4,9 +4,11 @@ import { z } from "zod"; import { tenantListContext } from "src/context/tenant-list.context"; import { AppConfig } from "src/app-config"; -const TransformInputSchema = z.object({ - target_tenant_id: z.coerce.number(), -}).and(z.discriminatedUnion("qtype", [ +const QuerySchema = z.object({ + tenant: z.string(), +}); + +const InputSchema = z.discriminatedUnion("qtype", [ z.object({ qtype: z.literal('last'), last: z.coerce.number(), @@ -16,28 +18,33 @@ const TransformInputSchema = z.object({ from: z.coerce.date(), to: z.coerce.date(), }) -])); +]); export const StartTransform: RouteHandlerMethod = async (request, reply) => { const auth = getAuth(request); + if (!auth.userId) return reply.status(404).send(); + + const parsedQuery = QuerySchema.safeParse(request.query); + if (!parsedQuery.success) return reply.status(400).send(); + + const { tenant: tenantKey } = parsedQuery.data; const { tenantList } = tenantListContext(); - if (!auth.userId) return reply.status(404).send(); - const parsedTransformInput = TransformInputSchema.safeParse(request.body); + const tenant = tenantList.find(tenant => tenant.name === tenantKey); + if (!tenant) return reply.status(400).send(); - if (!parsedTransformInput.success) return reply.view("component.log.html", { error: parsedTransformInput.error }); + const apiToken = await clerkClient.sessions.getToken(auth.sessionId, "dashboard") as string; - const extractInput = parsedTransformInput.data; - const tenant = tenantList.find(tenant => tenant.id === extractInput.target_tenant_id); - if (!tenant) return reply.view("component.log.html", { error: `Invalid target_tenant_id: ${extractInput.target_tenant_id}` }); - const apiToken = await clerkClient.sessions.getToken(auth.sessionId, "dashboard") as string; + const parsedInput = InputSchema.safeParse(request.body); + if (!parsedInput.success) return reply.status(400).send(); + const input = parsedInput.data; let to = new Date(); let from = new Date(to); - if (extractInput.qtype === 'last') from.setDate(to.getDate() - extractInput.last); - if (extractInput.qtype === 'between') { - from = extractInput.from; - to = extractInput.to; + if (input.qtype === 'last') from.setDate(to.getDate() - input.last); + if (input.qtype === 'between') { + from = input.from; + to = input.to; } const requestBody = JSON.stringify({ tenantId: tenant.id, from, to }); diff --git a/apps/app-htmx/src/views/component.repository.html b/apps/app-htmx/src/views/component.repository.html index 955cc3a3c..9386e0e88 100644 --- a/apps/app-htmx/src/views/component.repository.html +++ b/apps/app-htmx/src/views/component.repository.html @@ -1,4 +1,4 @@ -
+
{% if repo.forge == "gitlab" %} @@ -7,34 +7,4 @@ {% endif %}

{{repo.org}}/{{repo.name}}

-
- - - - - - - - -
-
- -
- - -
-
- -
-
\ No newline at end of file diff --git a/apps/app-htmx/src/views/page.home.html b/apps/app-htmx/src/views/page.home.html index eb172afbe..c62368584 100644 --- a/apps/app-htmx/src/views/page.home.html +++ b/apps/app-htmx/src/views/page.home.html @@ -1,56 +1,55 @@ {% extends "layout.app.html" %} {% block content %} -

{{targetTenant.name}}

+

{{tenant.name}}

+
+
+ + + +
+
+ +
+ + +
+
+ + +
+
+

Register Repository

+
- + /
+

Repositories

{% for repo in repos %} {% include "component.repository.html" %} {% endfor %}
-

Transform whole tenant

-
-
-

{{targetTenant.name}}

-
-
- - - -
-
- -
- - -
-
- -
-
-

Logs

From 43a8bd3edab4981437a2a78ba01245ff9d6b663f Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Thu, 8 Feb 2024 15:47:43 +0100 Subject: [PATCH 2/7] package-lock app-htmx @acme/ deps --- package-lock.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package-lock.json b/package-lock.json index 5c5cd1768..b36ad5043 100644 --- a/package-lock.json +++ b/package-lock.json @@ -67,6 +67,8 @@ "name": "@acme/app-htmx", "version": "0.1.0", "dependencies": { + "@acme/extract-functions": "*", + "@acme/extract-schema": "*", "@acme/source-control": "*", "@acme/super-schema": "*", "@clerk/fastify": "0.6.30", From dd9e7393a25274eb19d9abd715a6d2ab02bcd038 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Thu, 8 Feb 2024 15:48:22 +0100 Subject: [PATCH 3/7] better transform api response --- apps/stack/src/transform/transform-tenant.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/stack/src/transform/transform-tenant.ts b/apps/stack/src/transform/transform-tenant.ts index ee0a82685..0e276c249 100644 --- a/apps/stack/src/transform/transform-tenant.ts +++ b/apps/stack/src/transform/transform-tenant.ts @@ -114,6 +114,13 @@ export const apiHandler = ApiHandler(async (ev) => { const { tenantId, from, to } = inputValidation.data; + const tenants = getTenants(); + const tenant = tenants.find(tenant => tenant.id === tenantId); + if (!tenant) return { + statusCode: 404, + message: JSON.stringify({ error: "Tenant not found" }) + } + try { await sender.send({ tenantId @@ -134,6 +141,6 @@ export const apiHandler = ApiHandler(async (ev) => { return { statusCode: 200, - body: JSON.stringify({ message: `Transforming tenant ${tenantId} from ${from} to ${to}` }), + body: JSON.stringify({ message: `Transforming tenant "${tenant.name}" merge requests in period (${from.toISOString()}...${to.toISOString()})` }), }; }); From f240a2079dfed2a14740260da1a6992b3d28fa5f Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Thu, 8 Feb 2024 15:48:58 +0100 Subject: [PATCH 4/7] extract-tenants:apiHandler --- apps/stack/src/extract/extract-tenants.ts | 64 +++++++++++++++++++++++ apps/stack/stacks/ExtractStack.ts | 4 +- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/apps/stack/src/extract/extract-tenants.ts b/apps/stack/src/extract/extract-tenants.ts index fc86e6fda..348e219d3 100644 --- a/apps/stack/src/extract/extract-tenants.ts +++ b/apps/stack/src/extract/extract-tenants.ts @@ -6,6 +6,7 @@ import { getTenantDb } from "@stack/config/get-tenant-db"; import { namespaces, repositories } from "@acme/extract-schema"; import { eq } from "drizzle-orm"; import { repositorySenderHandler } from "./extract-repository"; +import { ApiHandler, useJsonBody } from "sst/node/api"; export const tenantSenderHandler = createMessageHandler({ queueId: 'ExtractQueue', @@ -78,3 +79,66 @@ export const cronHandler = async ()=> { }; +const contextSchema = z.object({ + authorizer: z.object({ + jwt: z.object({ + claims: z.object({ + sub: z.string(), + }), + }), + }), +}); + +const inputSchema = z.object({ + tenant: z.number(), + from: z.coerce.date(), + to: z.coerce.date() +}); + +export const apiHandler = ApiHandler(async (ev) => { + + const body = useJsonBody() as unknown; + + const lambdaContextValidation = contextSchema.safeParse(ev.requestContext); + + if (!lambdaContextValidation.success) { + console.log("Error: Authorization failed - ", lambdaContextValidation.error.issues); // TODO: compliance check, might be insufficient_scope or something + return { + statusCode: 401, + body: JSON.stringify({ message: "Unauthorized" }), + } + } + + const inputValidation = inputSchema.safeParse(body); + + if (!inputValidation.success) { + console.log("Error: Input validation failed - ", inputValidation.error.issues); + return { + statusCode: 400, + body: JSON.stringify({ error: inputValidation.error.toString() }), + } + } + + const { tenant: tenantId, from, to } = inputValidation.data; + + const tenants = getTenants(); + const tenant = tenants.find(tenant => tenant.id === tenantId); + if (!tenant) return { + statusCode: 404, + message: JSON.stringify({ error: "Tenant not found" }) + } + + await sender.sendAll([{ tenantId }], { + version: -1, + caller: 'extract-tenant:apiHandler', + timestamp: Date.now(), + userId: lambdaContextValidation.data.authorizer.jwt.claims.sub, + from, + to, + tenantId: -1, + }); + return { + statusCode: 200, + body: JSON.stringify({ message: `Extracting tenant ${tenant.name} repositories in period (${from.toISOString()}...${to.toISOString()})` }) + }; +}); diff --git a/apps/stack/stacks/ExtractStack.ts b/apps/stack/stacks/ExtractStack.ts index 3ff4a7676..67741552a 100644 --- a/apps/stack/stacks/ExtractStack.ts +++ b/apps/stack/stacks/ExtractStack.ts @@ -167,7 +167,7 @@ export function ExtractStack({ stack }: StackContext) { authorizer: "JwtAuthorizer", function: { bind: [ - bus, + extractQueue, TENANT_DATABASE_AUTH_TOKEN, SUPER_DATABASE_AUTH_TOKEN, SUPER_DATABASE_URL, @@ -189,7 +189,7 @@ export function ExtractStack({ stack }: StackContext) { }, }, routes: { - "POST /start": "src/extract/extract-repository.handler", + "POST /start": "src/extract/extract-tenants.apiHandler", }, }); From ed6a94e2ea6c7ecf0ac5f33901a563ed78275e00 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Thu, 8 Feb 2024 15:50:55 +0100 Subject: [PATCH 5/7] cleanup extract-tenants:apiHandler --- apps/stack/src/extract/extract-tenants.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/stack/src/extract/extract-tenants.ts b/apps/stack/src/extract/extract-tenants.ts index 348e219d3..0a9d4b56f 100644 --- a/apps/stack/src/extract/extract-tenants.ts +++ b/apps/stack/src/extract/extract-tenants.ts @@ -120,6 +120,7 @@ export const apiHandler = ApiHandler(async (ev) => { } const { tenant: tenantId, from, to } = inputValidation.data; + const { sub } = lambdaContextValidation.data.authorizer.jwt.claims; const tenants = getTenants(); const tenant = tenants.find(tenant => tenant.id === tenantId); @@ -132,7 +133,7 @@ export const apiHandler = ApiHandler(async (ev) => { version: -1, caller: 'extract-tenant:apiHandler', timestamp: Date.now(), - userId: lambdaContextValidation.data.authorizer.jwt.claims.sub, + userId: sub, from, to, tenantId: -1, From a349148e3b4cdc1efc3afa71e4ffa1f886c7cb40 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Thu, 8 Feb 2024 15:53:12 +0100 Subject: [PATCH 6/7] remove handler (api) from extract-repository --- apps/stack/src/extract/extract-repository.ts | 53 -------------------- 1 file changed, 53 deletions(-) diff --git a/apps/stack/src/extract/extract-repository.ts b/apps/stack/src/extract/extract-repository.ts index 3982edb43..8266d302e 100644 --- a/apps/stack/src/extract/extract-repository.ts +++ b/apps/stack/src/extract/extract-repository.ts @@ -5,7 +5,6 @@ import { GitlabSourceControl, GitHubSourceControl } from "@acme/source-control"; import { repositories, namespaces, RepositorySchema, NamespaceSchema } from "@acme/extract-schema"; import { instances } from "@acme/crawl-schema"; import { z } from "zod"; -import { ApiHandler, useJsonBody } from 'sst/node/api'; import { getClerkUserToken } from "./get-clerk-user-token"; import { setInstance } from "@acme/crawl-functions"; import { getTenantDb, type OmitDb } from "@stack/config/get-tenant-db"; @@ -70,58 +69,6 @@ const extractRepository = async (input: Input, userId: string) => { } -const contextSchema = z.object({ - authorizer: z.object({ - jwt: z.object({ - claims: z.object({ - sub: z.string(), - }), - }), - }), -}); - -export const handler = ApiHandler(async (ev) => { - - const body = useJsonBody() as unknown; - - const lambdaContextValidation = contextSchema.safeParse(ev.requestContext); - - if (!lambdaContextValidation.success) { - console.log("Error: Authorization failed - ", lambdaContextValidation.error.issues); // TODO: compliance check, might be insufficient_scope or something - return { - statusCode: 401, - body: JSON.stringify({ message: "Unauthorized" }), - } - } - - const inputValidation = inputSchema.safeParse(body); - - if (!inputValidation.success) { - console.log("Error: Input validation failed - ", inputValidation.error.issues); - return { - statusCode: 400, - body: JSON.stringify({ error: inputValidation.error.toString() }), - } - } - - const input = inputValidation.data; - const { sub } = lambdaContextValidation.data.authorizer.jwt.claims; - - try { - await extractRepository(input, sub); - } catch (error) { - return { - statusCode: 500, - body: JSON.stringify({ error: (error as Error).toString() }) - } - } - - return { - statusCode: 200, - body: JSON.stringify({ from: inputValidation.data.from, to: inputValidation.data.to }) - }; -}); - export const repositorySenderHandler = createMessageHandler({ queueId: 'ExtractQueue', kind: MessageKind.Repository, From 22e3636f6316f461e9ad417564ee5f892346d1c3 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Fri, 9 Feb 2024 11:38:19 +0100 Subject: [PATCH 7/7] sort deps --- apps/app-htmx/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/app-htmx/package.json b/apps/app-htmx/package.json index 90ba7b92e..71174f631 100644 --- a/apps/app-htmx/package.json +++ b/apps/app-htmx/package.json @@ -14,10 +14,10 @@ "esbuild": "^0.19.9" }, "dependencies": { + "@acme/extract-functions": "*", + "@acme/extract-schema": "*", "@acme/source-control": "*", "@acme/super-schema": "*", - "@acme/extract-schema": "*", - "@acme/extract-functions": "*", "@clerk/fastify": "0.6.30", "@fastify/formbody": "7.4.0", "@fastify/view": "8.2.0",