From c88e1a68c7cbeab745eca09a969eaa0b30a8f584 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Fri, 8 Dec 2023 13:25:02 +0100 Subject: [PATCH 01/10] add public-repos package --- package-lock.json | 8 ++++++++ packages/config/public-repos/index.ts | 21 +++++++++++++++++++++ packages/config/public-repos/package.json | 9 +++++++++ 3 files changed, 38 insertions(+) create mode 100644 packages/config/public-repos/index.ts create mode 100644 packages/config/public-repos/package.json diff --git a/package-lock.json b/package-lock.json index 3967241ad..6d473463a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -137,6 +137,10 @@ "resolved": "packages/schemas/extract", "link": true }, + "node_modules/@acme/public-repos": { + "resolved": "packages/config/public-repos", + "link": true + }, "node_modules/@acme/source-control": { "resolved": "packages/integrations/source-control", "link": true @@ -19644,6 +19648,10 @@ "eslint": "8.51.0" } }, + "packages/config/public-repos": { + "version": "0.1.0", + "license": "MIT" + }, "packages/config/tailwind": { "name": "@acme/tailwind-config", "version": "0.1.0", diff --git a/packages/config/public-repos/index.ts b/packages/config/public-repos/index.ts new file mode 100644 index 000000000..d2b425b11 --- /dev/null +++ b/packages/config/public-repos/index.ts @@ -0,0 +1,21 @@ +type GithubRepo = { + forgeType: 'github', + name: string, + owner: string, + webUrl: string, +} + +type UnknownRepo = { + forgeType: 'unknown', +} + +export type PublicRepo = GithubRepo | UnknownRepo; + +export const PUBLIC_REPOS = [ + { + forgeType: 'github', + name: 'mr-tool-monorepo', + owner: 'crocoder-dev', + webUrl: 'https://github.com/crocoder-dev/mr-tool-monorepo' + } +] satisfies PublicRepo[]; \ No newline at end of file diff --git a/packages/config/public-repos/package.json b/packages/config/public-repos/package.json new file mode 100644 index 000000000..0830d291a --- /dev/null +++ b/packages/config/public-repos/package.json @@ -0,0 +1,9 @@ +{ + "name": "@acme/public-repos", + "version": "0.1.0", + "main": "index.ts", + "license": "MIT", + "files": [ + "index.ts" + ] +} From 25a2e6e8c1754562df5630963d41899b51a115b9 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Fri, 8 Dec 2023 14:59:37 +0100 Subject: [PATCH 02/10] add app extract trigger for public repos --- apps/app/package.json | 1 + apps/app/src/app/page.tsx | 2 + .../src/components/public-repo-control.tsx | 78 +++++++++++++++++++ package-lock.json | 2 + 4 files changed, 83 insertions(+) create mode 100644 apps/app/src/components/public-repo-control.tsx diff --git a/apps/app/package.json b/apps/app/package.json index a47f43076..1b36624ef 100644 --- a/apps/app/package.json +++ b/apps/app/package.json @@ -15,6 +15,7 @@ "dependencies": { "@acme/eslint-config": "*", "@acme/tailwind-config": "*", + "@acme/public-repos": "*", "@clerk/nextjs": "^4.23.2", "@radix-ui/react-avatar": "^1.0.3", "@radix-ui/react-dropdown-menu": "^2.0.5", diff --git a/apps/app/src/app/page.tsx b/apps/app/src/app/page.tsx index 035ab9c18..e2938ff41 100644 --- a/apps/app/src/app/page.tsx +++ b/apps/app/src/app/page.tsx @@ -3,6 +3,7 @@ import { MainNav } from "~/components/ui/main-nav"; import { GenerateToken } from "~/components/generate-token"; import { ExtractStartForm } from "~/components/extract-start-form"; import { TransformStartForm } from "~/components/transform-start-form"; +import { PublicRepoControl } from "~/components/public-repo-control"; export default function Page() { return ( @@ -21,6 +22,7 @@ export default function Page() { + ); } diff --git a/apps/app/src/components/public-repo-control.tsx b/apps/app/src/components/public-repo-control.tsx new file mode 100644 index 000000000..35e60ce9b --- /dev/null +++ b/apps/app/src/components/public-repo-control.tsx @@ -0,0 +1,78 @@ +"use client" +import { useAuth } from '@clerk/nextjs'; +import { type ChangeEvent, useState } from 'react'; + +import { PUBLIC_REPOS, type PublicRepo } from "@acme/public-repos" + +type PublicRepoCardProps = { + repo: (PublicRepo & {forgeType: 'github'}) +} +const PublicRepoCard = ({ repo }: PublicRepoCardProps) => { + const { getToken } = useAuth(); + const [status, setStatus] = useState('---'); + const [daysAgo, setDaysAgo] = useState('1'); + + const handleSelectChange = (ev: ChangeEvent) => setDaysAgo(ev.target.value); + + + const onExtractClick = async ()=> { + if (!process.env.NEXT_PUBLIC_EXTRACT_API_URL) return console.error('Missing ENV variable: NEXT_PUBLIC_EXTRACT_API_URL'); + setStatus('...'); + + const token = await getToken({ template: 'dashboard' }); + if (!token) return; + + const now = new Date(); + const extractPeriodStart = new Date(now.getTime() - (Number(daysAgo) * 24 * 60 * 60 * 1000)); + + const requestBody = JSON.stringify({ + repositoryId: 0, + repositoryName: repo.name, + namespaceName: repo.owner, sourceControl: repo.forgeType, + from: extractPeriodStart, + to: now + }); + + const res = await fetch(process.env.NEXT_PUBLIC_EXTRACT_API_URL, { + method: 'post', + body: requestBody, + headers: { + 'Authorization': 'Bearer ' + token + } + }); + + setStatus(res.status.toString()); + } + + return ( +
+ +
{repo.name}
+ +

{repo.owner}

+ + + +

Status: {status}

+ + {/* NO PERIOD FOR TRANSFORM EXISTS, OR PER REPOSITORY */} + {/* */} +
+ ) +} + +export function PublicRepoControl() { + + return (<> +
+
Public Repositories
+ {PUBLIC_REPOS.filter(repo => repo.forgeType === 'github').map((repo, idx) => ())} +
+ ) +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 6d473463a..6e01dd65d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,6 +38,7 @@ "version": "0.1.0", "dependencies": { "@acme/eslint-config": "*", + "@acme/public-repos": "*", "@acme/tailwind-config": "*", "@clerk/nextjs": "^4.23.2", "@radix-ui/react-avatar": "^1.0.3", @@ -19649,6 +19650,7 @@ } }, "packages/config/public-repos": { + "name": "@acme/public-repos", "version": "0.1.0", "license": "MIT" }, From 26658c31a8fc9912e6ee3bd4636a39f83b567721 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Fri, 8 Dec 2023 14:59:53 +0100 Subject: [PATCH 03/10] disable dark theme bcuz clerk is weird --- apps/app/src/app/layout.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/app/src/app/layout.tsx b/apps/app/src/app/layout.tsx index 47567c35a..a64edd5a1 100644 --- a/apps/app/src/app/layout.tsx +++ b/apps/app/src/app/layout.tsx @@ -10,11 +10,11 @@ export default function RootLayout({ return ( - - + + {children} - - + + ); From 5c4a06ee0e43b9f8c899c2c7736b56e6140c726c Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Fri, 8 Dec 2023 17:02:48 +0100 Subject: [PATCH 04/10] add cron for period extract --- apps/stack/.sst/types/index.ts | 7 ++ apps/stack/package.json | 1 + .../extract/extract-periodic-public-repos.ts | 82 +++++++++++++++++++ .../stack/src/extract/get-clerk-user-token.ts | 5 ++ apps/stack/stacks/ExtractStack.ts | 23 ++++++ package-lock.json | 1 + 6 files changed, 119 insertions(+) create mode 100644 apps/stack/src/extract/extract-periodic-public-repos.ts diff --git a/apps/stack/.sst/types/index.ts b/apps/stack/.sst/types/index.ts index 450e4cefd..cd8a49733 100644 --- a/apps/stack/.sst/types/index.ts +++ b/apps/stack/.sst/types/index.ts @@ -67,6 +67,13 @@ declare module "sst/node/config" { value: string; } } +}import "sst/node/config"; +declare module "sst/node/config" { + export interface SecretResources { + "SYSTEM_GITHUB_TOKEN": { + value: string; + } + } }import "sst/node/event-bus"; declare module "sst/node/event-bus" { export interface EventBusResources { diff --git a/apps/stack/package.json b/apps/stack/package.json index 652eb8a03..136d5828a 100644 --- a/apps/stack/package.json +++ b/apps/stack/package.json @@ -24,6 +24,7 @@ "@acme/source-control": "*", "@acme/transform-functions": "*", "@acme/transform-schema": "*", + "@acme/public-repos": "*", "@aws-sdk/client-sqs": "^3.395.0", "@clerk/clerk-sdk-node": "^4.12.5", "@tsconfig/node16": "^16.1.1", diff --git a/apps/stack/src/extract/extract-periodic-public-repos.ts b/apps/stack/src/extract/extract-periodic-public-repos.ts new file mode 100644 index 000000000..2d1861603 --- /dev/null +++ b/apps/stack/src/extract/extract-periodic-public-repos.ts @@ -0,0 +1,82 @@ +import { extractRepositoryEvent } from "./events"; +import { getRepository } from "@acme/extract-functions"; +import type { Context, GetRepositorySourceControl, GetRepositoryEntities } from "@acme/extract-functions"; +import { GitHubSourceControl } from "@acme/source-control"; +import { repositories, namespaces } from "@acme/extract-schema"; +import { instances } from "@acme/crawl-schema"; +import { createClient } from '@libsql/client'; +import { drizzle } from 'drizzle-orm/libsql'; +import { Config } from "sst/node/config"; +import { overrideClerkUserToken } from "./get-clerk-user-token"; +import { setInstance } from "@acme/crawl-functions"; + +import { PUBLIC_REPOS } from "@acme/public-repos" + +const client = createClient({ + url: Config.EXTRACT_DATABASE_URL, + authToken: Config.EXTRACT_DATABASE_AUTH_TOKEN +}); + +const crawlClient = createClient({ + url: Config.CRAWL_DATABASE_URL, + authToken: Config.CRAWL_DATABASE_AUTH_TOKEN +}); + +const db = drizzle(client); + +const crawlDb = drizzle(crawlClient); + +const context: Context = { + entities: { + repositories, + namespaces, + }, + integrations: { + sourceControl: new GitHubSourceControl(Config.SYSTEM_GITHUB_TOKEN), + }, + db, +}; + +const sub = "__system"; +export const handler = async () => { + + if (Config.SYSTEM_GITHUB_TOKEN === "__disabled") { + console.log("System token is '__disabled', skipping periodic extract..."); + return; + } + + const githubRepos = PUBLIC_REPOS.filter(repo => repo.forgeType === 'github'); + + // ISSUES: need longer override cache + await overrideClerkUserToken(sub, 'oauth_github', Config.SYSTEM_GITHUB_TOKEN); + + // CONSIDER: config for what time ? what about period + const utcTodayAt10AM = new Date(); + utcTodayAt10AM.setUTCHours(10, 0, 0, 0); + const utcYesterdayAt10AM = new Date(utcTodayAt10AM); + utcYesterdayAt10AM.setHours(utcTodayAt10AM.getUTCHours() - 24); + + for (const repo of githubRepos) { + const { namespace, repository } = await getRepository({ externalRepositoryId: 0, repositoryName: repo.name, namespaceName: repo.owner }, context); + const { instanceId } = await setInstance({ repositoryId: repository.id, userId: sub }, { db: crawlDb, entities: { instances } }); + + await extractRepositoryEvent.publish( + { + repositoryId: repository.id, + namespaceId: namespace.id + }, + { + crawlId: instanceId, + caller: 'extract-repository', + timestamp: new Date().getTime(), + version: 1, + sourceControl: 'github', + userId: sub, + from: utcYesterdayAt10AM, + to: utcTodayAt10AM, + } + ); + + } + +} \ No newline at end of file diff --git a/apps/stack/src/extract/get-clerk-user-token.ts b/apps/stack/src/extract/get-clerk-user-token.ts index 0e5f53344..de5eaffcb 100644 --- a/apps/stack/src/extract/get-clerk-user-token.ts +++ b/apps/stack/src/extract/get-clerk-user-token.ts @@ -25,6 +25,11 @@ const fetchClerkUserToken = async (userId: string, provider: SupportedClerkOAuth return userOauthAccessTokenPayload.token; }; +export const overrideClerkUserToken = async (userId: string, provider:SupportedClerkOAuthProviders, userToken: string) => { + const userTokenCacheKey = `${userId}_${provider}`; + await redisClient.set(userTokenCacheKey, encodeUserTokenCacheValue(userToken), { ex: Number(Config.REDIS_USER_TOKEN_TTL) }); +} + export const getClerkUserToken = async (userId: string, provider: SupportedClerkOAuthProviders) => { const userTokenCacheKey = `${userId}_${provider}`; let encodedUserTokenCacheValue: string | null; diff --git a/apps/stack/stacks/ExtractStack.ts b/apps/stack/stacks/ExtractStack.ts index 3733aa31a..b176138f1 100644 --- a/apps/stack/stacks/ExtractStack.ts +++ b/apps/stack/stacks/ExtractStack.ts @@ -3,6 +3,7 @@ import { Config, EventBus, Queue, + Cron, type StackContext, } from "sst/constructs"; import { z } from "zod"; @@ -17,6 +18,7 @@ export function ExtractStack({ stack }: StackContext) { const REDIS_TOKEN = new Config.Secret(stack, "REDIS_TOKEN"); const REDIS_USER_TOKEN_TTL = new Config.Parameter(stack, "REDIS_USER_TOKEN_TTL", { value: (20 * 60).toString() }); const PER_PAGE = new Config.Parameter(stack, "PER_PAGE", { value: (30).toString() }); + const SYSTEM_GITHUB_TOKEN = new Config.Secret(stack, "SYSTEM_GITHUB_TOKEN"); // set to "__disabled" if you want to disable periodic extraction const bus = new EventBus(stack, "ExtractBus", { rules: { @@ -195,6 +197,27 @@ export function ExtractStack({ stack }: StackContext) { }, }); + const _cronExtractPeriodic = new Cron(stack, "CronExtractPeriodic", { + schedule: "cron(0 10 * * ? *)", + job: { + function: { + handler: "src/extract/extract-periodic-public-repos.handler", + bind: [ + bus, + EXTRACT_DATABASE_URL, + EXTRACT_DATABASE_AUTH_TOKEN, + CRAWL_DATABASE_URL, + CRAWL_DATABASE_AUTH_TOKEN, + CLERK_SECRET_KEY, + REDIS_URL, + REDIS_TOKEN, + REDIS_USER_TOKEN_TTL, + SYSTEM_GITHUB_TOKEN + ] + } + } + }); + stack.addOutputs({ ApiEndpoint: api.url, }); diff --git a/package-lock.json b/package-lock.json index 6e01dd65d..18686bd0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -73,6 +73,7 @@ "@acme/crawl-schema": "*", "@acme/extract-functions": "*", "@acme/extract-schema": "*", + "@acme/public-repos": "*", "@acme/source-control": "*", "@acme/transform-functions": "*", "@acme/transform-schema": "*", From 9529bc19f168d83720845a8c8398a14976ca0573 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Fri, 8 Dec 2023 17:03:17 +0100 Subject: [PATCH 05/10] move ui things ? idk --- apps/app/src/components/public-repo-control.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/app/src/components/public-repo-control.tsx b/apps/app/src/components/public-repo-control.tsx index 35e60ce9b..e42605d32 100644 --- a/apps/app/src/components/public-repo-control.tsx +++ b/apps/app/src/components/public-repo-control.tsx @@ -59,10 +59,10 @@ const PublicRepoCard = ({ repo }: PublicRepoCardProps) => { -

Status: {status}

{/* NO PERIOD FOR TRANSFORM EXISTS, OR PER REPOSITORY */} {/* */} +

Status: {status}

) } From fd0dc17f6a9abcc46a00e998c66857c6f48bb29f Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Mon, 11 Dec 2023 16:05:57 +0100 Subject: [PATCH 06/10] sort dependencies --- apps/app/package.json | 2 +- apps/stack/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/app/package.json b/apps/app/package.json index 1b36624ef..edaf9e5dd 100644 --- a/apps/app/package.json +++ b/apps/app/package.json @@ -14,8 +14,8 @@ }, "dependencies": { "@acme/eslint-config": "*", - "@acme/tailwind-config": "*", "@acme/public-repos": "*", + "@acme/tailwind-config": "*", "@clerk/nextjs": "^4.23.2", "@radix-ui/react-avatar": "^1.0.3", "@radix-ui/react-dropdown-menu": "^2.0.5", diff --git a/apps/stack/package.json b/apps/stack/package.json index 136d5828a..54d56a39e 100644 --- a/apps/stack/package.json +++ b/apps/stack/package.json @@ -21,10 +21,10 @@ "@acme/crawl-schema": "*", "@acme/extract-functions": "*", "@acme/extract-schema": "*", + "@acme/public-repos": "*", "@acme/source-control": "*", "@acme/transform-functions": "*", "@acme/transform-schema": "*", - "@acme/public-repos": "*", "@aws-sdk/client-sqs": "^3.395.0", "@clerk/clerk-sdk-node": "^4.12.5", "@tsconfig/node16": "^16.1.1", From 70debabcc4b66f4d1633b11b971db910ab15ff84 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Tue, 12 Dec 2023 12:31:10 +0100 Subject: [PATCH 07/10] refactor --- apps/app/package.json | 1 - apps/app/src/app/page.tsx | 2 - .../src/components/public-repo-control.tsx | 78 ---------- apps/stack/.sst/types/index.ts | 7 - apps/stack/package.json | 1 - .../extract/extract-periodic-public-repos.ts | 82 ----------- apps/stack/src/extract/extract-repository.ts | 133 +++++++++++------- apps/stack/stacks/ExtractStack.ts | 53 ++++--- package-lock.json | 11 -- packages/config/public-repos/index.ts | 21 --- packages/config/public-repos/package.json | 9 -- 11 files changed, 117 insertions(+), 281 deletions(-) delete mode 100644 apps/app/src/components/public-repo-control.tsx delete mode 100644 apps/stack/src/extract/extract-periodic-public-repos.ts delete mode 100644 packages/config/public-repos/index.ts delete mode 100644 packages/config/public-repos/package.json diff --git a/apps/app/package.json b/apps/app/package.json index b6f69b941..e6663bd6f 100644 --- a/apps/app/package.json +++ b/apps/app/package.json @@ -14,7 +14,6 @@ }, "dependencies": { "@acme/eslint-config": "*", - "@acme/public-repos": "*", "@acme/tailwind-config": "*", "@clerk/nextjs": "^4.23.2", "@radix-ui/react-avatar": "^1.0.3", diff --git a/apps/app/src/app/page.tsx b/apps/app/src/app/page.tsx index e2938ff41..035ab9c18 100644 --- a/apps/app/src/app/page.tsx +++ b/apps/app/src/app/page.tsx @@ -3,7 +3,6 @@ import { MainNav } from "~/components/ui/main-nav"; import { GenerateToken } from "~/components/generate-token"; import { ExtractStartForm } from "~/components/extract-start-form"; import { TransformStartForm } from "~/components/transform-start-form"; -import { PublicRepoControl } from "~/components/public-repo-control"; export default function Page() { return ( @@ -22,7 +21,6 @@ export default function Page() { - ); } diff --git a/apps/app/src/components/public-repo-control.tsx b/apps/app/src/components/public-repo-control.tsx deleted file mode 100644 index e42605d32..000000000 --- a/apps/app/src/components/public-repo-control.tsx +++ /dev/null @@ -1,78 +0,0 @@ -"use client" -import { useAuth } from '@clerk/nextjs'; -import { type ChangeEvent, useState } from 'react'; - -import { PUBLIC_REPOS, type PublicRepo } from "@acme/public-repos" - -type PublicRepoCardProps = { - repo: (PublicRepo & {forgeType: 'github'}) -} -const PublicRepoCard = ({ repo }: PublicRepoCardProps) => { - const { getToken } = useAuth(); - const [status, setStatus] = useState('---'); - const [daysAgo, setDaysAgo] = useState('1'); - - const handleSelectChange = (ev: ChangeEvent) => setDaysAgo(ev.target.value); - - - const onExtractClick = async ()=> { - if (!process.env.NEXT_PUBLIC_EXTRACT_API_URL) return console.error('Missing ENV variable: NEXT_PUBLIC_EXTRACT_API_URL'); - setStatus('...'); - - const token = await getToken({ template: 'dashboard' }); - if (!token) return; - - const now = new Date(); - const extractPeriodStart = new Date(now.getTime() - (Number(daysAgo) * 24 * 60 * 60 * 1000)); - - const requestBody = JSON.stringify({ - repositoryId: 0, - repositoryName: repo.name, - namespaceName: repo.owner, sourceControl: repo.forgeType, - from: extractPeriodStart, - to: now - }); - - const res = await fetch(process.env.NEXT_PUBLIC_EXTRACT_API_URL, { - method: 'post', - body: requestBody, - headers: { - 'Authorization': 'Bearer ' + token - } - }); - - setStatus(res.status.toString()); - } - - return ( -
- -
{repo.name}
- -

{repo.owner}

- - - - - {/* NO PERIOD FOR TRANSFORM EXISTS, OR PER REPOSITORY */} - {/* */} -

Status: {status}

-
- ) -} - -export function PublicRepoControl() { - - return (<> -
-
Public Repositories
- {PUBLIC_REPOS.filter(repo => repo.forgeType === 'github').map((repo, idx) => ())} -
- ) -} \ No newline at end of file diff --git a/apps/stack/.sst/types/index.ts b/apps/stack/.sst/types/index.ts index cd8a49733..450e4cefd 100644 --- a/apps/stack/.sst/types/index.ts +++ b/apps/stack/.sst/types/index.ts @@ -67,13 +67,6 @@ declare module "sst/node/config" { value: string; } } -}import "sst/node/config"; -declare module "sst/node/config" { - export interface SecretResources { - "SYSTEM_GITHUB_TOKEN": { - value: string; - } - } }import "sst/node/event-bus"; declare module "sst/node/event-bus" { export interface EventBusResources { diff --git a/apps/stack/package.json b/apps/stack/package.json index 6c36d5cce..149d14708 100644 --- a/apps/stack/package.json +++ b/apps/stack/package.json @@ -21,7 +21,6 @@ "@acme/crawl-schema": "*", "@acme/extract-functions": "*", "@acme/extract-schema": "*", - "@acme/public-repos": "*", "@acme/source-control": "*", "@acme/transform-functions": "*", "@acme/transform-schema": "*", diff --git a/apps/stack/src/extract/extract-periodic-public-repos.ts b/apps/stack/src/extract/extract-periodic-public-repos.ts deleted file mode 100644 index 2d1861603..000000000 --- a/apps/stack/src/extract/extract-periodic-public-repos.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { extractRepositoryEvent } from "./events"; -import { getRepository } from "@acme/extract-functions"; -import type { Context, GetRepositorySourceControl, GetRepositoryEntities } from "@acme/extract-functions"; -import { GitHubSourceControl } from "@acme/source-control"; -import { repositories, namespaces } from "@acme/extract-schema"; -import { instances } from "@acme/crawl-schema"; -import { createClient } from '@libsql/client'; -import { drizzle } from 'drizzle-orm/libsql'; -import { Config } from "sst/node/config"; -import { overrideClerkUserToken } from "./get-clerk-user-token"; -import { setInstance } from "@acme/crawl-functions"; - -import { PUBLIC_REPOS } from "@acme/public-repos" - -const client = createClient({ - url: Config.EXTRACT_DATABASE_URL, - authToken: Config.EXTRACT_DATABASE_AUTH_TOKEN -}); - -const crawlClient = createClient({ - url: Config.CRAWL_DATABASE_URL, - authToken: Config.CRAWL_DATABASE_AUTH_TOKEN -}); - -const db = drizzle(client); - -const crawlDb = drizzle(crawlClient); - -const context: Context = { - entities: { - repositories, - namespaces, - }, - integrations: { - sourceControl: new GitHubSourceControl(Config.SYSTEM_GITHUB_TOKEN), - }, - db, -}; - -const sub = "__system"; -export const handler = async () => { - - if (Config.SYSTEM_GITHUB_TOKEN === "__disabled") { - console.log("System token is '__disabled', skipping periodic extract..."); - return; - } - - const githubRepos = PUBLIC_REPOS.filter(repo => repo.forgeType === 'github'); - - // ISSUES: need longer override cache - await overrideClerkUserToken(sub, 'oauth_github', Config.SYSTEM_GITHUB_TOKEN); - - // CONSIDER: config for what time ? what about period - const utcTodayAt10AM = new Date(); - utcTodayAt10AM.setUTCHours(10, 0, 0, 0); - const utcYesterdayAt10AM = new Date(utcTodayAt10AM); - utcYesterdayAt10AM.setHours(utcTodayAt10AM.getUTCHours() - 24); - - for (const repo of githubRepos) { - const { namespace, repository } = await getRepository({ externalRepositoryId: 0, repositoryName: repo.name, namespaceName: repo.owner }, context); - const { instanceId } = await setInstance({ repositoryId: repository.id, userId: sub }, { db: crawlDb, entities: { instances } }); - - await extractRepositoryEvent.publish( - { - repositoryId: repository.id, - namespaceId: namespace.id - }, - { - crawlId: instanceId, - caller: 'extract-repository', - timestamp: new Date().getTime(), - version: 1, - sourceControl: 'github', - userId: sub, - from: utcYesterdayAt10AM, - to: utcTodayAt10AM, - } - ); - - } - -} \ No newline at end of file diff --git a/apps/stack/src/extract/extract-repository.ts b/apps/stack/src/extract/extract-repository.ts index ed602ca83..18783dda5 100644 --- a/apps/stack/src/extract/extract-repository.ts +++ b/apps/stack/src/extract/extract-repository.ts @@ -37,6 +37,50 @@ const context: Context = { db, }; +const inputSchema = z.object({ + repositoryId: z.number(), + repositoryName: z.string(), + namespaceName: z.string(), + sourceControl: z.literal("gitlab").or(z.literal("github")), + from: z.coerce.date(), + to: z.coerce.date() +}); + +type Input = z.infer; +const extractRepository = async (input: Input, userId: string) => { + const { repositoryId, repositoryName, namespaceName, sourceControl, from, to } = input; + + const sourceControlAccessToken = await getClerkUserToken(userId, `oauth_${sourceControl}`); + + if (sourceControl === "gitlab") { + context.integrations.sourceControl = new GitlabSourceControl(sourceControlAccessToken); + } else if (sourceControl === "github") { + context.integrations.sourceControl = new GitHubSourceControl(sourceControlAccessToken); + } + + const { repository, namespace } = await getRepository({ externalRepositoryId: repositoryId, repositoryName, namespaceName }, context); + + const { instanceId } = await setInstance({ repositoryId: repository.id, userId }, { db: crawlDb, entities: { instances } }); + + await extractRepositoryEvent.publish( + { + repositoryId: repository.id, + namespaceId: namespace.id + }, + { + crawlId: instanceId, + caller: 'extract-repository', + timestamp: new Date().getTime(), + version: 1, + sourceControl, + userId, + from, + to, + } + ); + +} + const contextSchema = z.object({ authorizer: z.object({ jwt: z.object({ @@ -49,17 +93,6 @@ const contextSchema = z.object({ type CTX = z.infer; -const inputSchema = z.object({ - repositoryId: z.number(), - repositoryName: z.string(), - namespaceName: z.string(), - sourceControl: z.literal("gitlab").or(z.literal("github")), - from: z.coerce.date(), - to: z.coerce.date() -}); - -type Input = z.infer; - export const handler = ApiHandler(async (ev) => { const body = useJsonBody() as unknown; @@ -76,7 +109,6 @@ export const handler = ApiHandler(async (ev) => { } let input: Input; - let sourceControlAccessToken: string; try { input = inputSchema.parse(body); @@ -90,48 +122,49 @@ export const handler = ApiHandler(async (ev) => { const { sub } = lambdaContext.authorizer.jwt.claims; - - const { repositoryId, repositoryName, namespaceName, sourceControl, from, to } = input; - - try { - sourceControlAccessToken = await getClerkUserToken(sub, `oauth_${sourceControl}`); - } catch (error) { - return { - statusCode: 500, - body: JSON.stringify({ error: (error as Error).message }), - } - } - - if (sourceControl === "gitlab") { - context.integrations.sourceControl = new GitlabSourceControl(sourceControlAccessToken); - } else if (sourceControl === "github") { - context.integrations.sourceControl = new GitHubSourceControl(sourceControlAccessToken); +try { + await extractRepository(input, sub); +} catch (error) { + return { + statusCode: 500, + body: JSON.stringify({ error: (error as Error).toString() }) } - - const { repository, namespace } = await getRepository({ externalRepositoryId: repositoryId, repositoryName, namespaceName }, context); - - const { instanceId } = await setInstance({ repositoryId: repository.id, userId: sub }, { db: crawlDb, entities: { instances } }); - - await extractRepositoryEvent.publish( - { - repositoryId: repository.id, - namespaceId: namespace.id - }, - { - crawlId: instanceId, - caller: 'extract-repository', - timestamp: new Date().getTime(), - version: 1, - sourceControl, - userId: sub, - from, - to, - } - ); - +} return { statusCode: 200, body: JSON.stringify({}) }; }); + +const CRON_ENV = z.object({ + CRON_USER_ID: z.string(), + PUBLIC_REPO_NAME: z.string(), + PUBLIC_REPO_OWNER: z.string(), +}) +export const cronHandler = async ()=> { + + const validEnv = CRON_ENV.safeParse(process.env); + + if (!validEnv.success) { + console.error("Invalid environment in lambda 'extract-repository.cronHandler':", ...validEnv.error.issues); + throw new Error("Invalid environment"); + } + + const { CRON_USER_ID, PUBLIC_REPO_NAME, PUBLIC_REPO_OWNER } = validEnv.data; + + const utcTodayAt10AM = new Date(); + utcTodayAt10AM.setUTCHours(10, 0, 0, 0); + const utcYesterdayAt10AM = new Date(utcTodayAt10AM); + utcYesterdayAt10AM.setHours(utcTodayAt10AM.getUTCHours() - 24); + + await extractRepository({ + namespaceName: PUBLIC_REPO_OWNER, + repositoryId: 0, + repositoryName: PUBLIC_REPO_NAME, + sourceControl: 'github', + from: utcYesterdayAt10AM, + to: utcTodayAt10AM, + }, CRON_USER_ID); + +} diff --git a/apps/stack/stacks/ExtractStack.ts b/apps/stack/stacks/ExtractStack.ts index b176138f1..83ffc3c02 100644 --- a/apps/stack/stacks/ExtractStack.ts +++ b/apps/stack/stacks/ExtractStack.ts @@ -18,7 +18,6 @@ export function ExtractStack({ stack }: StackContext) { const REDIS_TOKEN = new Config.Secret(stack, "REDIS_TOKEN"); const REDIS_USER_TOKEN_TTL = new Config.Parameter(stack, "REDIS_USER_TOKEN_TTL", { value: (20 * 60).toString() }); const PER_PAGE = new Config.Parameter(stack, "PER_PAGE", { value: (30).toString() }); - const SYSTEM_GITHUB_TOKEN = new Config.Secret(stack, "SYSTEM_GITHUB_TOKEN"); // set to "__disabled" if you want to disable periodic extraction const bus = new EventBus(stack, "ExtractBus", { rules: { @@ -160,9 +159,18 @@ export function ExtractStack({ stack }: StackContext) { const ENVSchema = z.object({ CLERK_JWT_ISSUER: z.string(), CLERK_JWT_AUDIENCE: z.string(), + PUBLIC_REPOS: z.string(), + CRON_USER_ID: z.string(), }); + const publicReposSchema = z.array( + z.object({ + owner: z.string(), + name: z.string(), + }) + ); const ENV = ENVSchema.parse(process.env); + const PUBLIC_REPOS = publicReposSchema.parse(JSON.parse(ENV.PUBLIC_REPOS)); const api = new Api(stack, "ExtractApi", { defaults: { @@ -197,25 +205,32 @@ export function ExtractStack({ stack }: StackContext) { }, }); - const _cronExtractPeriodic = new Cron(stack, "CronExtractPeriodic", { - schedule: "cron(0 10 * * ? *)", - job: { - function: { - handler: "src/extract/extract-periodic-public-repos.handler", - bind: [ - bus, - EXTRACT_DATABASE_URL, - EXTRACT_DATABASE_AUTH_TOKEN, - CRAWL_DATABASE_URL, - CRAWL_DATABASE_AUTH_TOKEN, - CLERK_SECRET_KEY, - REDIS_URL, - REDIS_TOKEN, - REDIS_USER_TOKEN_TTL, - SYSTEM_GITHUB_TOKEN - ] + PUBLIC_REPOS.forEach(publicRepo => { + new Cron(stack, `${publicRepo.name}_ExtractCron`, { + schedule: "cron(00 10 * * ? *)", + job: { + function: { + handler: "src/extract/extract-repository.cronHandler", + environment: { + CRON_USER_ID: ENV.CRON_USER_ID, + PUBLIC_REPO_OWNER: publicRepo.owner, + PUBLIC_REPO_NAME: publicRepo.name, + }, + bind: [ + bus, + EXTRACT_DATABASE_URL, + EXTRACT_DATABASE_AUTH_TOKEN, + CRAWL_DATABASE_URL, + CRAWL_DATABASE_AUTH_TOKEN, + CLERK_SECRET_KEY, + REDIS_URL, + REDIS_TOKEN, + REDIS_USER_TOKEN_TTL + ], + runtime: "nodejs18.x", + } } - } + }) }); stack.addOutputs({ diff --git a/package-lock.json b/package-lock.json index 0cf3acfef..2c58f68a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,7 +38,6 @@ "version": "0.1.0", "dependencies": { "@acme/eslint-config": "*", - "@acme/public-repos": "*", "@acme/tailwind-config": "*", "@clerk/nextjs": "^4.23.2", "@radix-ui/react-avatar": "^1.0.3", @@ -73,7 +72,6 @@ "@acme/crawl-schema": "*", "@acme/extract-functions": "*", "@acme/extract-schema": "*", - "@acme/public-repos": "*", "@acme/source-control": "*", "@acme/transform-functions": "*", "@acme/transform-schema": "*", @@ -139,10 +137,6 @@ "resolved": "packages/schemas/extract", "link": true }, - "node_modules/@acme/public-repos": { - "resolved": "packages/config/public-repos", - "link": true - }, "node_modules/@acme/source-control": { "resolved": "packages/integrations/source-control", "link": true @@ -21585,11 +21579,6 @@ "eslint": "8.55.0" } }, - "packages/config/public-repos": { - "name": "@acme/public-repos", - "version": "0.1.0", - "license": "MIT" - }, "packages/config/tailwind": { "name": "@acme/tailwind-config", "version": "0.1.0", diff --git a/packages/config/public-repos/index.ts b/packages/config/public-repos/index.ts deleted file mode 100644 index d2b425b11..000000000 --- a/packages/config/public-repos/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -type GithubRepo = { - forgeType: 'github', - name: string, - owner: string, - webUrl: string, -} - -type UnknownRepo = { - forgeType: 'unknown', -} - -export type PublicRepo = GithubRepo | UnknownRepo; - -export const PUBLIC_REPOS = [ - { - forgeType: 'github', - name: 'mr-tool-monorepo', - owner: 'crocoder-dev', - webUrl: 'https://github.com/crocoder-dev/mr-tool-monorepo' - } -] satisfies PublicRepo[]; \ No newline at end of file diff --git a/packages/config/public-repos/package.json b/packages/config/public-repos/package.json deleted file mode 100644 index 0830d291a..000000000 --- a/packages/config/public-repos/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "@acme/public-repos", - "version": "0.1.0", - "main": "index.ts", - "license": "MIT", - "files": [ - "index.ts" - ] -} From b9652f127d8b2ae8ea207be5abef8a184844d590 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Tue, 12 Dec 2023 12:32:22 +0100 Subject: [PATCH 08/10] revert layout changes --- apps/app/src/app/layout.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/app/src/app/layout.tsx b/apps/app/src/app/layout.tsx index a64edd5a1..47567c35a 100644 --- a/apps/app/src/app/layout.tsx +++ b/apps/app/src/app/layout.tsx @@ -10,11 +10,11 @@ export default function RootLayout({ return ( - - + + {children} - - + + ); From 56cf5900dfd4f43db7897eb997fba9496c5ade91 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Tue, 12 Dec 2023 12:33:18 +0100 Subject: [PATCH 09/10] revert overrideClerkUserToken --- apps/stack/src/extract/get-clerk-user-token.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/stack/src/extract/get-clerk-user-token.ts b/apps/stack/src/extract/get-clerk-user-token.ts index de5eaffcb..0e5f53344 100644 --- a/apps/stack/src/extract/get-clerk-user-token.ts +++ b/apps/stack/src/extract/get-clerk-user-token.ts @@ -25,11 +25,6 @@ const fetchClerkUserToken = async (userId: string, provider: SupportedClerkOAuth return userOauthAccessTokenPayload.token; }; -export const overrideClerkUserToken = async (userId: string, provider:SupportedClerkOAuthProviders, userToken: string) => { - const userTokenCacheKey = `${userId}_${provider}`; - await redisClient.set(userTokenCacheKey, encodeUserTokenCacheValue(userToken), { ex: Number(Config.REDIS_USER_TOKEN_TTL) }); -} - export const getClerkUserToken = async (userId: string, provider: SupportedClerkOAuthProviders) => { const userTokenCacheKey = `${userId}_${provider}`; let encodedUserTokenCacheValue: string | null; From a43488b0f69e5a9899ca5ed2bf7af7ae1506dc65 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Tue, 12 Dec 2023 15:06:18 +0100 Subject: [PATCH 10/10] add ALL the environment variables --- turbo.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/turbo.json b/turbo.json index c16fefc61..3e4d5786f 100644 --- a/turbo.json +++ b/turbo.json @@ -41,6 +41,12 @@ "CRAWL_DATABASE_URL", "CRAWL_DATABASE_AUTH_TOKEN", "NEXT_PUBLIC_EXTRACT_API_URL", - "NEXT_PUBLIC_TRANSFORM_API_URL" + "NEXT_PUBLIC_TRANSFORM_API_URL", + "PUBLIC_REPOS", + "CRON_USER_ID", + "PUBLIC_REPO_OWNER", + "PUBLIC_REPO_NAME", + "CLERK_JWT_ISSUER", + "CLERK_JWT_AUDIENCE" ] }