From 98643ee0b765b03e5e3e9b5bc501e27ae9121597 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Thu, 17 Aug 2023 11:12:36 +0200 Subject: [PATCH 1/4] coerce dates in zod db schemas --- packages/schemas/extract/src/members.ts | 11 +++++++++-- packages/schemas/extract/src/merge-requests.ts | 11 +++++++++-- packages/schemas/extract/src/namespaces.ts | 11 +++++++++-- packages/schemas/extract/src/repositories.ts | 11 +++++++++-- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/packages/schemas/extract/src/members.ts b/packages/schemas/extract/src/members.ts index 1d2d974bb..ede04d194 100644 --- a/packages/schemas/extract/src/members.ts +++ b/packages/schemas/extract/src/members.ts @@ -2,6 +2,7 @@ import type { InferModel } from 'drizzle-orm'; import { sql } from 'drizzle-orm'; import { sqliteTable, text, integer, uniqueIndex } from 'drizzle-orm/sqlite-core'; import { createInsertSchema, createSelectSchema } from 'drizzle-zod'; +import { z } from 'zod'; export const members = sqliteTable('members', { id: integer('id').primaryKey(), @@ -16,5 +17,11 @@ export const members = sqliteTable('members', { export type Member = InferModel; export type NewMember = InferModel; -export const MemberSchema = createSelectSchema(members); -export const NewMemberSchema = createInsertSchema(members); +export const MemberSchema = createSelectSchema(members, { + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), +}); +export const NewMemberSchema = createInsertSchema(members, { + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), +}); diff --git a/packages/schemas/extract/src/merge-requests.ts b/packages/schemas/extract/src/merge-requests.ts index 8bbda742a..8669c321c 100644 --- a/packages/schemas/extract/src/merge-requests.ts +++ b/packages/schemas/extract/src/merge-requests.ts @@ -2,6 +2,7 @@ import type { InferModel } from "drizzle-orm"; import { sql } from "drizzle-orm"; import { integer, sqliteTable, uniqueIndex } from "drizzle-orm/sqlite-core"; import { createInsertSchema } from "drizzle-zod"; +import { z } from "zod"; export const mergeRequests = sqliteTable( "merge_requests", @@ -24,5 +25,11 @@ export const mergeRequests = sqliteTable( export type MergeRequest = InferModel; export type NewMergeRequest = InferModel; -export const MergeRequestSchema = createInsertSchema(mergeRequests); -export const NewMergeRequestSchema = createInsertSchema(mergeRequests); +export const MergeRequestSchema = createInsertSchema(mergeRequests, { + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), +}); +export const NewMergeRequestSchema = createInsertSchema(mergeRequests, { + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), +}); diff --git a/packages/schemas/extract/src/namespaces.ts b/packages/schemas/extract/src/namespaces.ts index 094991ffa..d8c3db4eb 100644 --- a/packages/schemas/extract/src/namespaces.ts +++ b/packages/schemas/extract/src/namespaces.ts @@ -2,6 +2,7 @@ import type { InferModel } from 'drizzle-orm'; import { sql } from 'drizzle-orm'; import { sqliteTable, text, integer, uniqueIndex } from 'drizzle-orm/sqlite-core'; import { createInsertSchema, createSelectSchema } from 'drizzle-zod'; +import { z } from 'zod'; export const namespaces = sqliteTable('namespaces', { id: integer('id').primaryKey(), @@ -15,5 +16,11 @@ export const namespaces = sqliteTable('namespaces', { export type Namespace = InferModel; export type NewNamespace = InferModel; -export const NewNamespaceSchema = createInsertSchema(namespaces); -export const NamespaceSchema = createSelectSchema(namespaces); +export const NewNamespaceSchema = createInsertSchema(namespaces, { + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), +}); +export const NamespaceSchema = createSelectSchema(namespaces, { + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), +}); diff --git a/packages/schemas/extract/src/repositories.ts b/packages/schemas/extract/src/repositories.ts index 43a60eaf3..592201215 100644 --- a/packages/schemas/extract/src/repositories.ts +++ b/packages/schemas/extract/src/repositories.ts @@ -2,6 +2,7 @@ import type { InferModel } from 'drizzle-orm'; import { sql } from 'drizzle-orm'; import { sqliteTable, integer,text, uniqueIndex } from 'drizzle-orm/sqlite-core'; import { createInsertSchema, createSelectSchema } from 'drizzle-zod'; +import { z } from 'zod'; export const repositories = sqliteTable('repositories', { id: integer('id').primaryKey(), @@ -15,5 +16,11 @@ export const repositories = sqliteTable('repositories', { export type Repository = InferModel; export type NewRepository = InferModel; -export const NewRepositorySchema = createInsertSchema(repositories); -export const RepositorySchema = createSelectSchema(repositories); +export const NewRepositorySchema = createInsertSchema(repositories, { + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), +}); +export const RepositorySchema = createSelectSchema(repositories, { + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), +}); From 5f8ab57149269e49b27d1d3a75c4a1929c8ef6d9 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Thu, 17 Aug 2023 11:25:27 +0200 Subject: [PATCH 2/4] parse zod schema in queuehandler --- apps/extract-stack/src/create-message.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/apps/extract-stack/src/create-message.ts b/apps/extract-stack/src/create-message.ts index a989ab5d2..5689e5555 100644 --- a/apps/extract-stack/src/create-message.ts +++ b/apps/extract-stack/src/create-message.ts @@ -66,13 +66,18 @@ export function createMessage = { send: Send; sendAll: BatchSend + shapes: { contentShape: Shape, metadataShape: MetadataShape }; } type MessagePayload = { @@ -86,6 +91,10 @@ export function QueueHandler ) => Promise ) { + const schema = z.object({ + content: z.object(_sender.shapes.contentShape), + metadata: z.object(_sender.shapes.metadataShape) + }); /** * TODO: * - Do consumers always recieve batches ? @@ -94,7 +103,11 @@ export function QueueHandler { if (event.Records.length > 1) console.warn('WARNING: QueueHandler should process 1 message but got', event.Records.length); for (const record of event.Records) { - await cb(JSON.parse(record.body) as MessagePayload); + const parsed = schema.parse(JSON.parse(record.body) as unknown) as MessagePayload; + console.log('??????????????????????') + console.log(parsed); + console.log('??????????????????????') + await cb(parsed); } } } \ No newline at end of file From 6f8f13b9239b9cec69e0742a54cfec03524cbd54 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Thu, 17 Aug 2023 12:13:12 +0200 Subject: [PATCH 3/4] change event schema to only send ids in props --- apps/extract-stack/src/events.ts | 6 ++---- apps/extract-stack/src/extract-members.ts | 19 ++++++++++++++----- apps/extract-stack/src/extract-repository.ts | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/apps/extract-stack/src/events.ts b/apps/extract-stack/src/events.ts index 970348641..2fa953a09 100644 --- a/apps/extract-stack/src/events.ts +++ b/apps/extract-stack/src/events.ts @@ -1,14 +1,12 @@ import { EventBus } from "sst/node/event-bus"; import { z } from "zod"; -import { RepositorySchema } from "@acme/extract-schema"; import { MergeRequestSchema } from "@acme/extract-schema/src/merge-requests"; -import { NamespaceSchema } from "@acme/extract-schema/src/namespaces"; import { createEvent } from "./create-event"; const extractRepositoryEventSchema = z.object({ - repository: RepositorySchema, - namespace: z.nullable(NamespaceSchema), + repositoryId: z.number(), + namespaceId: z.nullable(z.number()), }); const metadataSchema = z.object({ diff --git a/apps/extract-stack/src/extract-members.ts b/apps/extract-stack/src/extract-members.ts index dedb76dbd..ba0cd8e66 100644 --- a/apps/extract-stack/src/extract-members.ts +++ b/apps/extract-stack/src/extract-members.ts @@ -5,7 +5,7 @@ import { createClient } from "@libsql/client"; import { drizzle } from "drizzle-orm/libsql"; import { getMembers } from "@acme/extract-functions"; import type { Context, GetMembersEntities, GetMembersSourceControl } from "@acme/extract-functions"; -import { members, repositoriesToMembers } from "@acme/extract-schema"; +import { members, namespaces, repositories, repositoriesToMembers } from "@acme/extract-schema"; import type { Namespace, Repository } from "@acme/extract-schema"; import { GitHubSourceControl, GitlabSourceControl } from "@acme/source-control"; import type { Pagination } from "@acme/source-control"; @@ -13,6 +13,7 @@ import { Config } from "sst/node/config"; import { extractMemberPageMessage } from "./messages"; import { QueueHandler } from "./create-message"; +import { eq } from "drizzle-orm"; const clerkClient = Clerk({ secretKey: Config.CLERK_SECRET_KEY }); const client = createClient({ url: Config.DATABASE_URL, authToken: Config.DATABASE_AUTH_TOKEN }); @@ -72,9 +73,17 @@ const extractMembersPage = async ({ namespace, repository, sourceControl, userId }; export const eventHandler = EventHandler(extractRepositoryEvent, async (ev) => { + if (!ev.properties.namespaceId) throw new Error("Missing namespaceId"); + + const repository = await db.select().from(repositories).where(eq(repositories.id, ev.properties.repositoryId)).get(); + const namespace = await db.select().from(namespaces).where(eq(namespaces.id, ev.properties.namespaceId)).get(); + + if (!repository) throw new Error("invalid repo id"); + if (!namespace) throw new Error("Invalid namespace id"); + const pagination = await extractMembersPage({ - namespace: ev.properties.namespace, - repository: ev.properties.repository, + namespace: namespace, + repository: repository, sourceControl: ev.metadata.sourceControl, userId: ev.metadata.userId, }); @@ -82,8 +91,8 @@ export const eventHandler = EventHandler(extractRepositoryEvent, async (ev) => { const arrayOfExtractMemberPageMessageContent: { repository: Repository, namespace: Namespace | null, pagination: Pagination }[] = []; for (let i = 2; i <= pagination.totalPages; i++) { arrayOfExtractMemberPageMessageContent.push({ - namespace: ev.properties.namespace, - repository: ev.properties.repository, + namespace: namespace, + repository: repository, pagination: { page: i, perPage: pagination.perPage, diff --git a/apps/extract-stack/src/extract-repository.ts b/apps/extract-stack/src/extract-repository.ts index 44b2d7f9c..6beb66e2d 100644 --- a/apps/extract-stack/src/extract-repository.ts +++ b/apps/extract-stack/src/extract-repository.ts @@ -106,7 +106,7 @@ export const handler = ApiHandler(async (ev) => { const { repository, namespace } = await getRepository({ externalRepositoryId: repositoryId, repositoryName, namespaceName }, context); - await extractRepositoryEvent.publish({ repository, namespace }, { caller: 'extract-repository', timestamp: new Date().getTime(), version: 1, sourceControl, userId: sub }); + await extractRepositoryEvent.publish({ repositoryId: repository.id, namespaceId: namespace?.id || null }, { caller: 'extract-repository', timestamp: new Date().getTime(), version: 1, sourceControl, userId: sub }); return { statusCode: 200, From a674b1f1f376c43914e376531e4c6f635296628b Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Thu, 17 Aug 2023 12:15:40 +0200 Subject: [PATCH 4/4] remove logs --- apps/extract-stack/src/create-message.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/extract-stack/src/create-message.ts b/apps/extract-stack/src/create-message.ts index 5689e5555..9636d6cab 100644 --- a/apps/extract-stack/src/create-message.ts +++ b/apps/extract-stack/src/create-message.ts @@ -104,9 +104,6 @@ export function QueueHandler 1) console.warn('WARNING: QueueHandler should process 1 message but got', event.Records.length); for (const record of event.Records) { const parsed = schema.parse(JSON.parse(record.body) as unknown) as MessagePayload; - console.log('??????????????????????') - console.log(parsed); - console.log('??????????????????????') await cb(parsed); } }