From aa165a3ed002d38111846f3e5d28a4967b02de0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ante=20Kocei=C4=87?= Date: Wed, 29 Nov 2023 09:12:29 +0100 Subject: [PATCH 1/7] Added filling user junk dimension --- .../transform/src/merge-request-metrics.ts | 79 ++++++++++++++++++- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/packages/functions/transform/src/merge-request-metrics.ts b/packages/functions/transform/src/merge-request-metrics.ts index 42e9fd60f..7b0556e23 100644 --- a/packages/functions/transform/src/merge-request-metrics.ts +++ b/packages/functions/transform/src/merge-request-metrics.ts @@ -282,6 +282,75 @@ type MapUsersToJunksArgs = { reviewers: transform.ForgeUser['id'][] } +function addUnique(newElement: number, currentArray: number[]) { + if (!currentArray.includes(newElement)) { + currentArray.push(newElement); + } +} + +async function getUserIds(timelineEvents: TimelineEventData[], extractDb: ExtractDatabase, transformDb: TransformDatabase, extractMergeRequestId: number) { + const reviewers: number[] = []; + const approvers: number[] = []; + const committers: number[] = []; + let mergedBy; + + console.log(extractMergeRequestId,'timelineEvents', timelineEvents); + + for (const timelineEvent of timelineEvents) { + if (timelineEvent.type === 'reviewed' && timelineEvent.actorId) { + const reviewer = await transformDb.select({ + id: transform.forgeUsers.id, + }).from(transform.forgeUsers) + .where(eq(transform.forgeUsers.externalId, timelineEvent.actorId)).get(); + if (reviewer) { + addUnique(reviewer.id, reviewers); + } + if (timelineEvent.data) { + if ((timelineEvent.data as string).includes('approved')) { + const approver = await transformDb.select({ + id: transform.forgeUsers.id, + }).from(transform.forgeUsers) + .where(eq(transform.forgeUsers.externalId, timelineEvent.actorId)).get(); + if (approver) { + addUnique(approver.id, approvers); + } + } + } + } else if (timelineEvent.type === 'committed') { + const data = JSON.parse(timelineEvent.data as string) as { committerName: string, committerEmail: string, committedDate: Date }; + const extractUserExternalId = await extractDb.select({ + id: extract.members.externalId, + }).from(extract.members) + .where(or( + eq(extract.members.username, data.committerName), + eq(extract.members.name, data.committerName)) + ).get(); + console.log(extractMergeRequestId, 'FINALE_TEST_IDS', data.committerName, extractUserExternalId); + if (extractUserExternalId) { + const committer = await transformDb.select({ + id: transform.forgeUsers.id, + }).from(transform.forgeUsers) + .where(eq(transform.forgeUsers.externalId, extractUserExternalId?.id)).get(); + if (committer) { + addUnique(committer.id, committers); + } + } + } else if (timelineEvent.type === 'merged' && timelineEvent.actorId) { + mergedBy = await transformDb.select({ + id: transform.forgeUsers.id, + }).from(transform.forgeUsers) + .where(eq(transform.forgeUsers.externalId, timelineEvent.actorId)).get(); + } + } + + return { + mergedBy: mergedBy?.id, + approvers, + committers, + reviewers, + }; +} + function mapUsersToJunk({ author, mergedBy, approvers, committers, reviewers }: MapUsersToJunksArgs, nullForgeUserId: number) { return { author: author || nullForgeUserId, @@ -584,6 +653,8 @@ export async function run(extractMergeRequestId: number, ctx: RunContext) { const timeline = runTimeline(extractData.mergeRequest, extractData.timelineEvents, extractData.notes); + const users = await getUserIds(extractData.timelineEvents, ctx.extractDatabase, ctx.transformDatabase, extractMergeRequestId); + const { dateId: nullDateId, userId: nullUserId, @@ -613,10 +684,10 @@ export async function run(extractMergeRequestId: number, ctx: RunContext) { eq(transform.forgeUsers.externalId, extractData.mergeRequest.authorExternalId || 0), eq(transform.forgeUsers.forgeType, extractData.repository.forgeType), )).get())?.id || null, // TODO: ??? - mergedBy: null, - approvers: [], - committers: [], - reviewers: [], + mergedBy: users.mergedBy, + approvers: users.approvers, + committers: users.committers, + reviewers: users.reviewers, }, nullUserId); const { id: transformRepositoryId } = await upsertRepository(ctx.transformDatabase, extractData.repository).get(); From bc62d021fb0361aafe306d5516fe41616c304bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ante=20Kocei=C4=87?= Date: Wed, 29 Nov 2023 09:24:18 +0100 Subject: [PATCH 2/7] Removed console.logs --- packages/functions/transform/src/merge-request-metrics.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/functions/transform/src/merge-request-metrics.ts b/packages/functions/transform/src/merge-request-metrics.ts index 7b0556e23..74cfc441d 100644 --- a/packages/functions/transform/src/merge-request-metrics.ts +++ b/packages/functions/transform/src/merge-request-metrics.ts @@ -294,8 +294,6 @@ async function getUserIds(timelineEvents: TimelineEventData[], extractDb: Extrac const committers: number[] = []; let mergedBy; - console.log(extractMergeRequestId,'timelineEvents', timelineEvents); - for (const timelineEvent of timelineEvents) { if (timelineEvent.type === 'reviewed' && timelineEvent.actorId) { const reviewer = await transformDb.select({ @@ -325,7 +323,6 @@ async function getUserIds(timelineEvents: TimelineEventData[], extractDb: Extrac eq(extract.members.username, data.committerName), eq(extract.members.name, data.committerName)) ).get(); - console.log(extractMergeRequestId, 'FINALE_TEST_IDS', data.committerName, extractUserExternalId); if (extractUserExternalId) { const committer = await transformDb.select({ id: transform.forgeUsers.id, From bc93a0d1b28117f1ee3f632e95dd774bddc6963d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ante=20Kocei=C4=87?= Date: Wed, 29 Nov 2023 09:32:01 +0100 Subject: [PATCH 3/7] Fixed lint --- packages/functions/transform/src/merge-request-metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/functions/transform/src/merge-request-metrics.ts b/packages/functions/transform/src/merge-request-metrics.ts index 74cfc441d..cfe3a2ad3 100644 --- a/packages/functions/transform/src/merge-request-metrics.ts +++ b/packages/functions/transform/src/merge-request-metrics.ts @@ -288,7 +288,7 @@ function addUnique(newElement: number, currentArray: number[]) { } } -async function getUserIds(timelineEvents: TimelineEventData[], extractDb: ExtractDatabase, transformDb: TransformDatabase, extractMergeRequestId: number) { +async function getUserIds(timelineEvents: TimelineEventData[], extractDb: ExtractDatabase, transformDb: TransformDatabase) { const reviewers: number[] = []; const approvers: number[] = []; const committers: number[] = []; From f8bdba07fa7f0061dc7f5ee50d0d2a87be8899d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ante=20Kocei=C4=87?= Date: Wed, 29 Nov 2023 09:36:14 +0100 Subject: [PATCH 4/7] Removed unneeded argument --- packages/functions/transform/src/merge-request-metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/functions/transform/src/merge-request-metrics.ts b/packages/functions/transform/src/merge-request-metrics.ts index cfe3a2ad3..b61fb12ba 100644 --- a/packages/functions/transform/src/merge-request-metrics.ts +++ b/packages/functions/transform/src/merge-request-metrics.ts @@ -650,7 +650,7 @@ export async function run(extractMergeRequestId: number, ctx: RunContext) { const timeline = runTimeline(extractData.mergeRequest, extractData.timelineEvents, extractData.notes); - const users = await getUserIds(extractData.timelineEvents, ctx.extractDatabase, ctx.transformDatabase, extractMergeRequestId); + const users = await getUserIds(extractData.timelineEvents, ctx.extractDatabase, ctx.transformDatabase); const { dateId: nullDateId, From 0c822deabaebcb48acfe77505c28f272e1ab99fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ante=20Kocei=C4=87?= Date: Wed, 29 Nov 2023 10:44:52 +0100 Subject: [PATCH 5/7] Replaced if-else with switch-case --- .../transform/src/merge-request-metrics.ts | 80 ++++++++++--------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/packages/functions/transform/src/merge-request-metrics.ts b/packages/functions/transform/src/merge-request-metrics.ts index b61fb12ba..433aff405 100644 --- a/packages/functions/transform/src/merge-request-metrics.ts +++ b/packages/functions/transform/src/merge-request-metrics.ts @@ -288,6 +288,13 @@ function addUnique(newElement: number, currentArray: number[]) { } } +async function getId(actorId: number, db: TransformDatabase) { + return await db.select({ + id: transform.forgeUsers.id, + }).from(transform.forgeUsers) + .where(eq(transform.forgeUsers.externalId, actorId)).get(); +} + async function getUserIds(timelineEvents: TimelineEventData[], extractDb: ExtractDatabase, transformDb: TransformDatabase) { const reviewers: number[] = []; const approvers: number[] = []; @@ -295,48 +302,43 @@ async function getUserIds(timelineEvents: TimelineEventData[], extractDb: Extrac let mergedBy; for (const timelineEvent of timelineEvents) { - if (timelineEvent.type === 'reviewed' && timelineEvent.actorId) { - const reviewer = await transformDb.select({ - id: transform.forgeUsers.id, - }).from(transform.forgeUsers) - .where(eq(transform.forgeUsers.externalId, timelineEvent.actorId)).get(); - if (reviewer) { - addUnique(reviewer.id, reviewers); - } - if (timelineEvent.data) { - if ((timelineEvent.data as string).includes('approved')) { - const approver = await transformDb.select({ - id: transform.forgeUsers.id, - }).from(transform.forgeUsers) - .where(eq(transform.forgeUsers.externalId, timelineEvent.actorId)).get(); - if (approver) { - addUnique(approver.id, approvers); + switch (timelineEvent.type) { + case 'reviewed': + if (!timelineEvent.actorId) { + break; + } + const reviewer = await getId(timelineEvent.actorId, transformDb); + if (reviewer) { + addUnique(reviewer.id, reviewers); + if (timelineEvent.data && (timelineEvent.data as string).includes('approved')) { + addUnique(reviewer?.id, approvers); } } - } - } else if (timelineEvent.type === 'committed') { - const data = JSON.parse(timelineEvent.data as string) as { committerName: string, committerEmail: string, committedDate: Date }; - const extractUserExternalId = await extractDb.select({ - id: extract.members.externalId, - }).from(extract.members) - .where(or( - eq(extract.members.username, data.committerName), - eq(extract.members.name, data.committerName)) - ).get(); - if (extractUserExternalId) { - const committer = await transformDb.select({ - id: transform.forgeUsers.id, - }).from(transform.forgeUsers) - .where(eq(transform.forgeUsers.externalId, extractUserExternalId?.id)).get(); - if (committer) { - addUnique(committer.id, committers); + break; + case 'committed': + const data = JSON.parse(timelineEvent.data as string) as { committerName: string, committerEmail: string, committedDate: Date }; + const extractUserExternalId = await extractDb.select({ + id: extract.members.externalId, + }).from(extract.members) + .where(or( + eq(extract.members.username, data.committerName), + eq(extract.members.name, data.committerName)) + ).get(); + if (extractUserExternalId) { + const committer = await getId(extractUserExternalId.id, transformDb); + if (committer) { + addUnique(committer.id, committers); + } } - } - } else if (timelineEvent.type === 'merged' && timelineEvent.actorId) { - mergedBy = await transformDb.select({ - id: transform.forgeUsers.id, - }).from(transform.forgeUsers) - .where(eq(transform.forgeUsers.externalId, timelineEvent.actorId)).get(); + break; + case 'merged': + if (!timelineEvent.actorId) { + break; + } + mergedBy = await getId(timelineEvent.actorId, transformDb); + break; + default: + break; } } From a053b58ba004a78b2ecc931811a4ae5098994d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ante=20Kocei=C4=87?= Date: Tue, 5 Dec 2023 09:29:23 +0100 Subject: [PATCH 6/7] Fixed Zod Error --- .../transform/src/merge-request-metrics.ts | 9 +++++---- .../source-control/src/github/index.ts | 16 ++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/functions/transform/src/merge-request-metrics.ts b/packages/functions/transform/src/merge-request-metrics.ts index 33f58fc08..239f16171 100644 --- a/packages/functions/transform/src/merge-request-metrics.ts +++ b/packages/functions/transform/src/merge-request-metrics.ts @@ -310,13 +310,13 @@ async function getUserIds(timelineEvents: TimelineEventData[], extractDb: Extrac const reviewer = await getId(timelineEvent.actorId, transformDb); if (reviewer) { addUnique(reviewer.id, reviewers); - if (timelineEvent.data && (timelineEvent.data as string).includes('approved')) { + if (timelineEvent.data && ((timelineEvent.data as extract.ReviewedEvent).state === 'approved')) { addUnique(reviewer?.id, approvers); } } break; case 'committed': - const data = JSON.parse(timelineEvent.data as string) as { committerName: string, committerEmail: string, committedDate: Date }; + const data = timelineEvent.data as extract.CommittedEvent; const extractUserExternalId = await extractDb.select({ id: extract.members.externalId, }).from(extract.members) @@ -543,7 +543,6 @@ type calcTimelineArgs = { } export function calculateTimeline(timelineMapKeys: TimelineMapKey[], timelineMap: Map, { authorExternalId }: calcTimelineArgs) { - const commitedEvents = timelineMapKeys.filter(key => key.type === 'committed'); commitedEvents.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); @@ -678,7 +677,9 @@ function runTimeline(mergeRequestData: MergeRequestData, timelineEvents: Timelin }); // TODO: can this be optimized with the map ? - const approved = timelineEvents.find(ev => ev.type === 'reviewed' && (JSON.parse(ev.data as string) as extract.ReviewedEvent).state === 'approved') !== undefined; + console.log('timelineEvents', timelineEvents); + + const approved = timelineEvents.find(ev => ev.type === 'reviewed' && (ev.data as extract.ReviewedEvent).state === 'approved') !== undefined; return { startedCodingAt, diff --git a/packages/integrations/source-control/src/github/index.ts b/packages/integrations/source-control/src/github/index.ts index 143a97c39..8630bb96a 100644 --- a/packages/integrations/source-control/src/github/index.ts +++ b/packages/integrations/source-control/src/github/index.ts @@ -366,10 +366,10 @@ export class GitHubSourceControl implements SourceControl { timestamp: new Date(assignedEvent.created_at), actorName: assignedEvent.actor.login, actorId: assignedEvent.actor.id, - data: JSON.stringify({ + data: { assigneeId: assignedEvent.assignee.id, assigneeName: assignedEvent.assignee.login, - }), + }, } satisfies NewTimelineEvents; case 'committed': const committedEvent = singleEvent as components["schemas"]["timeline-committed-event"] @@ -380,11 +380,11 @@ export class GitHubSourceControl implements SourceControl { timestamp: new Date(committedEvent.author.date), actorName: committedEvent.author.name, actorEmail: committedEvent.author.email, - data: JSON.stringify({ + data: { committerEmail: committedEvent.committer.email, committerName: committedEvent.committer.name, committedDate: new Date(committedEvent.committer.date), - }), + }, } satisfies NewTimelineEvents; case 'review_requested': case 'review_request_removed': @@ -396,10 +396,10 @@ export class GitHubSourceControl implements SourceControl { timestamp: new Date(requestedEvent.created_at), actorName: requestedEvent.actor.login, actorId: requestedEvent.actor.id, - data: JSON.stringify({ + data: { requestedReviewerId: requestedEvent.requested_reviewer?.id, requestedReviewerName: requestedEvent.requested_reviewer?.login, - }), + }, } satisfies NewTimelineEvents; case 'reviewed': const reviewedEvent = singleEvent as components["schemas"]["timeline-reviewed-event"] @@ -410,9 +410,9 @@ export class GitHubSourceControl implements SourceControl { timestamp: new Date(reviewedEvent.submitted_at as string), actorName: reviewedEvent.user.login, actorId: reviewedEvent.user.id, - data: JSON.stringify({ + data: { state: reviewedEvent.state, - }), + }, } satisfies NewTimelineEvents; default: const generalEvent = singleEvent as components["schemas"]["state-change-issue-event"]; From e099691541b9574eddd9f43682388fe41c2ff18e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ante=20Kocei=C4=87?= Date: Tue, 5 Dec 2023 10:16:52 +0100 Subject: [PATCH 7/7] Fixed typo --- .../transform/src/merge-request-metrics.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/functions/transform/src/merge-request-metrics.ts b/packages/functions/transform/src/merge-request-metrics.ts index 239f16171..630f00f9a 100644 --- a/packages/functions/transform/src/merge-request-metrics.ts +++ b/packages/functions/transform/src/merge-request-metrics.ts @@ -543,11 +543,12 @@ type calcTimelineArgs = { } export function calculateTimeline(timelineMapKeys: TimelineMapKey[], timelineMap: Map, { authorExternalId }: calcTimelineArgs) { - const commitedEvents = timelineMapKeys.filter(key => key.type === 'committed'); - commitedEvents.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); + + const committedEvents = timelineMapKeys.filter(key => key.type === 'committed'); + committedEvents.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); - const firstCommitEvent = commitedEvents[0] || null; - const lastCommitEvent = commitedEvents[commitedEvents.length - 1] || null; + const firstCommitEvent = committedEvents[0] || null; + const lastCommitEvent = committedEvents[committedEvents.length - 1] || null; const startedCodingAt = firstCommitEvent ? firstCommitEvent.timestamp : null; @@ -570,7 +571,7 @@ export function calculateTimeline(timelineMapKeys: TimelineMapKey[], timelineMap reviewedEventsBeforeLastCommitEvent.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); const firstReviewedEventBeforeLastCommitEvent = reviewedEventsBeforeLastCommitEvent[0]; if (firstReviewedEventBeforeLastCommitEvent) { - return [...commitedEvents].reverse().find(event => event.timestamp < firstReviewedEventBeforeLastCommitEvent.timestamp)?.timestamp || null; + return [...committedEvents].reverse().find(event => event.timestamp < firstReviewedEventBeforeLastCommitEvent.timestamp)?.timestamp || null; } return lastCommitEvent.timestamp; @@ -587,7 +588,7 @@ export function calculateTimeline(timelineMapKeys: TimelineMapKey[], timelineMap const firstReviewedEventAfterLastReadyForReviewEvent = reviewedEventsAfterLastReadyForReviewEvent[0] if (firstReviewedEventAfterLastReadyForReviewEvent) { - const temp = [...commitedEvents].reverse().find( + const temp = [...committedEvents].reverse().find( event => event.timestamp > lastReadyForReviewEvent.timestamp && event.timestamp < firstReviewedEventAfterLastReadyForReviewEvent.timestamp @@ -677,8 +678,6 @@ function runTimeline(mergeRequestData: MergeRequestData, timelineEvents: Timelin }); // TODO: can this be optimized with the map ? - console.log('timelineEvents', timelineEvents); - const approved = timelineEvents.find(ev => ev.type === 'reviewed' && (ev.data as extract.ReviewedEvent).state === 'approved') !== undefined; return {