From 826f8bd42c6c937d2728dc606e5a8d9ba97c3e16 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Wed, 16 Aug 2023 13:15:53 +0200 Subject: [PATCH 1/8] rename file --- apps/extract-stack/src/{extract-member.ts => extract-members.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename apps/extract-stack/src/{extract-member.ts => extract-members.ts} (100%) diff --git a/apps/extract-stack/src/extract-member.ts b/apps/extract-stack/src/extract-members.ts similarity index 100% rename from apps/extract-stack/src/extract-member.ts rename to apps/extract-stack/src/extract-members.ts From 6060a52506633259e4ba9e7b5c4e9b27652faa86 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Wed, 16 Aug 2023 13:16:39 +0200 Subject: [PATCH 2/8] remove useless try-catch block --- apps/extract-stack/src/extract-members.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/apps/extract-stack/src/extract-members.ts b/apps/extract-stack/src/extract-members.ts index 64f26f489..f119211cb 100644 --- a/apps/extract-stack/src/extract-members.ts +++ b/apps/extract-stack/src/extract-members.ts @@ -57,12 +57,7 @@ const extractMembersPage = async ({ namespace, repository, sourceControl, userId const page = paginationInfo?.page; const perPage = paginationInfo?.perPage; - try { - context.integrations.sourceControl = await initSourceControl(userId, sourceControl); - } catch (error) { - console.error(error); - throw error; - } + context.integrations.sourceControl = await initSourceControl(userId, sourceControl); const { paginationInfo: resultPaginationInfo } = await getMembers({ externalRepositoryId: repository.externalId, @@ -97,7 +92,7 @@ export const eventHandler = EventHandler(extractRepositoryEvent, async (ev) => { } }) } - + if (arrayOfExtractMemberPageMessageContent.length === 0) return console.log("No more pages left, no need to enqueue"); await extractMemberPageMessage.sendAll(arrayOfExtractMemberPageMessageContent, { From 10b03e87d64885b93efb4a71cd7982e86a33b4ad Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Wed, 16 Aug 2023 13:17:12 +0200 Subject: [PATCH 3/8] remove pre-set pagination parameters --- apps/extract-stack/src/extract-members.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/extract-stack/src/extract-members.ts b/apps/extract-stack/src/extract-members.ts index f119211cb..1f35283a7 100644 --- a/apps/extract-stack/src/extract-members.ts +++ b/apps/extract-stack/src/extract-members.ts @@ -77,7 +77,7 @@ export const eventHandler = EventHandler(extractRepositoryEvent, async (ev) => { repository: ev.properties.repository, sourceControl: ev.metadata.sourceControl, userId: ev.metadata.userId, - paginationInfo: { page: 1, perPage: 2, totalPages: 1000 }, + paginationInfo: null, }); const arrayOfExtractMemberPageMessageContent: { repository: Repository, namespace: Namespace | null, pagination: Pagination }[] = []; From 01ef8a73782df9ff9dc6cca0f1220d99905e2425 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Wed, 16 Aug 2023 13:17:51 +0200 Subject: [PATCH 4/8] change pagination optional type from null to undefined --- apps/extract-stack/src/extract-members.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/extract-stack/src/extract-members.ts b/apps/extract-stack/src/extract-members.ts index 1f35283a7..964c79e93 100644 --- a/apps/extract-stack/src/extract-members.ts +++ b/apps/extract-stack/src/extract-members.ts @@ -50,7 +50,7 @@ type ExtractMembersPageInput = { repository: Repository; sourceControl: "github" | "gitlab"; userId: string; - paginationInfo: Pagination | null; + paginationInfo: Pagination | undefined; } const extractMembersPage = async ({ namespace, repository, sourceControl, userId, paginationInfo }: ExtractMembersPageInput) => { @@ -77,7 +77,6 @@ export const eventHandler = EventHandler(extractRepositoryEvent, async (ev) => { repository: ev.properties.repository, sourceControl: ev.metadata.sourceControl, userId: ev.metadata.userId, - paginationInfo: null, }); const arrayOfExtractMemberPageMessageContent: { repository: Repository, namespace: Namespace | null, pagination: Pagination }[] = []; From 59f460468755a39ffe7ee8ff6e9be8d0a8ca11ba Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Wed, 16 Aug 2023 13:18:45 +0200 Subject: [PATCH 5/8] add default first page to gitlab integration --- packages/integrations/source-control/src/gitlab/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/integrations/source-control/src/gitlab/index.ts b/packages/integrations/source-control/src/gitlab/index.ts index 1da15a359..78ff3db88 100644 --- a/packages/integrations/source-control/src/gitlab/index.ts +++ b/packages/integrations/source-control/src/gitlab/index.ts @@ -34,7 +34,7 @@ export class GitlabSourceControl implements SourceControl { const { data, paginationInfo } = await this.api.ProjectMembers.all(externalRepositoryId, { includeInherited: true, perPage, - page, + page: page || 1, pagination: 'offset', showExpanded: true, }); From d5fddc36bfadb2c0aef83b5f793214f93649950f Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Wed, 16 Aug 2023 13:43:50 +0200 Subject: [PATCH 6/8] config stack to reflect name change --- apps/extract-stack/src/stack.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/extract-stack/src/stack.ts b/apps/extract-stack/src/stack.ts index e8eee5fb4..fcca3ca6b 100644 --- a/apps/extract-stack/src/stack.ts +++ b/apps/extract-stack/src/stack.ts @@ -47,7 +47,7 @@ export function ExtractStack({ stack }: StackContext) { CLERK_SECRET_KEY, DATABASE_AUTH_TOKEN, ], // Issue: need to bind bus because same file - handler: "src/extract-member.queueHandler", + handler: "src/extract-members.queueHandler", }, }); @@ -55,7 +55,7 @@ export function ExtractStack({ stack }: StackContext) { extractMember: { function: { bind: [bus, membersQueue, mergeRequestQueue], - handler: "src/extract-member.eventHandler", + handler: "src/extract-members.eventHandler", }, }, }); From ba4dd014b09317d74440432c71f70be45a8f13a6 Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Wed, 16 Aug 2023 14:03:18 +0200 Subject: [PATCH 7/8] add first page parameter to gitlab --- packages/integrations/source-control/src/gitlab/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/integrations/source-control/src/gitlab/index.ts b/packages/integrations/source-control/src/gitlab/index.ts index 78ff3db88..cd558ead4 100644 --- a/packages/integrations/source-control/src/gitlab/index.ts +++ b/packages/integrations/source-control/src/gitlab/index.ts @@ -31,10 +31,13 @@ export class GitlabSourceControl implements SourceControl { } async fetchMembers(externalRepositoryId: number, namespaceName: string, repositoryName: string, page?: number, perPage?: number): Promise<{ members: NewMember[], pagination: Pagination }> { + const isFirstPage = page === undefined && perPage === undefined; + const { data, paginationInfo } = await this.api.ProjectMembers.all(externalRepositoryId, { includeInherited: true, perPage, - page: page || 1, + page, + maxPages: isFirstPage ? 1 : undefined, pagination: 'offset', showExpanded: true, }); From 958424aade13392c211d49eda94392035e82b52d Mon Sep 17 00:00:00 2001 From: dejan-crocoder Date: Wed, 16 Aug 2023 14:44:03 +0200 Subject: [PATCH 8/8] fix optional parameter --- apps/extract-stack/src/extract-members.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/extract-stack/src/extract-members.ts b/apps/extract-stack/src/extract-members.ts index 964c79e93..dedb76dbd 100644 --- a/apps/extract-stack/src/extract-members.ts +++ b/apps/extract-stack/src/extract-members.ts @@ -50,7 +50,7 @@ type ExtractMembersPageInput = { repository: Repository; sourceControl: "github" | "gitlab"; userId: string; - paginationInfo: Pagination | undefined; + paginationInfo?: Pagination; } const extractMembersPage = async ({ namespace, repository, sourceControl, userId, paginationInfo }: ExtractMembersPageInput) => {