Extract Members Loose-ends

apps/extract-stack/src/extract-member.ts → apps/extract-stack/src/extract-members.ts

@@ -50,19 +50,14 @@ type ExtractMembersPageInput = {
   repository: Repository;
   sourceControl: "github" | "gitlab";
   userId: string;
-  paginationInfo: Pagination | null;
+  paginationInfo?: Pagination;
 }
 
 const extractMembersPage = async ({ namespace, repository, sourceControl, userId, paginationInfo }: ExtractMembersPageInput) => {
   const page = paginationInfo?.page;
   const perPage = paginationInfo?.perPage;
 
-  try {
-    context.integrations.sourceControl = await initSourceControl(userId, sourceControl);
-  } catch (error) {
-    console.error(error);
-    throw error;
-  }
+  context.integrations.sourceControl = await initSourceControl(userId, sourceControl);
 
   const { paginationInfo: resultPaginationInfo } = await getMembers({
     externalRepositoryId: repository.externalId,
@@ -82,7 +77,6 @@ export const eventHandler = EventHandler(extractRepositoryEvent, async (ev) => {
     repository: ev.properties.repository,
     sourceControl: ev.metadata.sourceControl,
     userId: ev.metadata.userId,
-    paginationInfo: { page: 1, perPage: 2, totalPages: 1000 },
   });
 
   const arrayOfExtractMemberPageMessageContent: { repository: Repository, namespace: Namespace | null, pagination: Pagination }[] = [];
@@ -97,7 +91,7 @@ export const eventHandler = EventHandler(extractRepositoryEvent, async (ev) => {
       }
     })
   }
-  
+
   if (arrayOfExtractMemberPageMessageContent.length === 0) return console.log("No more pages left, no need to enqueue");
 
   await extractMemberPageMessage.sendAll(arrayOfExtractMemberPageMessageContent, {

apps/extract-stack/src/stack.ts

@@ -47,15 +47,15 @@ export function ExtractStack({ stack }: StackContext) {
         CLERK_SECRET_KEY,
         DATABASE_AUTH_TOKEN,
       ], // Issue: need to bind bus because same file
-      handler: "src/extract-member.queueHandler",
+      handler: "src/extract-members.queueHandler",
     },
   });
 
   bus.addTargets(stack, "repository", {
     extractMember: {
       function: {
         bind: [bus, membersQueue, mergeRequestQueue],
-        handler: "src/extract-member.eventHandler",
+        handler: "src/extract-members.eventHandler",
       },
     },
   });

packages/integrations/source-control/src/gitlab/index.ts

@@ -31,10 +31,13 @@ export class GitlabSourceControl implements SourceControl {
   }
 
   async fetchMembers(externalRepositoryId: number, namespaceName: string, repositoryName: string, page?: number, perPage?: number): Promise<{ members: NewMember[], pagination: Pagination }> {
+    const isFirstPage = page === undefined && perPage === undefined;
+
     const { data, paginationInfo } = await this.api.ProjectMembers.all(externalRepositoryId, {
       includeInherited: true,
       perPage,
       page,
+      maxPages: isFirstPage ? 1 : undefined,
       pagination: 'offset',
       showExpanded: true,
     });