Merge pull request #250 from JupiterOne/NO-TICKET/catch-error

Increase running timer to 45 seconds and catch errors around batchIterateEntities
JupiterOne · Mar 22, 2024 · 01c1d6f · 01c1d6f
2 parents 922c1ab + c55f1a2
commit 01c1d6f
Show file tree

Hide file tree

Showing 10 changed files with 58 additions and 55 deletions.
diff --git a/src/steps/account.ts b/src/steps/account.ts
@@ -32,7 +32,7 @@ export async function fetchAccountDetails({
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
   let stepAnnouncer;
   if (accountFlagged) {
-    stepAnnouncer = new StepAnnouncer(Steps.ACCOUNT, 10, logger);
+    stepAnnouncer = new StepAnnouncer(Steps.ACCOUNT, logger);
   }
 
   const apiClient = createAPIClient(instance.config, logger);

diff --git a/src/steps/applicationCreation.ts b/src/steps/applicationCreation.ts
@@ -18,7 +18,7 @@ export async function buildUserCreatedApplication({
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
   let stepAnnouncer;
   if (accountFlagged) {
-    stepAnnouncer = new StepAnnouncer(Steps.APPLICATION_CREATION, 10, logger);
+    stepAnnouncer = new StepAnnouncer(Steps.APPLICATION_CREATION, logger);
   }
 
   const apiClient = createAPIClient(instance.config, logger);

diff --git a/src/steps/applications.ts b/src/steps/applications.ts
@@ -31,7 +31,7 @@ export async function fetchApplications({
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
   let stepAnnouncer;
   if (accountFlagged) {
-    stepAnnouncer = new StepAnnouncer(Steps.APPLICATIONS, 10, logger);
+    stepAnnouncer = new StepAnnouncer(Steps.APPLICATIONS, logger);
   }
 
   const apiClient = createAPIClient(instance.config, logger);

diff --git a/src/steps/devices.ts b/src/steps/devices.ts
@@ -26,7 +26,7 @@ export async function fetchDevices({
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
   let stepAnnouncer;
   if (accountFlagged) {
-    stepAnnouncer = new StepAnnouncer(Steps.DEVICES, 10, logger);
+    stepAnnouncer = new StepAnnouncer(Steps.DEVICES, logger);
   }
 
   const apiClient = createAPIClient(instance.config, logger);

diff --git a/src/steps/factorDevices.ts b/src/steps/factorDevices.ts
@@ -19,7 +19,7 @@ export async function fetchFactorDevices({
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
   let stepAnnouncer;
   if (accountFlagged) {
-    stepAnnouncer = new StepAnnouncer(Steps.MFA_DEVICES, 10, logger);
+    stepAnnouncer = new StepAnnouncer(Steps.MFA_DEVICES, logger);
   }
 
   const apiClient = createAPIClient(instance.config, logger);

diff --git a/src/steps/groups.ts b/src/steps/groups.ts
@@ -34,7 +34,7 @@ export async function fetchGroups({
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
   let stepAnnouncer;
   if (accountFlagged) {
-    stepAnnouncer = new StepAnnouncer(Steps.GROUPS, 10, logger);
+    stepAnnouncer = new StepAnnouncer(Steps.GROUPS, logger);
   }
 
   const apiClient = createAPIClient(instance.config, logger);
@@ -96,7 +96,6 @@ export async function buildAppUserGroupUserRelationships(
   if (accountFlagged) {
     stepAnnouncer = new StepAnnouncer(
       Steps.APP_USER_GROUP_USERS_RELATIONSHIP,
-      10,
       context.logger,
     );
   }
@@ -117,7 +116,6 @@ export async function buildUserGroupUserRelationships(
   if (accountFlagged) {
     stepAnnouncer = new StepAnnouncer(
       Steps.USER_GROUP_USERS_RELATIONSHIP,
-      10,
       context.logger,
     );
   }
@@ -137,7 +135,6 @@ async function buildGroupEntityToUserRelationships(
   if (accountFlagged) {
     stepAnnouncer = new StepAnnouncer(
       Steps.APP_USER_GROUP_USERS_RELATIONSHIP,
-      10,
       context.logger,
     );
   }
@@ -186,23 +183,27 @@ async function buildGroupEntityToUserRelationships(
     }
   }
 
-  await batchIterateEntities({
-    context,
-    batchSize: 1000,
-    filter: { _type: groupEntityType },
-    async iteratee(groupEntities) {
-      const usersForGroupEntities = await collectUsersForGroupEntities(
-        apiClient,
-        groupEntities,
-      );
-
-      for (const { groupEntity, users } of usersForGroupEntities) {
-        for (const user of users) {
-          await createGroupUserRelationshipWithJob(groupEntity, user);
+  try {
+    await batchIterateEntities({
+      context,
+      batchSize: 1000,
+      filter: { _type: groupEntityType },
+      async iteratee(groupEntities) {
+        const usersForGroupEntities = await collectUsersForGroupEntities(
+          apiClient,
+          groupEntities,
+        );
+
+        for (const { groupEntity, users } of usersForGroupEntities) {
+          for (const user of users) {
+            await createGroupUserRelationshipWithJob(groupEntity, user);
+          }
         }
-      }
-    },
-  });
+      },
+    });
+  } catch (err) {
+    logger.error({ err }, 'Failed to build group to user relationships');
+  }
 
   if (accountFlagged) {
     stepAnnouncer.finish();

diff --git a/src/steps/roles.ts b/src/steps/roles.ts
@@ -53,7 +53,7 @@ export async function fetchRoles({
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
   let stepAnnouncer;
   if (accountFlagged) {
-    stepAnnouncer = new StepAnnouncer(Steps.ROLES, 10, logger);
+    stepAnnouncer = new StepAnnouncer(Steps.ROLES, logger);
   }
 
   const apiClient = createAPIClient(instance.config, logger);

diff --git a/src/steps/rules.ts b/src/steps/rules.ts
@@ -26,7 +26,7 @@ export async function fetchRules({
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
   let stepAnnouncer;
   if (accountFlagged) {
-    stepAnnouncer = new StepAnnouncer(Steps.RULES, 10, logger);
+    stepAnnouncer = new StepAnnouncer(Steps.RULES, logger);
   }
 
   const apiClient = createAPIClient(instance.config, logger);

diff --git a/src/steps/users.ts b/src/steps/users.ts
@@ -26,7 +26,7 @@ export async function fetchUsers({
 }: IntegrationStepExecutionContext<IntegrationConfig>) {
   let stepAnnouncer;
   if (accountFlagged) {
-    stepAnnouncer = new StepAnnouncer(Steps.USERS, 10, logger);
+    stepAnnouncer = new StepAnnouncer(Steps.USERS, logger);
   }
 
   const apiClient = createAPIClient(instance.config, logger);

diff --git a/src/util/runningTimer.ts b/src/util/runningTimer.ts
@@ -1,7 +1,4 @@
-import {
-  IntegrationInfoEventName,
-  IntegrationLogger,
-} from '@jupiterone/integration-sdk-core';
+import { IntegrationLogger } from '@jupiterone/integration-sdk-core';
 
 class StepAnnouncer {
   private stepId: string;
@@ -12,58 +9,63 @@ class StepAnnouncer {
 
   constructor(
     stepId: string,
-    announceEvery: number,
     logger: IntegrationLogger,
+    announceEvery: number = 45,
   ) {
     this.stepId = stepId;
-    this.announceEvery = announceEvery * 1000; // Keep milliseconds for JS timers
+    this.announceEvery = announceEvery * 1000;
     this.logger = logger;
     this.startedAt = new Date();
-    this.start(); // Consider starting outside of the constructor for more control
+    this.start();
   }
 
   private getReadableHumanTime(): string {
     const elapsedSeconds = Math.floor(
       (new Date().getTime() - this.startedAt.getTime()) / 1000,
     );
-    const minutes = Math.floor(elapsedSeconds / 60);
+    const hours = Math.floor(elapsedSeconds / 3600);
+    const remainingMinutes = Math.floor((elapsedSeconds % 3600) / 60);
     const remainingSeconds = elapsedSeconds % 60;
-    let message =
-      minutes > 0 ? `${minutes} minute${minutes > 1 ? 's' : ''}` : '';
-    if (remainingSeconds > 0) {
-      message += message
-        ? ` ${remainingSeconds} second${remainingSeconds > 1 ? 's' : ''}`
-        : `${remainingSeconds} second${remainingSeconds > 1 ? 's' : ''}`;
+
+    const messageParts: string[] = [];
+
+    if (hours > 0) {
+      messageParts.push(`${hours} hour${hours > 1 ? 's' : ''}`);
+    }
+    if (remainingMinutes > 0) {
+      messageParts.push(
+        `${remainingMinutes} minute${remainingMinutes > 1 ? 's' : ''}`,
+      );
     }
-    return message || '0 seconds';
+    if (remainingSeconds > 0 || messageParts.length === 0) {
+      messageParts.push(
+        `${remainingSeconds} second${remainingSeconds > 1 ? 's' : ''}`,
+      );
+    }
+
+    return messageParts.join(' ');
   }
 
   public start(): void {
     if (this.intervalId === null) {
       this.intervalId = setInterval(() => this.announce(), this.announceEvery);
-      this.logger.publishInfoEvent({
-        description: `[${this.stepId}] has started.`,
-        name: IntegrationInfoEventName.Stats,
-      });
+      this.logger.info(`[${this.stepId}] has started.`);
     }
   }
 
   private announce(): void {
     const timeMessage = this.getReadableHumanTime();
-    this.logger.publishInfoEvent({
-      description: `[${this.stepId}] has been running for ${timeMessage}.`,
-      name: IntegrationInfoEventName.Stats,
-    });
+    const description = `[${this.stepId}] has been running for ${timeMessage}.`;
+    this.logger.info(description);
   }
 
   public finish(): void {
     if (this.intervalId !== null) {
       clearInterval(this.intervalId);
       this.intervalId = null;
-      this.logger.publishInfoEvent({
-        description: `[${this.stepId}] has finished after ${this.getReadableHumanTime()}.`,
-        name: IntegrationInfoEventName.Stats,
-      });
+
+      const description = `[${this.stepId}] has finished after ${this.getReadableHumanTime()}.`;
+      this.logger.info(description);
     }
   }
 }