fix: count events instead of loading them in memory (#3382)

Refactor project events to use count instead of loading the events in memory
Unleash · Mar 27, 2023 · 354e54a · 354e54a
1 parent 30c1ebc
commit 354e54a
Show file tree

Hide file tree

Showing 7 changed files with 67 additions and 28 deletions.
diff --git a/src/lib/db/event-store.ts b/src/lib/db/event-store.ts
@@ -207,6 +207,35 @@ class EventStore implements IEventStore {
         }
     }
 
+    async queryCount(operations: IQueryOperations[]): Promise<number> {
+        try {
+            let query: Knex.QueryBuilder = this.db.count().from(TABLE);
+
+            operations.forEach((operation) => {
+                if (operation.op === 'where') {
+                    query = this.where(query, operation.parameters);
+                }
+
+                if (operation.op === 'forFeatures') {
+                    query = this.forFeatures(query, operation.parameters);
+                }
+
+                if (operation.op === 'beforeDate') {
+                    query = this.beforeDate(query, operation.parameters);
+                }
+
+                if (operation.op === 'betweenDate') {
+                    query = this.betweenDate(query, operation.parameters);
+                }
+            });
+
+            const queryResult = await query.first();
+            return parseInt(queryResult.count || 0);
+        } catch (e) {
+            return 0;
+        }
+    }
+
     where(
         query: Knex.QueryBuilder,
         parameters: { [key: string]: string },

diff --git a/src/lib/server-impl.ts b/src/lib/server-impl.ts
@@ -43,7 +43,7 @@ async function createApp(
     const db = createDb(config);
     const stores = createStores(config, db);
     const services = createServices(stores, config, db);
-    scheduleServices(services, config);
+    scheduleServices(services);
 
     const metricsMonitor = createMetricsMonitor();
     const unleashSession = sessionDb(config, db);

diff --git a/src/lib/services/index.ts b/src/lib/services/index.ts
@@ -58,10 +58,7 @@ import {
 } from '../features/change-request-access-service/createChangeRequestAccessReadModel';
 
 // TODO: will be moved to scheduler feature directory
-export const scheduleServices = (
-    services: IUnleashServices,
-    config: IUnleashConfig,
-): void => {
+export const scheduleServices = (services: IUnleashServices): void => {
     const {
         schedulerService,
         apiTokenService,
@@ -94,12 +91,10 @@ export const scheduleServices = (
         hoursToMilliseconds(24),
     );
 
-    if (config.flagResolver.isEnabled('projectStatusApi')) {
-        schedulerService.schedule(
-            projectService.statusJob.bind(projectService),
-            hoursToMilliseconds(24),
-        );
-    }
+    schedulerService.schedule(
+        projectService.statusJob.bind(projectService),
+        hoursToMilliseconds(24),
+    );
 
     schedulerService.schedule(
         projectHealthService.setHealthRating.bind(projectHealthService),

diff --git a/src/lib/services/project-service.ts b/src/lib/services/project-service.ts
@@ -35,6 +35,7 @@ import {
     ProjectUserRemovedEvent,
     ProjectUserUpdateRoleEvent,
     RoleName,
+    IFlagResolver,
 } from '../types';
 import {
     IProjectQuery,
@@ -114,6 +115,8 @@ export default class ProjectService {
 
     private projectStatsStore: IProjectStatsStore;
 
+    private flagResolver: IFlagResolver;
+
     constructor(
         {
             projectStore,
@@ -157,6 +160,7 @@ export default class ProjectService {
         this.groupService = groupService;
         this.projectStatsStore = projectStatsStore;
         this.logger = config.getLogger('services/project-service.js');
+        this.flagResolver = config.flagResolver;
     }
 
     async getProjects(
@@ -664,20 +668,24 @@ export default class ProjectService {
     }
 
     async statusJob(): Promise<void> {
-        const projects = await this.store.getAll();
+        if (this.flagResolver.isEnabled('projectStatusApi')) {
+            const projects = await this.store.getAll();
 
-        const statusUpdates = await Promise.all(
-            projects.map((project) => this.getStatusUpdates(project.id)),
-        );
+            const statusUpdates = await Promise.all(
+                projects.map((project) => this.getStatusUpdates(project.id)),
+            );
 
-        await Promise.all(
-            statusUpdates.map((statusUpdate) => {
-                return this.projectStatsStore.updateProjectStats(
-                    statusUpdate.projectId,
-                    statusUpdate.updates,
-                );
-            }),
-        );
+            await Promise.all(
+                statusUpdates.map((statusUpdate) => {
+                    return this.projectStatsStore.updateProjectStats(
+                        statusUpdate.projectId,
+                        statusUpdate.updates,
+                    );
+                }),
+            );
+        } else {
+            this.logger.info('Project status API is disabled');
+        }
     }
 
     async getStatusUpdates(projectId: string): Promise<ICalculateStatus> {
@@ -726,7 +734,7 @@ export default class ProjectService {
 
         const [projectActivityCurrentWindow, projectActivityPastWindow] =
             await Promise.all([
-                this.eventStore.query([
+                this.eventStore.queryCount([
                     { op: 'where', parameters: { project: projectId } },
                     {
                         op: 'beforeDate',
@@ -736,7 +744,7 @@ export default class ProjectService {
                         },
                     },
                 ]),
-                this.eventStore.query([
+                this.eventStore.queryCount([
                     { op: 'where', parameters: { project: projectId } },
                     {
                         op: 'betweenDate',
@@ -792,9 +800,8 @@ export default class ProjectService {
                 createdPastWindow: createdPastWindow.length,
                 archivedCurrentWindow: archivedCurrentWindow.length,
                 archivedPastWindow: archivedPastWindow.length,
-                projectActivityCurrentWindow:
-                    projectActivityCurrentWindow.length,
-                projectActivityPastWindow: projectActivityPastWindow.length,
+                projectActivityCurrentWindow: projectActivityCurrentWindow,
+                projectActivityPastWindow: projectActivityPastWindow,
                 projectMembersAddedCurrentWindow:
                     projectMembersAddedCurrentWindow,
             },

diff --git a/src/lib/types/stores/event-store.ts b/src/lib/types/stores/event-store.ts
@@ -15,4 +15,5 @@ export interface IEventStore
     searchEvents(search: SearchEventsSchema): Promise<IEvent[]>;
     getMaxRevisionId(currentMax?: number): Promise<number>;
     query(operations: IQueryOperations[]): Promise<IEvent[]>;
+    queryCount(operations: IQueryOperations[]): Promise<number>;
 }
diff --git a/src/test/e2e/services/project-service.e2e.test.ts b/src/test/e2e/services/project-service.e2e.test.ts
@@ -1390,4 +1390,6 @@ test('should get correct amount of project members for current and past window',
 
     const result = await projectService.getStatusUpdates(project.id);
     expect(result.updates.projectMembersAddedCurrentWindow).toBe(5);
+    expect(result.updates.projectActivityCurrentWindow).toBe(6);
+    expect(result.updates.projectActivityPastWindow).toBe(0);
 });
diff --git a/src/test/fixtures/fake-event-store.ts b/src/test/fixtures/fake-event-store.ts
@@ -95,6 +95,11 @@ class FakeEventStore implements IEventStore {
         return [];
     }
 
+    async queryCount(operations: IQueryOperations[]): Promise<number> {
+        if (operations) return 0;
+        return 0;
+    }
+
     setMaxListeners(number: number): EventEmitter {
         return this.eventEmitter.setMaxListeners(number);
     }