Feat/project status monthly (#2986)

This PR takes the project status API a step further by adding the
capability of providing a date to control the selection. We are
currently making calculations based on a gliding 30 day window, updated
once a day. The initial database structure and method for updating the
UI is outlined in this PR.

src/lib/db/event-store.ts

@@ -17,7 +17,48 @@ const EVENT_COLUMNS = [
     'feature_name',
     'project',
     'environment',
-];
+] as const;
+
+export type IQueryOperations =
+    | IWhereOperation
+    | IBeforeDateOperation
+    | IBetweenDatesOperation
+    | IForFeaturesOperation;
+
+interface IWhereOperation {
+    op: 'where';
+    parameters: {
+        [key: string]: string;
+    };
+}
+
+interface IBeforeDateOperation {
+    op: 'beforeDate';
+    parameters: {
+        dateAccessor: string;
+        date: string;
+    };
+}
+
+interface IBetweenDatesOperation {
+    op: 'betweenDate';
+    parameters: {
+        dateAccessor: string;
+        range: string[];
+    };
+}
+
+interface IForFeaturesOperation {
+    op: 'forFeatures';
+    parameters: IForFeaturesParams;
+}
+
+interface IForFeaturesParams {
+    type: string;
+    projectId: string;
+    environments: string[];
+    features: string[];
+}
 
 export interface IEventTable {
     id: number;
@@ -120,25 +161,77 @@ class EventStore extends AnyEventEmitter implements IEventStore {
         return present;
     }
 
-    async getForFeatures(
-        features: string[],
-        environments: string[],
-        query: { type: string; projectId: string },
-    ): Promise<IEvent[]> {
+    async query(operations: IQueryOperations[]): Promise<IEvent[]> {
         try {
-            const rows = await this.db
-                .select(EVENT_COLUMNS)
-                .from(TABLE)
-                .where({ type: query.type, project: query.projectId })
-                .whereIn('feature_name', features)
-                .whereIn('environment', environments);
+            let query: Knex.QueryBuilder = this.select();
+
+            operations.forEach((operation) => {
+                if (operation.op === 'where') {
+                    query = this.where(query, operation.parameters);
+                }
+
+                if (operation.op === 'forFeatures') {
+                    query = this.forFeatures(query, operation.parameters);
+                }
 
+                if (operation.op === 'beforeDate') {
+                    query = this.beforeDate(query, operation.parameters);
+                }
+
+                if (operation.op === 'betweenDate') {
+                    query = this.betweenDate(query, operation.parameters);
+                }
+            });
+
+            const rows = await query;
             return rows.map(this.rowToEvent);
         } catch (e) {
             return [];
         }
     }
 
+    where(
+        query: Knex.QueryBuilder,
+        parameters: { [key: string]: string },
+    ): Knex.QueryBuilder {
+        return query.where(parameters);
+    }
+
+    beforeDate(
+        query: Knex.QueryBuilder,
+        parameters: { dateAccessor: string; date: string },
+    ): Knex.QueryBuilder {
+        return query.andWhere(parameters.dateAccessor, '>=', parameters.date);
+    }
+
+    betweenDate(
+        query: Knex.QueryBuilder,
+        parameters: { dateAccessor: string; range: string[] },
+    ): Knex.QueryBuilder {
+        if (parameters.range && parameters.range.length === 2) {
+            return query.andWhereBetween(parameters.dateAccessor, [
+                parameters.range[0],
+                parameters.range[1],
+            ]);
+        }
+
+        return query;
+    }
+
+    select(): Knex.QueryBuilder {
+        return this.db.select(EVENT_COLUMNS).from(TABLE);
+    }
+
+    forFeatures(
+        query: Knex.QueryBuilder,
+        parameters: IForFeaturesParams,
+    ): Knex.QueryBuilder {
+        return query
+            .where({ type: parameters.type, project: parameters.projectId })
+            .whereIn('feature_name', parameters.features)
+            .whereIn('environment', parameters.environments);
+    }
+
     async get(key: number): Promise<IEvent> {
         const row = await this.db(TABLE).where({ id: key }).first();
         return this.rowToEvent(row);

src/lib/db/feature-toggle-store.ts

@@ -111,6 +111,35 @@ export default class FeatureToggleStore implements IFeatureToggleStore {
         return rows.map(this.rowToFeature);
     }
 
+    async getByDate(queryModifiers: {
+        archived?: boolean;
+        project?: string;
+        date?: string;
+        range?: string[];
+        dateAccessor: string;
+    }): Promise<FeatureToggle[]> {
+        const { project, archived, dateAccessor } = queryModifiers;
+        let query = this.db
+            .select(FEATURE_COLUMNS)
+            .from(TABLE)
+            .where({ project })
+            .modify(FeatureToggleStore.filterByArchived, archived);
+
+        if (queryModifiers.date) {
+            query.andWhere(dateAccessor, '>=', queryModifiers.date);
+        }
+
+        if (queryModifiers.range && queryModifiers.range.length === 2) {
+            query.andWhereBetween(dateAccessor, [
+                queryModifiers.range[0],
+                queryModifiers.range[1],
+            ]);
+        }
+
+        const rows = await query;
+        return rows.map(this.rowToFeature);
+    }
+
     /**
      * Get projectId from feature filtered by name. Used by Rbac middleware
      * @deprecated

src/lib/db/index.ts

@@ -35,6 +35,7 @@ import { PublicSignupTokenStore } from './public-signup-token-store';
 import { FavoriteFeaturesStore } from './favorite-features-store';
 import { FavoriteProjectsStore } from './favorite-projects-store';
 import { AccountStore } from './account-store';
+import ProjectStatsStore from './project-stats-store';
 
 export const createStores = (
     config: IUnleashConfig,
@@ -113,6 +114,7 @@ export const createStores = (
             eventBus,
             getLogger,
         ),
+        projectStatsStore: new ProjectStatsStore(db, eventBus, getLogger),
     };
 };
 

src/lib/db/project-stats-store.ts

@@ -0,0 +1,52 @@
+import { Knex } from 'knex';
+import { Logger, LogProvider } from '../logger';
+
+import metricsHelper from '../util/metrics-helper';
+import { DB_TIME } from '../metric-events';
+import EventEmitter from 'events';
+import { IProjectStats } from 'lib/services/project-service';
+import { IProjectStatsStore } from 'lib/types/stores/project-stats-store-type';
+
+const TABLE = 'project_stats';
+
+class ProjectStatsStore implements IProjectStatsStore {
+    private db: Knex;
+
+    private logger: Logger;
+
+    private timer: Function;
+
+    constructor(db: Knex, eventBus: EventEmitter, getLogger: LogProvider) {
+        this.db = db;
+        this.logger = getLogger('project-stats-store.ts');
+        this.timer = (action) =>
+            metricsHelper.wrapTimer(eventBus, DB_TIME, {
+                store: 'project_stats',
+                action,
+            });
+    }
+
+    async updateProjectStats(
+        projectId: string,
+        status: IProjectStats,
+    ): Promise<void> {
+        await this.db(TABLE)
+            .insert({
+                avg_time_to_prod_current_window:
+                    status.avgTimeToProdCurrentWindow,
+                avg_time_to_prod_past_window: status.avgTimeToProdPastWindow,
+                project: projectId,
+                features_created_current_window: status.createdCurrentWindow,
+                features_created_past_window: status.createdPastWindow,
+                features_archived_current_window: status.archivedCurrentWindow,
+                features_archived_past_window: status.archivedPastWindow,
+                project_changes_current_window:
+                    status.projectActivityCurrentWindow,
+                project_changes_past_window: status.projectActivityPastWindow,
+            })
+            .onConflict('project')
+            .merge();
+    }
+}
+
+export default ProjectStatsStore;

src/lib/read-models/project-status/project-status.test.ts → src/lib/read-models/time-to-production/time-to-production.test.ts

@@ -1,6 +1,6 @@
 import { addDays, subDays } from 'date-fns';
 import { IEvent } from 'lib/types';
-import { ProjectStatus } from './project-status';
+import { TimeToProduction } from './time-to-production';
 
 const modifyEventCreatedAt = (events: IEvent[], days: number): IEvent[] => {
     return events.map((event) => {
@@ -97,7 +97,7 @@ const features = [
         type: 'release',
         project: 'average-time-to-prod',
         stale: false,
-        createdAt: new Date('2023-01-19T09:37:32.483Z'),
+        createdAt: new Date('2022-12-05T09:37:32.483Z'),
         lastSeenAt: null,
         impressionData: false,
         archivedAt: null,
@@ -143,7 +143,11 @@ const features = [
 
 describe('calculate average time to production', () => {
     test('should build a map of feature events', () => {
-        const projectStatus = new ProjectStatus(features, environments, events);
+        const projectStatus = new TimeToProduction(
+            features,
+            environments,
+            events,
+        );
 
         const featureEvents = projectStatus.getFeatureEvents();
 
@@ -153,15 +157,19 @@ describe('calculate average time to production', () => {
     });
 
     test('should calculate average correctly', () => {
-        const projectStatus = new ProjectStatus(features, environments, events);
+        const projectStatus = new TimeToProduction(
+            features,
+            environments,
+            events,
+        );
 
         const timeToProduction = projectStatus.calculateAverageTimeToProd();
 
-        expect(timeToProduction).toBe(9.75);
+        expect(timeToProduction).toBe(21);
     });
 
     test('should sort events by createdAt', () => {
-        const projectStatus = new ProjectStatus(features, environments, [
+        const projectStatus = new TimeToProduction(features, environments, [
             ...modifyEventCreatedAt(events, 5),
             ...events,
         ]);
@@ -192,7 +200,7 @@ describe('calculate average time to production', () => {
     });
 
     test('should not count events that are development environments', () => {
-        const projectStatus = new ProjectStatus(features, environments, [
+        const projectStatus = new TimeToProduction(features, environments, [
             createEvent('development', {
                 createdAt: subDays(new Date('2023-01-25T09:37:32.504Z'), 10),
             }),
@@ -203,6 +211,6 @@ describe('calculate average time to production', () => {
         ]);
 
         const timeToProduction = projectStatus.calculateAverageTimeToProd();
-        expect(timeToProduction).toBe(9.75);
+        expect(timeToProduction).toBe(21);
     });
 });

src/lib/read-models/project-status/project-status.ts → src/lib/read-models/time-to-production/time-to-production.ts

@@ -10,7 +10,7 @@ interface IFeatureTimeToProdData {
     events: IEvent[];
 }
 
-export class ProjectStatus {
+export class TimeToProduction {
     private features: FeatureToggle[];
 
     private productionEnvironments: IProjectEnvironment[];
@@ -31,16 +31,25 @@ export class ProjectStatus {
         const featureEvents = this.getFeatureEvents();
         const sortedFeatureEvents =
             this.sortFeatureEventsByCreatedAt(featureEvents);
+
         const timeToProdPerFeature =
             this.calculateTimeToProdForFeatures(sortedFeatureEvents);
+        if (timeToProdPerFeature.length) {
+            const sum = timeToProdPerFeature.reduce(
+                (acc, curr) => acc + curr,
+                0,
+            );
 
-        const sum = timeToProdPerFeature.reduce((acc, curr) => acc + curr, 0);
+            return Number(
+                (sum / Object.keys(sortedFeatureEvents).length).toFixed(1),
+            );
+        }
 
-        return sum / Object.keys(sortedFeatureEvents).length;
+        return 0;
     }
 
     getFeatureEvents(): IFeatureTimeToProdCalculationMap {
-        return this.filterEvents(this.events).reduce((acc, event) => {
+        return this.getProductionEvents(this.events).reduce((acc, event) => {
             if (acc[event.featureName]) {
                 acc[event.featureName].events.push(event);
             } else {
@@ -55,7 +64,7 @@ export class ProjectStatus {
         }, {});
     }
 
-    filterEvents(events: IEvent[]): IEvent[] {
+    getProductionEvents(events: IEvent[]): IEvent[] {
         return events.filter((event) => {
             const found = this.productionEnvironments.find(
                 (env) => env.name === event.environment,
@@ -74,6 +83,7 @@ export class ProjectStatus {
     ): number[] {
         return Object.keys(featureEvents).map((featureName) => {
             const feature = featureEvents[featureName];
+
             const earliestEvent = feature.events[0];
 
             const createdAtDate = new Date(feature.createdAt);

src/lib/services/index.ts

@@ -164,7 +164,7 @@ export const createServices = (
     if (config.flagResolver.isEnabled('projectStatusApi')) {
         const ONE_DAY = 1440;
         schedulerService.schedule(
-            projectService.statusJob.bind(projectHealthService),
+            projectService.statusJob.bind(projectService),
             minutesToMilliseconds(ONE_DAY),
         );
     }