fix: generate all hour buckets if missing (#2319)

src/lib/__snapshots__/create-config.test.ts.snap

@@ -73,6 +73,7 @@ exports[`should create default config 1`] = `
       "cloneEnvironment": false,
       "embedProxy": false,
       "embedProxyFrontend": false,
+      "fixHourMetrics": false,
       "publicSignup": false,
       "responseTimeWithAppName": false,
       "syncSSOGroups": false,
@@ -87,6 +88,7 @@ exports[`should create default config 1`] = `
       "cloneEnvironment": false,
       "embedProxy": false,
       "embedProxyFrontend": false,
+      "fixHourMetrics": false,
       "publicSignup": false,
       "responseTimeWithAppName": false,
       "syncSSOGroups": false,

src/lib/services/client-metrics/metrics-service-v2.ts

@@ -8,14 +8,20 @@ import {
     IClientMetricsStoreV2,
 } from '../../types/stores/client-metrics-store-v2';
 import { clientMetricsSchema } from './schema';
-import { hoursToMilliseconds, secondsToMilliseconds } from 'date-fns';
+import {
+    compareAsc,
+    hoursToMilliseconds,
+    secondsToMilliseconds,
+} from 'date-fns';
 import { IFeatureToggleStore } from '../../types/stores/feature-toggle-store';
 import { CLIENT_METRICS } from '../../types/events';
 import ApiUser from '../../types/api-user';
 import { ALL } from '../../types/models/api-token';
 import User from '../../types/user';
 import { collapseHourlyMetrics } from '../../util/collapseHourlyMetrics';
 import { LastSeenService } from './last-seen-service';
+import { generateHourBuckets } from '../../util/time-utils';
+import { IFlagResolver } from '../../types/experimental';
 
 export default class ClientMetricsServiceV2 {
     private config: IUnleashConfig;
@@ -30,6 +36,8 @@ export default class ClientMetricsServiceV2 {
 
     private lastSeenService: LastSeenService;
 
+    private flagResolver: IFlagResolver;
+
     private logger: Logger;
 
     constructor(
@@ -45,6 +53,7 @@ export default class ClientMetricsServiceV2 {
         this.clientMetricsStoreV2 = clientMetricsStoreV2;
         this.lastSeenService = lastSeenService;
         this.config = config;
+        this.flagResolver = config.flagResolver;
         this.logger = config.getLogger(
             '/services/client-metrics/client-metrics-service-v2.ts',
         );
@@ -149,13 +158,56 @@ export default class ClientMetricsServiceV2 {
     }
 
     async getClientMetricsForToggle(
-        toggleName: string,
-        hoursBack?: number,
+        featureName: string,
+        hoursBack: number = 24,
     ): Promise<IClientMetricsEnv[]> {
-        return this.clientMetricsStoreV2.getMetricsForFeatureToggle(
-            toggleName,
-            hoursBack,
-        );
+        const metrics =
+            await this.clientMetricsStoreV2.getMetricsForFeatureToggle(
+                featureName,
+                hoursBack,
+            );
+
+        if (this.flagResolver.isEnabled('fixHourMetrics')) {
+            const hours = generateHourBuckets(hoursBack);
+
+            const environments = [
+                ...new Set(metrics.map((x) => x.environment)),
+            ];
+
+            const applications = [
+                ...new Set(metrics.map((x) => x.appName)),
+            ].slice(0, 100);
+
+            const result = environments.flatMap((environment) =>
+                applications.flatMap((appName) =>
+                    hours.flatMap((hourBucket) => {
+                        const metric = metrics.find(
+                            (item) =>
+                                compareAsc(
+                                    hourBucket.timestamp,
+                                    item.timestamp,
+                                ) === 0 &&
+                                item.appName === appName &&
+                                item.environment === environment,
+                        );
+                        return (
+                            metric || {
+                                timestamp: hourBucket.timestamp,
+                                no: 0,
+                                yes: 0,
+                                appName,
+                                environment,
+                                featureName,
+                            }
+                        );
+                    }),
+                ),
+            );
+
+            return result.sort((a, b) => compareAsc(a.timestamp, b.timestamp));
+        } else {
+            return metrics;
+        }
     }
 
     resolveMetricsEnvironment(user: User | ApiUser, data: IClientApp): string {

src/lib/types/experimental.ts

@@ -38,6 +38,10 @@ export const defaultExperimentalOptions = {
             process.env.UNLEASH_EXPERIMENTAL_CLONE_ENVIRONMENT,
             false,
         ),
+        fixHourMetrics: parseEnvVarBoolean(
+            process.env.UNLEASH_EXPERIMENTAL_FIX_HOUR_METRICS,
+            false,
+        ),
     },
     externalResolver: { isEnabled: (): boolean => false },
 };

src/lib/util/time-utils.test.ts

@@ -0,0 +1,7 @@
+import { generateHourBuckets } from './time-utils';
+
+test('generateHourBuckets', () => {
+    const result = generateHourBuckets(24);
+
+    expect(result).toHaveLength(24);
+});

src/lib/util/time-utils.ts

@@ -0,0 +1,16 @@
+import { startOfHour, subHours } from 'date-fns';
+
+export interface HourBucket {
+    timestamp: Date;
+}
+
+export function generateHourBuckets(hours: number): HourBucket[] {
+    const start = startOfHour(new Date());
+
+    const result = [];
+
+    for (let i = 0; i < hours; i++) {
+        result.push({ timestamp: subHours(start, i) });
+    }
+    return result;
+}

src/server-dev.ts

@@ -41,6 +41,7 @@ process.nextTick(async () => {
                         syncSSOGroups: true,
                         changeRequests: true,
                         cloneEnvironment: true,
+                        fixHourMetrics: true,
                     },
                 },
                 authentication: {

src/test/config/test-config.ts

@@ -30,6 +30,7 @@ export function createTestConfig(config?: IUnleashOptions): IUnleashConfig {
                 syncSSOGroups: true,
                 changeRequests: true,
                 cloneEnvironment: true,
+                fixHourMetrics: true,
             },
         },
     };

src/test/e2e/api/admin/client-metrics.e2e.test.ts

@@ -79,18 +79,18 @@ test('should return raw metrics, aggregated on key', async () => {
         .expect('Content-Type', /json/)
         .expect(200);
 
-    expect(demo.data).toHaveLength(2);
-    expect(demo.data[0].environment).toBe('default');
-    expect(demo.data[0].yes).toBe(5);
-    expect(demo.data[0].no).toBe(4);
-    expect(demo.data[1].environment).toBe('test');
-    expect(demo.data[1].yes).toBe(1);
-    expect(demo.data[1].no).toBe(3);
+    expect(demo.data).toHaveLength(48);
+    expect(demo.data[46].environment).toBe('default');
+    expect(demo.data[46].yes).toBe(5);
+    expect(demo.data[46].no).toBe(4);
+    expect(demo.data[47].environment).toBe('test');
+    expect(demo.data[47].yes).toBe(1);
+    expect(demo.data[47].no).toBe(3);
 
-    expect(t2.data).toHaveLength(1);
-    expect(t2.data[0].environment).toBe('default');
-    expect(t2.data[0].yes).toBe(7);
-    expect(t2.data[0].no).toBe(104);
+    expect(t2.data).toHaveLength(24);
+    expect(t2.data[23].environment).toBe('default');
+    expect(t2.data[23].yes).toBe(7);
+    expect(t2.data[23].no).toBe(104);
 });
 
 test('should support the hoursBack query param for raw metrics', async () => {
@@ -141,10 +141,10 @@ test('should support the hoursBack query param for raw metrics', async () => {
     const hoursTooMany = await fetchHoursBack(999);
 
     expect(hours1.data).toHaveLength(1);
-    expect(hours24.data).toHaveLength(2);
-    expect(hours48.data).toHaveLength(3);
-    expect(hoursTooFew.data).toHaveLength(2);
-    expect(hoursTooMany.data).toHaveLength(2);
+    expect(hours24.data).toHaveLength(24);
+    expect(hours48.data).toHaveLength(48);
+    expect(hoursTooFew.data).toHaveLength(24);
+    expect(hoursTooMany.data).toHaveLength(24);
 });
 
 test('should return toggle summary', async () => {