diff --git a/src/lib/routes/client-api/metrics.test.ts b/src/lib/routes/client-api/metrics.test.ts index 9a579de5e44..a0568c4bd49 100644 --- a/src/lib/routes/client-api/metrics.test.ts +++ b/src/lib/routes/client-api/metrics.test.ts @@ -4,7 +4,7 @@ import getApp from '../../app'; import { createTestConfig } from '../../../test/config/test-config'; import { clientMetricsSchema } from '../../services/client-metrics/schema'; import { createServices } from '../../services'; -import { IUnleashOptions, IUnleashStores } from '../../types'; +import { IUnleashOptions, IUnleashServices, IUnleashStores } from '../../types'; async function getSetup(opts?: IUnleashOptions) { const stores = createStores(); @@ -16,6 +16,7 @@ async function getSetup(opts?: IUnleashOptions) { return { request: supertest(app), stores, + services, destroy: () => { services.versionService.destroy(); services.clientInstanceService.destroy(); @@ -26,6 +27,7 @@ async function getSetup(opts?: IUnleashOptions) { let request; let stores: IUnleashStores; +let services: IUnleashServices; let destroy; beforeEach(async () => { @@ -33,6 +35,7 @@ beforeEach(async () => { request = setup.request; stores = setup.stores; destroy = setup.destroy; + services = setup.services; }); afterEach(() => { @@ -202,6 +205,7 @@ test('should set lastSeen on toggle', async () => { }) .expect(202); + await services.lastSeenService.store(); const toggle = await stores.featureToggleStore.get('toggleLastSeen'); expect(toggle.lastSeenAt).toBeTruthy(); diff --git a/src/lib/services/client-metrics/last-seen-service.ts b/src/lib/services/client-metrics/last-seen-service.ts new file mode 100644 index 00000000000..a6c95b82f00 --- /dev/null +++ b/src/lib/services/client-metrics/last-seen-service.ts @@ -0,0 +1,58 @@ +import { secondsToMilliseconds } from 'date-fns'; +import { Logger } from '../../logger'; +import { IUnleashConfig } from '../../server-impl'; +import { IUnleashStores } from '../../types'; +import { IClientMetricsEnv } from '../../types/stores/client-metrics-store-v2'; +import { IFeatureToggleStore } from '../../types/stores/feature-toggle-store'; + +export class LastSeenService { + private timers: NodeJS.Timeout[] = []; + + private lastSeenToggles: Set = new Set(); + + private logger: Logger; + + private featureToggleStore: IFeatureToggleStore; + + constructor( + { featureToggleStore }: Pick, + config: IUnleashConfig, + lastSeenInterval = secondsToMilliseconds(30), + ) { + this.featureToggleStore = featureToggleStore; + this.logger = config.getLogger( + '/services/client-metrics/last-seen-service.ts', + ); + + this.timers.push( + setInterval(() => this.store(), lastSeenInterval).unref(), + ); + } + + async store(): Promise { + const count = this.lastSeenToggles.size; + if (count > 0) { + const lastSeenToggles = [...this.lastSeenToggles]; + this.lastSeenToggles = new Set(); + this.logger.debug( + `Updating last seen for ${lastSeenToggles.length} toggles`, + ); + await this.featureToggleStore.setLastSeen(lastSeenToggles); + } + return count; + } + + updateLastSeen(clientMetrics: IClientMetricsEnv[]): void { + clientMetrics + .filter( + (clientMetric) => clientMetric.yes > 0 || clientMetric.no > 0, + ) + .forEach((clientMetric) => + this.lastSeenToggles.add(clientMetric.featureName), + ); + } + + destroy(): void { + this.timers.forEach(clearInterval); + } +} diff --git a/src/lib/services/client-metrics/metrics-service-v2.ts b/src/lib/services/client-metrics/metrics-service-v2.ts index dfb4ce15d18..c2f9b03f7db 100644 --- a/src/lib/services/client-metrics/metrics-service-v2.ts +++ b/src/lib/services/client-metrics/metrics-service-v2.ts @@ -15,6 +15,7 @@ import ApiUser from '../../types/api-user'; import { ALL } from '../../types/models/api-token'; import User from '../../types/user'; import { collapseHourlyMetrics } from '../../util/collapseHourlyMetrics'; +import { LastSeenService } from './last-seen-service'; export default class ClientMetricsServiceV2 { private config: IUnleashConfig; @@ -27,6 +28,8 @@ export default class ClientMetricsServiceV2 { private featureToggleStore: IFeatureToggleStore; + private lastSeenService: LastSeenService; + private logger: Logger; constructor( @@ -35,10 +38,12 @@ export default class ClientMetricsServiceV2 { clientMetricsStoreV2, }: Pick, config: IUnleashConfig, + lastSeenService: LastSeenService, bulkInterval = secondsToMilliseconds(5), ) { this.featureToggleStore = featureToggleStore; this.clientMetricsStoreV2 = clientMetricsStoreV2; + this.lastSeenService = lastSeenService; this.config = config; this.logger = config.getLogger( '/services/client-metrics/client-metrics-service-v2.ts', @@ -62,30 +67,35 @@ export default class ClientMetricsServiceV2 { clientIp: string, ): Promise { const value = await clientMetricsSchema.validateAsync(data); - const toggleNames = Object.keys(value.bucket.toggles); - if (toggleNames.length > 0) { - await this.featureToggleStore.setLastSeen(toggleNames); - } + const toggleNames = Object.keys(value.bucket.toggles).filter( + (name) => + !( + value.bucket.toggles[name].yes === 0 && + value.bucket.toggles[name].no === 0 + ), + ); this.logger.debug(`got metrics from ${clientIp}`); - const clientMetrics: IClientMetricsEnv[] = toggleNames - .map((name) => ({ - featureName: name, - appName: value.appName, - environment: value.environment, - timestamp: value.bucket.start, //we might need to approximate between start/stop... - yes: value.bucket.toggles[name].yes, - no: value.bucket.toggles[name].no, - })) - .filter((item) => !(item.yes === 0 && item.no === 0)); + const clientMetrics: IClientMetricsEnv[] = toggleNames.map((name) => ({ + featureName: name, + appName: value.appName, + environment: value.environment, + timestamp: value.bucket.start, //we might need to approximate between start/stop... + yes: value.bucket.toggles[name].yes, + no: value.bucket.toggles[name].no, + })); if (this.config.flagResolver.isEnabled('batchMetrics')) { this.unsavedMetrics = collapseHourlyMetrics([ ...this.unsavedMetrics, ...clientMetrics, ]); + this.lastSeenService.updateLastSeen(clientMetrics); } else { + if (toggleNames.length > 0) { + await this.featureToggleStore.setLastSeen(toggleNames); + } await this.clientMetricsStoreV2.batchInsertMetrics(clientMetrics); } @@ -161,5 +171,6 @@ export default class ClientMetricsServiceV2 { destroy(): void { this.timers.forEach(clearInterval); + this.lastSeenService.destroy(); } } diff --git a/src/lib/services/index.ts b/src/lib/services/index.ts index eded5eebb08..be559687a36 100644 --- a/src/lib/services/index.ts +++ b/src/lib/services/index.ts @@ -35,6 +35,7 @@ import { ProxyService } from './proxy-service'; import EdgeService from './edge-service'; import PatService from './pat-service'; import { PublicSignupTokenService } from './public-signup-token-service'; +import { LastSeenService } from './client-metrics/last-seen-service'; export const createServices = ( stores: IUnleashStores, config: IUnleashConfig, @@ -43,7 +44,12 @@ export const createServices = ( const accessService = new AccessService(stores, config, groupService); const apiTokenService = new ApiTokenService(stores, config); const clientInstanceService = new ClientInstanceService(stores, config); - const clientMetricsServiceV2 = new ClientMetricsServiceV2(stores, config); + const lastSeenService = new LastSeenService(stores, config); + const clientMetricsServiceV2 = new ClientMetricsServiceV2( + stores, + config, + lastSeenService, + ); const contextService = new ContextService(stores, config); const emailService = new EmailService(config.email, config.getLogger); const eventService = new EventService(stores, config); @@ -147,6 +153,7 @@ export const createServices = ( edgeService, patService, publicSignupTokenService, + lastSeenService, }; }; diff --git a/src/lib/types/services.ts b/src/lib/types/services.ts index 61b39c379e8..b2c7da1ee09 100644 --- a/src/lib/types/services.ts +++ b/src/lib/types/services.ts @@ -33,6 +33,7 @@ import { ProxyService } from '../services/proxy-service'; import EdgeService from '../services/edge-service'; import PatService from '../services/pat-service'; import { PublicSignupTokenService } from '../services/public-signup-token-service'; +import { LastSeenService } from '../services/client-metrics/last-seen-service'; export interface IUnleashServices { accessService: AccessService; @@ -71,4 +72,5 @@ export interface IUnleashServices { openApiService: OpenApiService; clientSpecService: ClientSpecService; patService: PatService; + lastSeenService: LastSeenService; } diff --git a/src/test/e2e/api/client/metricsV2.e2e.test.ts b/src/test/e2e/api/client/metricsV2.e2e.test.ts index a2afc9673dc..6fabf4ca692 100644 --- a/src/test/e2e/api/client/metricsV2.e2e.test.ts +++ b/src/test/e2e/api/client/metricsV2.e2e.test.ts @@ -97,3 +97,53 @@ test('should pick up environment from token', async () => { expect(metrics[0].environment).toBe('test'); expect(metrics[0].appName).toBe('some-fancy-app'); }); + +test('should set lastSeen for toggles with metrics', async () => { + const start = Date.now(); + await app.services.featureToggleServiceV2.createFeatureToggle( + 'default', + { name: 't1' }, + 'tester', + ); + await app.services.featureToggleServiceV2.createFeatureToggle( + 'default', + { name: 't2' }, + 'tester', + ); + const token = await app.services.apiTokenService.createApiToken({ + type: ApiTokenType.CLIENT, + project: 'default', + environment: 'default', + username: 'tester', + }); + + await app.request + .post('/api/client/metrics') + .set('Authorization', token.secret) + .send({ + appName: 'some-fancy-app', + instanceId: '1', + bucket: { + start: Date.now(), + stop: Date.now(), + toggles: { + t1: { + yes: 100, + no: 50, + }, + t2: { + yes: 0, + no: 0, + }, + }, + }, + }) + .expect(202); + + await app.services.clientMetricsServiceV2.bulkAdd(); + await app.services.lastSeenService.store(); + const t1 = await db.stores.featureToggleStore.get('t1'); + const t2 = await db.stores.featureToggleStore.get('t2'); + expect(t1.lastSeenAt.getTime()).toBeGreaterThanOrEqual(start); + expect(t2.lastSeenAt).toBeDefined(); +}); diff --git a/src/test/e2e/services/last-seen-service.e2e.test.ts b/src/test/e2e/services/last-seen-service.e2e.test.ts new file mode 100644 index 00000000000..b0931039dcd --- /dev/null +++ b/src/test/e2e/services/last-seen-service.e2e.test.ts @@ -0,0 +1,131 @@ +import { createTestConfig } from '../../config/test-config'; +import dbInit from '../helpers/database-init'; +import { IUnleashStores } from '../../../lib/types/stores'; +import { LastSeenService } from '../../../lib/services/client-metrics/last-seen-service'; +import { IClientMetricsEnv } from '../../../lib/types/stores/client-metrics-store-v2'; + +let stores: IUnleashStores; +let db; +let config; + +beforeAll(async () => { + config = createTestConfig(); + db = await dbInit('last_seen_service_serial', config.getLogger); + stores = db.stores; +}); +beforeEach(async () => { + await stores.featureToggleStore.deleteAll(); +}); +afterAll(async () => { + await db.destroy(); +}); + +test('Should update last seen for known toggles', async () => { + const service = new LastSeenService(stores, config); + const time = Date.now(); + await stores.featureToggleStore.create('default', { name: 'ta1' }); + + const metrics: IClientMetricsEnv[] = [ + { + featureName: 'ta1', + appName: 'some-App', + environment: 'default', + timestamp: new Date(time), + yes: 1, + no: 0, + }, + { + featureName: 'ta2', + appName: 'some-App', + environment: 'default', + timestamp: new Date(time), + yes: 1, + no: 0, + }, + ]; + + service.updateLastSeen(metrics); + await service.store(); + + const t1 = await stores.featureToggleStore.get('ta1'); + + expect(t1.lastSeenAt.getTime()).toBeGreaterThan(time); + + service.destroy(); +}); + +test('Should not update last seen toggles with 0 metrics', async () => { + // jest.useFakeTimers(); + const service = new LastSeenService(stores, config, 30); + const time = Date.now(); + await stores.featureToggleStore.create('default', { name: 'tb1' }); + await stores.featureToggleStore.create('default', { name: 'tb2' }); + + const metrics: IClientMetricsEnv[] = [ + { + featureName: 'tb1', + appName: 'some-App', + environment: 'default', + timestamp: new Date(time), + yes: 1, + no: 0, + }, + { + featureName: 'tb2', + appName: 'some-App', + environment: 'default', + timestamp: new Date(time), + yes: 0, + no: 0, + }, + ]; + + service.updateLastSeen(metrics); + + // bypass interval waiting + await service.store(); + + const t1 = await stores.featureToggleStore.get('tb1'); + const t2 = await stores.featureToggleStore.get('tb2'); + + expect(t2.lastSeenAt).toBeNull(); + expect(t1.lastSeenAt.getTime()).toBeGreaterThanOrEqual(time); + + service.destroy(); +}); + +test('Should not update anything for 0 toggles', async () => { + // jest.useFakeTimers(); + const service = new LastSeenService(stores, config, 30); + const time = Date.now(); + await stores.featureToggleStore.create('default', { name: 'tb1' }); + await stores.featureToggleStore.create('default', { name: 'tb2' }); + + const metrics: IClientMetricsEnv[] = [ + { + featureName: 'tb1', + appName: 'some-App', + environment: 'default', + timestamp: new Date(time), + yes: 0, + no: 0, + }, + { + featureName: 'tb2', + appName: 'some-App', + environment: 'default', + timestamp: new Date(time), + yes: 0, + no: 0, + }, + ]; + + service.updateLastSeen(metrics); + + // bypass interval waiting + const count = await service.store(); + + expect(count).toBe(0); + + service.destroy(); +});