✨ [RUMF-1467] Collect user data telemetry (#1941)

Collect user data telemetry
Refacto common context

packages/core/src/domain/telemetry/telemetry.ts

@@ -32,6 +32,7 @@ export const enum TelemetryService {
 export interface Telemetry {
   setContextProvider: (provider: () => Context) => void
   observable: Observable<TelemetryEvent & Context>
+  enabled: boolean
 }
 
 const TELEMETRY_EXCLUDED_SITES: string[] = [INTAKE_SITE_US1_FED]
@@ -49,12 +50,13 @@ export function startTelemetry(telemetryService: TelemetryService, configuration
   let contextProvider: () => Context
   const observable = new Observable<TelemetryEvent & Context>()
 
-  telemetryConfiguration.telemetryEnabled = performDraw(configuration.telemetrySampleRate)
+  telemetryConfiguration.telemetryEnabled =
+    !includes(TELEMETRY_EXCLUDED_SITES, configuration.site) && performDraw(configuration.telemetrySampleRate)
   telemetryConfiguration.telemetryConfigurationEnabled =
     telemetryConfiguration.telemetryEnabled && performDraw(configuration.telemetryConfigurationSampleRate)
 
   onRawTelemetryEventCollected = (rawEvent: RawTelemetryEvent) => {
-    if (!includes(TELEMETRY_EXCLUDED_SITES, configuration.site) && telemetryConfiguration.telemetryEnabled) {
+    if (telemetryConfiguration.telemetryEnabled) {
       const event = toTelemetryEvent(telemetryService, rawEvent)
       observable.notify(event)
       sendToExtension('telemetry', event)
@@ -90,6 +92,7 @@ export function startTelemetry(telemetryService: TelemetryService, configuration
       contextProvider = provider
     },
     observable,
+    enabled: telemetryConfiguration.telemetryEnabled,
   }
 }
 

packages/core/src/index.ts

@@ -47,6 +47,7 @@ export {
   Payload,
   createHttpRequest,
   Batch,
+  BatchFlushEvent,
   canUseEventBridge,
   getEventBridge,
   startBatchWithReplica,
@@ -81,7 +82,7 @@ export * from './browser/addEventListener'
 export { initConsoleObservable, ConsoleLog } from './domain/console/consoleObservable'
 export { BoundedBuffer } from './tools/boundedBuffer'
 export { catchUserErrors } from './tools/catchUserErrors'
-export { createContextManager } from './tools/contextManager'
+export { createContextManager, ContextManager } from './tools/contextManager'
 export { limitModification } from './tools/limitModification'
 export { ContextHistory, ContextHistoryEntry, CLEAR_OLD_CONTEXTS_INTERVAL } from './tools/contextHistory'
 export { readBytesFromStream } from './tools/readBytesFromStream'

packages/core/src/tools/contextManager.spec.ts

@@ -73,4 +73,30 @@ describe('createContextManager', () => {
     manager.clearContext()
     expect(manager.getContext()).toEqual({})
   })
+
+  it('should compute the bytes count only if the context has been updated', () => {
+    const computeBytesCountStub = jasmine.createSpy('computeBytesCountStub').and.returnValue(1)
+    const manager = createContextManager(computeBytesCountStub)
+
+    manager.getBytesCount()
+
+    manager.remove('foo')
+    manager.getBytesCount()
+
+    manager.set({ foo: 'bar' })
+    manager.getBytesCount()
+
+    manager.removeContextProperty('foo')
+    manager.getBytesCount()
+
+    manager.setContext({ foo: 'bar' })
+    manager.getBytesCount()
+
+    manager.clearContext()
+    manager.getBytesCount()
+    const bytesCount = manager.getBytesCount()
+
+    expect(bytesCount).toEqual(1)
+    expect(computeBytesCountStub).toHaveBeenCalledTimes(6)
+  })
 })

packages/core/src/tools/contextManager.ts

@@ -1,45 +1,60 @@
-import { deepClone } from './utils'
-
+import { computeBytesCount, deepClone, jsonStringify } from './utils'
 import type { Context, ContextValue } from './context'
 
-export function createContextManager() {
+export type ContextManager = ReturnType<typeof createContextManager>
+
+export function createContextManager(computeBytesCountImpl = computeBytesCount) {
   let context: Context = {}
+  let bytesCountCache: number | undefined
 
   return {
+    getBytesCount: () => {
+      if (bytesCountCache === undefined) {
+        bytesCountCache = computeBytesCountImpl(jsonStringify(context)!)
+      }
+      return bytesCountCache
+    },
     /** @deprecated use getContext instead */
     get: () => context,
 
     /** @deprecated use setContextProperty instead */
     add: (key: string, value: any) => {
       context[key] = value as ContextValue
+      bytesCountCache = undefined
     },
 
     /** @deprecated renamed to removeContextProperty */
     remove: (key: string) => {
       delete context[key]
+      bytesCountCache = undefined
     },
 
     /** @deprecated use setContext instead */
     set: (newContext: object) => {
       context = newContext as Context
+      bytesCountCache = undefined
     },
 
     getContext: () => deepClone(context),
 
     setContext: (newContext: Context) => {
       context = deepClone(newContext)
+      bytesCountCache = undefined
     },
 
     setContextProperty: (key: string, property: any) => {
       context[key] = deepClone(property)
+      bytesCountCache = undefined
     },
 
     removeContextProperty: (key: string) => {
       delete context[key]
+      bytesCountCache = undefined
     },
 
     clearContext: () => {
       context = {}
+      bytesCountCache = undefined
     },
   }
 }

packages/core/src/tools/utils.spec.ts

@@ -4,6 +4,7 @@ import { display } from './display'
 import {
   arrayFrom,
   combine,
+  computeBytesCount,
   cssEscape,
   deepClone,
   elementMatches,
@@ -689,3 +690,13 @@ describe('matchList', () => {
     expect(display.error).toHaveBeenCalled()
   })
 })
+
+describe('computeBytesCount', () => {
+  it('should count the bytes of a message composed of 1 byte characters', () => {
+    expect(computeBytesCount('1234')).toEqual(4)
+  })
+
+  it('should count the bytes of a message composed of multiple bytes characters', () => {
+    expect(computeBytesCount('🪐')).toEqual(4)
+  })
+})

packages/core/src/tools/utils.ts

@@ -594,3 +594,19 @@ export function cssEscape(str: string) {
     return `\\${ch}`
   })
 }
+
+// eslint-disable-next-line no-control-regex
+const HAS_MULTI_BYTES_CHARACTERS = /[^\u0000-\u007F]/
+
+export function computeBytesCount(candidate: string): number {
+  // Accurate bytes count computations can degrade performances when there is a lot of events to process
+  if (!HAS_MULTI_BYTES_CHARACTERS.test(candidate)) {
+    return candidate.length
+  }
+
+  if (window.TextEncoder !== undefined) {
+    return new TextEncoder().encode(candidate).length
+  }
+
+  return new Blob([candidate]).size
+}

packages/core/src/transport/batch.spec.ts

@@ -3,6 +3,7 @@ import type { PageExitEvent } from '../browser/pageExitObservable'
 import { PageExitReason } from '../browser/pageExitObservable'
 import { Observable } from '../tools/observable'
 import { noop } from '../tools/utils'
+import type { BatchFlushEvent } from './batch'
 import { Batch } from './batch'
 import type { HttpRequest } from './httpRequest'
 
@@ -15,6 +16,7 @@ describe('batch', () => {
   let transport: HttpRequest
   let sendSpy: jasmine.Spy<HttpRequest['send']>
   let pageExitObservable: Observable<PageExitEvent>
+  let flushNotifySpy: jasmine.Spy<() => BatchFlushEvent>
 
   beforeEach(() => {
     transport = { send: noop } as unknown as HttpRequest
@@ -28,6 +30,7 @@ describe('batch', () => {
       FLUSH_TIMEOUT,
       pageExitObservable
     )
+    flushNotifySpy = spyOn(batch.flushObservable, 'notify')
   })
 
   it('should add context to message', () => {
@@ -51,14 +54,6 @@ describe('batch', () => {
     expect(transport.send).not.toHaveBeenCalled()
   })
 
-  it('should count the bytes of a message composed of 1 byte characters', () => {
-    expect(batch.computeBytesCount('1234')).toEqual(4)
-  })
-
-  it('should count the bytes of a message composed of multiple bytes characters', () => {
-    expect(batch.computeBytesCount('🪐')).toEqual(4)
-  })
-
   it('should flush when the message count limit is reached', () => {
     batch.add({ message: '1' })
     batch.add({ message: '2' })
@@ -189,4 +184,10 @@ describe('batch', () => {
     batch.flush()
     expect(sendSpy).toHaveBeenCalledTimes(2)
   })
+
+  it('should notify when the batch is flushed', () => {
+    batch.add({})
+    batch.flush()
+    expect(flushNotifySpy).toHaveBeenCalledOnceWith({ bufferBytesCount: 2, bufferMessagesCount: 1 })
+  })
 })

packages/core/src/transport/batch.ts

@@ -1,16 +1,19 @@
 import { display } from '../tools/display'
 import type { Context } from '../tools/context'
-import { jsonStringify, objectValues } from '../tools/utils'
+import { computeBytesCount, jsonStringify, objectValues } from '../tools/utils'
 import { monitor } from '../tools/monitor'
-import type { Observable } from '../tools/observable'
+import { Observable } from '../tools/observable'
 import type { PageExitEvent } from '../browser/pageExitObservable'
 import type { HttpRequest } from './httpRequest'
 
-// https://en.wikipedia.org/wiki/UTF-8
-// eslint-disable-next-line no-control-regex
-const HAS_MULTI_BYTES_CHARACTERS = /[^\u0000-\u007F]/
+export interface BatchFlushEvent {
+  bufferBytesCount: number
+  bufferMessagesCount: number
+}
 
 export class Batch {
+  flushObservable = new Observable<BatchFlushEvent>()
+
   private pushOnlyBuffer: string[] = []
   private upsertBuffer: { [key: string]: string } = {}
   private bufferBytesCount = 0
@@ -41,6 +44,11 @@ export class Batch {
       const messages = this.pushOnlyBuffer.concat(objectValues(this.upsertBuffer))
       const bytesCount = this.bufferBytesCount
 
+      this.flushObservable.notify({
+        bufferBytesCount: this.bufferBytesCount,
+        bufferMessagesCount: this.bufferMessagesCount,
+      })
+
       this.pushOnlyBuffer = []
       this.upsertBuffer = {}
       this.bufferBytesCount = 0
@@ -50,19 +58,6 @@ export class Batch {
     }
   }
 
-  computeBytesCount(candidate: string) {
-    // Accurate bytes count computations can degrade performances when there is a lot of events to process
-    if (!HAS_MULTI_BYTES_CHARACTERS.test(candidate)) {
-      return candidate.length
-    }
-
-    if (window.TextEncoder !== undefined) {
-      return new TextEncoder().encode(candidate).length
-    }
-
-    return new Blob([candidate]).size
-  }
-
   private addOrUpdate(message: Context, key?: string) {
     const { processedMessage, messageBytesCount } = this.process(message)
     if (messageBytesCount >= this.messageBytesLimit) {
@@ -86,7 +81,7 @@ export class Batch {
 
   private process(message: Context) {
     const processedMessage = jsonStringify(message)!
-    const messageBytesCount = this.computeBytesCount(processedMessage)
+    const messageBytesCount = computeBytesCount(processedMessage)
     return { processedMessage, messageBytesCount }
   }
 
@@ -107,7 +102,7 @@ export class Batch {
   private remove(key: string) {
     const removedMessage = this.upsertBuffer[key]
     delete this.upsertBuffer[key]
-    const messageBytesCount = this.computeBytesCount(removedMessage)
+    const messageBytesCount = computeBytesCount(removedMessage)
     this.bufferBytesCount -= messageBytesCount
     this.bufferMessagesCount -= 1
     if (this.bufferMessagesCount > 0) {

packages/core/src/transport/index.ts

@@ -1,4 +1,4 @@
 export { HttpRequest, createHttpRequest, Payload, RetryInfo } from './httpRequest'
-export { Batch } from './batch'
+export { Batch, BatchFlushEvent } from './batch'
 export { canUseEventBridge, getEventBridge, BrowserWindowWithEventBridge } from './eventBridge'
 export { startBatchWithReplica } from './startBatchWithReplica'

packages/logs/src/boot/logsPublicApi.spec.ts

@@ -145,8 +145,8 @@ describe('logs entry', () => {
     it('should have the current date, view and global context', () => {
       LOGS.setGlobalContextProperty('foo', 'bar')
 
-      const getCommonContext = startLogs.calls.mostRecent().args[2]
-      expect(getCommonContext()).toEqual({
+      const buildCommonContext = startLogs.calls.mostRecent().args[2]
+      expect(buildCommonContext()).toEqual({
         view: {
           referrer: document.referrer,
           url: window.location.href,
@@ -346,8 +346,8 @@ describe('logs entry', () => {
         const user = { id: 'foo', name: 'bar', email: 'qux', foo: { bar: 'qux' } }
         logsPublicApi.setUser(user)
 
-        const getCommonContext = startLogs.calls.mostRecent().args[2]
-        expect(getCommonContext().user).toEqual({
+        const buildCommonContext = startLogs.calls.mostRecent().args[2]
+        expect(buildCommonContext().user).toEqual({
           email: 'qux',
           foo: { bar: 'qux' },
           id: 'foo',
@@ -358,8 +358,8 @@ describe('logs entry', () => {
       it('should sanitize predefined properties', () => {
         const user = { id: null, name: 2, email: { bar: 'qux' } }
         logsPublicApi.setUser(user as any)
-        const getCommonContext = startLogs.calls.mostRecent().args[2]
-        expect(getCommonContext().user).toEqual({
+        const buildCommonContext = startLogs.calls.mostRecent().args[2]
+        expect(buildCommonContext().user).toEqual({
           email: '[object Object]',
           id: 'null',
           name: '2',
@@ -371,8 +371,8 @@ describe('logs entry', () => {
         logsPublicApi.setUser(user)
         logsPublicApi.clearUser()
 
-        const getCommonContext = startLogs.calls.mostRecent().args[2]
-        expect(getCommonContext().user).toEqual({})
+        const buildCommonContext = startLogs.calls.mostRecent().args[2]
+        expect(buildCommonContext().user).toEqual({})
       })
 
       it('should reject non object input', () => {

packages/logs/src/boot/logsPublicApi.ts

@@ -45,7 +45,7 @@ export function makeLogsPublicApi(startLogsImpl: StartLogs) {
   let handleLogStrategy: StartLogsResult['handleLog'] = (
     logsMessage: LogsMessage,
     logger: Logger,
-    savedCommonContext = deepClone(getCommonContext()),
+    savedCommonContext = deepClone(buildCommonContext()),
     date = timeStampNow()
   ) => {
     beforeInitLoggerLog.add(() => handleLogStrategy(logsMessage, logger, savedCommonContext, date))
@@ -54,7 +54,7 @@ export function makeLogsPublicApi(startLogsImpl: StartLogs) {
   let getInitConfigurationStrategy = (): InitConfiguration | undefined => undefined
   const mainLogger = new Logger((...params) => handleLogStrategy(...params))
 
-  function getCommonContext(): CommonContext {
+  function buildCommonContext(): CommonContext {
     return {
       view: {
         referrer: document.referrer,
@@ -88,7 +88,7 @@ export function makeLogsPublicApi(startLogsImpl: StartLogs) {
       ;({ handleLog: handleLogStrategy, getInternalContext: getInternalContextStrategy } = startLogsImpl(
         initConfiguration,
         configuration,
-        getCommonContext,
+        buildCommonContext,
         mainLogger
       ))