✨ Collect user data measures with telemetry

packages/core/src/domain/configuration/configuration.ts

@@ -116,7 +116,7 @@ export function validateAndBuildConfiguration(initConfiguration: InitConfigurati
         initConfiguration.beforeSend && catchUserErrors(initConfiguration.beforeSend, 'beforeSend threw an error:'),
       cookieOptions: buildCookieOptions(initConfiguration),
       sessionSampleRate: sessionSampleRate ?? 100,
-      telemetrySampleRate: initConfiguration.telemetrySampleRate ?? 20,
+      telemetrySampleRate: initConfiguration.telemetrySampleRate ?? 100,
       telemetryConfigurationSampleRate: initConfiguration.telemetryConfigurationSampleRate ?? 5,
       service: initConfiguration.service,
       silentMultipleInit: !!initConfiguration.silentMultipleInit,

packages/core/src/index.ts

@@ -47,6 +47,7 @@ export {
   Payload,
   createHttpRequest,
   Batch,
+  BatchFlushEvent,
   canUseEventBridge,
   getEventBridge,
   startBatchWithReplica,
@@ -81,7 +82,7 @@ export * from './browser/addEventListener'
 export { initConsoleObservable, ConsoleLog } from './domain/console/consoleObservable'
 export { BoundedBuffer } from './tools/boundedBuffer'
 export { catchUserErrors } from './tools/catchUserErrors'
-export { createContextManager } from './tools/contextManager'
+export { createContextManager, contextBytesCounter, ContextManager } from './tools/contextManager'
 export { limitModification } from './tools/limitModification'
 export { ContextHistory, ContextHistoryEntry, CLEAR_OLD_CONTEXTS_INTERVAL } from './tools/contextHistory'
 export { readBytesFromStream } from './tools/readBytesFromStream'

packages/core/src/tools/contextManager.ts

@@ -1,45 +1,71 @@
-import { deepClone } from './utils'
-
+import { computeBytesCount, deepClone, isEmptyObject, jsonStringify } from './utils'
 import type { Context, ContextValue } from './context'
 
+export type ContextManager = ReturnType<typeof createContextManager>
+
 export function createContextManager() {
   let context: Context = {}
+  const bytesCounter = contextBytesCounter()
 
   return {
+    getBytesCount: () => bytesCounter.compute(context),
     /** @deprecated use getContext instead */
     get: () => context,
 
     /** @deprecated use setContextProperty instead */
     add: (key: string, value: any) => {
       context[key] = value as ContextValue
+      bytesCounter.invalidate()
     },
 
     /** @deprecated renamed to removeContextProperty */
     remove: (key: string) => {
       delete context[key]
+      bytesCounter.invalidate()
     },
 
     /** @deprecated use setContext instead */
     set: (newContext: object) => {
       context = newContext as Context
+      bytesCounter.invalidate()
     },
 
     getContext: () => deepClone(context),
 
     setContext: (newContext: Context) => {
       context = deepClone(newContext)
+      bytesCounter.invalidate()
     },
 
     setContextProperty: (key: string, property: any) => {
       context[key] = deepClone(property)
+      bytesCounter.invalidate()
     },
 
     removeContextProperty: (key: string) => {
       delete context[key]
+      bytesCounter.invalidate()
     },
 
     clearContext: () => {
       context = {}
+      bytesCounter.invalidate()
+    },
+  }
+}
+
+export function contextBytesCounter() {
+  let bytesCount: number | undefined
+
+  return {
+    compute: (context: Context) => {
+      if (bytesCount === undefined) {
+        bytesCount = !isEmptyObject(context) ? computeBytesCount(jsonStringify(context)!) : 0
+      }
+      return bytesCount
+    },
+    invalidate: () => {
+      bytesCount = undefined
     },
   }
 }

packages/core/src/tools/utils.ts

@@ -594,3 +594,19 @@ export function cssEscape(str: string) {
     return `\\${ch}`
   })
 }
+
+// eslint-disable-next-line no-control-regex
+const HAS_MULTI_BYTES_CHARACTERS = /[^\u0000-\u007F]/
+
+export function computeBytesCount(candidate: string): number {
+  // Accurate bytes count computations can degrade performances when there is a lot of events to process
+  if (!HAS_MULTI_BYTES_CHARACTERS.test(candidate)) {
+    return candidate.length
+  }
+
+  if (window.TextEncoder !== undefined) {
+    return new TextEncoder().encode(candidate).length
+  }
+
+  return new Blob([candidate]).size
+}

packages/core/src/transport/batch.ts

@@ -2,15 +2,22 @@ import { display } from '../tools/display'
 import type { Context } from '../tools/context'
 import { jsonStringify, objectValues } from '../tools/utils'
 import { monitor } from '../tools/monitor'
-import type { Observable } from '../tools/observable'
+import { Observable } from '../tools/observable'
 import type { PageExitEvent } from '../browser/pageExitObservable'
 import type { HttpRequest } from './httpRequest'
 
 // https://en.wikipedia.org/wiki/UTF-8
 // eslint-disable-next-line no-control-regex
 const HAS_MULTI_BYTES_CHARACTERS = /[^\u0000-\u007F]/
 
+export interface BatchFlushEvent {
+  bufferBytesCount: number
+  bufferMessagesCount: number
+}
+
 export class Batch {
+  flushObservable = new Observable<BatchFlushEvent>()
+
   private pushOnlyBuffer: string[] = []
   private upsertBuffer: { [key: string]: string } = {}
   private bufferBytesCount = 0
@@ -41,6 +48,11 @@ export class Batch {
       const messages = this.pushOnlyBuffer.concat(objectValues(this.upsertBuffer))
       const bytesCount = this.bufferBytesCount
 
+      this.flushObservable.notify({
+        bufferBytesCount: this.bufferBytesCount,
+        bufferMessagesCount: this.bufferMessagesCount,
+      })
+
       this.pushOnlyBuffer = []
       this.upsertBuffer = {}
       this.bufferBytesCount = 0

packages/core/src/transport/index.ts

@@ -1,4 +1,4 @@
 export { HttpRequest, createHttpRequest, Payload, RetryInfo } from './httpRequest'
-export { Batch } from './batch'
+export { Batch, BatchFlushEvent } from './batch'
 export { canUseEventBridge, getEventBridge, BrowserWindowWithEventBridge } from './eventBridge'
 export { startBatchWithReplica } from './startBatchWithReplica'

packages/rum-core/src/boot/rumPublicApi.ts

@@ -157,6 +157,8 @@ export function makeRumPublicApi(
         hasReplay: recorderApi.isRecording() ? true : undefined,
       }),
       recorderApi,
+      globalContextManager,
+      userContextManager,
       initialViewOptions
     )
 

packages/rum-core/src/boot/startRum.ts

@@ -1,4 +1,4 @@
-import type { Observable, TelemetryEvent, RawError } from '@datadog/browser-core'
+import type { Observable, TelemetryEvent, RawError, ContextManager } from '@datadog/browser-core'
 import {
   sendToExtension,
   createPageExitObservable,
@@ -33,13 +33,16 @@ import type { RumConfiguration, RumInitConfiguration } from '../domain/configura
 import { serializeRumConfiguration } from '../domain/configuration'
 import type { ViewOptions } from '../domain/rumEventsCollection/view/trackViews'
 import { startFeatureFlagContexts } from '../domain/contexts/featureFlagContext'
+import { startUserDataMeasurement } from '../domain/startUserDataMeasurement'
 import type { RecorderApi } from './rumPublicApi'
 
 export function startRum(
   initConfiguration: RumInitConfiguration,
   configuration: RumConfiguration,
   getCommonContext: () => CommonContext,
   recorderApi: RecorderApi,
+  globalContextManager: ContextManager,
+  userContextManager: ContextManager,
   initialViewOptions?: ViewOptions
 ) {
   const lifeCycle = new LifeCycle()
@@ -65,12 +68,13 @@ export function startRum(
   const reportError = (error: RawError) => {
     lifeCycle.notify(LifeCycleEventType.RAW_ERROR_COLLECTED, { error })
   }
+  let batch
   if (!canUseEventBridge()) {
     const pageExitObservable = createPageExitObservable()
     pageExitObservable.subscribe((event) => {
       lifeCycle.notify(LifeCycleEventType.PAGE_EXITED, event)
     })
-    startRumBatch(configuration, lifeCycle, telemetry.observable, reportError, pageExitObservable)
+    batch = startRumBatch(configuration, lifeCycle, telemetry.observable, reportError, pageExitObservable)
   } else {
     startRumEventBridge(lifeCycle)
   }
@@ -90,6 +94,16 @@ export function startRum(
       getCommonContext,
       reportError
     )
+
+  if (batch) {
+    startUserDataMeasurement(
+      lifeCycle,
+      globalContextManager,
+      userContextManager,
+      featureFlagContexts,
+      batch.flushObservable
+    )
+  }
   addTelemetryConfiguration(serializeRumConfiguration(initConfiguration))
 
   startLongTaskCollection(lifeCycle, session)

packages/rum-core/src/domain/contexts/featureFlagContext.ts

@@ -1,5 +1,6 @@
 import type { RelativeTime, ContextValue, Context } from '@datadog/browser-core'
 import {
+  contextBytesCounter,
   deepClone,
   noop,
   isExperimentalFeatureEnabled,
@@ -15,6 +16,7 @@ export type FeatureFlagContext = Context
 
 export interface FeatureFlagContexts {
   findFeatureFlagEvaluations: (startTime?: RelativeTime) => FeatureFlagContext | undefined
+  getFeatureFlagBytesCount: () => number
   addFeatureFlagEvaluation: (key: string, value: ContextValue) => void
 }
 
@@ -30,26 +32,38 @@ export function startFeatureFlagContexts(lifeCycle: LifeCycle): FeatureFlagConte
   if (!isExperimentalFeatureEnabled('feature_flags')) {
     return {
       findFeatureFlagEvaluations: () => undefined,
+      getFeatureFlagBytesCount: () => 0,
       addFeatureFlagEvaluation: noop,
     }
   }
 
   const featureFlagContexts = new ContextHistory<FeatureFlagContext>(FEATURE_FLAG_CONTEXT_TIME_OUT_DELAY)
+  const bytesCounter = contextBytesCounter()
 
   lifeCycle.subscribe(LifeCycleEventType.VIEW_ENDED, ({ endClocks }) => {
     featureFlagContexts.closeActive(endClocks.relative)
   })
 
   lifeCycle.subscribe(LifeCycleEventType.VIEW_CREATED, ({ startClocks }) => {
     featureFlagContexts.add({}, startClocks.relative)
+    bytesCounter.invalidate()
   })
 
   return {
     findFeatureFlagEvaluations: (startTime?: RelativeTime) => featureFlagContexts.find(startTime),
+    getFeatureFlagBytesCount: () => {
+      const currentContext = featureFlagContexts.find()
+      if (!currentContext) {
+        return 0
+      }
+
+      return bytesCounter.compute(currentContext)
+    },
     addFeatureFlagEvaluation: (key: string, value: ContextValue) => {
       const currentContext = featureFlagContexts.find()
       if (currentContext) {
         currentContext[key] = deepClone(value)
+        bytesCounter.invalidate()
       }
     },
   }

packages/rum-core/src/domain/startUserDataMeasurement.ts

@@ -0,0 +1,85 @@
+import type { BatchFlushEvent, Context, ContextManager, Observable } from '@datadog/browser-core'
+import { ONE_SECOND, addTelemetryDebug, monitor } from '@datadog/browser-core'
+import { RumEventType } from '../rawRumEvent.types'
+import type { RumEvent } from '../rumEvent.types'
+import type { FeatureFlagContexts } from './contexts/featureFlagContext'
+import type { LifeCycle } from './lifeCycle'
+import { LifeCycleEventType } from './lifeCycle'
+
+export const MEASURES_FLUSH_INTERVAL = 10 * ONE_SECOND
+
+type Measure = {
+  min: number
+  max: number
+  sum: number
+}
+
+let batchCount = 0
+let batchBytesCount: Measure | undefined
+let batchMessagesCount: Measure | undefined
+let globalContextBytes: Measure | undefined
+let userContextBytes: Measure | undefined
+let featureFlagBytes: Measure | undefined
+
+export function startUserDataMeasurement(
+  lifeCycle: LifeCycle,
+  globalContextManager: ContextManager,
+  userContextManager: ContextManager,
+  featureFlagContexts: FeatureFlagContexts,
+  batchFlushObservable: Observable<BatchFlushEvent>
+) {
+  lifeCycle.subscribe(LifeCycleEventType.RUM_EVENT_COLLECTED, (event: RumEvent & Context) => {
+    globalContextBytes = computeMeasure(globalContextBytes, globalContextManager.getBytesCount())
+    userContextBytes = computeMeasure(userContextBytes, userContextManager.getBytesCount())
+
+    if ([RumEventType.VIEW, RumEventType.ERROR].includes(event.type as RumEventType)) {
+      featureFlagBytes = computeMeasure(featureFlagBytes, featureFlagContexts.getFeatureFlagBytesCount())
+    }
+  })
+
+  batchFlushObservable.subscribe(({ bufferBytesCount, bufferMessagesCount }) => {
+    batchCount += 1
+    batchBytesCount = computeMeasure(batchBytesCount, bufferBytesCount)
+    batchMessagesCount = computeMeasure(batchMessagesCount, bufferMessagesCount)
+  })
+
+  setInterval(monitor(sendMeasures), MEASURES_FLUSH_INTERVAL)
+}
+
+function sendMeasures() {
+  if (batchCount === 0) {
+    return
+  }
+
+  addTelemetryDebug('User contexts measures', {
+    batchCount,
+    batchBytesCount,
+    batchMessagesCount,
+    globalContextBytes,
+    userContextBytes,
+    featureFlagBytes,
+  })
+  clearMeasures()
+}
+
+function computeMeasure({ min, max, sum }: Measure = { min: Infinity, max: 0, sum: 0 }, newValue: number) {
+  // only measure when there is some data. It avoid collecting the measure of un unused feature (like global context, etc..).
+  if (sum === 0 && newValue === 0) {
+    return undefined
+  }
+
+  return {
+    sum: sum + newValue,
+    min: Math.min(min, newValue),
+    max: Math.max(max, newValue),
+  }
+}
+
+function clearMeasures() {
+  batchCount = 0
+  batchBytesCount = undefined
+  batchMessagesCount = undefined
+  globalContextBytes = undefined
+  userContextBytes = undefined
+  featureFlagBytes = undefined
+}