Context Spy

examples/README.md

@@ -50,6 +50,16 @@ pnpm watch
   - Whether or not to log the latest FPS sample to the console every 1 second.
   - After skipping the first 10 samples, every 100 samples after that will result
     in a statistics summary printed to the console.
+- `contextSpy` (boolean, default: "false")
+  - Whether or not to turn on the context spy and reporting
+  - The context spy intercepts all calls to the (WebGL) context and reports
+    how many calls to each function occurred during the last FPS sampling period
+    (1 second for these tests).
+  - Statistical results of every context call will be reported along with the
+    FPS statistics summary.
+  - `fps` must be enabled in order to see any reporting.
+  - Enabling the context spy has a serious impact on performance so only use it
+    when you need to extract context call information.
 - `ppr` (number, default: 1)
   - Device physical pixel ratio.
 - `multiplier` (number, default: 1)

examples/common/StatTracker.ts

@@ -0,0 +1,119 @@
+/*
+ * If not stated otherwise in this file or this component's LICENSE file the
+ * following copyright and licenses apply:
+ *
+ * Copyright 2023 Comcast Cable Communications Management, LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the License);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Simple utility class for capturing a set of sample groups and
+ * calculating statistics on them.
+ */
+export class StatTracker {
+  private data: Record<string, number[]> = {};
+
+  /**
+   * Clear all sample groups
+   */
+  reset() {
+    this.data = {};
+  }
+
+  /**
+   * Add a value to a sample group
+   *
+   * @param sampleGroup
+   * @param value
+   */
+  add(sampleGroup: string, value: number) {
+    if (!this.data[sampleGroup]) {
+      this.data[sampleGroup] = [];
+    }
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    this.data[sampleGroup]!.push(value);
+  }
+
+  /**
+   * Get the percentile value for a sample group
+   *
+   * @param sampleGroup
+   * @param percentile
+   * @returns
+   */
+  getPercentile(sampleGroup: string, percentile: number) {
+    const values = this.data[sampleGroup];
+    if (!values) {
+      return 0;
+    }
+    values.sort((a, b) => a - b);
+    const index = Math.floor((percentile / 100) * values.length);
+    return values[index]!;
+  }
+
+  /**
+   * Get the standard deviation for a sample group
+   *
+   * @param sampleGroup
+   * @returns
+   */
+  getStdDev(sampleGroup: string) {
+    const values = this.data[sampleGroup];
+    if (!values) {
+      return 0;
+    }
+    const mean = values.reduce((a, b) => a + b, 0) / values.length;
+    const variance =
+      values.map((value) => Math.pow(value - mean, 2)).reduce((a, b) => a + b) /
+      values.length;
+    return Math.sqrt(variance);
+  }
+
+  /**
+   * Get the average value for a sample group
+   *
+   * @param sampleGroup
+   * @returns
+   */
+  getAverage(sampleGroup: string) {
+    const values = this.data[sampleGroup];
+    if (!values) {
+      return 0;
+    }
+    return values.reduce((a, b) => a + b, 0) / values.length;
+  }
+
+  /**
+   * Get the sample count for a sample group
+   *
+   * @param sampleGroup
+   * @returns
+   */
+  getCount(sampleGroup: string) {
+    const values = this.data[sampleGroup];
+    if (!values) {
+      return 0;
+    }
+    return values.length;
+  }
+
+  /**
+   * Get the names of all the sample groups
+   *
+   * @returns
+   */
+  getSampleGroups() {
+    return Object.keys(this.data);
+  }
+}

examples/index.ts

@@ -24,11 +24,13 @@ import {
   type ICoreDriver,
   type NodeLoadedPayload,
   type RendererMainSettings,
+  type FpsUpdatePayload,
 } from '@lightningjs/renderer';
 import { assertTruthy } from '@lightningjs/renderer/utils';
 import coreWorkerUrl from './common/CoreWorker.js?importChunkUrl';
 import coreExtensionModuleUrl from './common/AppCoreExtension.js?importChunkUrl';
 import type { ExampleSettings } from './common/ExampleSettings.js';
+import { StatTracker } from './common/StatTracker.js';
 
 interface TestModule {
   default: (settings: ExampleSettings) => Promise<void>;
@@ -60,29 +62,13 @@ const defaultResolution = 720;
 const defaultPhysicalPixelRatio = 1;
 
 (async () => {
-  // URL params
-  // - driver: main | threadx (default: threadx)
-  // - test: <test name> (default: test)
-  // - resolution: <number> (default: 720)
-  //   - Resolution (height) of to render the test at (in logical pixels)
-  // - ppr: <number> (default: 1)
-  //   - Device physical pixel ratio
-  // - showOverlay: true | false (default: true)
-  // - fps: true | false (default: false)
-  //   - Log FPS to console every second
-  // - multiplier: <number> (default: 1)
-  //   - In tests that support it, multiply the number of objects created by
-  //     this number. Useful for performance tests.
-  // - finalizationRegistry: true | false (default: false)
-  //   - Use FinalizationRegistryTextureUsageTracker instead of
-  //     ManualCountTextureUsageTracker
-  // - automation: true | false (default: false)
-  //   - Run all tests in automation mode
+  // See README.md for details on the supported URL params
   const urlParams = new URLSearchParams(window.location.search);
   const automation = urlParams.get('automation') === 'true';
   const test = urlParams.get('test') || (automation ? null : 'test');
   const showOverlay = urlParams.get('overlay') !== 'false';
   const logFps = urlParams.get('fps') === 'true';
+  const enableContextSpy = urlParams.get('contextSpy') === 'true';
   const perfMultiplier = Number(urlParams.get('multiplier')) || 1;
   const resolution = Number(urlParams.get('resolution')) || 720;
   const physicalPixelRatio =
@@ -103,6 +89,7 @@ const defaultPhysicalPixelRatio = 1;
       logicalPixelRatio,
       physicalPixelRatio,
       logFps,
+      enableContextSpy,
       perfMultiplier,
     );
     return;
@@ -121,6 +108,7 @@ async function runTest(
   logicalPixelRatio: number,
   physicalPixelRatio: number,
   logFps: boolean,
+  enableContextSpy: boolean,
   perfMultiplier: number,
 ) {
   const testModule = testModules[getTestPath(test)];
@@ -139,6 +127,7 @@ async function runTest(
   const { renderer, appElement } = await initRenderer(
     driverName,
     logFps,
+    enableContextSpy,
     logicalPixelRatio,
     physicalPixelRatio,
     customSettings,
@@ -182,6 +171,7 @@ async function runTest(
 async function initRenderer(
   driverName: string,
   logFps: boolean,
+  enableContextSpy: boolean,
   logicalPixelRatio: number,
   physicalPixelRatio: number,
   customSettings?: Partial<RendererMainSettings>,
@@ -205,16 +195,17 @@ async function initRenderer(
       clearColor: 0x00000000,
       coreExtensionModule: coreExtensionModuleUrl,
       fpsUpdateInterval: logFps ? 1000 : 0,
+      enableContextSpy,
       ...customSettings,
     },
     'app',
     driver,
   );
 
   /**
-   * FPS sample captured
+   * Sample data captured
    */
-  const fpsSamples: number[] = [];
+  const samples: StatTracker = new StatTracker();
   /**
    * Number of samples to capture before calculating FPS stats
    */
@@ -228,39 +219,71 @@ async function initRenderer(
    */
   let fpsSampleIndex = 0;
   let fpsSamplesLeft = fpsSampleCount;
-  renderer.on('fpsUpdate', (target: RendererMain, fps: number) => {
-    const captureSample = fpsSampleIndex >= fpsSampleSkipCount;
-    if (captureSample) {
-      fpsSamples.push(fps);
-      fpsSamplesLeft--;
-      if (fpsSamplesLeft === 0) {
-        const sortedSamples = fpsSamples.sort((a, b) => a - b);
-        const averageFps =
-          fpsSamples.reduce((a, b) => a + b, 0) / fpsSamples.length;
-        const p01Fps = sortedSamples[Math.floor(fpsSamples.length * 0.01)]!;
-        const p05Fps = sortedSamples[Math.floor(fpsSamples.length * 0.05)]!;
-        const p25Fps = sortedSamples[Math.floor(fpsSamples.length * 0.25)]!;
-        const medianFps = sortedSamples[Math.floor(fpsSamples.length * 0.5)]!;
-        const stdDevFps = Math.sqrt(
-          fpsSamples.reduce((a, b) => a + (b - averageFps) ** 2, 0) /
-            fpsSamples.length,
-        );
-        console.log(`---------------------------------`);
-        console.log(`Average FPS: ${averageFps}`);
-        console.log(`Median FPS: ${medianFps}`);
-        console.log(`P01 FPS: ${p01Fps}`);
-        console.log(`P05 FPS: ${p05Fps}`);
-        console.log(`P25 FPS: ${p25Fps}`);
-        console.log(`Std Dev FPS: ${stdDevFps}`);
-        console.log(`Num samples: ${fpsSamples.length}`);
-        console.log(`---------------------------------`);
-        fpsSamples.length = 0;
-        fpsSamplesLeft = fpsSampleCount;
+  renderer.on(
+    'fpsUpdate',
+    (target: RendererMain, fpsData: FpsUpdatePayload) => {
+      const captureSample = fpsSampleIndex >= fpsSampleSkipCount;
+      if (captureSample) {
+        samples.add('fps', fpsData.fps);
+
+        if (fpsData.contextSpyData) {
+          let totalCalls = 0;
+          for (const key in fpsData.contextSpyData) {
+            const numCalls = fpsData.contextSpyData[key]!;
+            totalCalls += numCalls;
+            samples.add(key, numCalls);
+          }
+          samples.add('totalCalls', totalCalls);
+        }
+
+        fpsSamplesLeft--;
+        if (fpsSamplesLeft === 0) {
+          const averageFps = samples.getAverage('fps');
+          const p01Fps = samples.getPercentile('fps', 1);
+          const p05Fps = samples.getPercentile('fps', 5);
+          const p25Fps = samples.getPercentile('fps', 25);
+          const medianFps = samples.getPercentile('fps', 50);
+          const stdDevFps = samples.getStdDev('fps');
+          console.log(`---------------------------------`);
+          console.log(`Average FPS: ${averageFps}`);
+          console.log(`Median FPS: ${medianFps}`);
+          console.log(`P01 FPS: ${p01Fps}`);
+          console.log(`P05 FPS: ${p05Fps}`);
+          console.log(`P25 FPS: ${p25Fps}`);
+          console.log(`Std Dev FPS: ${stdDevFps}`);
+          console.log(`Num samples: ${samples.getCount('fps')}`);
+          console.log(`---------------------------------`);
+
+          // Print out median data for all context spy data
+          if (fpsData.contextSpyData) {
+            const contextKeys = samples
+              .getSampleGroups()
+              .filter((key) => key !== 'fps' && key !== 'totalCalls');
+            // Print out median data for all context spy data
+            for (const key of contextKeys) {
+              const median = samples.getPercentile(key, 50);
+              console.log(
+                `median(${key}) / median(fps): ${Math.round(
+                  median / medianFps,
+                )}`,
+              );
+            }
+            const medianTotalCalls = samples.getPercentile('totalCalls', 50);
+            console.log(
+              `median(totalCalls) / median(fps): ${Math.round(
+                medianTotalCalls / medianFps,
+              )}`,
+            );
+            console.log(`---------------------------------`);
+          }
+          samples.reset();
+          fpsSamplesLeft = fpsSampleCount;
+        }
       }
-    }
-    console.log(`FPS: ${fps} (samples left: ${fpsSamplesLeft})`);
-    fpsSampleIndex++;
-  });
+      console.log(`FPS: ${fpsData.fps} (samples left: ${fpsSamplesLeft})`);
+      fpsSampleIndex++;
+    },
+  );
 
   await renderer.init();
 
@@ -276,6 +299,7 @@ async function runAutomation(driverName: string, logFps: boolean) {
   const { renderer, appElement } = await initRenderer(
     driverName,
     logFps,
+    false,
     logicalPixelRatio,
     defaultPhysicalPixelRatio,
   );

src/common/CommonTypes.ts

@@ -94,3 +94,12 @@ export type NodeFailedEventHandler = (
   target: any,
   payload: NodeFailedPayload,
 ) => void;
+
+/**
+ * Event payload for when an FpsUpdate event is emitted by either the Stage or
+ * MainRenderer
+ */
+export interface FpsUpdatePayload {
+  fps: number;
+  contextSpyData: Record<string, number> | null;
+}