From 3602a59ac9d345d4a464b68a974a17cd5d9c7ab4 Mon Sep 17 00:00:00 2001 From: timamura Date: Thu, 28 Jul 2022 15:37:55 -0700 Subject: [PATCH] feat: use markers config to decide if end at lcp --- packages/cli/markdown/compare.md | 12 +-- packages/cli/markdown/record-har.md | 4 +- packages/cli/oclif.manifest.json | 3 +- packages/cli/src/command-config/tb-config.ts | 5 - packages/cli/src/commands/compare/index.ts | 10 -- packages/cli/src/helpers/flags.ts | 10 -- packages/cli/src/helpers/utils.ts | 2 +- packages/cli/tb-schema.json | 13 +-- packages/cli/test/commands/compare.test.ts | 66 ++---------- packages/core/package.json | 2 +- packages/core/src/create-trace-benchmark.ts | 2 - .../src/create-trace-navigation-benchmark.ts | 39 +++---- packages/core/src/index.ts | 7 +- .../src/metrics/extract-navigation-sample.ts | 33 ++++-- packages/core/src/trace/index.ts | 12 ++- packages/core/src/trace/utils.ts | 53 +++++++++ .../core/src/util/inject-mark-observer.ts | 101 ++++++------------ packages/core/test/test.ts | 54 +++++++++- packages/core/test/trace/trace-utils-test.ts | 55 ++++++++++ 19 files changed, 275 insertions(+), 208 deletions(-) create mode 100644 packages/core/test/trace/trace-utils-test.ts diff --git a/packages/cli/markdown/compare.md b/packages/cli/markdown/compare.md index c3fc8a0e..199e8fae 100644 --- a/packages/cli/markdown/compare.md +++ b/packages/cli/markdown/compare.md @@ -60,9 +60,6 @@ OPTIONS --isCIEnv=isCIEnv Provides a drastically slimmed down stdout report for CI workflows. However does NOT hide analysis. - --lcpRegex=lcpRegex - The regex pattern to identify the LCP candidate, if undefined, we use the first LCP candidate - --markers=markers (required) [default: domComplete] User Timing Markers @@ -89,12 +86,9 @@ OPTIONS --tbResultsFolder=tbResultsFolder (required) [default: ./tracerbench-results] The output folder path for all tracerbench results - - --traceEndAtLcp - Overrides the default loadEventEnd as trace end or the last marker in markers array as tracerbench ``` -_See code: [dist/src/commands/compare/index.ts](https://github.com/TracerBench/tracerbench/blob/v6.1.2/dist/src/commands/compare/index.ts)_ +_See code: [dist/src/commands/compare/index.ts](https://github.com/TracerBench/tracerbench/blob/v7.0.0/dist/src/commands/compare/index.ts)_ ## `tracerbench compare:analyze RESULTSFILE` @@ -124,7 +118,7 @@ OPTIONS threshold runs against. ``` -_See code: [dist/src/commands/compare/analyze.ts](https://github.com/TracerBench/tracerbench/blob/v6.1.2/dist/src/commands/compare/analyze.ts)_ +_See code: [dist/src/commands/compare/analyze.ts](https://github.com/TracerBench/tracerbench/blob/v7.0.0/dist/src/commands/compare/analyze.ts)_ ## `tracerbench compare:report` @@ -150,4 +144,4 @@ ALIASES $ tracerbench report ``` -_See code: [dist/src/commands/compare/report.ts](https://github.com/TracerBench/tracerbench/blob/v6.1.2/dist/src/commands/compare/report.ts)_ +_See code: [dist/src/commands/compare/report.ts](https://github.com/TracerBench/tracerbench/blob/v7.0.0/dist/src/commands/compare/report.ts)_ diff --git a/packages/cli/markdown/record-har.md b/packages/cli/markdown/record-har.md index 5b6acac3..9917faef 100644 --- a/packages/cli/markdown/record-har.md +++ b/packages/cli/markdown/record-har.md @@ -39,7 +39,7 @@ OPTIONS --url=url (required) URL to visit for record-har, auth, timings & trace commands ``` -_See code: [dist/src/commands/record-har/index.ts](https://github.com/TracerBench/tracerbench/blob/v6.1.2/dist/src/commands/record-har/index.ts)_ +_See code: [dist/src/commands/record-har/index.ts](https://github.com/TracerBench/tracerbench/blob/v7.0.0/dist/src/commands/record-har/index.ts)_ ## `tracerbench record-har:auth` @@ -73,4 +73,4 @@ OPTIONS --username=username (required) The username to login to the form ``` -_See code: [dist/src/commands/record-har/auth.ts](https://github.com/TracerBench/tracerbench/blob/v6.1.2/dist/src/commands/record-har/auth.ts)_ +_See code: [dist/src/commands/record-har/auth.ts](https://github.com/TracerBench/tracerbench/blob/v7.0.0/dist/src/commands/record-har/auth.ts)_ diff --git a/packages/cli/oclif.manifest.json b/packages/cli/oclif.manifest.json index cdd0f2da..b4c15da0 100644 --- a/packages/cli/oclif.manifest.json +++ b/packages/cli/oclif.manifest.json @@ -1,2 +1 @@ -{"version":"7.0.0","commands":{"compare:analyze":{"id":"compare:analyze","description":"Generates stdout report from the \"tracerbench compare\" command output, 'compare.json'","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"fidelity":{"name":"fidelity","type":"option","description":"Directly correlates to the number of samples per trace. eg. test,low,medium,high OR any number between 2-100","required":true,"default":"low"},"regressionThreshold":{"name":"regressionThreshold","type":"option","description":"The upper limit the experiment can regress slower in milliseconds. eg 50","required":true,"default":50},"isCIEnv":{"name":"isCIEnv","type":"option","description":"Provides a drastically slimmed down stdout report for CI workflows. However does NOT hide analysis.","required":true,"default":false},"regressionThresholdStat":{"name":"regressionThresholdStat","type":"option","description":"The statistic which the regression threshold runs against.","options":["estimator","ci-lower","ci-upper"],"default":"estimator"},"jsonReport":{"name":"jsonReport","type":"boolean","description":"Include a JSON file from the stdout report","allowNo":false}},"args":[{"name":"resultsFile","description":"The \"tracerbench compare\" command json output file","required":true}]},"compare":{"id":"compare","description":"Compare the performance delta between an experiment and control","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"hideAnalysis":{"name":"hideAnalysis","type":"boolean","description":"Hide the the analysis output in terminal","allowNo":false},"browserArgs":{"name":"browserArgs","type":"option","description":"(Default Recommended) Additional chrome flags for the TracerBench render benchmark. TracerBench includes many non-configurable defaults in this category.","required":true,"default":["--crash-dumps-dir=./tmp","--disable-background-timer-throttling","--disable-dev-shm-usage","--disable-cache","--disable-v8-idle-tasks","--disable-breakpad","--disable-notifications","--disable-hang-monitor","--safebrowsing-disable-auto-update","--ignore-certificate-errors","--v8-cache-options=none"]},"cpuThrottleRate":{"name":"cpuThrottleRate","type":"option","description":"CPU throttle multiplier","required":true,"default":0},"fidelity":{"name":"fidelity","type":"option","description":"Directly correlates to the number of samples per trace. eg. test,low,medium,high OR any number between 2-100","required":true,"default":"low"},"markers":{"name":"markers","type":"option","description":"User Timing Markers","required":true,"default":"domComplete"},"network":{"name":"network","type":"option","description":"Simulated network conditions.","required":true,"options":["none","offline","dialup","slow-2g","2g","slow-edge","edge","slow-3g","dsl","3g","fast-3g","4g","cable","LTE","FIOS"],"default":"none"},"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","required":true,"default":"./tracerbench-results"},"controlURL":{"name":"controlURL","type":"option","description":"Control URL to visit for compare command","required":false},"experimentURL":{"name":"experimentURL","type":"option","description":"Experiment URL to visit for compare command","required":false},"emulateDevice":{"name":"emulateDevice","type":"option","description":"Emulate a mobile device screen size.","options":["iphone-4","iphone-5se","iphone-678","iphone-678-plus","iphone-x","blackberry-z30","nexus-4","nexus-5","nexus-5x","nexus-6","nexus-6p","pixel-2","pixel-2-xl","lg-optimus-l70","nokia-n9","nokia-lumia-520","microsoft-lumia-550","microsoft-lumia-950","galaxy-s-iii","galaxy-s5","kindle-fire-hdx","ipad-mini","ipad","ipad-pro","blackberry-playbook","nexus-10","nexus-7","galaxy-note-3","galaxy-note-ii","laptop-with-touch","laptop-with-hidpi-screen","laptop-with-mdpi-screen"],"default":""},"emulateDeviceOrientation":{"name":"emulateDeviceOrientation","type":"option","description":"Expected to be either \"vertical\" or \"horizontal\". Dictates orientation of device screen.","options":["horizontal","vertical"],"default":"vertical"},"socksPorts":{"name":"socksPorts","type":"option","description":"Specify a socks proxy port as browser option for control and experiment"},"regressionThreshold":{"name":"regressionThreshold","type":"option","description":"The upper limit the experiment can regress slower in milliseconds. eg 50","default":50},"sampleTimeout":{"name":"sampleTimeout","type":"option","description":"The number of seconds to wait for a sample.","default":30},"config":{"name":"config","type":"option","description":"Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all."},"runtimeStats":{"name":"runtimeStats","type":"boolean","description":"Compare command output deep-dive stats during run.","allowNo":false},"report":{"name":"report","type":"boolean","description":"Generate a PDF report directly after running the compare command.","allowNo":false},"debug":{"name":"debug","type":"boolean","description":"Debug flag per command. Will output noisy command","allowNo":false},"headless":{"name":"headless","type":"boolean","description":"Run with headless chrome flags","allowNo":false},"isCIEnv":{"name":"isCIEnv","type":"option","description":"Provides a drastically slimmed down stdout report for CI workflows. However does NOT hide analysis.","default":false},"regressionThresholdStat":{"name":"regressionThresholdStat","type":"option","description":"The statistic which the regression threshold runs against.","options":["estimator","ci-lower","ci-upper"],"default":"estimator"},"traceEndAtLcp":{"name":"traceEndAtLcp","type":"boolean","description":"Overrides the default loadEventEnd as trace end or the last marker in markers array as tracerbench","allowNo":false},"lcpRegex":{"name":"lcpRegex","type":"option","description":"The regex pattern to identify the LCP candidate, if undefined, we use the first LCP candidate","required":false}},"args":[]},"compare:report":{"id":"compare:report","description":"Generates report files (PDF/HTML) from the \"tracerbench compare\" command output","pluginName":"tracerbench","pluginType":"core","aliases":["report"],"flags":{"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","required":true,"default":"./tracerbench-results"},"config":{"name":"config","type":"option","description":"Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all."},"plotTitle":{"name":"plotTitle","type":"option","description":"Specify the title of the report pdf/html files.","default":"TracerBench"},"isCIEnv":{"name":"isCIEnv","type":"option","description":"Provides a drastically slimmed down stdout report for CI workflows. However does NOT hide analysis.","default":false}},"args":[]},"record-har:auth":{"id":"record-har:auth","description":"Authenticate with a given login URL, username, password and retrieve auth cookies","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"url":{"name":"url","type":"option","description":"URL to visit for record-har, auth, timings & trace commands","required":true},"dest":{"name":"dest","type":"option","description":"The destination path for the generated file. Default process.cwd()","required":true,"default":""},"filename":{"name":"filename","type":"option","description":"The filename for the generated file","required":true,"default":"cookies"},"username":{"name":"username","type":"option","description":"The username to login to the form","required":true},"password":{"name":"password","type":"option","description":"The password to login to the form","required":true},"config":{"name":"config","type":"option","description":"Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all."},"headless":{"name":"headless","type":"boolean","description":"Run with headless chrome flags","allowNo":false},"screenshots":{"name":"screenshots","type":"boolean","description":"Include chrome screenshots from command execution","allowNo":false},"proxy":{"name":"proxy","type":"option","description":"Uses a specified proxy server, overrides system settings. Only affects HTTP and HTTPS requests.","required":false},"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","default":"./tracerbench-results"}},"args":[]},"record-har":{"id":"record-har","description":"Generates a HAR file from a URL","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"url":{"name":"url","type":"option","description":"URL to visit for record-har, auth, timings & trace commands","required":true},"dest":{"name":"dest","type":"option","description":"The destination path for the generated file. Default process.cwd()","required":true,"default":""},"cookiespath":{"name":"cookiespath","type":"option","description":"The path to a JSON file containing cookies to authenticate against the correlated URL","required":true,"default":""},"filename":{"name":"filename","type":"option","description":"The filename for the generated file","required":true,"default":"tracerbench"},"marker":{"name":"marker","type":"option","description":"The last marker before ending a HAR recording","required":true,"default":"loadEventEnd"},"config":{"name":"config","type":"option","description":"Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all."},"headless":{"name":"headless","type":"boolean","description":"Run with headless chrome flags","allowNo":false},"screenshots":{"name":"screenshots","type":"boolean","description":"Include chrome screenshots from command execution","allowNo":false},"proxy":{"name":"proxy","type":"option","description":"Uses a specified proxy server, overrides system settings. Only affects HTTP and HTTPS requests.","required":false},"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","default":"./tracerbench-results"}},"args":[]}}} - +{"version":"7.0.0","commands":{"compare:analyze":{"id":"compare:analyze","description":"Generates stdout report from the \"tracerbench compare\" command output, 'compare.json'","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"fidelity":{"name":"fidelity","type":"option","description":"Directly correlates to the number of samples per trace. eg. test,low,medium,high OR any number between 2-100","required":true,"default":"low"},"regressionThreshold":{"name":"regressionThreshold","type":"option","description":"The upper limit the experiment can regress slower in milliseconds. eg 50","required":true,"default":50},"isCIEnv":{"name":"isCIEnv","type":"option","description":"Provides a drastically slimmed down stdout report for CI workflows. However does NOT hide analysis.","required":true,"default":false},"regressionThresholdStat":{"name":"regressionThresholdStat","type":"option","description":"The statistic which the regression threshold runs against.","options":["estimator","ci-lower","ci-upper"],"default":"estimator"},"jsonReport":{"name":"jsonReport","type":"boolean","description":"Include a JSON file from the stdout report","allowNo":false}},"args":[{"name":"resultsFile","description":"The \"tracerbench compare\" command json output file","required":true}]},"compare":{"id":"compare","description":"Compare the performance delta between an experiment and control","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"hideAnalysis":{"name":"hideAnalysis","type":"boolean","description":"Hide the the analysis output in terminal","allowNo":false},"browserArgs":{"name":"browserArgs","type":"option","description":"(Default Recommended) Additional chrome flags for the TracerBench render benchmark. TracerBench includes many non-configurable defaults in this category.","required":true,"default":["--crash-dumps-dir=./tmp","--disable-background-timer-throttling","--disable-dev-shm-usage","--disable-cache","--disable-v8-idle-tasks","--disable-breakpad","--disable-notifications","--disable-hang-monitor","--safebrowsing-disable-auto-update","--ignore-certificate-errors","--v8-cache-options=none"]},"cpuThrottleRate":{"name":"cpuThrottleRate","type":"option","description":"CPU throttle multiplier","required":true,"default":0},"fidelity":{"name":"fidelity","type":"option","description":"Directly correlates to the number of samples per trace. eg. test,low,medium,high OR any number between 2-100","required":true,"default":"low"},"markers":{"name":"markers","type":"option","description":"User Timing Markers","required":true,"default":"domComplete"},"network":{"name":"network","type":"option","description":"Simulated network conditions.","required":true,"options":["none","offline","dialup","slow-2g","2g","slow-edge","edge","slow-3g","dsl","3g","fast-3g","4g","cable","LTE","FIOS"],"default":"none"},"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","required":true,"default":"./tracerbench-results"},"controlURL":{"name":"controlURL","type":"option","description":"Control URL to visit for compare command","required":false},"experimentURL":{"name":"experimentURL","type":"option","description":"Experiment URL to visit for compare command","required":false},"emulateDevice":{"name":"emulateDevice","type":"option","description":"Emulate a mobile device screen size.","options":["iphone-4","iphone-5se","iphone-678","iphone-678-plus","iphone-x","blackberry-z30","nexus-4","nexus-5","nexus-5x","nexus-6","nexus-6p","pixel-2","pixel-2-xl","lg-optimus-l70","nokia-n9","nokia-lumia-520","microsoft-lumia-550","microsoft-lumia-950","galaxy-s-iii","galaxy-s5","kindle-fire-hdx","ipad-mini","ipad","ipad-pro","blackberry-playbook","nexus-10","nexus-7","galaxy-note-3","galaxy-note-ii","laptop-with-touch","laptop-with-hidpi-screen","laptop-with-mdpi-screen"],"default":""},"emulateDeviceOrientation":{"name":"emulateDeviceOrientation","type":"option","description":"Expected to be either \"vertical\" or \"horizontal\". Dictates orientation of device screen.","options":["horizontal","vertical"],"default":"vertical"},"socksPorts":{"name":"socksPorts","type":"option","description":"Specify a socks proxy port as browser option for control and experiment"},"regressionThreshold":{"name":"regressionThreshold","type":"option","description":"The upper limit the experiment can regress slower in milliseconds. eg 50","default":50},"sampleTimeout":{"name":"sampleTimeout","type":"option","description":"The number of seconds to wait for a sample.","default":30},"config":{"name":"config","type":"option","description":"Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all."},"runtimeStats":{"name":"runtimeStats","type":"boolean","description":"Compare command output deep-dive stats during run.","allowNo":false},"report":{"name":"report","type":"boolean","description":"Generate a PDF report directly after running the compare command.","allowNo":false},"debug":{"name":"debug","type":"boolean","description":"Debug flag per command. Will output noisy command","allowNo":false},"headless":{"name":"headless","type":"boolean","description":"Run with headless chrome flags","allowNo":false},"isCIEnv":{"name":"isCIEnv","type":"option","description":"Provides a drastically slimmed down stdout report for CI workflows. However does NOT hide analysis.","default":false},"regressionThresholdStat":{"name":"regressionThresholdStat","type":"option","description":"The statistic which the regression threshold runs against.","options":["estimator","ci-lower","ci-upper"],"default":"estimator"}},"args":[]},"compare:report":{"id":"compare:report","description":"Generates report files (PDF/HTML) from the \"tracerbench compare\" command output","pluginName":"tracerbench","pluginType":"core","aliases":["report"],"flags":{"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","required":true,"default":"./tracerbench-results"},"config":{"name":"config","type":"option","description":"Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all."},"plotTitle":{"name":"plotTitle","type":"option","description":"Specify the title of the report pdf/html files.","default":"TracerBench"},"isCIEnv":{"name":"isCIEnv","type":"option","description":"Provides a drastically slimmed down stdout report for CI workflows. However does NOT hide analysis.","default":false}},"args":[]},"record-har:auth":{"id":"record-har:auth","description":"Authenticate with a given login URL, username, password and retrieve auth cookies","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"url":{"name":"url","type":"option","description":"URL to visit for record-har, auth, timings & trace commands","required":true},"dest":{"name":"dest","type":"option","description":"The destination path for the generated file. Default process.cwd()","required":true,"default":""},"filename":{"name":"filename","type":"option","description":"The filename for the generated file","required":true,"default":"cookies"},"username":{"name":"username","type":"option","description":"The username to login to the form","required":true},"password":{"name":"password","type":"option","description":"The password to login to the form","required":true},"config":{"name":"config","type":"option","description":"Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all."},"headless":{"name":"headless","type":"boolean","description":"Run with headless chrome flags","allowNo":false},"screenshots":{"name":"screenshots","type":"boolean","description":"Include chrome screenshots from command execution","allowNo":false},"proxy":{"name":"proxy","type":"option","description":"Uses a specified proxy server, overrides system settings. Only affects HTTP and HTTPS requests.","required":false},"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","default":"./tracerbench-results"}},"args":[]},"record-har":{"id":"record-har","description":"Generates a HAR file from a URL","pluginName":"tracerbench","pluginType":"core","aliases":[],"flags":{"url":{"name":"url","type":"option","description":"URL to visit for record-har, auth, timings & trace commands","required":true},"dest":{"name":"dest","type":"option","description":"The destination path for the generated file. Default process.cwd()","required":true,"default":""},"cookiespath":{"name":"cookiespath","type":"option","description":"The path to a JSON file containing cookies to authenticate against the correlated URL","required":true,"default":""},"filename":{"name":"filename","type":"option","description":"The filename for the generated file","required":true,"default":"tracerbench"},"marker":{"name":"marker","type":"option","description":"The last marker before ending a HAR recording","required":true,"default":"loadEventEnd"},"config":{"name":"config","type":"option","description":"Specify an alternative directory rather than the project root for the tbconfig.json. This explicit config will overwrite all."},"headless":{"name":"headless","type":"boolean","description":"Run with headless chrome flags","allowNo":false},"screenshots":{"name":"screenshots","type":"boolean","description":"Include chrome screenshots from command execution","allowNo":false},"proxy":{"name":"proxy","type":"option","description":"Uses a specified proxy server, overrides system settings. Only affects HTTP and HTTPS requests.","required":false},"tbResultsFolder":{"name":"tbResultsFolder","type":"option","description":"The output folder path for all tracerbench results","default":"./tracerbench-results"}},"args":[]}}} \ No newline at end of file diff --git a/packages/cli/src/command-config/tb-config.ts b/packages/cli/src/command-config/tb-config.ts index a1aa5909..eb75ed29 100644 --- a/packages/cli/src/command-config/tb-config.ts +++ b/packages/cli/src/command-config/tb-config.ts @@ -45,11 +45,6 @@ export interface ITBConfig { isCIEnv?: boolean | string; marker?: string; regressionThresholdStat?: RegressionThresholdStat; - //default to false, if it's true - //it overrides the default loadEventEnd or the last marker in markers array as trace end - traceEndAtLcp?: boolean; - //the regex pattern to the LCP candidate element user want to measure, if undefined, we use the first LCP candidate - lcpRegex?: string; } export interface IHARServer { diff --git a/packages/cli/src/commands/compare/index.ts b/packages/cli/src/commands/compare/index.ts index b8a0e8ca..5043f6d0 100644 --- a/packages/cli/src/commands/compare/index.ts +++ b/packages/cli/src/commands/compare/index.ts @@ -54,8 +54,6 @@ import { sampleTimeout, socksPorts, tbResultsFolder, - traceEndAtLcp, - lcpRegex, } from "../../helpers/flags"; import { chalkScheme, @@ -92,8 +90,6 @@ export interface ICompareFlags { report?: boolean; isCIEnv?: boolean; regressionThresholdStat: RegressionThresholdStat; - traceEndAtLcp?: boolean; - lcpRegex?: string; } export default class Compare extends TBBaseCommand { @@ -124,8 +120,6 @@ export default class Compare extends TBBaseCommand { headless, isCIEnv: isCIEnv(), regressionThresholdStat, - traceEndAtLcp, - lcpRegex: lcpRegex(), }; public compareFlags: ICompareFlags; public parsedConfig: ITBConfig = defaultFlagArgs; @@ -446,8 +440,6 @@ export default class Compare extends TBBaseCommand { captureV8RuntimeStats: this.compareFlags.runtimeStats, saveTraceAs: (group, i) => `${this.compareFlags.tbResultsFolder}/traces/${group}${i}.json`, - traceEndAtLcp: this.compareFlags.traceEndAtLcp, - lcpRegex: this.compareFlags.lcpRegex, }, }, ]; @@ -488,8 +480,6 @@ export default class Compare extends TBBaseCommand { captureV8RuntimeStats: this.compareFlags.runtimeStats, saveTraceAs: (group, i) => `${this.compareFlags.tbResultsFolder}/traces/${group}${i}.json`, - traceEndAtLcp: this.compareFlags.traceEndAtLcp, - lcpRegex: this.compareFlags.lcpRegex, }, }, ]; diff --git a/packages/cli/src/helpers/flags.ts b/packages/cli/src/helpers/flags.ts index ce0b4acd..f4c4d890 100644 --- a/packages/cli/src/helpers/flags.ts +++ b/packages/cli/src/helpers/flags.ts @@ -277,13 +277,3 @@ export const jsonReport = oclifFlags.boolean({ description: `Include a JSON file from the stdout report`, default: false, }); - -export const traceEndAtLcp = oclifFlags.boolean({ - description: `Overrides the default loadEventEnd as trace end or the last marker in markers array as tracerbench`, - default: false, -}); - -export const lcpRegex = oclifFlags.build({ - description: `The regex pattern to identify the LCP candidate, if undefined, we use the first LCP candidate`, - required: false, -}); diff --git a/packages/cli/src/helpers/utils.ts b/packages/cli/src/helpers/utils.ts index 480de11f..3687fd6b 100644 --- a/packages/cli/src/helpers/utils.ts +++ b/packages/cli/src/helpers/utils.ts @@ -88,7 +88,7 @@ export function parseMarkers(m: string | string[]): Marker[] { m = m.split(","); } - for (let i = 1; i < m.length; i++) { + for (let i = 1; i < m.length; i += 2) { a.push({ start: m[i - 1], label: m[i], diff --git a/packages/cli/tb-schema.json b/packages/cli/tb-schema.json index 6895ca67..5aae7a80 100644 --- a/packages/cli/tb-schema.json +++ b/packages/cli/tb-schema.json @@ -1,11 +1,9 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": { - }, + "additionalProperties": {}, "definitions": { "IBenchmarkEnvironmentOverride": { - "additionalProperties": { - }, + "additionalProperties": {}, "properties": { "cpuThrottle": { "type": "number" @@ -142,9 +140,6 @@ "boolean" ] }, - "lcpRegex": { - "type": "string" - }, "locations": { "type": "string" }, @@ -193,6 +188,7 @@ "responseEnd", "responseStart", "secureConnectionStart", + "serverTiming", "startTime", "toJSON", "transferSize", @@ -269,9 +265,6 @@ "tbResultsFolder": { "type": "string" }, - "traceEndAtLcp": { - "type": "boolean" - }, "traceFrame": { "type": "string" }, diff --git a/packages/cli/test/commands/compare.test.ts b/packages/cli/test/commands/compare.test.ts index d5851887..0d12d4ad 100644 --- a/packages/cli/test/commands/compare.test.ts +++ b/packages/cli/test/commands/compare.test.ts @@ -11,7 +11,8 @@ const fidelityLow = "10"; const emulateDevice = "iphone-4"; const regressionThreshold = "50"; const network = "FIOS"; -const lcpRegexPattern = "h1"; +const markers = 'navigationStart,load,jqueryLoaded,jquery,emberLoaded,application,startRouting,routing,willTransition,transition,largestContentfulPaint'; + describe("compare fixture: A/A", () => { test .stdout() @@ -82,65 +83,11 @@ describe("compare fixture: A/A CI", () => { ); }); -describe("compare regression: fixture: A/B trace end at specific LCP candidate", () => { - test - .stdout() - .it( - `runs compare --controlURL ${FIXTURE_APP.control} --experimentURL ${FIXTURE_APP.regression} --fidelity ${fidelityLow} --tbResultsFolder ${TB_RESULTS_FOLDER} --config ${FIXTURE_APP.regressionConfig} --regressionThreshold ${regressionThreshold} --headless --traceEndAtLcp --lcpRegex ${lcpRegexPattern}`, - async (ctx) => { - const results = await Compare.run([ - "--controlURL", - FIXTURE_APP.control, - "--experimentURL", - FIXTURE_APP.regression, - "--fidelity", - fidelityLow, - "--tbResultsFolder", - TB_RESULTS_FOLDER, - "--config", - FIXTURE_APP.regressionConfig, - "--regressionThreshold", - regressionThreshold, - "--lcpRegex", - lcpRegexPattern, - "--headless", - "--traceEndAtLcp", - ]); - - const resultsJSON: ICompareJSONResults = await JSON.parse(results); - expect(ctx.stdout).to.contain( - ` SUCCESS ${fidelityLow} test samples took` - ); - // confirm with headless flag is logging the trace stream - expect(ctx.stdout).to.contain(`duration phase estimated regression +`); - expect(ctx.stdout).to.contain( - ` ! ALERT Regression found exceeding the set regression threshold of ${regressionThreshold} ms` - ); - assert.isAbove( - parseInt(resultsJSON.benchmarkTableData[0].estimatorDelta, 10), - 500 - ); - assert.isAbove( - parseInt(resultsJSON.benchmarkTableData[0].confidenceInterval[0], 10), - 500 - ); - assert.isAbove( - parseInt(resultsJSON.benchmarkTableData[0].confidenceInterval[1], 10), - 500 - ); - // results are json and are significant - assert.isTrue(resultsJSON.areResultsSignificant); - // regression is over the threshold - assert.isFalse(resultsJSON.isBelowRegressionThreshold); - } - ); -}); - -describe("compare regression: fixture: A/B trace end at first LCP candidate", () => { +describe("compare regression: fixture: A/B trace end at LCP candidate after a marker", () => { test .stdout() .it( - `runs compare --controlURL ${FIXTURE_APP.control} --experimentURL ${FIXTURE_APP.regression} --fidelity ${fidelityLow} --tbResultsFolder ${TB_RESULTS_FOLDER} --config ${FIXTURE_APP.regressionConfig} --regressionThreshold ${regressionThreshold} --headless --traceEndAtLcp`, + `runs compare --controlURL ${FIXTURE_APP.control} --experimentURL ${FIXTURE_APP.regression} --fidelity ${fidelityLow} --tbResultsFolder ${TB_RESULTS_FOLDER} --config ${FIXTURE_APP.regressionConfig} --regressionThreshold ${regressionThreshold} --markers ${markers} --headless`, async (ctx) => { const results = await Compare.run([ "--controlURL", @@ -155,8 +102,9 @@ describe("compare regression: fixture: A/B trace end at first LCP candidate", () FIXTURE_APP.regressionConfig, "--regressionThreshold", regressionThreshold, - "--headless", - "--traceEndAtLcp", + "--markers", + markers, + "--headless" ]); const resultsJSON: ICompareJSONResults = await JSON.parse(results); diff --git a/packages/core/package.json b/packages/core/package.json index 0b9e0eae..ae354d36 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -32,7 +32,7 @@ "lint": "eslint -c .eslintrc.js --ext .ts .", "prepare": "yarn build", "report": "Rscript bin/report.R test/results/results.json", - "test": "yarn lint && yarn mocha", + "test": "yarn lint && yarn mocha \"test/test.ts\" \"test/trace/*.ts\"", "watch": "tsc -b -w" }, "dependencies": { diff --git a/packages/core/src/create-trace-benchmark.ts b/packages/core/src/create-trace-benchmark.ts index 15cd831d..8f9245c1 100644 --- a/packages/core/src/create-trace-benchmark.ts +++ b/packages/core/src/create-trace-benchmark.ts @@ -67,8 +67,6 @@ export interface TraceOptions { additionalCategories: string[]; additionalTrialCategories: string[]; saveTraceAs: SaveTraceAsFn; - traceEndAtLcp?: boolean; - lcpRegex?: string; } export type TraceFn = ( diff --git a/packages/core/src/create-trace-navigation-benchmark.ts b/packages/core/src/create-trace-navigation-benchmark.ts index b09e850b..ee740db9 100644 --- a/packages/core/src/create-trace-navigation-benchmark.ts +++ b/packages/core/src/create-trace-navigation-benchmark.ts @@ -6,13 +6,18 @@ import extractNavigationSample, { NavigationSample } from './metrics/extract-navigation-sample'; import type { Benchmark } from './run'; -import type { WaitForLCP, WaitForMark } from './util/inject-mark-observer'; +import type { WaitForMarkOrLCP } from './util/inject-mark-observer'; import injectMarkObserver, { injectLCPObserver } from './util/inject-mark-observer'; import navigate from './util/navigate'; import type { PageSetupOptions } from './util/setup-page'; import setupPage from './util/setup-page'; +import { + LCP_EVENT_NAME, + uniformLCPEventName, + isTraceEndAtLCP +} from './trace/utils'; export interface Marker { start: string; @@ -36,7 +41,7 @@ export default function createTraceNavigationBenchmark( label: 'load' }, { - start: 'loadEventEnd', + start: LCP_EVENT_NAME, label: 'paint' } ); @@ -46,34 +51,32 @@ export default function createTraceNavigationBenchmark( async (page, _i, _isTrial, raceCancel, trace) => { setupPage(page, raceCancel, options.pageSetupOptions); - let traceEndAtLcp = false; - let lcpRegex: string | undefined; - if (options.traceOptions) { - traceEndAtLcp = options.traceOptions.traceEndAtLcp ?? false; - lcpRegex = options.traceOptions.lcpRegex; + const markerList = uniformLCPEventName(markers); + const mLength = markerList.length; + + const { start: lastMarker } = markerList[mLength - 1]; + const traceEndAtLcp = isTraceEndAtLCP(markerList); + let priorMarker = 'navigationStart'; + if (traceEndAtLcp && mLength >= 2) { + priorMarker = markerList[mLength - 2].start; } - let waitForLcp: WaitForLCP; - let waitForMark: WaitForMark; - const { start: mark } = markers[markers.length - 1]; + let waitTraceEnd: WaitForMarkOrLCP; + if (traceEndAtLcp) { - waitForLcp = await injectLCPObserver(page, lcpRegex); + waitTraceEnd = await injectLCPObserver(page, priorMarker); } else { - waitForMark = await injectMarkObserver(page, mark); + waitTraceEnd = await injectMarkObserver(page, lastMarker); } return extractNavigationSample( buildModel( await trace(async (raceCancel) => { // do navigation await navigate(page, url, raceCancel); - if (traceEndAtLcp) { - await waitForLcp(raceCancel); - } else { - await waitForMark(raceCancel); - } + await waitTraceEnd(raceCancel); }) ), - markers + markerList ); }, options diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 0a1b5b6a..0d6529bc 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -45,5 +45,10 @@ export { authClient, getNewTab, createBrowser, - getTab + getTab, + LCP_EVENT_NAME, + LCP_EVENT_NAME_ALIAS, + isLCPEvent, + isTraceEndAtLCP, + uniformLCPEventName } from './trace'; diff --git a/packages/core/src/metrics/extract-navigation-sample.ts b/packages/core/src/metrics/extract-navigation-sample.ts index 5da09c8f..1739923f 100644 --- a/packages/core/src/metrics/extract-navigation-sample.ts +++ b/packages/core/src/metrics/extract-navigation-sample.ts @@ -8,6 +8,7 @@ import { } from '@tracerbench/trace-model'; import { Marker } from '../create-trace-navigation-benchmark'; +import { LCP_EVENT_NAME, isTraceEndAtLCP, isLCPEvent } from '../trace/utils'; export default function extractNavigationSample( trace: TraceModel, @@ -44,6 +45,8 @@ function findPhases(events: EventModel[], markers: Marker[]): PhaseSample[] { const phaseEvents: Array = []; let eventIdx = 0; let navigationStartArgs: NavigationStartArgs | undefined; + const traceEndAtLcp = isTraceEndAtLCP(markers); + // for each marker scan forward in the events to find it for (const marker of markers) { let markEvent: MarkEventModel | InstantEventModel | undefined; @@ -94,28 +97,46 @@ function findPhases(events: EventModel[], markers: Marker[]): PhaseSample[] { phaseEvents.push(markEvent); } } - let paintEvent: CompleteEventModel | undefined; + let paintEvent: CompleteEventModel | MarkEventModel | undefined; for (; eventIdx < events.length; eventIdx++) { const event = events[eventIdx]; - if (event.isComplete() && event.name === 'Paint') { + if ( + (event.isComplete() && event.name === 'Paint') || + (event.isMark() && event.name === LCP_EVENT_NAME && traceEndAtLcp) + ) { paintEvent = event; break; } } if (!paintEvent) { throw new Error( - `Could not find Paint event in trace after last mark "${ + `Could not find Paint event or ${LCP_EVENT_NAME} in trace after last mark "${ markers[markers.length - 1].start - }"` + }. The last marker should trigger a paint event."` ); } const phases: PhaseSample[] = []; const start = phaseEvents[0].start; + let end = phaseEvents[0].end; + for (let i = 0; i < phaseEvents.length; i++) { const marker = markers[i]; const markEvent = phaseEvents[i]; - const end = - i + 1 < phaseEvents.length ? phaseEvents[i + 1].start : paintEvent.end; + + if (i + 1 < phaseEvents.length) { + //skip LCP if user configure the LCP in the middle of marker list + if (isLCPEvent(markEvent.name)) { + continue; + } + end = phaseEvents[i + 1].start; + } else { + //skip if paint event is LCP event, do not double count the phases + if (isLCPEvent(paintEvent.name)) { + continue; + } + end = paintEvent.end; + } + phases.push({ phase: marker.label, start: markEvent.start - start, diff --git a/packages/core/src/trace/index.ts b/packages/core/src/trace/index.ts index 45ea7d8d..90d54574 100644 --- a/packages/core/src/trace/index.ts +++ b/packages/core/src/trace/index.ts @@ -5,4 +5,14 @@ export { IConditions, INetworkConditions } from './conditions'; -export { getBrowserArgs, createBrowser, getNewTab, getTab } from './utils'; +export { + getBrowserArgs, + createBrowser, + getNewTab, + getTab, + LCP_EVENT_NAME, + LCP_EVENT_NAME_ALIAS, + isLCPEvent, + isTraceEndAtLCP, + uniformLCPEventName +} from './utils'; diff --git a/packages/core/src/trace/utils.ts b/packages/core/src/trace/utils.ts index 8dcd26f4..ee467723 100644 --- a/packages/core/src/trace/utils.ts +++ b/packages/core/src/trace/utils.ts @@ -8,6 +8,7 @@ import type { Protocol } from 'devtools-protocol'; import { dirSync } from 'tmp'; import { IConditions, networkConditions } from './conditions'; +import { Marker } from '../create-trace-navigation-benchmark'; /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -232,3 +233,55 @@ function enforcePaintEvent() { } `; + +export const LCP_EVENT_NAME = 'largestContentfulPaint::Candidate'; +export const LCP_EVENT_NAME_ALIAS = 'largestContentfulPaint'; + +/** + * check if the last marker.start is largestContentfulPaint:Candidate + * That means user want trace to end at LCP + * @param markers - markers array + * @returns true if markers end at LCP event + */ +export function isTraceEndAtLCP(markers: Marker[]): boolean { + if (markers.length > 0) { + const { start: marker } = markers[markers.length - 1]; + return marker === LCP_EVENT_NAME; + } else { + return false; + } +} + +/** + * if the config or commandline has marker name as largestContentfulPaint + * convert it to the actual event name largestContentfulPaint:Candidate + * return a new marker list, Keep input markers immutable. + * @param markers - marker array + * @returns renamed marker array + */ +export function uniformLCPEventName(markers: Marker[]): Marker[] { + const renamedMarkers: Marker[] = []; + let renamedMarker: Marker; + markers.forEach((marker: Marker) => { + if (marker.start === LCP_EVENT_NAME_ALIAS) { + renamedMarker = { + start: LCP_EVENT_NAME, + label: marker.label + }; + } else { + renamedMarker = marker; + } + renamedMarkers.push(renamedMarker); + }); + + return renamedMarkers; +} + +/** + * check if an event name is largestContentfulPaint:Candidate + * @param marker - event name + * @returns - true or false + */ +export function isLCPEvent(marker: string): boolean { + return marker === LCP_EVENT_NAME; +} diff --git a/packages/core/src/util/inject-mark-observer.ts b/packages/core/src/util/inject-mark-observer.ts index 44a49d11..7448d7f2 100644 --- a/packages/core/src/util/inject-mark-observer.ts +++ b/packages/core/src/util/inject-mark-observer.ts @@ -5,16 +5,17 @@ import type { RaceCancellation } from 'race-cancellation'; import isNavigationTimingMark from './is-navigation-timing-mark'; -export type WaitForMark = (raceCancellation: RaceCancellation) => Promise; -export type WaitForLCP = (raceCancellation: RaceCancellation) => Promise; +export type WaitForMarkOrLCP = ( + raceCancellation: RaceCancellation +) => Promise; -const LCP_EVENT = 'largest-contentful-paint event'; +const LCP_EVENT_MSG = 'largest-contentful-paint event'; export default async function injectMarkObserver( page: ProtocolConnection, mark: string, variable = '__tracerbench' -): Promise { +): Promise { const scriptSource = isNavigationTimingMark(mark) ? navigationObserver(variable) : markObserver(mark, variable); @@ -24,56 +25,55 @@ export default async function injectMarkObserver( }); return (raceCancelation: RaceCancellation) => - waitForMark(page, variable, mark, raceCancelation); + waitForMarkOrLCP(page, variable, mark, raceCancelation); } export async function injectLCPObserver( page: ProtocolConnection, - elementPattern?: string, + priorMarker?: string, variable = '__tracerbenchLCP' -): Promise { - const scriptSource = lcpObserver(variable, elementPattern); +): Promise { + const scriptSource = lcpObserver(variable, priorMarker); await page.send('Page.addScriptToEvaluateOnLoad', { scriptSource }); return (raceCancelation: RaceCancellation) => - waitForLCP(page, variable, raceCancelation); + waitForMarkOrLCP(page, variable, LCP_EVENT_MSG, raceCancelation); } -function lcpObserver(variable: string, elementPattern?: string): string { +function lcpObserver(variable: string, priorMarker?: string): string { return `"use strict"; + var __tracerbenchPriorMarkerObserved = (typeof ${priorMarker} === 'undefined')? true : false; + if (!__tracerbenchPriorMarkerObserved){ + new PerformanceObserver((entryList, observer) => { + if (!__tracerbenchPriorMarkerObserved) { + var markerEntries = entryList.getEntriesByName(${priorMarker}); + if (markerEntries.length > 0) { + __tracerbenchPriorMarkerObserved = true; + observer.disconnect(); + } + } + }).observe({ type: 'mark' }); + } var ${variable} = self === top && opener === null && new Promise((resolve) => new PerformanceObserver((entryList, observer) => { - var entries = entryList.getEntries(); - var pattern = '${elementPattern}'; - for (var i = 0; i < entries.length; i++) { - if ( pattern !== 'undefined') { - var elm = entries[i].element.outerHTML;; - var regex = new RegExp(pattern); - if (regex.test(elm)){ - requestAnimationFrame(() => { - resolve(); - }); - } - } else { - requestAnimationFrame(() => { - resolve(); - }); - break; - } + var lcpEntries = entryList.getEntriesByType('largest-contentful-paint'); + if (lcpEntries.length > 0 && __tracerbenchPriorMarkerObserved) { + requestAnimationFrame(() => { + resolve(); + }); } observer.disconnect(); - }).observe({type: 'largest-contentful-paint', buffered: true}) + }).observe({ type: 'largest-contentful-paint', buffered: true }) );`; } function markObserver(mark: string, variable: string): string { return `"use strict"; - ${enforcePaintEventFn} var ${variable} = self === top && opener === null && @@ -81,14 +81,13 @@ function markObserver(mark: string, variable: string): string { new PerformanceObserver((records, observer) => { if (records.getEntriesByName(${JSON.stringify(mark)}).length > 0) { requestAnimationFrame(() => { - enforcePaintEvent(); - requestIdleCallback(() => { + requestAnimationFrame(() => { resolve(); }); }); observer.disconnect(); } - }).observe({ type: "mark" }) + }).observe({ type: 'mark' }) );`; } @@ -113,9 +112,10 @@ var ${variable} = );`; } -async function waitForLCP( +async function waitForMarkOrLCP( page: ProtocolConnection, expression: string, + waitType: string, raceCancelation: RaceCancellation ): Promise { let result: Protocol.Runtime.EvaluateResponse; @@ -132,42 +132,11 @@ async function waitForLCP( const { exceptionDetails } = result; if (exceptionDetails !== undefined) { - throw waitForMarkOrEventError(LCP_EVENT, { exceptionDetails }); - } - } catch (original) { - if (original instanceof Error) { - throw waitForMarkOrEventError(LCP_EVENT, { original }); - } else { - throw original; - } - } -} - -async function waitForMark( - page: ProtocolConnection, - expression: string, - mark: string, - raceCancelation: RaceCancellation -): Promise { - let result: Protocol.Runtime.EvaluateResponse; - try { - result = await page.send( - 'Runtime.evaluate', - { - expression, - awaitPromise: true, - returnByValue: true - }, - raceCancelation - ); - - const { exceptionDetails } = result; - if (exceptionDetails !== undefined) { - throw waitForMarkOrEventError(mark, { exceptionDetails }); + throw waitForMarkOrEventError(waitType, { exceptionDetails }); } } catch (original) { if (original instanceof Error) { - throw waitForMarkOrEventError(mark, { original }); + throw waitForMarkOrEventError(waitType, { original }); } else { throw original; } diff --git a/packages/core/test/test.ts b/packages/core/test/test.ts index fa1a4815..4fcbf4fc 100644 --- a/packages/core/test/test.ts +++ b/packages/core/test/test.ts @@ -2,6 +2,7 @@ import { writeFileSync, existsSync } from 'fs'; import mkdirp = require('mkdirp'); import { resolve, dirname, join } from 'path'; import { createTraceNavigationBenchmark, run } from '@tracerbench/core'; +import { Marker } from '@tracerbench/core'; import findUp = require('find-up'); import build from './build/index'; import * as execa from 'execa'; @@ -82,7 +83,7 @@ describe('Benchmark', function () { } }); - it('should work', async function () { + it('should work when last marker triggers a paint event', async function () { const markers = [ { start: 'fetchStart', label: 'jquery' }, { start: 'jqueryLoaded', label: 'ember' }, @@ -133,14 +134,15 @@ describe('Benchmark', function () { writeFileSync(join(resultDir, 'results.json'), JSON.stringify(results)); }); - it('should end trace at LCP', async function () { + it('should end trace at LCP if in markers list', async function () { //no need to add markers in render phase, we can still extract paint event const markers = [ { start: 'fetchStart', label: 'jquery' }, { start: 'jqueryLoaded', label: 'ember' }, { start: 'emberLoaded', label: 'application' }, { start: 'startRouting', label: 'routing' }, - { start: 'willTransition', label: 'transition' } + { start: 'willTransition', label: 'transition' }, + { start: 'largestContentfulPaint::Candidate', label: 'paint' }, ]; const benchmarks = tests.map(({ name, url }) => createTraceNavigationBenchmark(name, url, markers, { @@ -149,8 +151,50 @@ describe('Benchmark', function () { traceOptions: { saveTraceAs: (group, i) => join(resultDir, `trace-${group}-${i}.json`), - traceEndAtLcp: true, - lcpRegex: 'h1' + } + }) + ); + const start = Date.now(); + const results = await run( + benchmarks, + 4, + (elasped, completed, remaining, group, iteration) => { + if (completed > 0) { + const average = elasped / completed; + const remainingSecs = Math.round((remaining * average) / 1000); + console.log( + '%s %s %s seconds remaining', + group.padStart(15), + iteration.toString().padStart(3), + `about ${remainingSecs}`.padStart(10) + ); + } else { + console.log( + '%s %s', + group.padStart(15), + iteration.toString().padStart(3) + ); + } + } + ); + console.log( + 'completed in %d seconds', + Math.round((Date.now() - start) / 1000) + ); + + writeFileSync(join(resultDir, 'results.json'), JSON.stringify(results)); + }); + + it('should end trace at LCP by default if no markers configured', async function () { + //no need to add markers in render phase, we can still extract paint event + const markers: Marker[] = []; + const benchmarks = tests.map(({ name, url }) => + createTraceNavigationBenchmark(name, url, markers, { + spawnOptions: browserOpts, + pageSetupOptions: {}, + traceOptions: { + saveTraceAs: (group, i) => + join(resultDir, `trace-${group}-${i}.json`), } }) ); diff --git a/packages/core/test/trace/trace-utils-test.ts b/packages/core/test/trace/trace-utils-test.ts new file mode 100644 index 00000000..4f3d9024 --- /dev/null +++ b/packages/core/test/trace/trace-utils-test.ts @@ -0,0 +1,55 @@ +import { assert } from 'chai'; +import { + LCP_EVENT_NAME_ALIAS, + isTraceEndAtLCP, + isLCPEvent, + uniformLCPEventName, +} from '@tracerbench/core'; + +describe('Trace utils', function() { + const markersWithAlias = [ + { start: 'fetchStart', label: 'jquery' }, + { start: 'jqueryLoaded', label: 'ember' }, + { start: 'emberLoaded', label: 'application' }, + { start: 'startRouting', label: 'routing' }, + { start: 'willTransition', label: 'transition' }, + { start: 'largestContentfulPaint', label: 'paint' }, + ]; + const markers = [ + { start: 'fetchStart', label: 'jquery' }, + { start: 'jqueryLoaded', label: 'ember' }, + { start: 'emberLoaded', label: 'application' }, + { start: 'startRouting', label: 'routing' }, + { start: 'willTransition', label: 'transition' }, + { start: 'largestContentfulPaint::Candidate', label: 'paint' }, + ]; + + const markerWithoutLCP = [ + { start: 'fetchStart', label: 'jquery' }, + { start: 'jqueryLoaded', label: 'ember' }, + { start: 'emberLoaded', label: 'application' }, + { start: 'startRouting', label: 'routing' }, + { start: 'willTransition', label: 'transition' }, + { start: 'didTransition', label: 'render' }, + { start: 'renderEnd', label: 'afterRender' } + ]; + + it('uniformLCPEventName should uniform LCP event name', function() { + const renamedMarkerList = uniformLCPEventName(markersWithAlias); + assert.deepEqual(renamedMarkerList, markers, 'should rename marker names'); + assert.equal(markersWithAlias[markersWithAlias.length -1].start, + LCP_EVENT_NAME_ALIAS, + 'should not change the original marker list'); + }); + + it('isTraceEndAtLCP should validate last marker end at LCP', function() { + assert.equal(isTraceEndAtLCP(markers), true, 'should identify marker list end at LCP event'); + assert.equal(isTraceEndAtLCP(markerWithoutLCP), false, + 'false if marker list does not end at LCP'); + }); + + it('isLCPEvent should validate event name is LCP', function() { + assert.ok(isLCPEvent(markers[markers.length -1].start), 'should validate actual LCP event name'); + assert.ok(!isLCPEvent(markersWithAlias[markersWithAlias.length - 1].start), 'alias LCP name should return false'); + }) +}); \ No newline at end of file