Skip to content

Commit

Permalink
fix: smoke tests, change default cpu throttle to 0
Browse files Browse the repository at this point in the history
  • Loading branch information
tsubomi imamura committed Apr 11, 2022
1 parent 1949bc4 commit 83dfade
Show file tree
Hide file tree
Showing 11 changed files with 497 additions and 35 deletions.
2 changes: 1 addition & 1 deletion packages/cli/markdown/compare.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ OPTIONS
Control URL to visit for compare command
--cpuThrottleRate=cpuThrottleRate
(required) [default: 2] CPU throttle multiplier
(required) [default: 0] CPU throttle multiplier
--debug
Debug flag per command. Will output noisy command
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/oclif.manifest.json

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion packages/cli/src/command-config/default-flag-args.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export type PerformanceTimingMark = keyof PerformanceNavigationTiming;
// chrome-debugging-client#defaultFlags.ts
export const defaultFlagArgs: ITBConfig = {
plotTitle: "TracerBench",
cpuThrottleRate: 2,
cpuThrottleRate: 0,
fidelity: "low",
markers: "domComplete",
marker: "loadEventEnd",
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/src/commands/record-har/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ export default class RecordHAR extends TBBaseCommand {
let { browserArgs } = this.parsedConfig;
const conditions: IConditions = {
network: network ? network : "none",
cpu: cpuThrottleRate ? parseInt(cpuThrottleRate as string, 10) : 1,
cpu: cpuThrottleRate ? parseInt(cpuThrottleRate as string, 10) : 0,
};
let cookies = [
{
Expand Down
12 changes: 4 additions & 8 deletions packages/cli/test/commands/compare.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ describe("compare fixture: A/A", () => {
test
.stdout()
.it(
`runs compare --controlURL ${FIXTURE_APP.control} --experimentURL ${FIXTURE_APP.control} --fidelity ${fidelity} --tbResultsFolder ${TB_RESULTS_FOLDER} --cpuThrottleRate=1 --config ${FIXTURE_APP.controlConfig} --network ${network} --headless --debug --report`,
`runs compare --controlURL ${FIXTURE_APP.control} --experimentURL ${FIXTURE_APP.control} --fidelity ${fidelity} --tbResultsFolder ${TB_RESULTS_FOLDER} --config ${FIXTURE_APP.controlConfig} --network ${network} --headless --debug --report`,
async (ctx) => {
const results = await Compare.run([
"--controlURL",
Expand All @@ -30,7 +30,6 @@ describe("compare fixture: A/A", () => {
FIXTURE_APP.controlConfig,
"--network",
network,
"--cpuThrottleRate=1",
"--headless",
"--debug",
"--report"
Expand All @@ -54,7 +53,7 @@ describe("compare fixture: A/A CI", () => {
test
.stdout()
.it(
`runs compare --controlURL ${FIXTURE_APP.control} --experimentURL ${FIXTURE_APP.control} --fidelity ${fidelity} --tbResultsFolder ${TB_RESULTS_FOLDER} --cpuThrottleRate=1 --config ${FIXTURE_APP.controlConfig} --headless --isCIEnv=true`,
`runs compare --controlURL ${FIXTURE_APP.control} --experimentURL ${FIXTURE_APP.control} --fidelity ${fidelity} --tbResultsFolder ${TB_RESULTS_FOLDER} --config ${FIXTURE_APP.controlConfig} --headless --isCIEnv=true`,
async (ctx) => {
await Compare.run([
"--controlURL",
Expand All @@ -67,7 +66,6 @@ describe("compare fixture: A/A CI", () => {
TB_RESULTS_FOLDER,
"--config",
FIXTURE_APP.controlConfig,
"--cpuThrottleRate=1",
"--headless",
"--isCIEnv=true"
]);
Expand All @@ -87,7 +85,7 @@ describe("compare regression: fixture: A/B", () => {
test
.stdout()
.it(
`runs compare --controlURL ${FIXTURE_APP.control} --experimentURL ${FIXTURE_APP.regression} --fidelity ${fidelityLow} --tbResultsFolder ${TB_RESULTS_FOLDER} --config ${FIXTURE_APP.regressionConfig} --regressionThreshold ${regressionThreshold} --cpuThrottleRate=1 --headless`,
`runs compare --controlURL ${FIXTURE_APP.control} --experimentURL ${FIXTURE_APP.regression} --fidelity ${fidelityLow} --tbResultsFolder ${TB_RESULTS_FOLDER} --config ${FIXTURE_APP.regressionConfig} --regressionThreshold ${regressionThreshold} --headless`,
async (ctx) => {
const results = await Compare.run([
"--controlURL",
Expand All @@ -102,7 +100,6 @@ describe("compare regression: fixture: A/B", () => {
FIXTURE_APP.regressionConfig,
"--regressionThreshold",
regressionThreshold,
"--cpuThrottleRate=1",
"--headless"
]);

Expand Down Expand Up @@ -139,7 +136,7 @@ describe("compare mobile horizontal: fixture: A/A", () => {
test
.stdout()
.it(
`runs compare --controlURL ${FIXTURE_APP.control} --experimentURL ${FIXTURE_APP.experiment} --fidelity ${fidelity} --tbResultsFolder ${TB_RESULTS_FOLDER} --emulateDevice ${emulateDevice} --emulateDeviceOrientation horizontal --cpuThrottleRate=6 --headless`,
`runs compare --controlURL ${FIXTURE_APP.control} --experimentURL ${FIXTURE_APP.experiment} --fidelity ${fidelity} --tbResultsFolder ${TB_RESULTS_FOLDER} --emulateDevice ${emulateDevice} --emulateDeviceOrientation horizontal --headless`,
async (ctx) => {
await Compare.run([
"--controlURL",
Expand All @@ -154,7 +151,6 @@ describe("compare mobile horizontal: fixture: A/A", () => {
emulateDevice,
"--emulateDeviceOrientation",
"horizontal",
"--cpuThrottleRate=6",
"--headless"
]);

Expand Down
2 changes: 1 addition & 1 deletion packages/cli/test/tbconfig_base.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"$schema": "https://raw.githubusercontent.com/TracerBench/tracerbench/master/packages/cli/tb-schema.json",
// the title of the report pdf file
"plotTitle": "tbconfig_base file",
"cpuThrottleRate": 2,
"cpuThrottleRate": 0,
"tbResultsFolder": "../tracerbench-results",
"controlURL": "https://www.tracerbench.com/",
"experimentURL": "https://www.tracerbench.com/",
Expand Down
4 changes: 4 additions & 0 deletions packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,21 +57,25 @@
},
"devDependencies": {
"@types/chai": "^4.2.17",
"@types/dashdash": "^1.14.1",
"@types/debug": "^4.1.7",
"@types/express": "^4.17.13",
"@types/glob": "^7.1.3",
"@types/mkdirp": "^1.0.1",
"@types/mocha": "^9.0.0",
"@types/tar": "^4.0.4",
"@typescript-eslint/eslint-plugin": "^4.22.1",
"@typescript-eslint/parser": "^4.22.1",
"chai": "^4.3.4",
"dashdash": "^2.0.0",
"eslint": "^7.25.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-filenames": "^1.3.2",
"eslint-plugin-import": "^2.22.1",
"eslint-plugin-oclif": "^0.1.0",
"eslint-plugin-prettier": "^3.4.0",
"eslint-plugin-simple-import-sort": "7.0.0",
"express": "^4.17.3",
"find-up": "^5.0.0",
"jquery": "^3.6.0",
"mkdirp": "^1.0.4",
Expand Down
6 changes: 6 additions & 0 deletions packages/core/test/killTestServer.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
set -e
set -o pipefail

pid=$(ps -ef|grep -i "test-server"|sort -k5,5|awk '{print $2}'|head -1)
#echo "stop test server $pid"
kill -9 $pid
60 changes: 60 additions & 0 deletions packages/core/test/test-server.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import * as express from 'express';
import { createParser } from 'dashdash';
import { Results, Option } from 'dashdash';

(async function() {
const app = express();

type ServerOpt = {
port: number;
staticAssetFolder: string;
}
async function startServer(): Promise<void> {
const { port, staticAssetFolder } = initServerOption();
app.use(express.static(staticAssetFolder));

return new Promise((resolve) => {
app.listen(port, () => {
console.log(`Ready to serve test pages at http://localhost:${port}`);
resolve();
});
});
}

function initServerOption(): ServerOpt {
const options: Option[] = [
{
name: 'port',
type: 'number',
help: 'server port to listen to',
}, {
name: 'staticAssetFolder',
type: 'string',
help: 'directory of static assets',
default: 'build'
},
{
names: ['help', 'h'],
type: 'bool',
help: 'Print this help and exit.'
}
];

const parser = createParser({ options: options });
let opts: Results;
let serverOpt: ServerOpt;
try {
opts = parser.parse(process.argv);
serverOpt = {
port: opts.port,
staticAssetFolder: opts.staticAssetFolder
}
} catch (e) {
console.error('server start up option error: %s', (e as Error).message);
process.exit(1);
}
return serverOpt;
}

await startServer();
})();
50 changes: 41 additions & 9 deletions packages/core/test/test.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
import { writeFileSync, existsSync } from 'fs';
import mkdirp = require('mkdirp');
import { resolve, dirname, join } from 'path';
import { pathToFileURL } from 'url';
import { createTraceNavigationBenchmark, run } from '@tracerbench/core';
import findUp = require('find-up');
import build from './build/index';
import * as execa from 'execa';
import type { ChromeSpawnOptions } from 'chrome-debugging-client';

const channels = ['alpha', 'beta', 'release'];
const browserOpts: Partial<ChromeSpawnOptions> = {
headless: true,
stdio: 'ignore'
stdio: 'ignore',
};

const [ NODE_PATH ] = process.argv;
let packageRoot: string;
describe('Benchmark', function () {
describe('smoke test', function () {
this.timeout('10m');
Expand All @@ -22,32 +24,64 @@ describe('Benchmark', function () {
name: string;
url: string;
}[];

before(async () => {
const packageJson = await findUp('package.json');
if (packageJson === undefined) {
throw new Error(`failed to find-up package.json from ${__dirname}`);
}
const packageRoot = dirname(packageJson);
packageRoot = dirname(packageJson);

const needsBuild: string[] = [];

const port = 3000;
const BASE_URL = `http://localhost:${port}/`;
tests = channels.map((name) => {
const path = resolve(packageRoot, `dist/test/${name}/index.html`);
if (!existsSync(path)) {
needsBuild.push(name);
}
const url = pathToFileURL(path).toString();
const url = `${BASE_URL}${name}/index.html`;
return { name, url };
});

const staticAssetFolder: string = resolve(packageRoot, 'dist/test');
await build(needsBuild, packageRoot);
const serverPath = resolve(packageRoot, 'dist/test/test-server');
const serverArgs = [
serverPath,
'--port',
'3000',
'--staticAssetFolder',
staticAssetFolder,
'&'
];

try {
await execa(NODE_PATH, serverArgs, { shell: true, stdio: "ignore"});
console.log(`Ready to serve test pages at http://localhost:${port}`);
} catch(err) {
if (err && err instanceof Error) {
console.warn(`Failed to start test server ${err.message}`);
}
}

resultDir = resolve(packageRoot, 'test/results');

mkdirp.sync(resultDir);
});

after(async() => {
try {
await execa(resolve(packageRoot, './test/killTestServer.sh'), {
shell:true
});
console.log('Stopped test server');
} catch(err) {
if (err && err instanceof Error) {
console.warn(`Failed to terminate test server, need to stop it manually. ${err.message}`);
}
}
});

it('should work', async function () {
const markers = [
{ start: 'fetchStart', label: 'jquery' },
Expand All @@ -61,9 +95,7 @@ describe('Benchmark', function () {
const benchmarks = tests.map(({ name, url }) =>
createTraceNavigationBenchmark(name, url, markers, {
spawnOptions: browserOpts,
pageSetupOptions: {
cpuThrottlingRate: 4
},
pageSetupOptions: {},
traceOptions: {
saveTraceAs: (group, i) =>
join(resultDir, `trace-${group}-${i}.json`)
Expand Down
Loading

0 comments on commit 83dfade

Please sign in to comment.