Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 18 additions & 14 deletions scripts/bench/benchmark.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
'use strict';

const Lighthouse = require('lighthouse');
const ChromeLauncher = require('lighthouse/lighthouse-cli/chrome-launcher.js').ChromeLauncher;
const ChromeLauncher = require('lighthouse/lighthouse-cli/chrome-launcher.js')
.ChromeLauncher;
const stats = require('stats-analysis');
const config = require('lighthouse/lighthouse-core/config/perf.json');
const spawn = require('child_process').spawn;
Expand All @@ -14,18 +15,22 @@ function wait(val) {
}

async function runScenario(benchmark, launcher) {
const results = await Lighthouse(`http://localhost:8080/${benchmark}/`, {
output: 'json',
disableCpuThrottling: false,
disableNetworkThrottling: false,
}, config);
const results = await Lighthouse(
`http://localhost:8080/${benchmark}/`,
{
output: 'json',
disableCpuThrottling: false,
disableNetworkThrottling: false,
},
config
);
const perfMarkings = results.audits['user-timings'].extendedInfo.value;
const entries = perfMarkings
.filter(marker => !marker.isMark)
.map(({ duration, name }) => ({
entry: name,
time: duration,
}));
.filter(marker => !marker.isMark)
.map(({duration, name}) => ({
entry: name,
time: duration,
}));
entries.push({
entry: 'First Meaningful Paint',
time: results.audits['first-meaningful-paint'].rawValue,
Expand Down Expand Up @@ -55,7 +60,7 @@ function calculateAverages(runs) {
const averages = [];

runs.forEach((entries, x) => {
entries.forEach(({ entry, time }, i) => {
entries.forEach(({entry, time}, i) => {
if (i >= averages.length) {
data.push([time]);
averages.push({
Expand Down Expand Up @@ -83,7 +88,7 @@ async function initChrome() {
if (platform === 'linux') {
process.env.XVFBARGS = '-screen 0, 1024x768x16';
process.env.LIGHTHOUSE_CHROMIUM_PATH = 'chromium-browser';
const child = spawn('xvfb start', [{ detached: true, stdio: ['ignore'] }]);
const child = spawn('xvfb start', [{detached: true, stdio: ['ignore']}]);
child.unref();
// wait for chrome to load then continue
await wait(3000);
Expand Down Expand Up @@ -128,4 +133,3 @@ async function runBenchmark(benchmark, headless) {
}

module.exports = runBenchmark;

40 changes: 22 additions & 18 deletions scripts/bench/build.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,9 @@
const Git = require('nodegit');
const rimraf = require('rimraf');
const ncp = require('ncp').ncp;
const {
existsSync,
} = require('fs');
const {existsSync} = require('fs');
const exec = require('child_process').exec;
const {
join,
} = require('path');
const {join} = require('path');

const reactUrl = 'https://github.com/facebook/react.git';

Expand All @@ -18,14 +14,16 @@ function cleanDir() {
}

function executeCommand(command) {
return new Promise(_resolve => exec(command, (error) => {
if (!error) {
_resolve();
} else {
console.error(error);
process.exit(1);
}
}));
return new Promise(_resolve =>
exec(command, error => {
if (!error) {
_resolve();
} else {
console.error(error);
process.exit(1);
}
})
);
}

function asyncCopyTo(from, to) {
Expand All @@ -51,9 +49,10 @@ async function buldAllBundles(reactPath = getDefaultReactPath()) {

async function buildBenchmark(reactPath = getDefaultReactPath(), benchmark) {
// get the build.js from the benchmark directory and execute it
await require(
join(__dirname, 'benchmarks', benchmark, 'build.js')
)(reactPath, asyncCopyTo);
await require(join(__dirname, 'benchmarks', benchmark, 'build.js'))(
reactPath,
asyncCopyTo
);
}

function getBundleResults(reactPath = getDefaultReactPath()) {
Expand All @@ -69,7 +68,12 @@ async function getMergeBaseFromLocalGitRepo(localRepo) {
);
}

async function buildBenchmarkBundlesFromGitRepo(commitId, skipBuild, url = reactUrl, clean) {
async function buildBenchmarkBundlesFromGitRepo(
commitId,
skipBuild,
url = reactUrl,
clean
) {
let repo;
const remoteRepoDir = getDefaultReactPath();

Expand Down
47 changes: 23 additions & 24 deletions scripts/bench/runner.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
'use strict';

const {
readdirSync,
statSync,
} = require('fs');
const { join } = require('path');
const {readdirSync, statSync} = require('fs');
const {join} = require('path');
const runBenchmark = require('./benchmark');
const {
buildAllBundles,
Expand All @@ -18,8 +15,8 @@ const printResults = require('./stats');
const serveBenchmark = require('./server');

function getBenchmarkNames() {
return readdirSync(join(__dirname, 'benchmarks')).filter(
file => statSync(join(__dirname, 'benchmarks', file)).isDirectory()
return readdirSync(join(__dirname, 'benchmarks')).filter(file =>
statSync(join(__dirname, 'benchmarks', file)).isDirectory()
);
}

Expand All @@ -43,13 +40,16 @@ async function runBenchmarks(reactPath) {
const benchmarkName = benchmarkNames[i];

if (
!benchmarkFilter
||
!benchmarkFilter ||
(benchmarkFilter && benchmarkName.indexOf(benchmarkFilter) !== -1)
) {
console.log(chalk.gray(`- Building benchmark "${chalk.white(benchmarkName)}"...`));
console.log(
chalk.gray(`- Building benchmark "${chalk.white(benchmarkName)}"...`)
);
await buildBenchmark(reactPath, benchmarkName);
console.log(chalk.gray(`- Running benchmark "${chalk.white(benchmarkName)}"...`));
console.log(
chalk.gray(`- Running benchmark "${chalk.white(benchmarkName)}"...`)
);
results[benchmarkName] = await runBenchmark(benchmarkName, headless);
}
}
Expand All @@ -68,14 +68,13 @@ async function benchmarkRemoteMaster() {

if (!commit || typeof commit !== 'string') {
commit = await getMergeBaseFromLocalGitRepo(join(__dirname, '..', '..'));
console.log(chalk.gray(`- Merge base commit ${chalk.white(commit.tostrS())}`));
console.log(
chalk.gray(`- Merge base commit ${chalk.white(commit.tostrS())}`)
);
}
return {
// we build the bundles from the React repo
bundles: await buildBenchmarkBundlesFromGitRepo(
commit,
skipBuild
),
bundles: await buildBenchmarkBundlesFromGitRepo(commit, skipBuild),
// we use these bundles to run the benchmarks
benchmarks: await runBenchmarks(),
};
Expand All @@ -95,8 +94,8 @@ async function benchmarkLocal(reactPath) {

async function runLocalBenchmarks(showResults) {
console.log(
chalk.white.bold('Running benchmarks for ')
+ chalk.green.bold('Local (Current Branch)')
chalk.white.bold('Running benchmarks for ') +
chalk.green.bold('Local (Current Branch)')
);
const localResults = await benchmarkLocal(join(__dirname, '..', '..'));

Expand All @@ -108,8 +107,8 @@ async function runLocalBenchmarks(showResults) {

async function runRemoteBenchmarks(showResults) {
console.log(
chalk.white.bold('Running benchmarks for ')
+ chalk.yellow.bold('Remote (Merge Base)')
chalk.white.bold('Running benchmarks for ') +
chalk.yellow.bold('Remote (Merge Base)')
);
const remoteMasterResults = await benchmarkRemoteMaster();

Expand All @@ -121,10 +120,10 @@ async function runRemoteBenchmarks(showResults) {

async function compareLocalToMaster() {
console.log(
chalk.white.bold('Comparing ')
+ chalk.green.bold('Local (Current Branch)')
+ chalk.white.bold(' to ')
+ chalk.yellow.bold('Remote (Merge Base)')
chalk.white.bold('Comparing ') +
chalk.green.bold('Local (Current Branch)') +
chalk.white.bold(' to ') +
chalk.yellow.bold('Remote (Merge Base)')
);
const localResults = await runLocalBenchmarks(false);
const remoteMasterResults = await runRemoteBenchmarks(false);
Expand Down
15 changes: 8 additions & 7 deletions scripts/bench/server.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,8 @@

const http2Server = require('http2');
const httpServer = require('http-server');
const {
existsSync,
statSync,
createReadStream,
} = require('fs');
const { join } = require('path');
const {existsSync, statSync, createReadStream} = require('fs');
const {join} = require('path');
const argv = require('minimist')(process.argv.slice(2));
const mime = require('mime');

Expand All @@ -21,7 +17,12 @@ function sendFile(filename, response) {

function createHTTP2Server(benchmark) {
const server = http2Server.createServer({}, (request, response) => {
const filename = join(__dirname, 'benchmarks', benchmark, request.url).replace(/\?.*/g, '');
const filename = join(
__dirname,
'benchmarks',
benchmark,
request.url
).replace(/\?.*/g, '');

if (existsSync(filename) && statSync(filename).isFile()) {
sendFile(filename, response);
Expand Down
50 changes: 24 additions & 26 deletions scripts/bench/stats.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,20 +31,19 @@ function calculateMeanAndSdOfRatioFromDeltaMethod(
semControl,
semTest
) {
const mean = (
((meanTest - meanControl) / meanControl) -
(Math.pow(semControl, 2) * meanTest / Math.pow(meanControl, 3))
);
const variance = (
const mean =
(meanTest - meanControl) / meanControl -
Math.pow(semControl, 2) * meanTest / Math.pow(meanControl, 3);
const variance =
Math.pow(semTest / meanControl, 2) +
(Math.pow(semControl * meanTest, 2) / Math.pow(meanControl, 4))
);
Math.pow(semControl * meanTest, 2) / Math.pow(meanControl, 4);
return [mean, Math.sqrt(variance)];
}

function addBenchmarkResults(table, localResults, remoteMasterResults) {
const benchmarks = Object.keys(
(localResults && localResults.benchmarks) || (remoteMasterResults && remoteMasterResults.benchmarks)
(localResults && localResults.benchmarks) ||
(remoteMasterResults && remoteMasterResults.benchmarks)
);
benchmarks.forEach(benchmark => {
const rowHeader = [chalk.white.bold(benchmark)];
Expand All @@ -59,31 +58,32 @@ function addBenchmarkResults(table, localResults, remoteMasterResults) {
}
table.push(rowHeader);

const measurements = (
(localResults && localResults.benchmarks[benchmark].averages)
||
(remoteMasterResults && remoteMasterResults.benchmarks[benchmark].averages)
);
const measurements =
(localResults && localResults.benchmarks[benchmark].averages) ||
(remoteMasterResults &&
remoteMasterResults.benchmarks[benchmark].averages);
measurements.forEach((measurement, i) => {
const row = [
chalk.gray(measurement.entry),
];
const row = [chalk.gray(measurement.entry)];
let remoteMean;
let remoteSem;
if (remoteMasterResults) {
remoteMean = remoteMasterResults.benchmarks[benchmark].averages[i].mean;
remoteSem = remoteMasterResults.benchmarks[benchmark].averages[i].sem;
// https://en.wikipedia.org/wiki/1.96 gives a 99% confidence interval.
const ci95 = remoteSem * 1.96;
row.push(chalk.white(+remoteMean.toFixed(2) + ' ms +- ' + ci95.toFixed(2)));
row.push(
chalk.white(+remoteMean.toFixed(2) + ' ms +- ' + ci95.toFixed(2))
);
}
let localMean;
let localSem;
if (localResults) {
localMean = localResults.benchmarks[benchmark].averages[i].mean;
localSem = localResults.benchmarks[benchmark].averages[i].sem;
const ci95 = localSem * 1.96;
row.push(chalk.white(+localMean.toFixed(2) + ' ms +- ' + ci95.toFixed(2)));
row.push(
chalk.white(+localMean.toFixed(2) + ' ms +- ' + ci95.toFixed(2))
);
}
if (localResults && remoteMasterResults) {
row.push(percentChange(remoteMean, localMean, remoteSem, localSem));
Expand All @@ -107,17 +107,15 @@ function addBundleSizeComparions(table, localResults, remoteMasterResults) {
table.push(bundlesRowHeader);

const bundles = Object.keys(
(localResults && localResults.bundles.bundleSizes)
||
(remoteMasterResults && remoteMasterResults.bundles.bundleSizes)
(localResults && localResults.bundles.bundleSizes) ||
(remoteMasterResults && remoteMasterResults.bundles.bundleSizes)
);
bundles.forEach(bundle => {
const row = [
chalk.gray(bundle),
];
const row = [chalk.gray(bundle)];
let remoteSize = 0;
if (remoteMasterResults) {
const remoteBundle = remoteSize = remoteMasterResults.bundles.bundleSizes[bundle];
const remoteBundle = (remoteSize =
remoteMasterResults.bundles.bundleSizes[bundle]);

if (remoteBundle) {
remoteSize = remoteSize.size;
Expand Down Expand Up @@ -152,7 +150,7 @@ function printResults(localResults, remoteMasterResults) {
if (localResults && remoteMasterResults) {
head.push('');
}
const table = new Table({ head });
const table = new Table({head});

addBundleSizeComparions(table, localResults, remoteMasterResults);
addBenchmarkResults(table, localResults, remoteMasterResults);
Expand Down
2 changes: 1 addition & 1 deletion scripts/prettier/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ const config = {
},
scripts: {
patterns: ['scripts/**/*.js'],
ignore: ['**/bench/**'],
ignore: ['scripts/bench/benchmarks/**'],
options: {
'trailing-comma': 'es5',
},
Expand Down