fix(benchmarks): recreate setup for running benchmarks

Author: tbosch

build.sh

@@ -69,7 +69,8 @@ for PACKAGE in \
   http \
   router \
   upgrade \
-  compiler-cli
+  compiler-cli \
+  benchpress
 do
   PWD=`pwd`
   SRCDIR=${PWD}/modules/@angular/${PACKAGE}
@@ -101,7 +102,7 @@ do
     find ${DESTDIR} -type f -name '*.d.ts' -print0 | xargs -0 sed -i -E 's/^( +)abstract ([[:alnum:]]+\:)/\1\2/g'
   fi
 
-  if [[ ${PACKAGE} != compiler-cli ]]; then
+  if [[ ${PACKAGE} != compiler-cli && ${PACKAGE} != benchpress ]]; then
 
     echo "======      BUNDLING: ${SRCDIR} ====="
     mkdir ${DESTDIR}/bundles
@@ -128,6 +129,3 @@ do
 
   fi
 done
-
-echo "======      COMPILING: \$(npm bin)/tsc -p benchpress/tsconfig.json ====="
-$(npm bin)/tsc -p ./modules/benchpress/tsconfig.json

gulpfile.js

@@ -12,7 +12,7 @@ const os = require('os');
 
 const srcsToFmt =
     ['tools/**/*.ts', 'modules/@angular/**/*.ts', '!tools/public_api_guard/**/*.d.ts',
-    'modules/benchpress/**/*.ts', 'modules/playground/**/*.ts', 'modules/benchmarks/**/*.ts'];
+    'modules/playground/**/*.ts', 'modules/benchmarks/**/*.ts', 'modules/e2e_util/**/*.ts'];
 
 gulp.task('format:enforce', () => {
   const format = require('gulp-clang-format');

modules/@angular/benchpress/tsconfig-es5.json

@@ -7,7 +7,8 @@
         "sourceMap": true,
         "baseUrl": ".",
         "paths": {
-          "@angular/core": ["../../../dist/packages-dist/core"]
+          "@angular/core": ["../../../dist/packages-dist/core"],
+          "selenium-webdriver": ["../../../node_modules/@types/selenium-webdriver/index.d.ts"]
         },
         "experimentalDecorators": true,
         "rootDir": ".",
@@ -20,7 +21,7 @@
         "index.ts",
         "../../../node_modules/@types/node/index.d.ts",
         "../../../node_modules/@types/jasmine/index.d.ts",
-        "../../node_modules/@types/protractor/index.d.ts",
-        "../../node_modules/@types/selenium-webdriver/index.d.ts"
+        "../../../node_modules/@types/protractor/index.d.ts",
+        "../../../node_modules/zone.js/dist/zone.js.d.ts"
     ]
 }

modules/benchmarks/README.md

@@ -1,12 +1,24 @@
 # How to run the benchmarks locally
 
 ## Run in the browser
-$ build.sh
+$ build.sh (only needed 1x to copy over third party resources)
 $ cp -r ./modules/benchmarks ./dist/all/
 $ ./node_modules/.bin/tsc -p modules --emitDecoratorMetadata -w
 $ gulp serve
 $ open http://localhost:8000/all/benchmarks/src/tree/ng2/index.html?bundles=false
 
 ## Run e2e tests
 $ export NODE_PATH=$(pwd)/dist/all:$(pwd)/dist/tools
-$ ./node_modules/.bin/protractor protractor-js-new-world.conf.js --specs=dist/all/benchmarks/e2e_test/tree_perf.js
+$ ./node_modules/.bin/protractor protractor-e2e.conf.js --specs=dist/all/benchmarks/e2e_test/tree_spec.js
+
+Options for protractor with `protractor-e2e.conf.js`:
+- `--bundles=true`: use prebuilt bundles
+- `--ng-help`: show all available options
+
+## Run benchmarks tests
+$ export NODE_PATH=$(pwd)/dist/all:$(pwd)/dist/tools
+$ ./node_modules/.bin/protractor protractor-perf.conf.js --specs=dist/all/benchmarks/e2e_test/tree_perf.js
+
+Options for protractor with `protractor-perf.conf.js`:
+- `--bundles=true`: use prebuilt bundles
+- `--ng-help`: show all available options

modules/benchmarks/e2e_test/tree_perf.ts

@@ -6,32 +6,47 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
-import {verifyNoBrowserErrors} from 'e2e_util/e2e_util';
+import {runBenchmark, verifyNoBrowserErrors} from 'e2e_util/perf_util';
 
-const useBundles = false;
-
-describe('tree benchmark', function() {
+describe('tree benchmark', () => {
 
   afterEach(verifyNoBrowserErrors);
 
-  it('should work for the baseline', function() {
-    browser.ignoreSynchronization = true;
-    browser.get(`all/benchmarks/src/tree/baseline/index.html?bundles=${useBundles}`);
-    $('#createDom').click();
-    expect($('baseline').getText()).toContain('0');
+  it('should work for the baseline', function(done) {
+    runBenchmark({
+      id: 'deepTree.baseline',
+      url: 'all/benchmarks/src/tree/baseline/index.html',
+      ignoreBrowserSynchronization: true,
+      params: [{name: 'depth', value: 9}],
+      work: () => {
+        $('#createDom').click();
+        $('#destroyDom').click();
+      }
+    }).then(done, done.fail);
   });
 
-  it('should work for ng2', function() {
-    browser.get(`all/benchmarks/src/tree/ng2/index.html?bundles=${useBundles}`);
-    $('#createDom').click();
-    expect($('app').getText()).toContain('0');
+  it('should work for ng2', function(done) {
+    runBenchmark({
+      id: 'deepTree.ng2',
+      url: 'all/benchmarks/src/tree/ng2/index.html',
+      params: [{name: 'depth', value: 9}],
+      work: () => {
+        $('#createDom').click();
+        $('#destroyDom').click();
+      }
+    }).then(done, done.fail)
   });
 
-  it('should work for polymer', function() {
-    browser.ignoreSynchronization = true;
-    browser.get(`all/benchmarks/src/tree/polymer/index.html?bundles=${useBundles}`);
-    $('#createDom').click();
-    expect($('#app').getText()).toContain('0');
+  it('should work for polymer', function(done) {
+    runBenchmark({
+      id: 'deepTree.polymer',
+      url: 'all/benchmarks/src/tree/polymer/index.html',
+      ignoreBrowserSynchronization: true,
+      params: [{name: 'depth', value: 9}],
+      work: () => {
+        $('#createDom').click();
+        $('#destroyDom').click();
+      }
+    }).then(done, done.fail)
   });
-
 });

modules/benchmarks/e2e_test/tree_spec.ts

@@ -0,0 +1,43 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {openBrowser, verifyNoBrowserErrors} from 'e2e_util/e2e_util';
+
+const useBundles = false;
+
+describe('tree benchmark', function() {
+
+  afterEach(verifyNoBrowserErrors);
+
+  it('should work for the baseline', function() {
+    openBrowser({
+      url: 'all/benchmarks/src/tree/baseline/index.html',
+      ignoreBrowserSynchronization: true,
+    });
+    $('#createDom').click();
+    expect($('baseline').getText()).toContain('0');
+  });
+
+  it('should work for ng2', function() {
+    openBrowser({
+      url: 'all/benchmarks/src/tree/ng2/index.html',
+    });
+    $('#createDom').click();
+    expect($('app').getText()).toContain('0');
+  });
+
+  it('should work for polymer', function() {
+    openBrowser({
+      url: 'all/benchmarks/src/tree/polymer/index.html',
+      ignoreBrowserSynchronization: true,
+    });
+    $('#createDom').click();
+    expect($('#app').getText()).toContain('0');
+  });
+
+});

modules/e2e_util/e2e_util.ts

@@ -5,13 +5,43 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
-
+const yargs = require('yargs');
 import * as webdriver from 'selenium-webdriver';
 
+let cmdArgs: {'bundles': boolean};
 
 declare var browser: any;
 declare var expect: any;
 
+export function readCommandLine(extraOptions?: {[key: string]: any}) {
+  const options: {[key: string]: any} = {
+    'bundles': {describe: 'Whether to use the angular bundles or not.', default: false}
+  };
+  for (var key in extraOptions) {
+    options[key] = extraOptions[key];
+  }
+
+  cmdArgs = yargs.usage('Angular e2e test options.').options(options).help('ng-help').wrap(40).argv;
+  return cmdArgs;
+}
+
+export function openBrowser(config: {
+  url: string,
+  params?: {name: string, value: any}[],
+  ignoreBrowserSynchronization?: boolean
+}) {
+  if (config.ignoreBrowserSynchronization) {
+    browser.ignoreSynchronization = true;
+  }
+  var params = config.params || [];
+  params = params.concat([{name: 'bundles', value: cmdArgs.bundles}]);
+
+  var urlParams: string[] = [];
+  params.forEach((param) => { urlParams.push(param.name + '=' + param.value); });
+  var url = encodeURI(config.url + '?' + urlParams.join('&'));
+  browser.get(url);
+}
+
 /**
  * @experimental This API will be moved to Protractor.
  */

modules/e2e_util/perf_util.ts

@@ -5,70 +5,73 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
-
 export {verifyNoBrowserErrors} from './e2e_util';
 
-var benchpress = (global as any /** TODO #9100 */)['benchpress'];
-var bind = benchpress.bind;
-var Options = benchpress.Options;
+const yargs = require('yargs');
+const nodeUuid = require('node-uuid');
+import * as fs from 'fs-extra';
 
-export function runClickBenchmark(config: any /** TODO #9100 */) {
-  browser.ignoreSynchronization = !config.waitForAngular2;
-  var buttons =
-      config.buttons.map(function(selector: any /** TODO #9100 */) { return $(selector); });
-  config.work = function() {
-    buttons.forEach(function(button: any /** TODO #9100 */) { button.click(); });
-  };
-  return runBenchmark(config);
-}
+import {SeleniumWebDriverAdapter, Options, JsonFileReporter, Validator, RegressionSlopeValidator, ConsoleReporter, SizeValidator, MultiReporter, MultiMetric, Runner, Provider} from '@angular/benchpress';
+import {readCommandLine as readE2eCommandLine, openBrowser} from './e2e_util';
+
+let cmdArgs: {'sample-size': number, 'force-gc': boolean, 'dryrun': boolean, 'bundles': boolean};
+let runner: Runner;
 
-export function runBenchmark(config: any /** TODO #9100 */) {
-  return getScaleFactor(browser.params.benchmark.scaling).then(function(scaleFactor) {
-    var description = {};
-    var urlParams: any[] /** TODO #9100 */ = [];
-    if (config.params) {
-      config.params.forEach(function(param: any /** TODO #9100 */) {
-        var name = param.name;
-        var value = applyScaleFactor(param.value, scaleFactor, param.scale);
-        urlParams.push(name + '=' + value);
-        (description as any /** TODO #9100 */)[name] = value;
-      });
-    }
-    var url = encodeURI(config.url + '?' + urlParams.join('&'));
-    return browser.get(url).then(function() {
-      return (global as any /** TODO #9100 */)['benchpressRunner'].sample({
-        id: config.id,
-        execute: config.work,
-        prepare: config.prepare,
-        microMetrics: config.microMetrics,
-        providers: [{provide: Options.SAMPLE_DESCRIPTION, useValue: description}]
-      });
-    });
+export function readCommandLine() {
+  cmdArgs = <any>readE2eCommandLine({
+    'sample-size': {describe: 'Used for perf: sample size.', default: 20},
+    'force-gc': {describe: 'Used for perf: force gc.', default: false, type: 'boolean'},
+    'dryrun': {describe: 'If true, only run performance benchmarks once.', default: false},
+    'bundles': {describe: 'Whether to use the angular bundles or not.', default: false}
   });
+  runner = createBenchpressRunner();
 }
 
-function getScaleFactor(possibleScalings: any /** TODO #9100 */) {
-  return browser.executeScript('return navigator.userAgent').then(function(userAgent: string) {
-    var scaleFactor = 1;
-    possibleScalings.forEach(function(entry: any /** TODO #9100 */) {
-      if (userAgent.match(entry.userAgent)) {
-        scaleFactor = entry.value;
-      }
-    });
-    return scaleFactor;
+export function runBenchmark(config: {
+  id: string,
+  url: string,
+  params: {name: string, value: any}[],
+  ignoreBrowserSynchronization?: boolean,
+  microMetrics?: {[key: string]: string},
+  work?: () => void,
+  prepare?: () => void,
+}): Promise<any> {
+  openBrowser(config);
+
+  var description: {[key: string]: any} = {'bundles': cmdArgs.bundles};
+  config.params.forEach((param) => { description[param.name] = param.value; });
+  return runner.sample({
+    id: config.id,
+    execute: config.work,
+    prepare: config.prepare,
+    microMetrics: config.microMetrics,
+    providers: [{provide: Options.SAMPLE_DESCRIPTION, useValue: description}]
   });
 }
 
-function applyScaleFactor(
-    value: any /** TODO #9100 */, scaleFactor: any /** TODO #9100 */,
-    method: any /** TODO #9100 */) {
-  if (method === 'log2') {
-    return value + Math.log(scaleFactor) / Math.LN2;
-  } else if (method === 'sqrt') {
-    return value * Math.sqrt(scaleFactor);
-  } else if (method === 'linear') {
-    return value * scaleFactor;
+function createBenchpressRunner(): Runner {
+  let runId = nodeUuid.v1();
+  if (process.env.GIT_SHA) {
+    runId = process.env.GIT_SHA + ' ' + runId;
+  }
+  const resultsFolder = './dist/benchmark_results';
+  fs.ensureDirSync(resultsFolder);
+  let providers: Provider[] = [
+    SeleniumWebDriverAdapter.PROTRACTOR_PROVIDERS,
+    {provide: Options.FORCE_GC, useValue: cmdArgs['force-gc']},
+    {provide: Options.DEFAULT_DESCRIPTION, useValue: {'runId': runId}}, JsonFileReporter.PROVIDERS,
+    {provide: JsonFileReporter.PATH, useValue: resultsFolder}
+  ];
+  if (!cmdArgs['dryrun']) {
+    providers.push({provide: Validator, useExisting: RegressionSlopeValidator});
+    providers.push(
+        {provide: RegressionSlopeValidator.SAMPLE_SIZE, useValue: cmdArgs['sample-size']});
+    providers.push(MultiReporter.provideWith([ConsoleReporter, JsonFileReporter]));
   } else {
-    return value;
+    providers.push({provide: Validator, useExisting: SizeValidator});
+    providers.push({provide: SizeValidator.SAMPLE_SIZE, useValue: 1});
+    providers.push(MultiReporter.provideWith([]));
+    providers.push(MultiMetric.provideWith([]));
   }
+  return new Runner(providers);
 }

protractor-e2e.conf.js

@@ -1,3 +1,7 @@
+// Make sure that the command line is read as the first thing
+// as this could exit node if the help script should be printed.
+require('./dist/all/e2e_util/e2e_util').readCommandLine();
+
 var BROWSER_OPTIONS = {
   LocalChrome: {
     'browserName': 'chrome'
@@ -11,8 +15,6 @@ var BROWSER_OPTIONS = {
   }
 };
 
-
-
 exports.config = {
   allScriptsTimeout: 11000,
   specs: [