WIP feat(bench press): add iOS support

.gitignore

@@ -22,4 +22,8 @@ pubspec.lock
 # Don't check in secret files
 *secret.js
 
+# Don't check in downloaded jar files or log files
+*.jar
+*.log
+
 /docs/bower_components/

README.md

@@ -74,6 +74,32 @@ Angular specific command line options when running protractor:
 Angular specific command line options when running protractor (e.g. force gc, ...):
 `protractor protractor-{js|dart2js}-conf.js --ng-help`
 
+### Test on mobile devices
+
+## iOS:
+
+1. Download [latest iphone-driver](http://ios-driver-ci.ebaystratus.com/userContent/ios-server-standalone-0.6.6-SNAPSHOT.jar),
+   see `scripts/ci/install_io_driver.sh`.
+2. create a wifi network on which the iOS device can see the server
+3. Run `java -jar ios-server-standalone-0.6.6-SNAPSHOT.jar -real`
+4. `protractor protractor-js.conf.js --browsers=SafariIos --seleniumAddress=http://localhost:5555/wd/hub --serverAddress=yourServerName`
+
+Notes:
+* ios-driver does not yet support iOS 8.x native devices, so only use iOS 7.x for them
+* ios-driver does support iOS 8.x for the iOS simulator
+
+## Android:
+
+1. get root access so that you can control the CPU speed of the device
+  * e.g. flash a so called user-debug build of Android
+2. connect your Android device via usb
+3. run `./scripts/ci/init_android.sh`. This will:
+  * set up a reverse proxy so that the browser on Android can use `localhost`
+  * set the cpu speed to a fixed value
+  * prevent the device from sleeping
+4. `protractor protractor-js.conf.js --browsers=ChromeAndroid`
+
+
 ### Examples
 
 To see the examples, first build the project as described above.

modules/benchpress/src/runner.js

@@ -72,9 +72,6 @@ var _DEFAULT_BINDINGS = [
   WebDriverExtension.bindTo([ChromeDriverExtension, IOsDriverExtension]),
   Metric.bindTo(MultiMetric),
 
-  bind(Options.CAPABILITIES).toAsyncFactory(
-    (adapter) => adapter.capabilities(), [WebDriverAdapter]
-  ),
   bind(Options.USER_AGENT).toAsyncFactory(
     (adapter) => adapter.executeScript('return window.navigator.userAgent;'), [WebDriverAdapter]
   )

modules/benchpress/src/sample_options.js

@@ -14,8 +14,6 @@ export class Options {
   // TODO(tbosch): use static initializer when our transpiler supports it
   static get EXECUTE() { return _EXECUTE; }
   // TODO(tbosch): use static initializer when our transpiler supports it
-  static get CAPABILITIES() { return _CAPABILITIES; }
-  // TODO(tbosch): use static initializer when our transpiler supports it
   static get USER_AGENT() { return _USER_AGENT; }
   // TODO(tbosch): use static initializer when our transpiler supports it
   /**
@@ -31,6 +29,5 @@ var _SAMPLE_DESCRIPTION = new OpaqueToken('Options.sampleDescription');
 var _FORCE_GC = new OpaqueToken('Options.forceGc');
 var _PREPARE = new OpaqueToken('Options.prepare');
 var _EXECUTE = new OpaqueToken('Options.execute');
-var _CAPABILITIES = new OpaqueToken('Options.capabilities');
 var _USER_AGENT = new OpaqueToken('Options.userAgent');
 var _MICRO_ITERATIONS = new OpaqueToken('Options.microIterations');

modules/benchpress/src/web_driver_extension.js

@@ -20,19 +20,19 @@ export class WebDriverExtension {
         [Injector]
       ),
       bind(WebDriverExtension).toFactory(
-        (children, capabilities) => {
+        (children, userAgent) => {
           var delegate;
           ListWrapper.forEach(children, (extension) => {
-            if (extension.supports(capabilities)) {
+            if (extension.supports(userAgent)) {
               delegate = extension;
             }
           });
           if (isBlank(delegate)) {
-            throw new BaseException('Could not find a delegate for given capabilities!');
+            throw new BaseException(`Could not find a delegate for userAgent ${userAgent}!`);
           }
           return delegate;
         },
-        [_CHILDREN, Options.CAPABILITIES]
+        [_CHILDREN, Options.USER_AGENT]
       )
     ];
   }
@@ -64,7 +64,7 @@ export class WebDriverExtension {
     throw new BaseException('NYI');
   }
 
-  supports(capabilities:StringMap):boolean {
+  supports(userAgent:string):boolean {
     return true;
   }
 }

modules/benchpress/src/webdriver/chrome_driver_extension.js

@@ -111,8 +111,8 @@ export class ChromeDriverExtension extends WebDriverExtension {
     return normalizedEvents;
   }
 
-  supports(capabilities:StringMap):boolean {
-    return StringWrapper.equals(capabilities['browserName'].toLowerCase(), 'chrome');
+  supports(userAgent:string):boolean {
+    return isPresent(RegExpWrapper.firstMatch(_USER_AGENT_REGEX, userAgent));
   }
 }
 
@@ -142,6 +142,8 @@ function normalizeEvent(chromeEvent, data) {
   return result;
 }
 
+var _USER_AGENT_REGEX = RegExpWrapper.create('\\bChrome\\b');
+
 var _BINDINGS = [
   bind(ChromeDriverExtension).toFactory(
     (driver) => new ChromeDriverExtension(driver),

modules/benchpress/src/webdriver/ios_driver_extension.js

@@ -37,8 +37,11 @@ export class IOsDriverExtension extends WebDriverExtension {
   }
 
   // See https://github.com/WebKit/webkit/tree/master/Source/WebInspectorUI/Versions
+  // for the protocol.
+  // See https://github.com/WebKit/webkit/blob/master/Source/WebCore/inspector/InspectorTimelineAgent.cpp
+  // for which records are actually fired by Safari.
   readPerfLog() {
-    // TODO(tbosch): Bug in IOsDriver: Need to execute at least one command
+    // TODO(tbosch): Bug in WebKit: Need to execute at least one command
     // so that the browser logs can be read out!
     return this._driver.executeScript('1+1')
       .then( (_) => this._driver.logs('performance') )
@@ -64,7 +67,6 @@ export class IOsDriverExtension extends WebDriverExtension {
       var data = record['data'];
       var startTime = record['startTime'];
       var endTime = record['endTime'];
-
       if (StringWrapper.equals(type, 'FunctionCall') &&
           (isBlank(data) || !StringWrapper.equals(data['scriptName'], 'InjectedScript'))) {
         ListWrapper.push(events, createStartEvent('script', startTime));
@@ -99,8 +101,8 @@ export class IOsDriverExtension extends WebDriverExtension {
     return events;
   }
 
-  supports(capabilities:StringMap):boolean {
-    return StringWrapper.equals(capabilities['browserName'].toLowerCase(), 'safari');
+  supports(userAgent:string):boolean {
+    return isPresent(RegExpWrapper.firstMatch(_USER_AGENT_REGEX, userAgent));
   }
 }
 
@@ -136,6 +138,8 @@ function createMarkEndEvent(name, time) {
   return createEvent('e', name, time);
 }
 
+var _USER_AGENT_REGEX = RegExpWrapper.create('\\bMobile\\b.*\\bSafari\\b');
+
 var _BINDINGS = [
   bind(IOsDriverExtension).toFactory(
     (driver) => new IOsDriverExtension(driver),

modules/benchpress/test/web_driver_extension_spec.js

@@ -7,26 +7,26 @@ import { PromiseWrapper } from 'angular2/src/facade/async';
 import { WebDriverExtension, bind, Injector, Options } from 'benchpress/common';
 
 export function main() {
-  function createExtension(ids, caps) {
+  function createExtension(ids, userAgent) {
     return new Injector([
       ListWrapper.map(ids, (id) => bind(id).toValue(new MockExtension(id)) ),
-      bind(Options.CAPABILITIES).toValue(caps),
+      bind(Options.USER_AGENT).toValue(userAgent),
       WebDriverExtension.bindTo(ids)
     ]).asyncGet(WebDriverExtension);
   }
 
   describe('WebDriverExtension.bindTo', () => {
 
-    it('should bind the extension that matches the capabilities', (done) => {
-      createExtension(['m1', 'm2', 'm3'], {'browser': 'm2'}).then( (m) => {
+    it('should bind the extension that matches the userAgent', (done) => {
+      createExtension(['m1', 'm2', 'm3'], 'm2').then( (m) => {
         expect(m.id).toEqual('m2');
         done();
       });
     });
 
     it('should throw if there is no match', (done) => {
       PromiseWrapper.catchError(
-        createExtension(['m1'], {'browser': 'm2'}),
+        createExtension(['m1'], 'm2'),
         (err) => {
           expect(isPresent(err)).toBe(true);
           done();
@@ -45,7 +45,7 @@ class MockExtension extends WebDriverExtension {
     this.id = id;
   }
 
-  supports(capabilities:StringMap):boolean {
-    return StringWrapper.equals(capabilities['browser'], this.id);
+  supports(userAgent:string):boolean {
+    return StringWrapper.equals(userAgent, this.id);
   }
 }

modules/benchpress/test/webdriver/chrome_driver_extension_spec.js

@@ -208,13 +208,8 @@ export function main() {
       });
 
       it('should match chrome browsers', () => {
-        expect(createExtension().supports({
-          'browserName': 'chrome'
-        })).toBe(true);
-
-        expect(createExtension().supports({
-          'browserName': 'Chrome'
-        })).toBe(true);
+        expect(createExtension().supports('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) '
+          +'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.74 Safari/537.36')).toBe(true);
       });
 
     });

modules/benchpress/test/webdriver/ios_driver_extension_spec.js

@@ -156,13 +156,8 @@ export function main() {
       });
 
       it('should match safari browsers', () => {
-        expect(createExtension().supports({
-          'browserName': 'safari'
-        })).toBe(true);
-
-        expect(createExtension().supports({
-          'browserName': 'Safari'
-        })).toBe(true);
+        expect(createExtension().supports('Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) '+
+          'AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53')).toBe(true);
       });
 
     });

modules/examples/e2e_test/benchpress/smoke_test_perf.es6

@@ -0,0 +1,31 @@
+// Note: This does not use the common utilities for perf tests
+// in Angular2 to show how to write a simple standalone perf test
+// with benchpress.
+var benchpress = require('benchpress/benchpress');
+var runner = createRunner();
+
+describe('benchpress', () => {
+
+  fit('should work', (done) => {
+    browser.get('examples/src/benchpress/index.html');
+
+    runner.sample({
+      id: 'benchpress smoke test',
+      execute: () => {
+        $('button').click();
+        expect($('#log').getText()).toBe('hi');
+      }
+    }).then(done, done.fail);
+
+  });
+
+});
+
+function createRunner() {
+  return new benchpress.Runner([
+    benchpress.SeleniumWebDriverAdapter.PROTRACTOR_BINDINGS,
+    benchpress.bind(benchpress.Options.FORCE_GC).toValue(false),
+    benchpress.Validator.bindTo(benchpress.RegressionSlopeValidator),
+    benchpress.bind(benchpress.RegressionSlopeValidator.SAMPLE_SIZE).toValue(3)
+  ]);
+}

protractor-dart2js.conf.js

@@ -1,7 +1,7 @@
 var data = module.exports = require('./protractor-shared.js');
 var config = data.config;
 
-config.baseUrl = 'http://localhost:8002/';
+config.baseUrl = 'http://'+config.serverAddress+':8002/';
 
 config.exclude.push(
   'dist/js/cjs/examples/e2e_test/sourcemap/sourcemap_spec.js',

protractor-js.conf.js

@@ -1,7 +1,7 @@
 var data = module.exports = require('./protractor-shared.js');
 var config = data.config;
 
-config.baseUrl = 'http://localhost:8001/';
+config.baseUrl = 'http://'+config.serverAddress+':8001/';
 // TODO: remove exclusion when JS verison of scrolling benchmark is available
 config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/naive_infinite_scroll_spec.js');
 config.exclude.push('dist/js/cjs/benchmarks_external/e2e_test/naive_infinite_scroll_perf.js');