From b4d0f0f54603ad2cadf1aaefba3b756c2418ecdb Mon Sep 17 00:00:00 2001 From: Chirayu Krishnappa Date: Fri, 6 Jun 2014 12:15:44 -0700 Subject: [PATCH] chore(reporter): record lexer benchmark results This initial version sends benchmark metrics to the backend server. Visit https://ng-dash.appspot.com/commit/fb2aa60a584dd4b2bbb3d30e8fcd4c588f8669ed or https://ng-dash.appspot.com/all to see an example. (github: https://github.com/chirayuk/ng-dash) In this version, only Dart code run from the command line is able to record such reports. The code in benchmark/_reporter.dart handles sending the data to the server. A future version will mode this code into a locally running proxy server enabling browser based Dart code to send such reports while still including the information from the build environment. Reading data from the report server does not require any auth. See a simple example at https://github.com/chirayuk/ng-dash/commit/ex_01 Writing data requires authentication. Specifically, the REST server requires two cookies set: user_email and user_secret. (user_email does not have to be a valid e-mail just like AppEngine user e-mails.) Of these, user_secret is meant to be secret and should be safeguarded. For Travis, user_email is "travis-ci.org" and user_secret is encrypted in the .travis.yml. Refer http://docs.travis-ci.com/user/build-configuration/#Secure-environment-variables for details on how this is done. --- .travis.yml | 4 + benchmark/_reporter.dart | 179 ++++++++++++++++++++++++++++++++++++++ benchmark/lexer_perf.dart | 31 +++++-- 3 files changed, 208 insertions(+), 6 deletions(-) create mode 100644 benchmark/_reporter.dart diff --git a/.travis.yml b/.travis.yml index a9c3b4c2d..28b3fa5f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,6 +41,10 @@ env: - BROWSER_PROVIDER_READY_FILE=/tmp/sauce-connect-ready - SAUCE_USERNAME=angular-ci - SAUCE_ACCESS_KEY=9b988f434ff8-fbca-8aa4-4ae3-35442987 + # Reporting benchmarks to ng-dash.appspot.com + - NGDASH_BASE_URL=https://ng-dash.appspot.com + - NGDASH_USER_EMAIL=travis-ci.org + - secure: "n3KJsLLXEh1wlLRTF2wWvnDBAL+sOg+Mf/gc/Ub9/zCpXLDd1LP76hWBH/d7TCaC0oH5dnyD3ugV6PtJ7VEPRBZp72IbuNZzj8Ui8SisXVd0aos4u7s7X5NVwcxobhxd8Csoi5QPT31w8iT6qaC9VSXnYM3EEGqeppRqRBu6Hkg=" branches: except: diff --git a/benchmark/_reporter.dart b/benchmark/_reporter.dart new file mode 100644 index 000000000..8b134d487 --- /dev/null +++ b/benchmark/_reporter.dart @@ -0,0 +1,179 @@ +library _reporter; + +import 'dart:io'; +import 'dart:convert' show UTF8, JSON; + + +String getBaseUrl() { + String ngDashBaseUrl = Platform.environment["NGDASH_BASE_URL"]; + if (ngDashBaseUrl == null || ngDashBaseUrl.isEmpty) { + ngDashBaseUrl = "http://ng-dash.gae.localhost"; + } + return ngDashBaseUrl; +} + + +class ResultData { + final String name; + final String description; + final Map dimensions = {}; + final Map metrics = {}; + final List children = []; + + ResultData(this.name, this.description); + + ResultData newChild(String name, String description) { + ResultData child = new ResultData(name, description); + children.add(child); + return child; + } + + toJson() => { + "name": name, + "description": description, + "dimensions_json": JSON.encode(dimensions), + "metrics_json": JSON.encode(metrics), + "children": children.map((i) => i.toJson()).toList(), + }; +} + + +Map getTravisDimension() { + if (Platform.environment["TRAVIS"] != "true") { + throw "getTravisDimension(): Not called on TRAVIS"; + } + Map result = {}; + // Ref: http://docs.travis-ci.com/user/ci-environment/ + for (String envVar in const [ + "TRAVIS_BRANCH", // The name of the branch currently being built. + "TRAVIS_BUILD_ID", // The id of the current build that Travis CI uses internally. + "TRAVIS_BUILD_NUMBER", // The number of the current build (for example, "4"). + "TRAVIS_COMMIT", // The commit that the current build is testing. + "TRAVIS_COMMIT_RANGE", // The range of commits that were included in the push or pull request. + "TRAVIS_JOB_ID", // The id of the current job that Travis CI uses internally. + "TRAVIS_JOB_NUMBER", // The number of the current job (for example, "4.1"). + "TRAVIS_PULL_REQUEST", // The pull request number if the current job is a pull request, "false" if it's not a pull request. + "TRAVIS_REPO_SLUG", // "owner_name/repo_name" (e.g. "travis-ci/travis-build"). + "TRAVIS_OS_NAME", // Name of the operating system built on. (e.g. linux or osx) + "TRAVIS_TAG", // (optional) tag name for current build if relevant. + ]) { + String value = Platform.environment[envVar]; + if (value != null && value.isNotEmpty) { + result[envVar.substring("TRAVIS_".length).toLowerCase()] = value; + } + } + return result; +} + + +String getOsType() { + if (Platform.isMacOS) { + return "OSX"; + } else if (Platform.isLinux) { + return "Linux"; + } else if (Platform.isWindows) { + return "Windows"; + } else if (Platform.Android) { + return "Android"; + } +} + + +class Reporter { + ResultData data = new ResultData("", ""); + final String baseUrl = getBaseUrl(); + String commitSha = ""; + String treeSha = ""; + String reportId = null; + List cookies; + + Reporter() { + var dimensions = data.dimensions; + dimensions["project"] = "AngularDart"; + dimensions["dart"] = { + "full_version": Platform.version, + "version": Platform.version.split(" ")[0], + }; + dimensions["os"] = { + "type": getOsType(), + }; + if (Platform.environment["TRAVIS"] == "true") { + dimensions["travis"] = getTravisDimension(); + commitSha = dimensions["travis"]["commit"]; + } + + // Auth cookies + var user_email = Platform.environment["NGDASH_USER_EMAIL"]; + var user_secret = Platform.environment["NGDASH_USER_SECRET"]; + if (user_email == null || user_email.isEmpty || + user_secret == null || user_secret.isEmpty) { + throw "Please set NGDASH_USER_EMAIL and NGDASH_USER_SECRET credentials."; + } + cookies = [new Cookie("user_email", user_email), + new Cookie("user_secret", user_secret)]; + + if (commitSha.isEmpty) { + throw "Could not detect the commit SHA. (non-travis detection not implemented yet.)"; + } + } + + + // The following machinery is there just to serialize saving to the server. + // If we're already in the process of saving the results, then just mark that + // we need to save again. This is also particularly important because the + // first time, we create a new report, and in all subsequent calls, we update + // that same report using the report ID that was received when we created it. + var _messageQueue = []; + var _isQueueProcessing = true; + + _sendNextMessage() { + if (_messageQueue.isEmpty) { + _isQueueProcessing = true; + return; + } + + String requestData = JSON.encode(_messageQueue.removeAt(0)); + _isQueueProcessing = false; + Function onRequest = (HttpClientRequest request) { + request + ..headers.contentType = ContentType.JSON + ..cookies.addAll(cookies) + ..write(requestData); + return request.close(); + }; + + if (reportId == null) { + new HttpClient().postUrl(Uri.parse("${baseUrl}/api/run")) + .then(onRequest) + .then((HttpClientResponse response) { + List parts = []; + response.transform(UTF8.decoder).listen(parts.add, onDone: () { + // Extract reportId to use in future requests. + reportId = JSON.decode(parts.join(""))["id"]; + _sendNextMessage(); + }); + }); + } else { + new HttpClient().putUrl(Uri.parse("${baseUrl}/api/run/id=$reportId")) + .then(onRequest) + .then((HttpClientResponse response) { + response.transform(UTF8.decoder).listen(null, onDone: _sendNextMessage); + }); + } + } + + + void saveReport() { + _messageQueue.add(this); + if (_isQueueProcessing) { + _sendNextMessage(); + } + } + + + toJson() => { + "commit_sha": commitSha, + "tree_sha": treeSha, + "data": data.toJson(), + }; +} diff --git a/benchmark/lexer_perf.dart b/benchmark/lexer_perf.dart index 80aff438d..1ba607e11 100644 --- a/benchmark/lexer_perf.dart +++ b/benchmark/lexer_perf.dart @@ -1,20 +1,39 @@ library lexer_perf; import '_perf.dart'; +import '_reporter.dart'; import 'package:angular/core/parser/lexer.dart'; + +Reporter reporter = new Reporter(); +var lexerStats = reporter.data.newChild("lexer", "lexer benchmarks"); + + +report(name, fn) { + var stats = lexerStats.newChild(name, ""); + var metrics = statMeasure(fn); + stats.metrics["ops_per_sec"] = metrics.mean_ops_sec; + stats.metrics["variance"] = metrics.variance; + print('$name: => $metrics'); + reporter.saveReport(); +} + + main() { Lexer lexer = new Lexer(); - time('ident', () => + + report('ident', () => lexer.call('ctrl foo baz ctrl.foo ctrl.bar ctrl.baz')); - time('ident-path', () => + report('ident-path', () => lexer.call('a.b a.b.c a.b.c.d a.b.c.d.e.f')); - time('num', () => + report('num', () => lexer.call('1 23 34 456 12341234 12351235')); - time('num-double', () => + report('num-double', () => lexer.call('.0 .1 .12 0.123 0.1234')); - time('string', () => + report('string', () => lexer.call("'quick brown dog and fox say what'")); - time('string-escapes', () => + report('string-escapes', () => lexer.call("quick '\\' brown \u1234 dog and fox\n\rsay what'")); + + reporter.saveReport(); }