From d4732ac5c2cb9dd030fbd37ca5008dab73dfc2bb Mon Sep 17 00:00:00 2001 From: William Lachance Date: Mon, 29 Feb 2016 14:41:40 -0500 Subject: [PATCH] Bug 1252203 - Add some release blocker features to the e10s dashboard * Allow to filter results to exclude results which wouldn't block the release * Link to wiki page outlining blocker criteria --- ui/js/controllers/perf/e10s.js | 40 ++++++++++++++++++++++++++---- ui/js/directives/perf/compare.js | 15 +++++++---- ui/js/perf.js | 11 +++++++- ui/js/perfapp.js | 2 +- ui/partials/perf/comparetable.html | 6 +++++ ui/partials/perf/e10s.html | 22 ++++++++++------ 6 files changed, 76 insertions(+), 20 deletions(-) diff --git a/ui/js/controllers/perf/e10s.js b/ui/js/controllers/perf/e10s.js index 03bcc1a8d64..050f362af9f 100644 --- a/ui/js/controllers/perf/e10s.js +++ b/ui/js/controllers/perf/e10s.js @@ -7,6 +7,33 @@ perf.controller('e10sCtrl', [ ThOptionCollectionModel, PhSeries, PhCompare, thServiceDomain) { var projectName = 'mozilla-inbound'; var interval = 86400*2; + var blockers = { + "cart summary": 2.0, + "damp summary": 2.0, + "dromaeo_css summary": 2.0, + "dromaeo_dom summary": 2.0, + "glterrain summary": 5.0, + "kraken summary": 2.0, + "sessionrestore": 5.0, + "sessionrestore_no_auto_restore": 5.0, + "tart summary": 5.0, + "tcanvasmark summary": 5.0, + "tp5o % Processor Time": 2.0, + "tp5o Main_RSS": 2.0, + "tp5o Modified Page List Bytes": 2.0, + "tp5o Private Bytes": 2.0, + "tp5o XRes": 2.0, + "tp5o responsiveness": 2.0, + "tp5o summary": 5.0, + "tp5o_scroll summary": 2.0, + "tpaint": 5.0, + "tps summary": 5.0, + "tresize": 5.0, + "ts_paint": 2.0, + "tscrollx": 2.0, + "tsvgr_opacity summary": 5.0, + "tsvgx summary": 5.0 + }; $scope.testList = []; $scope.dataLoading = true; @@ -18,17 +45,20 @@ perf.controller('e10sCtrl', [ showOnlyImportant: Boolean($stateParams.showOnlyImportant !== undefined && parseInt($stateParams.showOnlyImportant)), showOnlyConfident: Boolean($stateParams.showOnlyConfident !== undefined && - parseInt($stateParams.showOnlyConfident)) + parseInt($stateParams.showOnlyConfident)), + showOnlyBlockers: Boolean($stateParams.showOnlyBlockers !== undefined && + parseInt($stateParams.showOnlyBlockers)) }; - $scope.$watchGroup(['filterOptions.filter', 'filterOptions.showOnlyImportant', - 'filterOptions.showOnlyConfident'], + 'filterOptions.showOnlyConfident', + 'filterOptions.showOnlyBlockers'], function() { $state.transitionTo('e10s', { filter: $scope.filterOptions.filter, showOnlyImportant: Boolean($scope.filterOptions.showOnlyImportant) ? 1 : undefined, - showOnlyConfident: Boolean($scope.filterOptions.showOnlyConfident) ? 1 : undefined + showOnlyConfident: Boolean($scope.filterOptions.showOnlyConfident) ? 1 : undefined, + showOnlyBlockers: Boolean($scope.filterOptions.showOnlyBlockers) ? 1 : undefined }, { location: true, inherit: true, @@ -90,7 +120,7 @@ perf.controller('e10sCtrl', [ if (e10sSig && baseSig) { var cmap = PhCompare.getCounterMap( testName, resultsMap['base'][baseSig], - resultsMap['e10s'][e10sSig]); + resultsMap['e10s'][e10sSig], blockers); cmap.name = platform + ' ' + (platform === 'osx-10-10' ? 'opt' : 'pgo'); cmap.links = [{ title: 'graph', diff --git a/ui/js/directives/perf/compare.js b/ui/js/directives/perf/compare.js index b9fd6d880ba..acfbbf482d4 100644 --- a/ui/js/directives/perf/compare.js +++ b/ui/js/directives/perf/compare.js @@ -13,7 +13,8 @@ treeherder.directive( compareResults: '=', testList: '=', filterOptions: '=', - filterByFramework: '@' + filterByFramework: '@', + releaseBlockerCriteria: '@' }, link: function(scope, element, attrs) { if (!scope.baseTitle) { @@ -28,10 +29,12 @@ treeherder.directive( return !matchText || item.toLowerCase().indexOf(matchText.toLowerCase()) > (-1); } function shouldBeShown(result) { - return (!scope.filterByFramework || _.isUndefined(scope.filterOptions.framework) || + return (!scope.filterByFramework || + _.isUndefined(scope.filterOptions.framework) || result.frameworkId === scope.filterOptions.framework.id) && (!scope.filterOptions.showOnlyImportant || result.isMeaningful) && - (!scope.filterOptions.showOnlyConfident || result.isConfident); + (!scope.filterOptions.showOnlyConfident || result.isConfident) && + (!scope.filterOptions.showOnlyBlockers || result.isBlocker); } function filterResult(results, key) { if (_.isUndefined(scope.filterOptions.filter)) { @@ -60,9 +63,11 @@ treeherder.directive( } scope.$watchGroup([ - 'filterOptions.framework', 'filterOptions.filter', + 'filterOptions.framework', + 'filterOptions.filter', 'filterOptions.showOnlyImportant', - 'filterOptions.showOnlyConfident'], + 'filterOptions.showOnlyConfident', + 'filterOptions.showOnlyBlockers'], function() { updateFilteredTestList(); }); diff --git a/ui/js/perf.js b/ui/js/perf.js index 7d5e4391301..cbc4b7aa9d0 100644 --- a/ui/js/perf.js +++ b/ui/js/perf.js @@ -322,6 +322,7 @@ perf.factory('PhCompare', [ '$q', '$http', 'thServiceDomain', 'PhSeries', // - .newIsBetter // is new result better or worse (even if unsure) // - .isImprovement // is new result better + we're confident about it // - .isRegression // is new result worse + we're confident about it + // - .isBlocker // new result matches "blocker" criteria // - .delta // - .deltaPercentage // - .confidence // t-test value @@ -332,7 +333,7 @@ perf.factory('PhCompare', [ '$q', '$http', 'thServiceDomain', 'PhSeries', // - .className // - .magnitude // - .marginDirection - getCounterMap: function getDisplayLineData(testName, originalData, newData) { + getCounterMap: function getDisplayLineData(testName, originalData, newData, blockerCriteria) { function removeZeroes(values) { return _.filter(values, function(v){ @@ -433,6 +434,14 @@ perf.factory('PhCompare', [ '$q', '$http', 'thServiceDomain', 'PhSeries', } cmap.isRegression = (cmap.className == 'compare-regression'); cmap.isImprovement = (cmap.className == 'compare-improvement'); + if (!_.isUndefined(blockerCriteria) && + !_.isUndefined(blockerCriteria[testName]) && + cmap.isRegression && + cmap.deltaPercentage > blockerCriteria[testName]) { + cmap.isBlocker = true; + } else { + cmap.isBlocker = false; + } cmap.isMeaningful = (cmap.className != ""); cmap.isComplete = (cmap.originalRuns.length && cmap.newRuns.length); diff --git a/ui/js/perfapp.js b/ui/js/perfapp.js index 2222c2a1079..3a4b583984f 100644 --- a/ui/js/perfapp.js +++ b/ui/js/perfapp.js @@ -35,7 +35,7 @@ perf.config(function($compileProvider, $httpProvider, $stateProvider, $urlRouter }).state('e10s', { title: 'e10s talos dashboard', templateUrl: 'partials/perf/e10s.html', - url: '/e10s?&filter&showOnlyImportant&showOnlyConfident', + url: '/e10s?&filter&showOnlyImportant&showOnlyConfident&showOnlyBlockers', controller: 'e10sCtrl' }); diff --git a/ui/partials/perf/comparetable.html b/ui/partials/perf/comparetable.html index 2fe58a5bf88..481349ee0ec 100644 --- a/ui/partials/perf/comparetable.html +++ b/ui/partials/perf/comparetable.html @@ -19,6 +19,12 @@ Hide uncertain results +
+ +
diff --git a/ui/partials/perf/e10s.html b/ui/partials/perf/e10s.html index 1b9a1112b38..ba5bf01323a 100644 --- a/ui/partials/perf/e10s.html +++ b/ui/partials/perf/e10s.html @@ -11,13 +11,19 @@

Perfherder e10s dashboard

will take a while to be reflected in results. When in doubt, check the graphs by hovering over each line.

- - +

For more information on what is considered "acceptable" in terms of a Talos + regression, see + + the official e10s release criteria. +

+ +