Skip to content
This repository has been archived by the owner on Nov 10, 2017. It is now read-only.

Commit

Permalink
Bug 1142680 - Write a compare-talos equivalent - items 1,2,7,8
Browse files Browse the repository at this point in the history
  • Loading branch information
@jmaher
jmaher committed Apr 30, 2015
1 parent cdd533f commit 571a8c1
Show file tree
Hide file tree
Showing 5 changed files with 161 additions and 56 deletions.
28 changes: 16 additions & 12 deletions webapp/app/css/perf.css
View file
Expand Up @@ -165,21 +165,21 @@ to { opacity: 0; }
}

.subtest-header {
background-image: linear-gradient(to bottom, #6A7B86, #424F5A);
color: white;
font-weight: bold;
background-image: linear-gradient(to bottom, #6A7B86, #424F5A);
color: white;
font-weight: bold;
}

.subtest-improvement {
background-color: green;
color: white;
font-weight: bold;
background: green;
color: white;
font-weight: bold;
}

.subtest-regression {
background-color: red;
color: white;
font-weight: bold;
background: red;
color: white;
font-weight: bold;
}

.subtest-empty {
Expand Down Expand Up @@ -227,17 +227,21 @@ to { opacity: 0; }
}

.compare-improvement {
background-color: green;
background-image: linear-gradient(to bottom, #80BB2E, #547D1C);
color: white;
font-weight: bold;
}

.compare-regression {
background-color: red;
background-image: linear-gradient(to bottom, #B20000, #940000);
color: white;
font-weight: bold;
}


.compare-notsure {
background-image: linear-gradient(to bottom, #FF9500, #FFCB00);
}

.bar-improvement {
background-color: green;
margin-left: 50%;
Expand Down
68 changes: 52 additions & 16 deletions webapp/app/js/compareperf.js
View file
Expand Up @@ -59,7 +59,7 @@ perf.controller('CompareResultsCtrl', [
var originalResultsMap = resultMaps[$scope.originalResultSetID];
var newResultsMap = resultMaps[$scope.newResultSetID];

// Optimization - collect all data in a single pass
// Optimization - we collected all data in a single pass
if (newResultsMap) {
$scope.dataLoading = false;
displayResults(originalResultsMap, newResultsMap);
Expand All @@ -80,7 +80,8 @@ perf.controller('CompareResultsCtrl', [
newSeriesData.seriesList,
timeRange,
[$scope.newResultSetID]);
}).then(function(newResultsMap) {
}).then(function(resultMaps) {
var newResultsMap = resultMaps[$scope.newResultSetID];
$scope.dataLoading = false;
displayResults(originalResultsMap, newResultsMap);
});
Expand Down Expand Up @@ -116,6 +117,7 @@ perf.controller('CompareResultsCtrl', [

cmap.detailsLink = detailsLink;
cmap.name = platform;
cmap.hideMinorChanges = $scope.hideMinorChanges;
$scope.compareResults[testName].push(cmap);
});
});
Expand Down Expand Up @@ -143,8 +145,9 @@ perf.controller('CompareResultsCtrl', [
});
}

var optionCollectionMap = {};
$scope.dataLoading = true;
var optionCollectionMap = {};
$scope.getCompareClasses = PhCompare.getCompareClasses;

$http.get(thServiceDomain + '/api/optioncollectionhash').then(
function(response) {
Expand Down Expand Up @@ -221,11 +224,23 @@ perf.controller('CompareSubtestResultsCtrl', [
$scope.titles[testName] = testName.replace('summary ', '');
$scope.compareResults[testName] = [];

$scope.pageList.sort();
$scope.pageList.forEach(function(page) {
var oldSig = _.find(Object.keys(rawResultsMap), function (sig) {
return (rawResultsMap[sig].name == page)});
var newSig = _.find(Object.keys(newRawResultsMap), function (sig) {
return (newRawResultsMap[sig].name == page)});
var mapsigs = [];
[rawResultsMap, newRawResultsMap].forEach(function(resultsMap) {
// If no data for a given platform, or test, display N/A in table
if (resultsMap) {
var tempsig = _.find(Object.keys(resultsMap), function (sig) {
return (resultsMap[sig].name == page)});
} else {
var tempsig = 'undefined';
resultsMap = {};
resultsMap[tempsig] = {};
}
mapsigs.push(tempsig);
});
var oldSig = mapsigs[0];
var newSig = mapsigs[1];

var cmap = PhCompare.getCounterMap(testName, rawResultsMap[oldSig], newRawResultsMap[newSig]);

Expand All @@ -250,12 +265,15 @@ perf.controller('CompareSubtestResultsCtrl', [

cmap.detailsLink = detailsLink;
cmap.name = page;
cmap.hideMinorChanges = $scope.hideMinorChanges;
$scope.compareResults[testName].push(cmap);
});
});
}

$scope.dataLoading = true;
var optionCollectionMap = {};
$scope.getCompareClasses = PhCompare.getCompareClasses;

$http.get(thServiceDomain + '/api/optioncollectionhash').then(
function(response) {
Expand All @@ -272,6 +290,7 @@ perf.controller('CompareSubtestResultsCtrl', [
$scope.originalRevision = $stateParams.originalRevision;
$scope.originalSignature = $stateParams.originalSignature;
$scope.newSignature = $stateParams.newSignature;
$scope.hideMinorChanges = Boolean($stateParams.hideMinorChanges);
if (!$scope.originalProject ||
!$scope.newProject ||
!$scope.originalRevision ||
Expand Down Expand Up @@ -311,34 +330,51 @@ perf.controller('CompareSubtestResultsCtrl', [
}).then(function(seriesMaps) {
var originalSeriesMap = seriesMaps[$scope.originalResultSetID];
var newSeriesMap = seriesMaps[$scope.newResultSetID];
Object.keys(originalSeriesMap).forEach(function(series) {
if (!_.contains($scope.pageList, originalSeriesMap[series].name)) {
$scope.pageList.push(originalSeriesMap[series].name);
[originalSeriesMap, newSeriesMap].forEach(function (seriesMap) {
// If there is no data for a given signature, handle it gracefully
if (seriesMap) {
Object.keys(seriesMap).forEach(function(series) {
if (!_.contains($scope.pageList, seriesMap[series].name)) {
$scope.pageList.push(seriesMap[series].name);
}
});
}
});

// Optimization- collect all data in a single pass
if (newSeriesMap) {
$scope.dataLoading = false;
displayResults(originalSeriesMap, newSeriesMap, timeRange);
return;
}

PhSeries.getSubtestSummaries($scope.newProject,
timeRange,
optionCollectionMap,
$scope.originalSignature).then(
$scope.newSignature).then(
function (newSeriesData) {
$scope.platformList = _.union($scope.platformList,
newSeriesData.platformList).sort();
$scope.testList = _.union($scope.testList,
newSeriesData.testList).sort();

return PhCompare.getResultsMap($scope.newProject,
newSeriesData.seriesList,
timeRange,
[$scope.newResultSetID]);
}).then(function(newSeriesMaps) {
var newSeriesMap = newSeriesMaps[$scope.newResultSetID];
Object.keys(newSeriesMap).forEach(function(series) {
if (!_.contains($scope.pageList, newSeriesMap[series].name)) {
$scope.pageList.push(newSeriesMap[series].name);
}
});
// There is a chance that we haven't received data for the given signature/resultSet yet
if (newSeriesMap) {
Object.keys(newSeriesMap).forEach(function(series) {
if (!_.contains($scope.pageList, newSeriesMap[series].name)) {
$scope.pageList.push(newSeriesMap[series].name);
}
});
} else {
newSeriesMap = {};
}
$scope.dataLoading = false;
displayResults(originalSeriesMap, newSeriesMap, timeRange);
});
});
Expand Down
83 changes: 69 additions & 14 deletions webapp/app/js/perf.js
View file
Expand Up @@ -73,7 +73,7 @@ perf.factory('PhSeries', ['$http', 'thServiceDomain', function($http, thServiceD
var seriesList = [];
var platformList = [];
var subtestSignatures = [];
var suiteName = "";
var suiteName;

//Given a signature, find the series and get subtest signatures
var seriesSummary = _.find(lists.seriesList,
Expand All @@ -99,10 +99,15 @@ perf.factory('PhSeries', ['$http', 'thServiceDomain', function($http, thServiceD
}
});

var testList = [];
if (suiteName) {
testList = [suiteName];
}

return {
seriesList: seriesList,
platformList: platformList,
testList: [suiteName]
testList: testList
};
});
},
Expand Down Expand Up @@ -166,31 +171,77 @@ perf.factory('isReverseTest', [ function() {
perf.factory('PhCompare', [ '$q', '$http', 'thServiceDomain', 'PhSeries',
'math', 'isReverseTest', 'phTimeRanges',
function($q, $http, thServiceDomain, PhSeries, math, isReverseTest, phTimeRanges) {
var getClassName = function(baselineMin, baselineMax, baselineAvg, currentAvg, test) {
var range = math.trimFloat((baselineMax - baselineMin) / 2);
if (isReverseTest(test)) {
if (currentAvg < baselineAvg - range) {
if (currentAvg < baselineMin) {
return "compare-regression";
}
// Still more than the min value we got out of the baseline, so we could
// be OK still, but there could be a regression here too.
return "compare-notsure";
}
else if (currentAvg > baselineMax) {
return "compare-improvement";
}
return "";
}

// We have a 'smaller is better' test.
if (currentAvg > baselineAvg + range) {
if (currentAvg > baselineMax) {
return "compare-regression";
}
// Still less than the max value we got out of the baseline, so we could be
// OK still, but there could be a regression here.
return "compare-notsure";
}
else if (currentAvg < baselineMin) {
return "compare-improvement";
}
return "";
};

return {
getCompareClasses: function(cr, type) {
if (cr.hideMinorChanges && cr.isMinor) return 'subtest-empty';
if (cr.isEmpty) return 'subtest-empty';
if (type == 'row') return '';
if (type == 'bar' && cr.isRegression) return 'bar-regression';
if (type == 'bar' && cr.isImprovement) return 'bar-improvement';
if (type == 'bar') return '';
return cr.className;
},

getCounterMap: function(testName, originalData, newData) {
var cmap = {originalGeoMean: NaN, originalRuns: 0, originalStddev: NaN,
newGeoMean: NaN, newRuns: 0, newStddev: NaN, delta: NaN,
deltaPercentage: NaN, barGraphMargin: 0, isEmpty: false,
var cmap = {originalGeoMean: 0, originalRuns: 0, originalStddev: 0,
newGeoMean: 0, newRuns: 0, newStddev: 0, delta: 0,
deltaPercentage: 0, barGraphMargin: 0, isEmpty: false,
isRegression: false, isImprovement: false, isMinor: true};

if (originalData) {
cmap.originalGeoMean = originalData.geomean;
cmap.originalRuns = originalData.runs;
cmap.originalStddev = originalData.stddev;
cmap.originalStddevPct = ((originalData.stddev / originalData.geomean) * 100);
cmap.originalMin = originalData.minVal;
cmap.originalMax = originalData.maxVal;
}
if (newData) {
cmap.newGeoMean = newData.geomean;
cmap.newRuns = newData.runs;
cmap.newStddev = newData.stddev;
cmap.newStddevPct = ((newData.stddev / newData.geomean) * 100);
cmap.newMin = newData.minVal;
cmap.newMax = newData.maxVal;
}

if ((cmap.originalRuns == 0 && cmap.newRuns == 0) ||
(testName == 'tp5n summary opt')) {
// We don't generate numbers for tp5n, just counters
cmap.isEmpty = true;
} else {
} else if (cmap.newGeoMean > 0 && cmap.originalGeoMean > 0) {
cmap.delta = (cmap.newGeoMean - cmap.originalGeoMean);
cmap.deltaPercentage = (cmap.delta / cmap.originalGeoMean * 100);
cmap.barGraphMargin = 50 - Math.min(50, Math.abs(Math.round(cmap.deltaPercentage) / 2));
Expand All @@ -207,13 +258,9 @@ perf.factory('PhCompare', [ '$q', '$http', 'thServiceDomain', 'PhSeries',
}
}

if (cmap.deltaPercentage > 2.0) {
cmap.isMinor = false;
isReverseTest(testName) ? cmap.isImprovement = true : cmap.isRegression = true;
} else if (cmap.deltaPercentage < -2.0) {
cmap.isMinor = false;
isReverseTest(testName) ? cmap.isRegression = true : cmap.isImprovement = true;
}
cmap.className = getClassName(cmap.originalMin, cmap.originalMax, cmap.originalGeoMean, cmap.newGeoMean, testName);
cmap.isRegression = (cmap.className == 'compare-regression')
cmap.isImprovement = (cmap.className == 'compare-improvement')
}
return cmap;
},
Expand Down Expand Up @@ -263,6 +310,8 @@ perf.factory('PhCompare', [ '$q', '$http', 'thServiceDomain', 'PhSeries',
var sigma = math.stddev(means, avg);

resultsMap[resultSetId][data.series_signature] = {geomean: avg,
minVal: Math.min.apply(Math, means),
maxVal: Math.max.apply(Math, means),
stddev: sigma,
runs: means.length,
name: seriesData.name,
Expand Down Expand Up @@ -297,6 +346,12 @@ perf.factory('math', [ function() {
return Math.sqrt(
values.map(function (v) { return Math.pow(v - avg, 2); })
.reduce(function (a, b) { return a + b; }) / (values.length - 1));
},

trimFloat: function(number) {
if (number === undefined)
return 'N/A';
return Math.round(number * 100) / 100;
}
};
}]);
Expand All @@ -305,7 +360,7 @@ perf.factory('math', [ function() {
perf.filter('displayPrecision', function() {
return function(input) {
if (!input) {
return "NaN";
return "N/A";
}

return parseFloat(input).toFixed(2);
Expand Down
15 changes: 9 additions & 6 deletions webapp/app/partials/perf/comparectrl.html
View file
Expand Up @@ -23,15 +23,18 @@ <h4>Compare all data for revision {{newProject}}: {{newRevision}} to {{originalP
<tr class="subtest-header">
<td>{{titles[testName]}}</td><td>Old Geomean</td><td>Old StdDev</td><td>New Geomean</td><td>New StdDev</td><td>Delta</td><td>Delta %</td><td></td>
</tr>
<tr ng-class="{'subtest-empty': compareResult.isEmpty, 'subtest-empty': (hideMinorChanges && compareResult.isMinor)}" ng-repeat="compareResult in compareResults[testName]">
<tr ng-class="getCompareClasses(compareResult, 'row')" ng-repeat="compareResult in compareResults[testName]">
<td>{{compareResult.name}} (<a ng-href="{{compareResult.detailsLink}}">Details</a>)</td>
<td ng-attr-title="runs: {{compareResult.originalRuns}}">{{compareResult.originalGeoMean|displayPrecision}}</td>
<td>+/-{{compareResult.originalStddev|displayPrecision}} ({{compareResult.originalStddevPct|displayPrecision}}%)</td>
<td ng-if="compareResult.originalStddev">+/-{{compareResult.originalStddev|displayPrecision}} ({{compareResult.originalStddevPct|displayPrecision}}%)</td>
<td ng-if="!compareResult.originalStddev">N/A</td>
<td ng-attr-title="runs: {{compareResult.newRuns}}">{{compareResult.newGeoMean|displayPrecision}}</td>
<td>+/-{{compareResult.newStddev|displayPrecision}} ({{compareResult.newStddevPct|displayPrecision}}%)</td>
<td ng-class="{'subtest-regression': compareResult.isRegression, 'subtest-improvement': compareResult.isImprovement}">{{compareResult.delta|displayPrecision}}</td>
<td ng-class="{'subtest-regression': compareResult.isRegression, 'subtest-improvement': compareResult.isImprovement}">{{compareResult.deltaPercentage|displayPrecision}}%</td>
<td width="20%"><div ng-class="{'bar-regression': compareResult.isRegression, 'bar-improvement': compareResult.isImprovement}" style="margin-{{compareResult.marginDirection}}: {{compareResult.barGraphMargin}}%;"></div></td>
<td ng-if="compareResult.newStddev">+/-{{compareResult.newStddev|displayPrecision}} ({{compareResult.newStddevPct|displayPrecision}}%)</td>
<td ng-if="!compareResult.newStddev">N/A</td>
<td ng-class="getCompareClasses(compareResult)">{{compareResult.delta|displayPrecision}}</td>
<td ng-class="getCompareClasses(compareResult)">{{compareResult.deltaPercentage|displayPrecision}}%</td>
<td ng-if="compareResult.delta" width="20%"><div ng-class="getCompareClasses(compareResult, 'bar')" style="margin-{{compareResult.marginDirection}}: {{compareResult.barGraphMargin}}%;"></div></td>
<td ng-if="!compareResult.delta"></td>
</tr>
</tbody>
</table>
Expand Down

0 comments on commit 571a8c1

Please sign in to comment.