Skip to content
Permalink
Browse files
Identify, rather than skip, builds where run-webkit-tests exited earl…
…y due to too many failures

Fixes <http://webkit.org/b/61441> TestFailures page should show when run-webkit-tests
started exiting early due to too many crashes, timeouts, or failures

Reviewed by David Kilzer.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
(Builder.prototype.getNumberOfFailingTests): Rather than returning -1 ("unknown") when
run-webkit-tests exits early due to too many failures, pass that information in another
argument to the callback.
(Builder.prototype.startFetchingBuildHistory): Updated the documentation comment to reflect
the new structure of the object passed to the callback.
(Builder.prototype._getFailingTests): Updated to expect a tooManyFailures boolean from
getNumberOfFailingTests and to pass that along to our own callbacks.
(Builder.prototype._incorporateBuildHistory): Updated to expect a tooManyFailures boolean
from _getFailingTests and to store that value in the history object.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css:
(.info): Added.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js:
(ViewController.prototype._displayBuilder): Updated for change in structure to the history
object and to add a note when run-webkit-tests exited early due to too many failures.
(ViewController.prototype._displayTesters): Renamed testersAndFailureCounts to
latestBuildInfos since each entry now contains more than just the tester and failure count.
Now displays a message for testers where the latest build exited early due to too many
failures. Updated to expect a tooManyFailures boolean from getNumberOfFailingTests and to
store that value in latestBuildInfos.

Canonical link: https://commits.webkit.org/76850@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@87299 268f45cc-cd09-0410-ab3c-d52691b4dbfc
  • Loading branch information
aroben committed May 25, 2011
1 parent 5a5b8cd commit 4ed5d784dc29c8259a2b6099e1800b031a49d088
@@ -102,30 +102,27 @@ Builder.prototype = {
var layoutTestStep = data.steps.findFirst(function(step) { return step.name === 'layout-test'; });
if (!layoutTestStep) {
self._cache[cacheKey] = -1;
callback(self._cache[cacheKey]);
callback(self._cache[cacheKey], false);
return;
}

if (!('isStarted' in layoutTestStep)) {
// run-webkit-tests never even ran.
self._cache[cacheKey] = -1;
callback(self._cache[cacheKey]);
callback(self._cache[cacheKey], false);
return;
}

if (!('results' in layoutTestStep) || layoutTestStep.results[0] === 0) {
// All tests passed.
self._cache[cacheKey] = 0;
callback(self._cache[cacheKey]);
callback(self._cache[cacheKey], false);
return;
}

if (/^Exiting early/.test(layoutTestStep.results[1][0])) {
// Too many tests crashed or timed out. We can't know for sure how many failed.
self._cache[cacheKey] = -1;
callback(self._cache[cacheKey]);
return;
}
var tooManyFailures = false;
if (/^Exiting early/.test(layoutTestStep.results[1][0]))
tooManyFailures = true;

var failureCount = layoutTestStep.results[1].reduce(function(sum, outputLine) {
var match = /^(\d+) test cases/.exec(outputLine);
@@ -135,19 +132,29 @@ Builder.prototype = {
}, 0);

self._cache[cacheKey] = failureCount;
callback(failureCount);
callback(failureCount, tooManyFailures);
});
},

/*
* Preiodically calls callback until all current failures have been explained. Callback is
* passed an object like the following:
* {
* 'r2_1 (1)': {
* 'css1/basic/class_as_selector2.html': 'fail',
* 'r12347 (681)': {
* 'tooManyFailures': false,
* 'tests': {
* 'css1/basic/class_as_selector2.html': 'fail',
* },
* },
* 'r12346 (680)': {
* 'tooManyFailures': false,
* 'tests': {},
* },
* 'r1_1 (0)': {
* 'css1/basic/class_as_selector.html': 'crash',
* 'r12345 (679)': {
* 'tooManyFailures': false,
* 'tests': {
* 'css1/basic/class_as_selector.html': 'crash',
* },
* },
* },
* Each build contains just the failures that a) are still occuring on the bots, and b) were new
@@ -233,16 +240,16 @@ Builder.prototype = {
var buildNumber = this.buildbot.parseBuildName(buildName).buildNumber;

var self = this;
self.getNumberOfFailingTests(buildNumber, function(failingTestCount) {
self.getNumberOfFailingTests(buildNumber, function(failingTestCount, tooManyFailures) {
if (failingTestCount < 0) {
// The number of failing tests couldn't be determined.
errorCallback(tests);
errorCallback(tests, tooManyFailures);
return;
}

if (!failingTestCount) {
// All tests passed.
callback(tests);
callback(tests, tooManyFailures);
return;
}

@@ -277,11 +284,11 @@ Builder.prototype = {
tests[name] = 'webprocess crash';
});

callback(tests);
callback(tests, tooManyFailures);
},
function(xhr) {
// We failed to fetch results.html. run-webkit-tests must have aborted early.
errorCallback(tests);
errorCallback(tests, tooManyFailures);
});
});
},
@@ -290,19 +297,22 @@ Builder.prototype = {
var previousBuildName = Object.keys(history).last();
var nextBuildName = buildNames[buildIndex];

this._getFailingTests(nextBuildName, function(tests) {
history[nextBuildName] = {};
this._getFailingTests(nextBuildName, function(tests, tooManyFailures) {
history[nextBuildName] = {
tooManyFailures: tooManyFailures,
tests: {},
};

for (var testName in tests) {
if (previousBuildName) {
if (!(testName in history[previousBuildName]))
if (!(testName in history[previousBuildName].tests))
continue;
delete history[previousBuildName][testName];
delete history[previousBuildName].tests[testName];
}
history[nextBuildName][testName] = tests[testName];
history[nextBuildName].tests[testName] = tests[testName];
}

callback(Object.keys(history[nextBuildName]).length);
callback(Object.keys(history[nextBuildName].tests).length);
},
function(tests) {
// Some tests failed, but we couldn't fetch results.html (perhaps because the test
@@ -1,11 +1,17 @@
ol {
list-style-type: none;
}

dt {
float: left;
font-weight: bold;
margin-right: 3px;
}

dt::after {
content: ':';
}

.info {
font-style: italic;
}
@@ -49,7 +49,7 @@ ViewController.prototype = {
builder.startFetchingBuildHistory(function(history) {
var list = document.createElement('ol');
Object.keys(history).forEach(function(buildName, buildIndex, buildNameArray) {
if (!Object.keys(history[buildName]).length)
if (!Object.keys(history[buildName].tests).length)
return;
var dlItems = [
[document.createTextNode('Failed'), self._domForBuildName(builder, buildName)],
@@ -61,10 +61,17 @@ ViewController.prototype = {
item.appendChild(createDefinitionList(dlItems));
list.appendChild(item);

if (history[buildName].tooManyFailures) {
var p = document.createElement('p');
p.className = 'info';
p.appendChild(document.createTextNode('run-webkit-tests exited early due to too many failures/crashes/timeouts'));
item.appendChild(p);
}

var testList = document.createElement('ol');
for (var testName in history[buildName]) {
for (var testName in history[buildName].tests) {
var testItem = document.createElement('li');
testItem.appendChild(self._domForFailedTest(builder, buildName, testName, history[buildName][testName]));
testItem.appendChild(self._domForFailedTest(builder, buildName, testName, history[buildName].tests[testName]));
testList.appendChild(testItem);
}
item.appendChild(testList);
@@ -81,23 +88,23 @@ ViewController.prototype = {

_displayTesters: function() {
var list = document.createElement('ul');
var testersAndFailureCounts = [];
var latestBuildInfos = [];

function updateList() {
testersAndFailureCounts.sort(function(a, b) { return a.tester.name.localeCompare(b.tester.name) });
latestBuildInfos.sort(function(a, b) { return a.tester.name.localeCompare(b.tester.name) });
while (list.firstChild)
list.removeChild(list.firstChild);
testersAndFailureCounts.forEach(function(testerAndFailureCount) {
var tester = testerAndFailureCount.tester;
var failureCount = testerAndFailureCount.failureCount;

latestBuildInfos.forEach(function(buildInfo) {
var link = document.createElement('a');
link.href = '#/' + tester.name;
link.appendChild(document.createTextNode(tester.name));
link.href = '#/' + buildInfo.tester.name;
link.appendChild(document.createTextNode(buildInfo.tester.name));

var item = document.createElement('li');
item.appendChild(link);
item.appendChild(document.createTextNode(' (' + failureCount + ' failing tests)'));
if (buildInfo.tooManyFailures)
item.appendChild(document.createTextNode(' (too many failures/crashes/timeouts)'));
else
item.appendChild(document.createTextNode(' (' + buildInfo.failureCount + ' failing tests)'));
list.appendChild(item);
});
}
@@ -107,10 +114,10 @@ ViewController.prototype = {
tester.getMostRecentCompletedBuildNumber(function(buildNumber) {
if (buildNumber < 0)
return;
tester.getNumberOfFailingTests(buildNumber, function(failureCount) {
tester.getNumberOfFailingTests(buildNumber, function(failureCount, tooManyFailures) {
if (failureCount <= 0)
return;
testersAndFailureCounts.push({ tester: tester, failureCount: failureCount });
latestBuildInfos.push({ tester: tester, failureCount: failureCount, tooManyFailures: tooManyFailures });
updateList();
});
});
@@ -1,3 +1,35 @@
2011-05-25 Adam Roben <aroben@apple.com>

Identify, rather than skip, builds where run-webkit-tests exited early due to too many failures

Fixes <http://webkit.org/b/61441> TestFailures page should show when run-webkit-tests
started exiting early due to too many crashes, timeouts, or failures

Reviewed by David Kilzer.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
(Builder.prototype.getNumberOfFailingTests): Rather than returning -1 ("unknown") when
run-webkit-tests exits early due to too many failures, pass that information in another
argument to the callback.
(Builder.prototype.startFetchingBuildHistory): Updated the documentation comment to reflect
the new structure of the object passed to the callback.
(Builder.prototype._getFailingTests): Updated to expect a tooManyFailures boolean from
getNumberOfFailingTests and to pass that along to our own callbacks.
(Builder.prototype._incorporateBuildHistory): Updated to expect a tooManyFailures boolean
from _getFailingTests and to store that value in the history object.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css:
(.info): Added.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js:
(ViewController.prototype._displayBuilder): Updated for change in structure to the history
object and to add a note when run-webkit-tests exited early due to too many failures.
(ViewController.prototype._displayTesters): Renamed testersAndFailureCounts to
latestBuildInfos since each entry now contains more than just the tester and failure count.
Now displays a message for testers where the latest build exited early due to too many
failures. Updated to expect a tooManyFailures boolean from getNumberOfFailingTests and to
store that value in latestBuildInfos.

2011-05-24 Keishi Hattori <keishi@webkit.org>

Reviewed by Kent Tamura.

0 comments on commit 4ed5d78

Please sign in to comment.