Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
MotionMark scores are super sensitive to a single long frame
https://bugs.webkit.org/show_bug.cgi?id=220847
<rdar://problem/74152743>

Reviewed by Maciej Stachowiak and Jon Lee.

Currently, "ramp" tests have three phases. The middle phase is where they try to determine a maximum reasonable
complexity, and the third one is where they try various complexities between 0 and the maximum. The calculation
of this maximum reasonable complexity is currently very sensitive to outlier frame times. If there is a single
outlier frame time, the failure mode is to assume that the maximum complexity is ~10. So, the solution is to
ignore outlier frame times during this first phase, and to ensure that there are at least 9 frames measured that
have non-outlier times.

* MotionMark/tests/resources/main.js:
(filterOutOutliers):
(_measureAndResetInterval):
(update):
(registerFrameTime):
(intervalHasConcluded):
(start):
(didFinishInterval):

Canonical link: https://commits.webkit.org/234319@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@273122 268f45cc-cd09-0410-ab3c-d52691b4dbfc
  • Loading branch information
litherum committed Feb 19, 2021
1 parent b8e51f6 commit 2239fea
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 10 deletions.
24 changes: 24 additions & 0 deletions PerformanceTests/ChangeLog
@@ -1,3 +1,27 @@
2021-02-18 Myles C. Maxfield <mmaxfield@apple.com>

MotionMark scores are super sensitive to a single long frame
https://bugs.webkit.org/show_bug.cgi?id=220847
<rdar://problem/74152743>

Reviewed by Maciej Stachowiak and Jon Lee.

Currently, "ramp" tests have three phases. The middle phase is where they try to determine a maximum reasonable
complexity, and the third one is where they try various complexities between 0 and the maximum. The calculation
of this maximum reasonable complexity is currently very sensitive to outlier frame times. If there is a single
outlier frame time, the failure mode is to assume that the maximum complexity is ~10. So, the solution is to
ignore outlier frame times during this first phase, and to ensure that there are at least 9 frames measured that
have non-outlier times.

* MotionMark/tests/resources/main.js:
(filterOutOutliers):
(_measureAndResetInterval):
(update):
(registerFrameTime):
(intervalHasConcluded):
(start):
(didFinishInterval):

2021-01-28 Myles C. Maxfield <mmaxfield@apple.com>

MotionMark focus test can cause extreme variance in whichever test runs directly after it
Expand Down
69 changes: 59 additions & 10 deletions PerformanceTests/MotionMark/tests/resources/main.js
Expand Up @@ -110,14 +110,33 @@ Controller = Utilities.createClass(
return comment in this._marks;
},

filterOutOutliers: function(array)
{
if (array.length == 0)
return [];

array.sort();
var q1 = array[Math.min(Math.round(array.length * 1 / 4), array.length - 1)];
var q3 = array[Math.min(Math.round(array.length * 3 / 4), array.length - 1)];
var interquartileRange = q3 - q1;
var minimum = q1 - interquartileRange * 1.5;
var maximum = q3 + interquartileRange * 1.5;
return array.filter(x => x >= minimum && x <= maximum);
},

_measureAndResetInterval: function(currentTimestamp)
{
var sampleCount = this._sampler.sampleCount;
var averageFrameLength = 0;

if (this._intervalEndTimestamp) {
var intervalStartTimestamp = this._sampler.samples[0][this._intervalStartIndex];
averageFrameLength = (currentTimestamp - intervalStartTimestamp) / (sampleCount - this._intervalStartIndex);
var durations = [];
for (var i = Math.max(this._intervalStartIndex, 1); i < sampleCount; ++i) {
durations.push(this._sampler.samples[0][i] - this._sampler.samples[0][i - 1]);
}
var filteredDurations = this.filterOutOutliers(durations);
if (filteredDurations.length > 0)
averageFrameLength = filteredDurations.reduce((a, b) => a + b, 0) / filteredDurations.length;
}

this._intervalStartIndex = sampleCount;
Expand All @@ -138,21 +157,34 @@ Controller = Utilities.createClass(
this._frameLengthEstimator.sample(lastFrameLength);
frameLengthEstimate = this._frameLengthEstimator.estimate;
}
} else if (timestamp >= this._intervalEndTimestamp) {
var intervalStartTimestamp = this._sampler.samples[0][this._intervalStartIndex];
intervalAverageFrameLength = this._measureAndResetInterval(timestamp);
if (this._isFrameLengthEstimatorEnabled) {
this._frameLengthEstimator.sample(intervalAverageFrameLength);
frameLengthEstimate = this._frameLengthEstimator.estimate;
} else {
this.registerFrameTime(lastFrameLength);
if (this.intervalHasConcluded(timestamp)) {
var intervalStartTimestamp = this._sampler.samples[0][this._intervalStartIndex];
intervalAverageFrameLength = this._measureAndResetInterval(timestamp);
if (this._isFrameLengthEstimatorEnabled) {
this._frameLengthEstimator.sample(intervalAverageFrameLength);
frameLengthEstimate = this._frameLengthEstimator.estimate;
}
didFinishInterval = true;
this.didFinishInterval(timestamp, stage, intervalAverageFrameLength);
this._frameLengthEstimator.reset();
}
didFinishInterval = true;
this.didFinishInterval(timestamp, stage, intervalAverageFrameLength);
}

this._sampler.record(timestamp, stage.complexity(), frameLengthEstimate);
this.tune(timestamp, stage, lastFrameLength, didFinishInterval, intervalAverageFrameLength);
},

registerFrameTime: function(lastFrameLength)
{
},

intervalHasConcluded: function(timestamp)
{
return timestamp >= this._intervalEndTimestamp;
},

didFinishInterval: function(timestamp, stage, intervalAverageFrameLength)
{
},
Expand Down Expand Up @@ -336,6 +368,8 @@ RampController = Utilities.createSubclass(Controller,
tierFastTestLength: 250,
// If the engine is under stress, let the test run a little longer to let the measurement settle
tierSlowTestLength: 750,
// Tier intervals must have this number of non-outlier frames in order to end.
numberOfFramesRequiredInInterval: 9,

rampWarmupLength: 200,

Expand All @@ -355,10 +389,25 @@ RampController = Utilities.createSubclass(Controller,
Controller.prototype.start.call(this, startTimestamp, stage);
this._rampStartTimestamp = 0;
this.intervalSamplingLength = 100;
this._frameTimeHistory = [];
},

registerFrameTime: function(lastFrameLength)
{
this._frameTimeHistory.push(lastFrameLength);
},

intervalHasConcluded: function(timestamp)
{
if (!Controller.prototype.intervalHasConcluded.call(this, timestamp))
return false;

return this._finishedTierSampling || this.filterOutOutliers(this._frameTimeHistory).length > this.numberOfFramesRequiredInInterval;
},

didFinishInterval: function(timestamp, stage, intervalAverageFrameLength)
{
this._frameTimeHistory = [];
if (!this._finishedTierSampling) {
if (this._tierStartTimestamp > 0 && timestamp < this._tierStartTimestamp + this.tierFastTestLength)
return;
Expand Down

0 comments on commit 2239fea

Please sign in to comment.