MotionMark scores are super sensitive to a single long frame

https://bugs.webkit.org/show_bug.cgi?id=220847
<rdar://problem/74152743>

Reviewed by Maciej Stachowiak and Jon Lee.

Currently, "ramp" tests have three phases. The middle phase is where they try to determine a maximum reasonable
complexity, and the third one is where they try various complexities between 0 and the maximum. The calculation
of this maximum reasonable complexity is currently very sensitive to outlier frame times. If there is a single
outlier frame time, the failure mode is to assume that the maximum complexity is ~10. So, the solution is to
ignore outlier frame times during this first phase, and to ensure that there are at least 9 frames measured that
have non-outlier times.

* MotionMark/tests/resources/main.js:
(filterOutOutliers):
(_measureAndResetInterval):
(update):
(registerFrameTime):
(intervalHasConcluded):
(start):
(didFinishInterval):

Canonical link: https://commits.webkit.org/234319@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@273122 268f45cc-cd09-0410-ab3c-d52691b4dbfc

PerformanceTests/ChangeLog

@@ -1,3 +1,27 @@
+2021-02-18  Myles C. Maxfield  <mmaxfield@apple.com>
+
+        MotionMark scores are super sensitive to a single long frame
+        https://bugs.webkit.org/show_bug.cgi?id=220847
+        <rdar://problem/74152743>
+
+        Reviewed by Maciej Stachowiak and Jon Lee.
+
+        Currently, "ramp" tests have three phases. The middle phase is where they try to determine a maximum reasonable
+        complexity, and the third one is where they try various complexities between 0 and the maximum. The calculation
+        of this maximum reasonable complexity is currently very sensitive to outlier frame times. If there is a single
+        outlier frame time, the failure mode is to assume that the maximum complexity is ~10. So, the solution is to
+        ignore outlier frame times during this first phase, and to ensure that there are at least 9 frames measured that
+        have non-outlier times.
+
+        * MotionMark/tests/resources/main.js:
+        (filterOutOutliers):
+        (_measureAndResetInterval):
+        (update):
+        (registerFrameTime):
+        (intervalHasConcluded):
+        (start):
+        (didFinishInterval):
+
 2021-01-28  Myles C. Maxfield  <mmaxfield@apple.com>
 
         MotionMark focus test can cause extreme variance in whichever test runs directly after it

PerformanceTests/MotionMark/tests/resources/main.js

@@ -110,14 +110,33 @@ Controller = Utilities.createClass(
         return comment in this._marks;
     },
 
+    filterOutOutliers: function(array)
+    {
+        if (array.length == 0)
+            return [];
+
+        array.sort();
+        var q1 = array[Math.min(Math.round(array.length * 1 / 4), array.length - 1)];
+        var q3 = array[Math.min(Math.round(array.length * 3 / 4), array.length - 1)];
+        var interquartileRange = q3 - q1;
+        var minimum = q1 - interquartileRange * 1.5;
+        var maximum = q3 + interquartileRange * 1.5;
+        return array.filter(x => x >= minimum && x <= maximum);
+    },
+
     _measureAndResetInterval: function(currentTimestamp)
     {
         var sampleCount = this._sampler.sampleCount;
         var averageFrameLength = 0;
 
         if (this._intervalEndTimestamp) {
-            var intervalStartTimestamp = this._sampler.samples[0][this._intervalStartIndex];
-            averageFrameLength = (currentTimestamp - intervalStartTimestamp) / (sampleCount - this._intervalStartIndex);
+            var durations = [];
+            for (var i = Math.max(this._intervalStartIndex, 1); i < sampleCount; ++i) {
+                durations.push(this._sampler.samples[0][i] - this._sampler.samples[0][i - 1]);
+            }
+            var filteredDurations = this.filterOutOutliers(durations);
+            if (filteredDurations.length > 0)
+                averageFrameLength = filteredDurations.reduce((a, b) => a + b, 0) / filteredDurations.length;
         }
 
         this._intervalStartIndex = sampleCount;
@@ -138,21 +157,34 @@ Controller = Utilities.createClass(
                 this._frameLengthEstimator.sample(lastFrameLength);
                 frameLengthEstimate = this._frameLengthEstimator.estimate;
             }
-        } else if (timestamp >= this._intervalEndTimestamp) {
-            var intervalStartTimestamp = this._sampler.samples[0][this._intervalStartIndex];
-            intervalAverageFrameLength = this._measureAndResetInterval(timestamp);
-            if (this._isFrameLengthEstimatorEnabled) {
-                this._frameLengthEstimator.sample(intervalAverageFrameLength);
-                frameLengthEstimate = this._frameLengthEstimator.estimate;
+        } else {
+            this.registerFrameTime(lastFrameLength);
+            if (this.intervalHasConcluded(timestamp)) {
+                var intervalStartTimestamp = this._sampler.samples[0][this._intervalStartIndex];
+                intervalAverageFrameLength = this._measureAndResetInterval(timestamp);
+                if (this._isFrameLengthEstimatorEnabled) {
+                    this._frameLengthEstimator.sample(intervalAverageFrameLength);
+                    frameLengthEstimate = this._frameLengthEstimator.estimate;
+                }
+                didFinishInterval = true;
+                this.didFinishInterval(timestamp, stage, intervalAverageFrameLength);
+                this._frameLengthEstimator.reset();
             }
-            didFinishInterval = true;
-            this.didFinishInterval(timestamp, stage, intervalAverageFrameLength);
         }
 
         this._sampler.record(timestamp, stage.complexity(), frameLengthEstimate);
         this.tune(timestamp, stage, lastFrameLength, didFinishInterval, intervalAverageFrameLength);
     },
 
+    registerFrameTime: function(lastFrameLength)
+    {
+    },
+
+    intervalHasConcluded: function(timestamp)
+    {
+        return timestamp >= this._intervalEndTimestamp;
+    },
+
     didFinishInterval: function(timestamp, stage, intervalAverageFrameLength)
     {
     },
@@ -336,6 +368,8 @@ RampController = Utilities.createSubclass(Controller,
     tierFastTestLength: 250,
     // If the engine is under stress, let the test run a little longer to let the measurement settle
     tierSlowTestLength: 750,
+    // Tier intervals must have this number of non-outlier frames in order to end.
+    numberOfFramesRequiredInInterval: 9,
 
     rampWarmupLength: 200,
 
@@ -355,10 +389,25 @@ RampController = Utilities.createSubclass(Controller,
         Controller.prototype.start.call(this, startTimestamp, stage);
         this._rampStartTimestamp = 0;
         this.intervalSamplingLength = 100;
+        this._frameTimeHistory = [];
+    },
+
+    registerFrameTime: function(lastFrameLength)
+    {
+        this._frameTimeHistory.push(lastFrameLength);
+    },
+
+    intervalHasConcluded: function(timestamp)
+    {
+        if (!Controller.prototype.intervalHasConcluded.call(this, timestamp))
+            return false;
+
+        return this._finishedTierSampling || this.filterOutOutliers(this._frameTimeHistory).length > this.numberOfFramesRequiredInInterval;
     },
 
     didFinishInterval: function(timestamp, stage, intervalAverageFrameLength)
     {
+        this._frameTimeHistory = [];
         if (!this._finishedTierSampling) {
             if (this._tierStartTimestamp > 0 && timestamp < this._tierStartTimestamp + this.tierFastTestLength)
                 return;