From f81de146d87830d8e40f3bcad7926f738402cf85 Mon Sep 17 00:00:00 2001
From: Philip Stoev <pstoev@materialize.io>
Date: Tue, 15 Feb 2022 11:31:39 +0100
Subject: [PATCH] feature-benchmark: Discard the first measurement obtained

Discard the first measurement obtained in case it has been tainted
somehow by cold caches or other adverse circumstances.
---
 .../materialize/feature_benchmark/filter.py     | 17 +++++++++++++----
 test/feature-benchmark/mzcompose.py             | 12 ++++++++----
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/misc/python/materialize/feature_benchmark/filter.py b/misc/python/materialize/feature_benchmark/filter.py
index bfe7273fc2d7..3b0a5240d417 100644
--- a/misc/python/materialize/feature_benchmark/filter.py
+++ b/misc/python/materialize/feature_benchmark/filter.py
@@ -13,13 +13,11 @@
 
 
 class Filter:
-    pass
-
-
-class RemoveOutliers(Filter):
     def __init__(self) -> None:
         self._data: List[float] = []
 
+
+class RemoveOutliers(Filter):
     def filter(self, measurement: float) -> bool:
         self._data.append(measurement)
 
@@ -37,3 +35,14 @@ def filter(self, measurement: float) -> bool:
 class NoFilter(Filter):
     def filter(self, measurement: float) -> bool:
         return False
+
+
+class FilterFirst(Filter):
+    def filter(self, measurement: float) -> bool:
+        self._data.append(measurement)
+
+        if len(self._data) == 1:
+            print("Discarding first measurement.")
+            return True
+        else:
+            return False
diff --git a/test/feature-benchmark/mzcompose.py b/test/feature-benchmark/mzcompose.py
index 62b0004dc26c..3cfdf04c0c9d 100644
--- a/test/feature-benchmark/mzcompose.py
+++ b/test/feature-benchmark/mzcompose.py
@@ -25,7 +25,7 @@
     RelativeThresholdComparator,
 )
 from materialize.feature_benchmark.executor import Docker
-from materialize.feature_benchmark.filter import Filter, NoFilter
+from materialize.feature_benchmark.filter import Filter, FilterFirst, NoFilter
 from materialize.feature_benchmark.termination import (
     NormalDistributionOverlap,
     ProbForMin,
@@ -48,8 +48,12 @@
 #
 
 
-def make_filter() -> Filter:
-    return NoFilter()
+def make_filter(args: argparse.Namespace) -> Filter:
+    # Discard the first run unless a small --max-runs limit is explicitly set
+    if args.max_runs <= 5:
+        return NoFilter()
+    else:
+        return FilterFirst()
 
 
 def make_termination_conditions(args: argparse.Namespace) -> List[TerminationCondition]:
@@ -125,7 +129,7 @@ def run_one_scenario(
                 scenario=scenario,
                 scale=args.scale,
                 executor=executor,
-                filter=make_filter(),
+                filter=make_filter(args),
                 termination_conditions=make_termination_conditions(args),
                 aggregation=make_aggregation(),
             )