Merge pull request #10688 from philip-stoev/feature-benchmark-filter-…

…first

feature-benchmark: Discard the first measurement obtained

misc/python/materialize/feature_benchmark/filter.py

@@ -13,13 +13,11 @@
 
 
 class Filter:
-    pass
-
-
-class RemoveOutliers(Filter):
     def __init__(self) -> None:
         self._data: List[float] = []
 
+
+class RemoveOutliers(Filter):
     def filter(self, measurement: float) -> bool:
         self._data.append(measurement)
 
@@ -37,3 +35,14 @@ def filter(self, measurement: float) -> bool:
 class NoFilter(Filter):
     def filter(self, measurement: float) -> bool:
         return False
+
+
+class FilterFirst(Filter):
+    def filter(self, measurement: float) -> bool:
+        self._data.append(measurement)
+
+        if len(self._data) == 1:
+            print("Discarding first measurement.")
+            return True
+        else:
+            return False

test/feature-benchmark/mzcompose.py

@@ -25,7 +25,7 @@
     RelativeThresholdComparator,
 )
 from materialize.feature_benchmark.executor import Docker
-from materialize.feature_benchmark.filter import Filter, NoFilter
+from materialize.feature_benchmark.filter import Filter, FilterFirst, NoFilter
 from materialize.feature_benchmark.termination import (
     NormalDistributionOverlap,
     ProbForMin,
@@ -48,8 +48,12 @@
 #
 
 
-def make_filter() -> Filter:
-    return NoFilter()
+def make_filter(args: argparse.Namespace) -> Filter:
+    # Discard the first run unless a small --max-runs limit is explicitly set
+    if args.max_runs <= 5:
+        return NoFilter()
+    else:
+        return FilterFirst()
 
 
 def make_termination_conditions(args: argparse.Namespace) -> List[TerminationCondition]:
@@ -125,7 +129,7 @@ def run_one_scenario(
                 scenario=scenario,
                 scale=args.scale,
                 executor=executor,
-                filter=make_filter(),
+                filter=make_filter(args),
                 termination_conditions=make_termination_conditions(args),
                 aggregation=make_aggregation(),
             )