Add deviations to compare API response

conbench · May 20, 2021 · a847221 · a847221
1 parent f7a8b6c
commit a847221
Show file tree

Hide file tree

Showing 4 changed files with 88 additions and 0 deletions.
diff --git a/conbench/api/_comparator.py b/conbench/api/_comparator.py
@@ -158,6 +158,7 @@ def formatted(self):
             "threshold": fmt(self.threshold) + "%",
             "regression": self.regression,
             "improvement": self.improvement,
+            "deviations": fmt(self.deviations),
             "baseline_z_score": fmt(self.baseline_z_score),
             "contender_z_score": fmt(self.contender_z_score),
             "baseline_regression_z": self.baseline_regression_z,
@@ -187,6 +188,7 @@ def compare(self):
             "threshold": fmt(self.threshold),
             "regression": self.regression,
             "improvement": self.improvement,
+            "deviations": fmt(self.deviations),
             "baseline_z_score": fmt(self.baseline_z_score),
             "contender_z_score": fmt(self.contender_z_score),
             "baseline_regression_z": self.baseline_regression_z,

diff --git a/conbench/api/_examples.py b/conbench/api/_examples.py
@@ -114,6 +114,7 @@ def _api_compare_entity(benchmark_ids, batch_ids, run_ids, batch, benchmark, tag
         "threshold": "5.000%",
         "regression": False,
         "improvement": False,
+        "deviations": "2.000",
         "baseline_z_score": "0.000",
         "contender_z_score": "0.000",
         "baseline_regression_z": False,
@@ -151,6 +152,7 @@ def _api_compare_list(
             "threshold": "5.000%",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -176,6 +178,7 @@ def _api_compare_list(
             "threshold": "5.000%",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,

diff --git a/conbench/entities/distribution.py b/conbench/entities/distribution.py
@@ -146,3 +146,52 @@ def update_distribution(repository, sha, summary, limit):
             )
         )
         conn.commit()
+
+
+q = """SELECT
+run_id,
+summary.case_id,
+summary.machine_id,
+machine.name AS machine_name,
+summary.unit,
+summary.time_unit,
+summary.min,
+summary.max,
+summary.mean,
+summary.median,
+commit.timestamp AS commit_timestamp,
+commit.repository AS commit_repository,
+(summary.mean - distribution.mean_mean) / distribution.mean_sd AS mean_z,
+(summary.min - distribution.min_mean) / distribution.min_sd AS min_z,
+(summary.max - distribution.max_mean) / distribution.max_sd AS max_z,
+(summary.median - distribution.median_mean) / distribution.median_sd AS median_z,
+commit.timestamp AS commit_timestamp,
+commit.repository AS commit_repository,
+distribution.*
+FROM summary
+INNER JOIN run
+ON summary.run_id = run.id
+INNER JOIN commit
+ON commit.id = run.commit_id
+INNER JOIN machine
+ON summary.machine_id = machine.id
+LEFT JOIN distribution
+ON summary.case_id = distribution.case_id AND machine.name = distribution.machine_name
+WHERE run.name = 'commit: {{to_compare_sha}}'"""
+
+
+def get_z_score(repository, sha, case_id, context_id, machine_hash, mean):
+    result = list(
+        Session.query(Distribution.mean_mean, Distribution.mean_sd).filter(
+            Distribution.repository == repository,
+            Distribution.sha == sha,
+            Distribution.case_id == case_id,
+            Distribution.context_id == context_id,
+            Distribution.machine_hash == machine_hash,
+        )
+    )
+    if result:
+        distribution_mean = result[0]["mean_mean"]
+        distribution_sd = result[0]["mean_sd"]
+        return (mean - distribution_mean) / distribution_sd
+    return None
diff --git a/conbench/tests/api/test_comparator.py b/conbench/tests/api/test_comparator.py
@@ -35,6 +35,7 @@ def test_compare_no_change():
         "threshold": "5.000",
         "regression": False,
         "improvement": False,
+        "deviations": "2.000",
         "baseline_z_score": "0.000",
         "contender_z_score": "0.000",
         "baseline_regression_z": False,
@@ -60,6 +61,7 @@ def test_compare_no_change():
         "threshold": "5.000%",
         "regression": False,
         "improvement": False,
+        "deviations": "2.000",
         "baseline_z_score": "0.000",
         "contender_z_score": "0.000",
         "baseline_regression_z": False,
@@ -114,6 +116,7 @@ def test_compare_regression():
         "threshold": "5.000",
         "regression": True,
         "improvement": False,
+        "deviations": "2.000",
         "baseline_z_score": "-3.000",
         "contender_z_score": "-3.000",
         "baseline_regression_z": True,
@@ -139,6 +142,7 @@ def test_compare_regression():
         "threshold": "5.000%",
         "regression": True,
         "improvement": False,
+        "deviations": "2.000",
         "baseline_z_score": "-3.000",
         "contender_z_score": "-3.000",
         "baseline_regression_z": True,
@@ -193,6 +197,7 @@ def test_compare_regression_but_under_threshold():
         "threshold": "5.000",
         "regression": False,
         "improvement": False,
+        "deviations": "2.000",
         "baseline_z_score": "-2.000",
         "contender_z_score": "-2.000",
         "baseline_regression_z": False,
@@ -218,6 +223,7 @@ def test_compare_regression_but_under_threshold():
         "threshold": "5.000%",
         "regression": False,
         "improvement": False,
+        "deviations": "2.000",
         "baseline_z_score": "-2.000",
         "contender_z_score": "-2.000",
         "baseline_regression_z": False,
@@ -272,6 +278,7 @@ def test_compare_improvement():
         "threshold": "5.000",
         "regression": False,
         "improvement": True,
+        "deviations": "2.000",
         "baseline_z_score": "3.000",
         "contender_z_score": "3.000",
         "baseline_regression_z": False,
@@ -297,6 +304,7 @@ def test_compare_improvement():
         "threshold": "5.000%",
         "regression": False,
         "improvement": True,
+        "deviations": "2.000",
         "baseline_z_score": "3.000",
         "contender_z_score": "3.000",
         "baseline_regression_z": False,
@@ -351,6 +359,7 @@ def test_compare_improvement_but_under_threshold():
         "threshold": "5.000",
         "regression": False,
         "improvement": False,
+        "deviations": "2.000",
         "baseline_z_score": "2.000",
         "contender_z_score": "2.000",
         "baseline_regression_z": False,
@@ -376,6 +385,7 @@ def test_compare_improvement_but_under_threshold():
         "threshold": "5.000%",
         "regression": False,
         "improvement": False,
+        "deviations": "2.000",
         "baseline_z_score": "2.000",
         "contender_z_score": "2.000",
         "baseline_regression_z": False,
@@ -459,6 +469,7 @@ def test_compare_list():
             "threshold": "5.000",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -484,6 +495,7 @@ def test_compare_list():
             "threshold": "5.000",
             "regression": False,
             "improvement": True,
+            "deviations": "2.000",
             "baseline_z_score": "3.000",
             "contender_z_score": "-3.000",
             "baseline_regression_z": True,
@@ -511,6 +523,7 @@ def test_compare_list():
             "threshold": "5.000%",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -536,6 +549,7 @@ def test_compare_list():
             "threshold": "5.000%",
             "regression": False,
             "improvement": True,
+            "deviations": "2.000",
             "baseline_z_score": "3.000",
             "contender_z_score": "-3.000",
             "baseline_regression_z": True,
@@ -609,6 +623,7 @@ def test_compare_list_missing_contender():
             "threshold": "5.000",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -634,6 +649,7 @@ def test_compare_list_missing_contender():
             "threshold": "5.000",
             "regression": False,
             "improvement": True,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -661,6 +677,7 @@ def test_compare_list_missing_contender():
             "threshold": "5.000%",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -686,6 +703,7 @@ def test_compare_list_missing_contender():
             "threshold": "5.000%",
             "regression": False,
             "improvement": True,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -760,6 +778,7 @@ def test_compare_list_empty_contender():
             "threshold": "5.000",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -785,6 +804,7 @@ def test_compare_list_empty_contender():
             "threshold": "5.000",
             "regression": False,
             "improvement": True,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -812,6 +832,7 @@ def test_compare_list_empty_contender():
             "threshold": "5.000%",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -837,6 +858,7 @@ def test_compare_list_empty_contender():
             "threshold": "5.000%",
             "regression": False,
             "improvement": True,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -910,6 +932,7 @@ def test_compare_list_missing_baseline():
             "threshold": "5.000",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -935,6 +958,7 @@ def test_compare_list_missing_baseline():
             "threshold": "5.000",
             "regression": False,
             "improvement": True,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -962,6 +986,7 @@ def test_compare_list_missing_baseline():
             "threshold": "5.000%",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -987,6 +1012,7 @@ def test_compare_list_missing_baseline():
             "threshold": "5.000%",
             "regression": False,
             "improvement": True,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -1061,6 +1087,7 @@ def test_compare_list_empty_baseline():
             "threshold": "5.000",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -1086,6 +1113,7 @@ def test_compare_list_empty_baseline():
             "threshold": "5.000",
             "regression": False,
             "improvement": True,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -1113,6 +1141,7 @@ def test_compare_list_empty_baseline():
             "threshold": "5.000%",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -1138,6 +1167,7 @@ def test_compare_list_empty_baseline():
             "threshold": "5.000%",
             "regression": False,
             "improvement": True,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -1182,6 +1212,7 @@ def test_compare_list_empty_pair():
             "threshold": "5.000",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -1207,6 +1238,7 @@ def test_compare_list_empty_pair():
             "threshold": "5.000",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -1234,6 +1266,7 @@ def test_compare_list_empty_pair():
             "threshold": "5.000%",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,
@@ -1259,6 +1292,7 @@ def test_compare_list_empty_pair():
             "threshold": "5.000%",
             "regression": False,
             "improvement": False,
+            "deviations": "2.000",
             "baseline_z_score": "0.000",
             "contender_z_score": "0.000",
             "baseline_regression_z": False,