Merge b411d71 into 0574870

conbench · Aug 23, 2021 · 865d283 · 865d283
2 parents 0574870 + b411d71
commit 865d283
Show file tree

Hide file tree

Showing 14 changed files with 89 additions and 68 deletions.
diff --git a/conbench/api/_examples.py b/conbench/api/_examples.py
@@ -64,7 +64,7 @@ def _api_benchmark_entity(summary_id, context_id, case_id, batch_id, run_id, nam
             "q1": "0.006500",
             "q3": "0.036942",
             "stdev": "0.049194",
-            "z_score": "0.000000",
+            "z_score": None,
             "z_regression": False,
             "z_improvement": False,
         },
@@ -122,8 +122,8 @@ def _api_compare_entity(benchmark_ids, batch_ids, run_ids, batch, benchmark, tag
         "regression": False,
         "improvement": False,
         "threshold_z": "5.000",
-        "baseline_z_score": "0.000",
-        "contender_z_score": "0.000",
+        "baseline_z_score": None,
+        "contender_z_score": None,
         "baseline_z_regression": False,
         "baseline_z_improvement": False,
         "contender_z_regression": False,
@@ -161,8 +161,8 @@ def _api_compare_list(
             "regression": False,
             "improvement": False,
             "threshold_z": "5.000",
-            "baseline_z_score": "0.000",
-            "contender_z_score": "0.000",
+            "baseline_z_score": None,
+            "contender_z_score": None,
             "baseline_z_regression": False,
             "baseline_z_improvement": False,
             "contender_z_regression": False,
@@ -188,8 +188,8 @@ def _api_compare_list(
             "regression": False,
             "improvement": False,
             "threshold_z": "5.000",
-            "baseline_z_score": "0.000",
-            "contender_z_score": "0.000",
+            "baseline_z_score": None,
+            "contender_z_score": None,
             "baseline_z_regression": False,
             "baseline_z_improvement": False,
             "contender_z_regression": False,

diff --git a/conbench/entities/_comparator.py b/conbench/entities/_comparator.py
@@ -21,11 +21,15 @@ def _less_is_better(unit):
 
 
 def z_regression(z_score, threshold_z=None):
+    if z_score is None:
+        return False
     threshold_z = threshold_z if threshold_z else Z_SCORE
     return -z_score > threshold_z
 
 
 def z_improvement(z_score, threshold_z=None):
+    if z_score is None:
+        return False
     threshold_z = threshold_z if threshold_z else Z_SCORE
     return z_score > threshold_z
 
@@ -53,7 +57,7 @@ def __init__(
         self.value = decimal.Decimal(value)
         self.tags = tags
         self.language = language
-        self.z_score = decimal.Decimal(z_score)
+        self.z_score = decimal.Decimal(z_score) if z_score is not None else None
 
 
 class BenchmarkComparator:
@@ -129,13 +133,13 @@ def improvement(self):
     @property
     def baseline_z_score(self):
         if self.baseline is None:
-            return 0.0
+            return None
         return self.baseline.z_score
 
     @property
     def contender_z_score(self):
         if self.contender is None:
-            return 0.0
+            return None
         return self.contender.z_score
 
     @property

diff --git a/conbench/entities/distribution.py b/conbench/entities/distribution.py
@@ -152,7 +152,7 @@ def set_z_scores(summaries):
         return
 
     for summary in summaries:
-        summary.z_score = 0
+        summary.z_score = None
 
     first = summaries[0]
     parent_commit = Commit.first(
@@ -186,9 +186,8 @@ def set_z_scores(summaries):
     lookup = {f"{d.case_id}-{d.context_id}": d for d in distributions}
 
     for summary in summaries:
-        summary.z_score = 0
         d = lookup.get(f"{summary.case_id}-{summary.context_id}")
         if d and d.mean_sd:
             summary.z_score = (summary.mean - d.mean_mean) / d.mean_sd
-        if _less_is_better(summary.unit) and summary.z_score != 0:
+        if _less_is_better(summary.unit) and summary.z_score:
             summary.z_score = summary.z_score * -1
diff --git a/conbench/entities/summary.py b/conbench/entities/summary.py
@@ -172,6 +172,7 @@ def _dump(self, summary):
         data = [result for _, result in by_iteration_data]
         by_iteration_times = sorted([(x.iteration, x.result) for x in summary.times])
         times = [result for _, result in by_iteration_times]
+        z_score = self.decimal_fmt.format(summary.z_score) if summary.z_score else None
         case = summary.case
         tags = {"id": case.id, "name": case.name}
         tags.update(case.tags)
@@ -195,7 +196,7 @@ def _dump(self, summary):
                 "q1": self.decimal_fmt.format(summary.q1),
                 "q3": self.decimal_fmt.format(summary.q3),
                 "iqr": self.decimal_fmt.format(summary.iqr),
-                "z_score": self.decimal_fmt.format(summary.z_score),
+                "z_score": z_score,
                 "z_regression": z_regression(summary.z_score),
                 "z_improvement": z_improvement(summary.z_score),
             },

diff --git a/conbench/templates/batch.html b/conbench/templates/batch.html
@@ -72,7 +72,11 @@
                          <div>{{ benchmark.display_name }}</div>
                      </a></td>
                      <td>{{ benchmark.display_mean }}</td>
-                     <td>{{ benchmark.stats.z_score }}</td>
+                     {% if benchmark.stats.z_score is not none %}
+                       <td>{{ benchmark.stats.z_score }}</td>
+                     {% else %}
+                       <td></td>
+                     {% endif %}
                 </tr>
                 {% endfor %}
             </tbody>

diff --git a/conbench/templates/benchmark-entity.html b/conbench/templates/benchmark-entity.html
@@ -67,14 +67,18 @@
           {% for k,v in benchmark.stats.items() %}
             <li class="list-group-item" style="overflow-y: auto;">
               <b>{{ k }}</b>
-              <div align="right" style="display:inline-block; float: right;">{{ v }}</div>
+              {% if v is not none %}
+                <div align="right" style="display:inline-block; float: right;">{{ v }}</div>
+              {% endif %}
             </li>
           {% endfor %}
             <li class="list-group-item active">Tags</li>
           {% for k,v in benchmark.tags.items() %}
             <li class="list-group-item" style="overflow-y: auto;">
               <b>{{ k }}</b>
-              <div align="right" style="display:inline-block; float: right;">{{ v }}</div>
+              {% if v is not none %}
+                <div align="right" style="display:inline-block; float: right;">{{ v }}</div>
+              {% endif %}
             </li>
           {% endfor %}
         </ul>

diff --git a/conbench/templates/compare-entity.html b/conbench/templates/compare-entity.html
@@ -43,7 +43,9 @@
                     {% for c in comparisons %}
                     <tr>
                          <td>
-                           {{ c.contender_z_score }} z
+                           {% if c.contender_z_score %}
+                             {{ c.contender_z_score }} z
+                           {% endif %}
                            {% if c.contender_z_regression %}
                            <span class="glyphicon glyphicon-arrow-down"></span></b>
                            {% endif %}
@@ -154,16 +156,18 @@
           {% for k,v in baseline.stats.items() %}
             <li class="list-group-item" style="overflow-y: auto;">
               <b>{{ k }}</b>
-              <div align="right" style="display:inline-block; float: right;">
-              {{ v }}
-              </div>
+              {% if v is not none %}
+                <div align="right" style="display:inline-block; float: right;">{{ v }}</div>
+              {% endif %}
             </li>
           {% endfor %}
           <li class="list-group-item active">Tags</li>
           {% for k,v in baseline.tags.items() %}
             <li class="list-group-item" style="overflow-y: auto;">
               <b>{{ k }}</b>
-              <div align="right" style="display:inline-block; float: right;">{{ v }}</div>
+              {% if v is not none %}
+                <div align="right" style="display:inline-block; float: right;">{{ v }}</div>
+              {% endif %}
             </li>
           {% endfor %}
           <li class="list-group-item active">Machine</li>
@@ -263,16 +267,18 @@
           {% for k,v in contender.stats.items() %}
             <li class="list-group-item" style="overflow-y: auto;">
               <b>{{ k }}</b>
-              <div align="right" style="display:inline-block; float: right;">
-              {{ v }}
-              </div>
+              {% if v is not none %}
+                <div align="right" style="display:inline-block; float: right;">{{ v }}</div>
+              {% endif %}
             </li>
           {% endfor %}
           <li class="list-group-item active">Tags</li>
           {% for k,v in contender.tags.items() %}
             <li class="list-group-item" style="overflow-y: auto;">
               <b>{{ k }}</b>
-              <div align="right" style="display:inline-block; float: right;">{{ v }}</div>
+              {% if v is not none %}
+                <div align="right" style="display:inline-block; float: right;">{{ v }}</div>
+              {% endif %}
             </li>
           {% endfor %}
           <li class="list-group-item active">Machine</li>

diff --git a/conbench/templates/compare-list.html b/conbench/templates/compare-list.html
@@ -26,6 +26,7 @@
     </nav>
 
     {% if baseline_run and contender_run %}
+      {% if baseline_run.commit.message and contender_run.commit.message %}
     <div class="row">
       <div class="col-md-12">
         <ul class="list-group">
@@ -39,6 +40,7 @@
          </ul>
       </div>
     </div>
+      {% endif %}
     {% endif %}
 
     <div class="row">
@@ -75,7 +77,7 @@
                 <tbody>
                     {% for c in comparisons %}
                     <tr>
-                         {% if c.contender is not none and c.baseline is not none %}
+                         {% if c.contender_z_score is not none %}
                            <td>{{ c.contender_z_score }}</td>
                          {% else %}
                            <td></td>

diff --git a/conbench/templates/run.html b/conbench/templates/run.html
@@ -42,7 +42,11 @@
                          <div>{{ benchmark.display_name }}</div>
                      </a></td>
                      <td>{{ benchmark.display_mean }}</td>
-                     <td>{{ benchmark.stats.z_score }}</td>
+                     {% if benchmark.stats.z_score is not none %}
+                       <td>{{ benchmark.stats.z_score }}</td>
+                     {% else %}
+                       <td></td>
+                     {% endif %}
                 </tr>
                 {% endfor %}
             </tbody>

diff --git a/conbench/tests/api/_expected_docs.py b/conbench/tests/api/_expected_docs.py
@@ -94,7 +94,7 @@
                                 "unit": "s",
                                 "z_improvement": False,
                                 "z_regression": False,
-                                "z_score": "0.000000",
+                                "z_score": None,
                             },
                             "tags": {
                                 "compression": "snappy",
@@ -162,7 +162,7 @@
                                 "unit": "s",
                                 "z_improvement": False,
                                 "z_regression": False,
-                                "z_score": "0.000000",
+                                "z_score": None,
                             },
                             "tags": {
                                 "compression": "snappy",
@@ -231,7 +231,7 @@
                                     "unit": "s",
                                     "z_improvement": False,
                                     "z_regression": False,
-                                    "z_score": "0.000000",
+                                    "z_score": None,
                                 },
                                 "tags": {
                                     "compression": "snappy",
@@ -307,7 +307,7 @@
                             "baseline_run_id": "some-run-uuid-1",
                             "baseline_z_improvement": False,
                             "baseline_z_regression": False,
-                            "baseline_z_score": "0.000",
+                            "baseline_z_score": None,
                             "batch": "file-read",
                             "benchmark": "snappy, nyctaxi_sample, parquet, arrow",
                             "change": "0.000%",
@@ -317,7 +317,7 @@
                             "contender_run_id": "some-run-uuid-2",
                             "contender_z_improvement": False,
                             "contender_z_regression": False,
-                            "contender_z_score": "0.000",
+                            "contender_z_score": None,
                             "improvement": False,
                             "language": "Python",
                             "less_is_better": True,
@@ -349,7 +349,7 @@
                                 "baseline_run_id": "some-run-uuid-1",
                                 "baseline_z_improvement": False,
                                 "baseline_z_regression": False,
-                                "baseline_z_score": "0.000",
+                                "baseline_z_score": None,
                                 "batch": "file-read",
                                 "benchmark": "snappy, nyctaxi_sample, parquet, arrow",
                                 "change": "0.000%",
@@ -359,7 +359,7 @@
                                 "contender_run_id": "some-run-uuid-2",
                                 "contender_z_improvement": False,
                                 "contender_z_regression": False,
-                                "contender_z_score": "0.000",
+                                "contender_z_score": None,
                                 "improvement": False,
                                 "language": "Python",
                                 "less_is_better": True,
@@ -383,7 +383,7 @@
                                 "baseline_run_id": "some-run-uuid-1",
                                 "baseline_z_improvement": False,
                                 "baseline_z_regression": False,
-                                "baseline_z_score": "0.000",
+                                "baseline_z_score": None,
                                 "batch": "file-write",
                                 "benchmark": "snappy, nyctaxi_sample, parquet, arrow",
                                 "change": "0.000%",
@@ -393,7 +393,7 @@
                                 "contender_run_id": "some-run-uuid-2",
                                 "contender_z_improvement": False,
                                 "contender_z_regression": False,
-                                "contender_z_score": "0.000",
+                                "contender_z_score": None,
                                 "improvement": False,
                                 "language": "Python",
                                 "less_is_better": True,

diff --git a/conbench/tests/api/test_compare.py b/conbench/tests/api/test_compare.py
@@ -75,7 +75,7 @@ def test_compare(self, client):
                 "contender": "20.000 s",
                 "change": "-566.667%",
                 "regression": True,
-                "baseline_z_score": "0.000",
+                "baseline_z_score": None,
                 "contender_z_score": "-{:.3f}".format(_fixtures.Z_SCORE_UP),
                 "contender_z_regression": True,
             }