Move run_id, batch_id, run_name, timestamp out of stats object (#106)

conbench · Jul 14, 2021 · 1cff7d1 · 1cff7d1
1 parent 9d2b569
commit 1cff7d1
Show file tree

Hide file tree

Showing 22 changed files with 145 additions and 146 deletions.
diff --git a/README.md b/README.md
@@ -241,6 +241,9 @@ $ conbench addition
 
 Benchmark result:
 {
+    "run_id": "c7e5280e65d24ec19d64a7636cef1bd4",
+    "batch_id": "c7e5280e65d24ec19d64a7636cef1bd4",
+    "timestamp": "2021-06-21T22:18:16.752993+00:00",
     "context": {
         "benchmark_language": "Python",
         "benchmark_language_version": "Python 3.9.2"
@@ -266,7 +269,6 @@ Benchmark result:
         "os_version": "10.16"
     },
     "stats": {
-        "batch_id": "c7e5280e65d24ec19d64a7636cef1bd4",
         "data": [
             "0.000003"
         ],
@@ -278,11 +280,9 @@ Benchmark result:
         "min": "0.000003",
         "q1": "0.000003",
         "q3": "0.000003",
-        "run_id": "c7e5280e65d24ec19d64a7636cef1bd4",
         "stdev": 0,
         "time_unit": "s",
         "times": [],
-        "timestamp": "2021-06-21T22:18:16.752993+00:00",
         "unit": "s"
     },
     "tags": {
@@ -356,6 +356,9 @@ $ conbench external --iterations=3
 
 Benchmark result:
 {
+    "run_id": "8058dde1491b49e5bd514646797c2a20",
+    "batch_id": "8058dde1491b49e5bd514646797c2a20",
+    "timestamp": "2021-06-21T22:16:54.786499+00:00",
     "context": {
         "benchmark_language": "C++"
     },
@@ -380,7 +383,6 @@ Benchmark result:
         "os_version": "10.16"
     },
     "stats": {
-        "batch_id": "8058dde1491b49e5bd514646797c2a20",
         "data": [
             "100.000000",
             "200.000000",
@@ -394,15 +396,13 @@ Benchmark result:
         "min": "100.000000",
         "q1": "150.000000",
         "q3": "250.000000",
-        "run_id": "8058dde1491b49e5bd514646797c2a20",
         "stdev": "100.000000",
         "time_unit": "s",
         "times": [
             "0.100000",
             "0.200000",
             "0.300000"
         ],
-        "timestamp": "2021-06-21T22:16:54.786499+00:00",
         "unit": "i/s"
     },
     "tags": {
@@ -501,6 +501,9 @@ $ conbench matrix --all=true
 
 Benchmark result:
 {
+    "run_id": "d509f6e80ed440a09af60fe1847dc033",
+    "batch_id": "d509f6e80ed440a09af60fe1847dc033",
+    "timestamp": "2021-06-22T18:39:53.805714+00:00",
     "context": {
         "benchmark_language": "Python",
         "benchmark_language_version": "Python 3.9.2"
@@ -526,7 +529,6 @@ Benchmark result:
         "os_version": "10.16"
     },
     "stats": {
-        "batch_id": "d509f6e80ed440a09af60fe1847dc033",
         "data": [
             "0.000010"
         ],
@@ -538,11 +540,9 @@ Benchmark result:
         "min": "0.000010",
         "q1": "0.000010",
         "q3": "0.000010",
-        "run_id": "d509f6e80ed440a09af60fe1847dc033",
         "stdev": 0,
         "time_unit": "s",
         "times": [],
-        "timestamp": "2021-06-22T18:39:53.805714+00:00",
         "unit": "s"
     },
     "tags": {
@@ -554,6 +554,9 @@ Benchmark result:
 
 Benchmark result:
 {
+    "run_id": "d509f6e80ed440a09af60fe1847dc033",
+    "batch_id": "d509f6e80ed440a09af60fe1847dc033",
+    "timestamp": "2021-06-22T18:39:53.830928+00:00",
     "context": {
         "benchmark_language": "Python",
         "benchmark_language_version": "Python 3.9.2"
@@ -579,7 +582,6 @@ Benchmark result:
         "os_version": "10.16"
     },
     "stats": {
-        "batch_id": "d509f6e80ed440a09af60fe1847dc033",
         "data": [
             "0.000006"
         ],
@@ -591,11 +593,9 @@ Benchmark result:
         "min": "0.000006",
         "q1": "0.000006",
         "q3": "0.000006",
-        "run_id": "d509f6e80ed440a09af60fe1847dc033",
         "stdev": 0,
         "time_unit": "s",
         "times": [],
-        "timestamp": "2021-06-22T18:39:53.830928+00:00",
         "unit": "s"
     },
     "tags": {
@@ -607,6 +607,9 @@ Benchmark result:
 
 Benchmark result:
 {
+    "run_id": "d509f6e80ed440a09af60fe1847dc033",
+    "batch_id": "d509f6e80ed440a09af60fe1847dc033",
+    "timestamp": "2021-06-22T18:39:53.843815+00:00",
     "context": {
         "benchmark_language": "Python",
         "benchmark_language_version": "Python 3.9.2"
@@ -632,7 +635,6 @@ Benchmark result:
         "os_version": "10.16"
     },
     "stats": {
-        "batch_id": "d509f6e80ed440a09af60fe1847dc033",
         "data": [
             "0.000007"
         ],
@@ -644,11 +646,9 @@ Benchmark result:
         "min": "0.000007",
         "q1": "0.000007",
         "q3": "0.000007",
-        "run_id": "d509f6e80ed440a09af60fe1847dc033",
         "stdev": 0,
         "time_unit": "s",
         "times": [],
-        "timestamp": "2021-06-22T18:39:53.843815+00:00",
         "unit": "s"
     },
     "tags": {

diff --git a/conbench/api/_examples.py b/conbench/api/_examples.py
@@ -25,9 +25,10 @@ def _api_user_entity(user):
 def _api_benchmark_entity(summary_id, context_id, case_id, batch_id, run_id, name):
     return {
         "id": summary_id,
+        "run_id": run_id,
+        "batch_id": batch_id,
+        "timestamp": "2020-11-25T21:02:42.706806",
         "stats": {
-            "batch_id": batch_id,
-            "run_id": run_id,
             "data": [
                 "0.099094",
                 "0.037129",
@@ -63,7 +64,6 @@ def _api_benchmark_entity(summary_id, context_id, case_id, batch_id, run_id, nam
             "q1": "0.006500",
             "q3": "0.036942",
             "stdev": "0.049194",
-            "timestamp": "2020-11-25T21:02:42.706806",
             "z_score": "0.000000",
             "z_regression": False,
             "z_improvement": False,

diff --git a/conbench/app/_util.py b/conbench/app/_util.py
@@ -30,8 +30,7 @@ def set_display_language(benchmark, contexts):
 
 
 def set_display_time(benchmark):
-    t = benchmark["stats"]["timestamp"]
-    benchmark["display_timestamp"] = display_time(t)
+    benchmark["display_timestamp"] = display_time(benchmark["timestamp"])
 
 
 def set_display_mean(benchmark):

diff --git a/conbench/app/benchmarks.py b/conbench/app/benchmarks.py
@@ -159,8 +159,7 @@ def _get_benchmark_and_run(self, benchmark_id):
         benchmark = self.get_display_benchmark(benchmark_id)
         run = None
         if benchmark is not None:
-            run_id = benchmark["stats"]["run_id"]
-            run = self.get_display_run(run_id)
+            run = self.get_display_run(benchmark["run_id"])
         return benchmark, run
 
 

diff --git a/conbench/app/compare.py b/conbench/app/compare.py
@@ -31,12 +31,11 @@ def page(self, comparisons, regressions, improvements, baseline_id, contender_id
             baseline = self.get_display_benchmark(baseline_id)
             contender = self.get_display_benchmark(contender_id)
             plot = self._get_plot(baseline, contender)
-            b_stats, c_stats = baseline["stats"], contender["stats"]
-            baseline_run_id = b_stats["run_id"]
-            contender_run_id = c_stats["run_id"]
+            baseline_run_id = baseline["run_id"]
+            contender_run_id = baseline["run_id"]
             compare = f"{baseline_run_id}...{contender_run_id}"
             compare_runs_url = f.url_for("app.compare-runs", compare_ids=compare)
-            compare = f'{b_stats["batch_id"]}...{c_stats["batch_id"]}'
+            compare = f'{baseline["batch_id"]}...{baseline["batch_id"]}'
             compare_batches_url = f.url_for("app.compare-batches", compare_ids=compare)
 
         if comparisons:

diff --git a/conbench/entities/summary.py b/conbench/entities/summary.py
@@ -103,8 +103,8 @@ def create(data):
             )
 
         # create if not exists
-        run_id = data["stats"]["run_id"]
-        run_name = stats.pop("run_name", None)
+        run_id = data["run_id"]
+        run_name = data.pop("run_name", None)
         run = Run.first(id=run_id)
         if not run:
             run = Run.create(
@@ -116,6 +116,9 @@ def create(data):
                 }
             )
 
+        stats["run_id"] = data["run_id"]
+        stats["batch_id"] = data["batch_id"]
+        stats["timestamp"] = data["timestamp"]
         stats["case_id"] = case.id
         stats["context_id"] = context.id
         summary = Summary(**stats)
@@ -148,10 +151,6 @@ class SummaryCreate(marshmallow.Schema):
     times = marshmallow.fields.List(marshmallow.fields.Decimal, required=True)
     unit = marshmallow.fields.String(required=True)
     time_unit = marshmallow.fields.String(required=True)
-    batch_id = marshmallow.fields.String(required=True)
-    run_id = marshmallow.fields.String(required=True)
-    run_name = marshmallow.fields.String(required=False)
-    timestamp = marshmallow.fields.DateTime(required=True)
     iterations = marshmallow.fields.Integer(required=True)
     min = marshmallow.fields.Decimal(required=False)
     max = marshmallow.fields.Decimal(required=False)
@@ -180,14 +179,15 @@ def _dump(self, summary):
         tags.update(case.tags)
         return {
             "id": summary.id,
+            "run_id": summary.run_id,
+            "batch_id": summary.batch_id,
+            "timestamp": summary.timestamp.isoformat(),
             "tags": tags,
             "stats": {
                 "data": [self.decimal_fmt.format(x) for x in data],
                 "times": [self.decimal_fmt.format(x) for x in times],
                 "unit": summary.unit,
                 "time_unit": summary.time_unit,
-                "batch_id": summary.batch_id,
-                "run_id": summary.run_id,
                 "iterations": summary.iterations,
                 "min": self.decimal_fmt.format(summary.min),
                 "max": self.decimal_fmt.format(summary.max),
@@ -197,7 +197,6 @@ def _dump(self, summary):
                 "q1": self.decimal_fmt.format(summary.q1),
                 "q3": self.decimal_fmt.format(summary.q3),
                 "iqr": self.decimal_fmt.format(summary.iqr),
-                "timestamp": summary.timestamp.isoformat(),
                 "z_score": self.decimal_fmt.format(summary.z_score),
                 "z_regression": z_regression(summary.z_score),
                 "z_improvement": z_improvement(summary.z_score),
@@ -221,6 +220,10 @@ class SummarySerializer:
 
 
 class _BenchmarkFacadeSchemaCreate(marshmallow.Schema):
+    run_id = marshmallow.fields.String(required=True)
+    run_name = marshmallow.fields.String(required=False)
+    batch_id = marshmallow.fields.String(required=True)
+    timestamp = marshmallow.fields.DateTime(required=True)
     machine_info = marshmallow.fields.Nested(MachineSchema().create, required=True)
     stats = marshmallow.fields.Nested(SummarySchema().create, required=True)
     tags = marshmallow.fields.Dict(required=True)

diff --git a/conbench/runner.py b/conbench/runner.py
@@ -163,26 +163,32 @@ def record(self, result, name, **kwargs):
         tags, context, github, options, output = self._init(kwargs)
 
         tags["name"] = name
-        timestamp = _now_formatted()
-        run_id = options.get("run_id")
-        run_name = options.get("run_name")
         stats = self._stats(
             result["data"],
             result["unit"],
             result.get("times", []),
             result.get("time_unit", "s"),
-            timestamp,
-            run_id,
-            self.batch_id,
-            run_name,
         )
+
+        run_id = options.get("run_id")
+        if run_id is None:
+            run_id = self.batch_id
+
         benchmark = {
+            "run_id": run_id,
+            "batch_id": self.batch_id,
+            "timestamp": _now_formatted(),
             "stats": stats,
             "machine_info": self.machine_info,
             "context": context,
             "tags": tags,
             "github": github,
         }
+
+        run_name = options.get("run_name")
+        if run_name is not None:
+            benchmark["run_name"] = run_name
+
         self.publish(benchmark)
         return benchmark, output
 
@@ -250,7 +256,7 @@ def _get_timing_options(self, options):
         }
 
     @staticmethod
-    def _stats(data, unit, times, time_unit, timestamp, run_id, batch_id, run_name):
+    def _stats(data, unit, times, time_unit):
         fmt = "{:.6f}"
 
         def _format(f, data, min_length=0):
@@ -261,18 +267,12 @@ def _format(f, data, min_length=0):
 
         q1, q3 = np.percentile(data, [25, 75])
 
-        if not run_id:
-            run_id = batch_id
-
         result = {
             "data": [fmt.format(x) for x in data],
             "times": [fmt.format(x) for x in times],
             "unit": unit,
             "time_unit": time_unit,
             "iterations": len(data),
-            "timestamp": timestamp,
-            "batch_id": batch_id,
-            "run_id": run_id,
             "mean": _format(statistics.mean, data),
             "median": _format(statistics.median, data),
             "min": _format(min, data),
@@ -283,9 +283,6 @@ def _format(f, data, min_length=0):
             "iqr": fmt.format(q3 - q1),
         }
 
-        if run_name is not None:
-            result["run_name"] = run_name
-
         return result
 
     def execute_r_command(self, r_command, quiet=True):

diff --git a/conbench/templates/batch.html b/conbench/templates/batch.html
@@ -5,12 +5,12 @@
       <ol class="breadcrumb">
         {% if benchmarks %}
         <li class="breadcrumb-item active">
-          <a href="{{ url_for('app.run', run_id=benchmarks[0].stats.run_id) }}">Run</a>
+          <a href="{{ url_for('app.run', run_id=benchmarks[0].run_id) }}">Run</a>
         </li>
         {% endif %}
         <li class="breadcrumb-item active">Batch</li>
         {% if benchmarks %}
-        <li class="breadcrumb-item active" aria-current="page">{{ benchmarks[0].stats.batch_id }}</li>
+        <li class="breadcrumb-item active" aria-current="page">{{ benchmarks[0].batch_id }}</li>
         {% endif %}
       </ol>
     </nav>