Drop sha and repository & enhance history API & drop distribution API

conbench · Jul 29, 2021 · 0b2d170 · 0b2d170
1 parent ffd38a6
commit 0b2d170
Show file tree

Hide file tree

Showing 11 changed files with 34 additions and 353 deletions.
diff --git a/conbench/api/__init__.py b/conbench/api/__init__.py
@@ -11,7 +11,6 @@
 from .commits import *  # noqa
 from .compare import *  # noqa
 from .contexts import *  # noqa
-from .distribution import *  # noqa
 from .history import *  # noqa
 from .index import *  # noqa
 from .machines import *  # noqa

diff --git a/conbench/api/_docs.py b/conbench/api/_docs.py
@@ -70,7 +70,6 @@ def _201_created(example, schema=None):
 spec.components.response("CompareList", _200_ok(ex.COMPARE_LIST))
 spec.components.response("ContextEntity", _200_ok(ex.CONTEXT_ENTITY))
 spec.components.response("ContextList", _200_ok([ex.CONTEXT_ENTITY]))
-spec.components.response("DistributionList", _200_ok([ex.DISTRIBUTION_ENTITY]))
 spec.components.response("HistoryList", _200_ok([ex.HISTORY_ENTITY]))
 spec.components.response("MachineEntity", _200_ok(ex.MACHINE_ENTITY))
 spec.components.response("MachineList", _200_ok([ex.MACHINE_ENTITY]))
@@ -89,7 +88,6 @@ def _201_created(example, schema=None):
     {"name": "Commits", "description": "Benchmarked commits"},
     {"name": "Comparisons", "description": "Benchmark comparisons"},
     {"name": "Contexts", "description": "Benchmark contexts"},
-    {"name": "Distribution", "description": "Benchmark distribution"},
     {"name": "History", "description": "Benchmark history"},
     {"name": "Machines", "description": "Benchmark machines"},
     {"name": "Runs", "description": "Benchmark runs"},

diff --git a/conbench/api/_examples.py b/conbench/api/_examples.py
@@ -224,27 +224,6 @@ def _api_context_entity(context_id, links=True):
     return result
 
 
-def _api_distribution_entity(
-    distribution_id,
-    case_id,
-    context_id,
-    commit_id,
-):
-    result = {
-        "id": distribution_id,
-        "case_id": case_id,
-        "context_id": context_id,
-        "commit_id": commit_id,
-        "machine_hash": "diana-2-4-17179869184",
-        "unit": "s",
-        "mean_mean": "0.036369",
-        "mean_sd": "0.000000",
-        "first_timestamp": "2021-02-25T01:02:51",
-        "last_timestamp": "2021-02-25T01:02:51",
-    }
-    return result
-
-
 def _api_history_entity(benchmark_id, case_id, context_id):
     return {
         "benchmark_id": benchmark_id,
@@ -253,6 +232,8 @@ def _api_history_entity(benchmark_id, case_id, context_id):
         "machine_hash": "diana-2-4-17179869184",
         "unit": "s",
         "mean": "0.036369",
+        "distribution_mean": "0.036369",
+        "distribution_stdev": "0.000000",
         "repository": "https://github.com/apache/arrow",
         "sha": "02addad336ba19a654f9c857ede546331be7b631",
         "timestamp": "2021-02-25T01:02:51",
@@ -360,12 +341,6 @@ def _api_run_entity(run_id, commit_id, machine_id, now, baseline_id):
     ],
 )
 CONTEXT_ENTITY = _api_context_entity("some-context-uuid-1")
-DISTRIBUTION_ENTITY = _api_distribution_entity(
-    "some-distribution-uuid-1",
-    "some-case-uuid-1",
-    "some-context-uuid-1",
-    "some-commit-uuid-1",
-)
 HISTORY_ENTITY = _api_history_entity(
     "some-benchmark-uuid-1",
     "some-case-uuid-1",

diff --git a/conbench/api/distribution.py b/conbench/api/distribution.py
diff --git a/conbench/app/_plots.py b/conbench/app/_plots.py
@@ -9,10 +9,9 @@
 class TimeSeriesPlotMixin:
     def _get_history_plot(self, benchmark):
         history = self._get_history(benchmark)
-        distribution = self._get_distribution(benchmark)
         return json.dumps(
             bokeh.embed.json_item(
-                time_series_plot(history, distribution, benchmark["id"]),
+                time_series_plot(history, benchmark["id"]),
                 "plot-history",
             )
         )
@@ -24,13 +23,6 @@ def _get_history(self, benchmark):
             return []
         return response.json
 
-    def _get_distribution(self, benchmark):
-        response = self.api_get("api.distribution", benchmark_id=benchmark["id"])
-        if response.status_code != 200:
-            self.flash("Error getting distribution.")
-            return []
-        return response.json
-
 
 def get_display_unit(unit):
     if unit == "s":
@@ -101,17 +93,10 @@ def simple_bar_plot(benchmarks, height=400, width=400):
     return p
 
 
-def time_series_plot(history, distribution, benchmark_id, height=250, width=1000):
-    dist_by_sha = {d["sha"]: d for d in distribution}
-    for h in history:
-        dist = dist_by_sha.get(h["sha"])
-        if dist:
-            h["mean_mean"] = dist["mean_mean"]
-            h["mean_sd"] = dist["mean_sd"]
-
+def time_series_plot(history, benchmark_id, height=250, width=1000):
     unit = get_display_unit(history[0]["unit"])
     current = [h for h in history if h["benchmark_id"] == benchmark_id]
-    with_dist = [h for h in history if h.get("mean_mean")]
+    with_dist = [h for h in history if h["distribution_mean"]]
 
     times = [h["mean"] for h in history]
     commits = [h["message"] for h in history]
@@ -121,15 +106,15 @@ def time_series_plot(history, distribution, benchmark_id, height=250, width=1000
     commits_x = [c["message"] for c in current]
     dates_x = [dateutil.parser.isoparse(c["timestamp"]) for c in current]
 
-    times_mean = [w["mean_mean"] for w in with_dist]
+    times_mean = [w["distribution_mean"] for w in with_dist]
     commits_mean = [w["message"] for w in with_dist]
     dates_mean = [dateutil.parser.isoparse(w["timestamp"]) for w in with_dist]
 
     alert_min, alert_max = [], []
     for w in with_dist:
-        alert = 5 * float(w["mean_sd"])
-        alert_min.append(float(w["mean_mean"]) - alert)
-        alert_max.append(float(w["mean_mean"]) + alert)
+        alert = 5 * float(w["distribution_stdev"])
+        alert_min.append(float(w["distribution_mean"]) - alert)
+        alert_max.append(float(w["distribution_mean"]) + alert)
 
     source_data = dict(x=dates, y=times, commit=commits)
     source = bokeh.models.ColumnDataSource(data=source_data)

diff --git a/conbench/entities/distribution.py b/conbench/entities/distribution.py
@@ -7,7 +7,6 @@
 from ..entities._entity import (
     Base,
     EntityMixin,
-    EntitySerializer,
     generate_uuid,
     NotNull,
     Nullable,
@@ -54,55 +53,6 @@ class Distribution(Base, EntityMixin):
 s.Index("distribution_machine_hash_index", Distribution.machine_hash)
 
 
-class _Serializer(EntitySerializer):
-    decimal_fmt = "{:.6f}"
-
-    def _dump(self, distribution):
-        standard_deviation = distribution.mean_sd if distribution.mean_sd else 0
-        result = {
-            "id": distribution.id,
-            "case_id": distribution.case_id,
-            "context_id": distribution.context_id,
-            "commit_id": distribution.commit_id,
-            "machine_hash": distribution.machine_hash,
-            "unit": distribution.unit,
-            "mean_mean": self.decimal_fmt.format(distribution.mean_mean),
-            "mean_sd": self.decimal_fmt.format(standard_deviation),
-            "first_timestamp": distribution.first_timestamp.isoformat(),
-            "last_timestamp": distribution.last_timestamp.isoformat(),
-        }
-        return result
-
-
-class DistributionSerializer:
-    one = _Serializer()
-    many = _Serializer(many=True)
-
-
-def get_distribution_history(case_id, context_id, machine_hash):
-    return (
-        Session.query(
-            Distribution.id,
-            Distribution.case_id,
-            Distribution.context_id,
-            Distribution.commit_id,
-            Distribution.machine_hash,
-            Distribution.unit,
-            Distribution.mean_mean,
-            Distribution.mean_sd,
-            Distribution.first_timestamp,
-            Distribution.last_timestamp,
-        )
-        .filter(
-            Distribution.case_id == case_id,
-            Distribution.context_id == context_id,
-            Distribution.machine_hash == machine_hash,
-        )
-        .order_by(Distribution.first_timestamp.asc())
-        .all()
-    )
-
-
 def get_commit_index(repository):
     ordered = (
         Session.query(Commit.id, Commit.sha, Commit.timestamp)

diff --git a/conbench/entities/history.py b/conbench/entities/history.py
@@ -1,6 +1,7 @@
 from ..db import Session
 from ..entities._entity import EntitySerializer
 from ..entities.commit import Commit
+from ..entities.distribution import Distribution
 from ..entities.machine import Machine
 from ..entities.run import Run
 from ..entities.summary import Summary
@@ -10,6 +11,7 @@ class _Serializer(EntitySerializer):
     decimal_fmt = "{:.6f}"
 
     def _dump(self, history):
+        standard_deviation = history.mean_sd if history.mean_sd else 0
         return {
             "benchmark_id": history.id,
             "case_id": history.case_id,
@@ -22,6 +24,8 @@ def _dump(self, history):
             "message": history.message,
             "timestamp": history.timestamp.isoformat(),
             "run_name": history.name,
+            "distribution_mean": self.decimal_fmt.format(history.mean_mean),
+            "distribution_stdev": self.decimal_fmt.format(standard_deviation),
         }
 
 
@@ -43,15 +47,20 @@ def get_history(case_id, context_id, machine_hash):
             Commit.repository,
             Commit.message,
             Commit.timestamp,
+            Distribution.mean_mean,
+            Distribution.mean_sd,
             Run.name,
         )
         .join(Run, Run.id == Summary.run_id)
         .join(Machine, Machine.id == Run.machine_id)
         .join(Commit, Commit.id == Run.commit_id)
+        .join(Distribution, Distribution.commit_id == Commit.id)
         .filter(
             Summary.case_id == case_id,
             Summary.context_id == context_id,
             Machine.hash == machine_hash,
+            Distribution.case_id == case_id,
+            Distribution.context_id == context_id,
         )
         .order_by(Commit.timestamp.asc())
         .all()

diff --git a/conbench/tests/api/_expected_docs.py b/conbench/tests/api/_expected_docs.py
@@ -456,27 +456,6 @@
                 },
                 "description": "OK",
             },
-            "DistributionList": {
-                "content": {
-                    "application/json": {
-                        "example": [
-                            {
-                                "case_id": "some-case-uuid-1",
-                                "commit_id": "some-commit-uuid-1",
-                                "context_id": "some-context-uuid-1",
-                                "first_timestamp": "2021-02-25T01:02:51",
-                                "id": "some-distribution-uuid-1",
-                                "last_timestamp": "2021-02-25T01:02:51",
-                                "machine_hash": "diana-2-4-17179869184",
-                                "mean_mean": "0.036369",
-                                "mean_sd": "0.000000",
-                                "unit": "s",
-                            }
-                        ]
-                    }
-                },
-                "description": "OK",
-            },
             "HistoryList": {
                 "content": {
                     "application/json": {
@@ -485,6 +464,8 @@
                                 "benchmark_id": "some-benchmark-uuid-1",
                                 "case_id": "some-case-uuid-1",
                                 "context_id": "some-context-uuid-1",
+                                "distribution_mean": "0.036369",
+                                "distribution_stdev": "0.000000",
                                 "machine_hash": "diana-2-4-17179869184",
                                 "mean": "0.036369",
                                 "message": "ARROW-11771: [Developer][Archery] Move benchmark tests (so CI runs them)",
@@ -1134,25 +1115,6 @@
                 "tags": ["Contexts"],
             }
         },
-        "/api/distribution/{benchmark_id}/": {
-            "get": {
-                "description": "Get benchmark distribution history.",
-                "parameters": [
-                    {
-                        "in": "path",
-                        "name": "benchmark_id",
-                        "required": True,
-                        "schema": {"type": "string"},
-                    }
-                ],
-                "responses": {
-                    "200": {"$ref": "#/components/responses/DistributionList"},
-                    "401": {"$ref": "#/components/responses/401"},
-                    "404": {"$ref": "#/components/responses/404"},
-                },
-                "tags": ["Distribution"],
-            }
-        },
         "/api/docs.json": {},
         "/api/history/{benchmark_id}/": {
             "get": {
@@ -1375,7 +1337,6 @@
         {"description": "Benchmarked commits", "name": "Commits"},
         {"description": "Benchmark comparisons", "name": "Comparisons"},
         {"description": "Benchmark contexts", "name": "Contexts"},
-        {"description": "Benchmark distribution", "name": "Distribution"},
         {"description": "Benchmark history", "name": "History"},
         {"description": "Benchmark machines", "name": "Machines"},
         {"description": "Benchmark runs", "name": "Runs"},