Revert: a run can have many contexts (#42)

conbench/api/_examples.py

@@ -219,18 +219,21 @@ def _api_machine_entity(machine_id, links=True):
     return result
 
 
-def _api_run_entity(run_id, commit_id, context_id, machine_id, now):
-    return {
+def _api_run_entity(run_id, commit_id, machine_id, now, baseline_id):
+    result = {
         "id": run_id,
         "name": "pull request: 9564",
         "timestamp": now,
         "commit": _api_commit_entity(commit_id),
-        "context": _api_context_entity(context_id, links=False),
         "machine": _api_machine_entity(machine_id, links=False),
         "links": {
             "self": "http://localhost/api/runs/%s/" % run_id,
         },
     }
+    if baseline_id:
+        baseline_url = "http://localhost/api/runs/%s/" % baseline_id
+        result["links"]["baseline"] = baseline_url
+    return result
 
 
 BENCHMARK_ENTITY = _api_benchmark_entity(
@@ -285,24 +288,24 @@ def _api_run_entity(run_id, commit_id, context_id, machine_id, now):
 RUN_ENTITY = _api_run_entity(
     "some-run-uuid-1",
     "some-commit-uuid-1",
-    "some-context-uuid-1",
     "some-machine-uuid-1",
     "2021-02-04T17:22:05.225583",
+    "some-run-uuid-0",
 )
 RUN_LIST = [
     _api_run_entity(
         "some-run-uuid-1",
         "some-commit-uuid-1",
-        "some-context-uuid-1",
         "some-machine-uuid-1",
         "2021-02-04T17:22:05.225583",
+        None,
     ),
     _api_run_entity(
         "some-run-uuid-2",
         "some-commit-uuid-1",
-        "some-context-uuid-1",
         "some-machine-uuid-1",
         "2021-03-04T17:18:05.715583",
+        None,
     ),
 ]
 USER_ENTITY = _api_user_entity(FakeUser1())

conbench/api/runs.py

@@ -1,9 +1,6 @@
-import flask as f
-
 from ..api import rule
 from ..api._endpoint import ApiEndpoint
 from ..entities._entity import NotFound
-from ..entities.commit import Commit
 from ..entities.run import Run, RunSerializer
 
 
@@ -47,27 +44,10 @@ def get(self):
         responses:
             "200": "RunList"
             "401": "401"
-        parameters:
-          - in: query
-            name: sha
-            schema:
-              type: string
-          - in: query
-            name: machine_id
-            schema:
-              type: string
         tags:
           - Runs
         """
-        sha = f.request.args.get("sha")
-        machine_id = f.request.args.get("machine_id")
-        if sha and machine_id:
-            runs = Run.search(
-                filters=[Run.machine_id == machine_id, Commit.sha == sha],
-                joins=[Commit],
-            )
-        else:
-            runs = Run.all(order_by=Run.timestamp.desc(), limit=500)
+        runs = Run.all(order_by=Run.timestamp.desc(), limit=500)
         return self.serializer.many.dump(runs)
 
 

conbench/app/benchmarks.py

@@ -61,19 +61,14 @@ def get_display_run(self, run_id):
         self._augment(run)
         return run
 
-    def get_display_baseline_run(self, sha, machine_id):
-        response = self.api_get("api.runs", sha=sha, machine_id=machine_id)
+    def get_display_baseline_run(self, run_url):
+        run, response = self._get_run_by_url(run_url)
         if response.status_code != 200:
             self.flash("Error getting run.")
             return None
 
-        runs = response.json
-        if runs:
-            run = runs[0]
-            self._augment(run)
-            return run
-
-        return None
+        self._augment(run)
+        return run
 
     def get_display_runs(self):
         runs, response = self._get_runs()
@@ -99,6 +94,10 @@ def _get_run(self, run_id):
         response = self.api_get("api.run", run_id=run_id)
         return response.json, response
 
+    def _get_run_by_url(self, run_url):
+        response = self.api_get_url(run_url)
+        return response.json, response
+
     def _get_runs(self):
         response = self.api_get("api.runs")
         return response.json, response

conbench/app/runs.py

@@ -25,10 +25,9 @@ def page(self, benchmarks, baseline_run, contender_run):
     def get(self, run_id):
         contender_run, baseline_run = self.get_display_run(run_id), None
         if contender_run:
-            parent = contender_run["commit"]["parent_sha"]
-            baseline_run = self.get_display_baseline_run(
-                parent, contender_run["machine"]["id"]
-            )
+            baseline_url = contender_run["links"].get("baseline")
+            if baseline_url:
+                baseline_run = self.get_display_baseline_run(baseline_url)
 
         benchmarks, response = self._get_benchmarks(run_id)
         if response.status_code != 200:

conbench/entities/_entity.py

@@ -2,7 +2,7 @@
 import uuid
 
 import flask as f
-from sqlalchemy import Column
+from sqlalchemy import Column, distinct
 from sqlalchemy.orm import declarative_base
 from sqlalchemy.orm.exc import NoResultFound
 
@@ -26,6 +26,11 @@ class EntityMixin:
     def __repr__(self):
         return f"<{self.__class__.__name__} {self.id}>"
 
+    @classmethod
+    def distinct(cls, column, filters):
+        q = Session.query(distinct(column))
+        return q.filter(*filters).all()
+
     @classmethod
     def search(cls, filters, joins=None):
         q = Session.query(cls)

conbench/entities/run.py

@@ -3,8 +3,7 @@
 from sqlalchemy.orm import relationship
 
 from ..entities._entity import Base, EntityMixin, EntitySerializer, NotNull, Nullable
-from ..entities.commit import CommitSerializer
-from ..entities.context import ContextSerializer
+from ..entities.commit import Commit, CommitSerializer
 from ..entities.machine import MachineSerializer
 
 
@@ -15,31 +14,53 @@ class Run(Base, EntityMixin):
     timestamp = NotNull(s.DateTime(timezone=False), server_default=s.sql.func.now())
     commit_id = NotNull(s.String(50), s.ForeignKey("commit.id"))
     commit = relationship("Commit", lazy="joined")
-    context_id = NotNull(s.String(50), s.ForeignKey("context.id"))
-    context = relationship("Context", lazy="joined")
     machine_id = NotNull(s.String(50), s.ForeignKey("machine.id"))
     machine = relationship("Machine", lazy="joined")
 
+    def get_baseline_id(self):
+        from ..entities.summary import Summary
+
+        parent = self.commit.parent
+        runs = Run.search(
+            filters=[Run.machine_id == self.machine_id, Commit.sha == parent],
+            joins=[Commit],
+        )
+        run_contexts = Summary.distinct(
+            Summary.context_id, filters=[Summary.run_id == self.id]
+        )
+
+        # TODO: What if there are multiple matches? Pick by date?
+        for run in runs:
+            baseline_contexts = Summary.distinct(
+                Summary.context_id, filters=[Summary.run_id == run.id]
+            )
+            if set(run_contexts) == set(baseline_contexts):
+                return run.id
+
+        return None
+
 
 class _Serializer(EntitySerializer):
     def _dump(self, run):
         commit = CommitSerializer().one.dump(run.commit)
-        context = ContextSerializer().one.dump(run.context) if run.context else {}
         machine = MachineSerializer().one.dump(run.machine)
         commit.pop("links", None)
-        context.pop("links", None)
         machine.pop("links", None)
         result = {
             "id": run.id,
             "name": run.name,
             "timestamp": run.timestamp.isoformat(),
             "commit": commit,
-            "context": context,
             "machine": machine,
             "links": {
                 "self": f.url_for("api.run", run_id=run.id, _external=True),
             },
         }
+        if not self.many:
+            baseline_id, baseline_url = run.get_baseline_id(), None
+            if baseline_id:
+                baseline_url = f.url_for("api.run", run_id=baseline_id, _external=True)
+            result["links"]["baseline"] = baseline_url
         return result
 
 

conbench/entities/summary.py

@@ -126,7 +126,6 @@ def create(data):
                     "id": run_id,
                     "name": run_name,
                     "commit_id": commit.id,
-                    "context_id": context.id,
                     "machine_id": machine.id,
                 }
             )

conbench/tests/api/_expected_docs.py

@@ -460,19 +460,10 @@
                                 "timestamp": "2021-02-25T01:02:51",
                                 "url": "https://github.com/apache/arrow/commit/02addad336ba19a654f9c857ede546331be7b631",
                             },
-                            "context": {
-                                "arrow_compiler_flags": "-fPIC -arch x86_64 -arch x86_64 -std=c++11 -Qunused-arguments -fcolor-diagnostics -O3 -DNDEBUG",
-                                "arrow_compiler_id": "AppleClang",
-                                "arrow_compiler_version": "11.0.0.11000033",
-                                "arrow_git_revision": "02addad336ba19a654f9c857ede546331be7b631",
-                                "arrow_version": "2.0.0",
-                                "benchmark_language": "Python",
-                                "benchmark_language_version": "Python 3.8.5",
-                                "id": "some-context-uuid-1",
-                            },
                             "id": "some-run-uuid-1",
                             "links": {
-                                "self": "http://localhost/api/runs/some-run-uuid-1/"
+                                "baseline": "http://localhost/api/runs/some-run-uuid-0/",
+                                "self": "http://localhost/api/runs/some-run-uuid-1/",
                             },
                             "machine": {
                                 "architecture_name": "x86_64",
@@ -516,16 +507,6 @@
                                     "timestamp": "2021-02-25T01:02:51",
                                     "url": "https://github.com/apache/arrow/commit/02addad336ba19a654f9c857ede546331be7b631",
                                 },
-                                "context": {
-                                    "arrow_compiler_flags": "-fPIC -arch x86_64 -arch x86_64 -std=c++11 -Qunused-arguments -fcolor-diagnostics -O3 -DNDEBUG",
-                                    "arrow_compiler_id": "AppleClang",
-                                    "arrow_compiler_version": "11.0.0.11000033",
-                                    "arrow_git_revision": "02addad336ba19a654f9c857ede546331be7b631",
-                                    "arrow_version": "2.0.0",
-                                    "benchmark_language": "Python",
-                                    "benchmark_language_version": "Python 3.8.5",
-                                    "id": "some-context-uuid-1",
-                                },
                                 "id": "some-run-uuid-1",
                                 "links": {
                                     "self": "http://localhost/api/runs/some-run-uuid-1/"
@@ -564,16 +545,6 @@
                                     "timestamp": "2021-02-25T01:02:51",
                                     "url": "https://github.com/apache/arrow/commit/02addad336ba19a654f9c857ede546331be7b631",
                                 },
-                                "context": {
-                                    "arrow_compiler_flags": "-fPIC -arch x86_64 -arch x86_64 -std=c++11 -Qunused-arguments -fcolor-diagnostics -O3 -DNDEBUG",
-                                    "arrow_compiler_id": "AppleClang",
-                                    "arrow_compiler_version": "11.0.0.11000033",
-                                    "arrow_git_revision": "02addad336ba19a654f9c857ede546331be7b631",
-                                    "arrow_version": "2.0.0",
-                                    "benchmark_language": "Python",
-                                    "benchmark_language_version": "Python 3.8.5",
-                                    "id": "some-context-uuid-1",
-                                },
                                 "id": "some-run-uuid-2",
                                 "links": {
                                     "self": "http://localhost/api/runs/some-run-uuid-2/"
@@ -1062,10 +1033,6 @@
         "/api/runs/": {
             "get": {
                 "description": "Get a list of runs.",
-                "parameters": [
-                    {"in": "query", "name": "sha", "schema": {"type": "string"}},
-                    {"in": "query", "name": "machine_id", "schema": {"type": "string"}},
-                ],
                 "responses": {
                     "200": {"$ref": "#/components/responses/RunList"},
                     "401": {"$ref": "#/components/responses/401"},

conbench/tests/api/test_runs.py

@@ -1,24 +1,28 @@
 import copy
 import urllib
+import uuid
 
 from ...api._examples import _api_run_entity
 from ...entities.summary import Summary
 from ...tests.api import _asserts
 from ...tests.api.test_benchmarks import VALID_PAYLOAD
 
 
-def _expected_entity(run):
+def _expected_entity(run, baseline_id=None):
     return _api_run_entity(
         run.id,
         run.commit_id,
-        run.context_id,
         run.machine_id,
         run.timestamp.isoformat(),
+        baseline_id,
     )
 
 
-def create_benchmark_summary():
+def create_benchmark_summary(parent_sha=None):
     data = copy.deepcopy(VALID_PAYLOAD)
+    if parent_sha:
+        data["run"]["commit"] = parent_sha
+        data["stats"]["run_id"] = uuid.uuid4().hex
     summary = Summary.create(data)
     return summary
 
@@ -27,15 +31,19 @@ class TestRunGet(_asserts.GetEnforcer):
     url = "/api/runs/{}/"
     public = True
 
-    def _create(self):
-        summary = create_benchmark_summary()
-        return summary.run
+    def _create(self, baseline=False):
+        contender = create_benchmark_summary()
+        if baseline:
+            parent_sha = "4beb514d071c9beec69b8917b5265e77ade22fb3"
+            baseline = create_benchmark_summary(parent_sha)
+            return contender.run, baseline.run
+        return contender.run
 
     def test_get_run(self, client):
         self.authenticate(client)
-        run = self._create()
+        run, baseline = self._create(baseline=True)
         response = client.get(f"/api/runs/{run.id}/")
-        self.assert_200_ok(response, _expected_entity(run))
+        self.assert_200_ok(response, _expected_entity(run, baseline.id))
 
 
 class TestRunList(_asserts.ListEnforcer):

migrations/versions/662175f2e6c6_backfill_parent.py

@@ -9,9 +9,7 @@
 
 from alembic import op
 import requests
-
-
-from conbench.entities.commit import Commit
+from sqlalchemy import MetaData
 
 
 # revision identifiers, used by Alembic.
@@ -22,8 +20,10 @@
 
 
 def upgrade():
-    commit_table = Commit.__table__
     connection = op.get_bind()
+    meta = MetaData()
+    meta.reflect(bind=connection)
+    commit_table = meta.tables["commit"]
 
     token, session = os.getenv("GITHUB_API_TOKEN"), None
     if token: