Unknown and no commit contexts (#137)

conbench/app/benchmarks.py

@@ -90,7 +90,9 @@ def _augment(self, run):
         self._display_time(run, "timestamp")
         self._display_time(run["commit"], "timestamp")
         repository = run["commit"]["repository"]
-        repository_name = repository.split("github.com/")[1]
+        repository_name = repository
+        if "github.com/" in repository:
+            repository_name = repository.split("github.com/")[1]
         run["commit"]["display_repository"] = repository_name
 
     def _display_time(self, obj, field):

conbench/entities/commit.py

@@ -1,3 +1,4 @@
+import datetime
 import functools
 import json
 import os
@@ -29,6 +30,52 @@ class Commit(Base, EntityMixin):
     author_avatar = Nullable(s.String(100))
     timestamp = NotNull(s.DateTime(timezone=False))
 
+    @staticmethod
+    def create_no_context():
+        commit = Commit.first(sha="none")
+        if not commit:
+            now = datetime.datetime.now(datetime.timezone.utc)
+            commit = Commit.create(
+                {
+                    "sha": "none",
+                    "repository": "none",
+                    "parent": "",
+                    "timestamp": now,
+                    "message": "none",
+                    "author_name": "none",
+                }
+            )
+        return commit
+
+    @staticmethod
+    def create_unknown_context(sha, repository):
+        now = datetime.datetime.now(datetime.timezone.utc)
+        return Commit.create(
+            {
+                "sha": sha,
+                "repository": repository,
+                "parent": "unknown",
+                "timestamp": now,
+                "message": "unknown",
+                "author_name": "unknown",
+            }
+        )
+
+    @staticmethod
+    def create_github_context(sha, repository, github):
+        return Commit.create(
+            {
+                "sha": sha,
+                "repository": repository,
+                "parent": github["parent"],
+                "timestamp": github["date"],
+                "message": github["message"],
+                "author_name": github["author_name"],
+                "author_login": github["author_login"],
+                "author_avatar": github["author_avatar"],
+            }
+        )
+
 
 s.Index(
     "commit_index",
@@ -67,6 +114,9 @@ class CommitSerializer:
 
 
 def get_github_commit(repository, sha):
+    if not repository or not sha:
+        return {}
+
     github = GitHub()
     name = repository.split("github.com/")[1]
     commit = github.get_commit(name, sha)

conbench/entities/summary.py

@@ -1,4 +1,3 @@
-import datetime
 import decimal
 
 import flask as f
@@ -85,36 +84,20 @@ def create(data):
         if not context:
             context = Context.create({"tags": data["context"]})
 
+        sha, repository = None, None
+        if "github" in data:
+            sha, repository = data["github"]["commit"], data["github"]["repository"]
+
         # create if not exists
-        sha, repository = data["github"]["commit"], data["github"]["repository"]
         commit = Commit.first(sha=sha)
         if not commit:
             github = get_github_commit(repository, sha)
             if github:
-                commit = Commit.create(
-                    {
-                        "sha": sha,
-                        "repository": repository,
-                        "parent": github["parent"],
-                        "timestamp": github["date"],
-                        "message": github["message"],
-                        "author_name": github["author_name"],
-                        "author_login": github["author_login"],
-                        "author_avatar": github["author_avatar"],
-                    }
-                )
+                commit = Commit.create_github_context(sha, repository, github)
+            elif sha and repository:
+                commit = Commit.create_unknown_context(sha, repository)
             else:
-                now = datetime.datetime.now(datetime.timezone.utc)
-                commit = Commit.create(
-                    {
-                        "sha": sha,
-                        "repository": repository,
-                        "parent": "unknown",
-                        "timestamp": now,
-                        "message": "unknown",
-                        "author_name": "unknown",
-                    }
-                )
+                commit = Commit.create_no_context()
 
         # create if not exists
         run_id = data["run_id"]
@@ -247,7 +230,7 @@ class _BenchmarkFacadeSchemaCreate(marshmallow.Schema):
     stats = marshmallow.fields.Nested(SummarySchema().create, required=True)
     tags = marshmallow.fields.Dict(required=True)
     context = marshmallow.fields.Dict(required=True)
-    github = marshmallow.fields.Nested(GitHubCreate(), required=True)
+    github = marshmallow.fields.Nested(GitHubCreate(), required=False)
 
 
 class BenchmarkFacadeSchema:

conbench/tests/api/_expected_docs.py

@@ -741,7 +741,6 @@
                 "required": [
                     "batch_id",
                     "context",
-                    "github",
                     "machine_info",
                     "run_id",
                     "stats",

conbench/tests/api/test_benchmarks.py

@@ -306,7 +306,6 @@ class TestBenchmarkPost(_asserts.PostEnforcer):
         "stats",
         "tags",
         "context",
-        "github",
     ]
 
     def test_create_benchmark(self, client):
@@ -360,6 +359,81 @@ def test_nested_schema_validation(self, client):
         }
         self.assert_400_bad_request(response, message)
 
+    def test_create_no_commit_context(self, client):
+        self.authenticate(client)
+        data = copy.deepcopy(self.valid_payload)
+        data["run_id"] = _uuid()
+        del data["github"]
+
+        # create benchmark without commit context
+        response = client.post("/api/benchmarks/", json=data)
+        new_id = response.json["id"]
+        summary = Summary.one(id=new_id)
+        assert summary.run.commit.sha == "none"
+        location = "http://localhost/api/benchmarks/%s/" % new_id
+        self.assert_201_created(response, _expected_entity(summary), location)
+
+        # create another benchmark without commit context
+        # (test duplicate key duplicate key -- commit_index)
+        response = client.post("/api/benchmarks/", json=data)
+        new_id = response.json["id"]
+        summary = Summary.one(id=new_id)
+        assert summary.run.commit.sha == "none"
+        location = "http://localhost/api/benchmarks/%s/" % new_id
+        self.assert_201_created(response, _expected_entity(summary), location)
+
+    def test_create_empty_commit_context(self, client):
+        self.authenticate(client)
+        data = copy.deepcopy(self.valid_payload)
+        data["run_id"] = _uuid()
+        data["github"]["commit"] = ""
+        data["github"]["repository"] = ""
+
+        # create benchmark without commit context
+        response = client.post("/api/benchmarks/", json=data)
+        new_id = response.json["id"]
+        summary = Summary.one(id=new_id)
+        assert summary.run.commit.sha == "none"
+        location = "http://localhost/api/benchmarks/%s/" % new_id
+        self.assert_201_created(response, _expected_entity(summary), location)
+
+        # create another benchmark without commit context
+        # (test duplicate key duplicate key -- commit_index)
+        response = client.post("/api/benchmarks/", json=data)
+        new_id = response.json["id"]
+        summary = Summary.one(id=new_id)
+        assert summary.run.commit.sha == "none"
+        location = "http://localhost/api/benchmarks/%s/" % new_id
+        self.assert_201_created(response, _expected_entity(summary), location)
+
+    def test_create_unknown_commit_context(self, client):
+        self.authenticate(client)
+        data = copy.deepcopy(self.valid_payload)
+        data["run_id"] = _uuid()
+        data["github"]["commit"] = "unknown commit"
+        data["github"]["repository"] = "github.com/apache/arrow"
+
+        # create benchmark with unknown commit context
+        response = client.post("/api/benchmarks/", json=data)
+        new_id = response.json["id"]
+        summary = Summary.one(id=new_id)
+        assert summary.run.commit.sha == "unknown commit"
+        assert summary.run.commit.repository == "github.com/apache/arrow"
+        assert summary.run.commit.parent == "unknown"
+        location = "http://localhost/api/benchmarks/%s/" % new_id
+        self.assert_201_created(response, _expected_entity(summary), location)
+
+        # create another benchmark with unknown commit context
+        # (test duplicate key duplicate key -- commit_index)
+        response = client.post("/api/benchmarks/", json=data)
+        new_id = response.json["id"]
+        summary = Summary.one(id=new_id)
+        assert summary.run.commit.sha == "unknown commit"
+        assert summary.run.commit.repository == "github.com/apache/arrow"
+        assert summary.run.commit.parent == "unknown"
+        location = "http://localhost/api/benchmarks/%s/" % new_id
+        self.assert_201_created(response, _expected_entity(summary), location)
+
     def test_create_benchmark_distribution(self, client):
         self.authenticate(client)
         data = copy.deepcopy(self.valid_payload)

conbench/tests/entities/test_commit.py

@@ -12,6 +12,16 @@
 this_dir = os.path.abspath(os.path.dirname(__file__))
 
 
+def test_get_github_commit_none():
+    repo = "https://github.com/apache/arrow"
+    sha = "3decc46119d583df56c7c66c77cf2803441c4458"
+
+    assert get_github_commit(None, None) == {}
+    assert get_github_commit("", "") == {}
+    assert get_github_commit(repo, None) == {}
+    assert get_github_commit(None, sha) == {}
+
+
 @pytest.mark.slow
 def test_get_github_commit():
     # NOTE: This integration test intentionally hits GitHub.

conbench/util.py

@@ -47,7 +47,7 @@ def _post(self, url, data, expected):
                 self.session.mount("https://", adapter)
             start = time.time()
             response = self.session.post(url, json=data)
-            print("Time to POST ", url, time.time() - start)
+            print("Time to POST", url, time.time() - start)
             if response.status_code != expected:
                 self._unexpected_response("POST", response, url)
         except requests.exceptions.ConnectionError:

setup.py

@@ -7,7 +7,7 @@
 
 setuptools.setup(
     name="conbench",
-    version="1.2.0",
+    version="1.3.0",
     description="Continuous Benchmarking (CB) Framework",
     long_description=long_description,
     long_description_content_type="text/markdown",