Merge 7d9c39d into 328bf6b

conbench/entities/commit.py

@@ -1,4 +1,9 @@
+import functools
+import json
+import os
+
 import dateutil.parser
+import requests
 import sqlalchemy as s
 
 from ..entities._entity import (
@@ -53,12 +58,117 @@ class CommitSerializer:
     many = _Serializer(many=True)
 
 
-def parse_commit(commit):
-    return {
-        "parent": commit["parents"][0]["sha"],
-        "date": dateutil.parser.isoparse(commit["commit"]["author"]["date"]),
-        "message": commit["commit"]["message"].split("\n")[0],
-        "author_name": commit["commit"]["author"]["name"],
-        "author_login": commit["author"]["login"] if commit["author"] else None,
-        "author_avatar": commit["author"]["avatar_url"] if commit["author"] else None,
-    }
+GITHUB = "https://api.github.com"
+this_dir = os.path.abspath(os.path.dirname(__file__))
+
+
+def get_github_commit(repository, sha):
+    github = GitHub()
+    name = repository.split("github.com/")[1]
+    commit = github.get_commit(name, sha)
+    parent = commit["parent"]
+    commits = github.get_commits(name, parent)
+    if parent in commits:
+        return commit
+    else:
+        # This is a pull request, find the parent of the first commit.
+        # TODO: This will fail if the pull request has more than 50 commits.
+        # It will also give up if it can't find the parent after 50 tries
+        # (which could happen for a really old pull request).
+        parent = commit["parent"]
+        for _ in range(50):
+            other = github.get_commit(name, parent)
+            if other["parent"] in commits:
+                commit["parent"] = other["parent"]
+                return commit
+            else:
+                parent = other["parent"]
+    return {}
+
+
+class GitHub:
+    def __init__(self):
+        self.test_shas = {
+            "02addad336ba19a654f9c857ede546331be7b631": "github_child.json",
+            "4beb514d071c9beec69b8917b5265e77ade22fb3": "github_parent.json",
+            "6d703c4c7b15be630af48d5e9ef61628751674b2": "github_grandparent.json",
+        }
+        self.test_commits = [
+            "02addad336ba19a654f9c857ede546331be7b631",
+            "4beb514d071c9beec69b8917b5265e77ade22fb3",
+            "6d703c4c7b15be630af48d5e9ef61628751674b2",
+            "81e9417eb68171e03a304097ae86e1fd83307130",
+        ]
+
+    def get_commits(self, name, sha):
+        if sha in self.test_commits:
+            return self.test_commits
+
+        commits = []
+
+        # Grabs the last 1000 commits to the main branch. TODO: If the pull
+        # request is old, the parent may not be in the last 1000 commits.
+        for branch in ["master", "main"]:
+            url = f"{GITHUB}/repos/{name}/commits?sha={branch}&per_page=100"
+            response = self._get_response(url)
+            if response:
+                commits = self._parse_commits(response)
+                if sha in commits:
+                    return commits
+                for page in range(2, 11):
+                    url = f"{GITHUB}/repos/{name}/commits?sha={branch}&per_page=100&page={page}"
+                    response = self._get_response(url)
+                    if response:
+                        commits.extend(self._parse_commits(response))
+                        if sha in commits:
+                            return commits
+            if commits:
+                break
+
+        return commits
+
+    def get_commit(self, name, sha):
+        if sha in self.test_commits:
+            response = self.test_commit(sha)
+        else:
+            url = f"{GITHUB}/repos/{name}/commits/{sha}"
+            response = self._get_response(url)
+        return self._parse_commit(response) if response else None
+
+    @functools.cached_property
+    def session(self):
+        token, session = os.getenv("GITHUB_API_TOKEN"), None
+        if token:
+            session = requests.Session()
+            session.headers = {"Authorization": f"Bearer {token}"}
+        return session
+
+    def test_commit(self, sha):
+        fixture = f"../tests/entities/{self.test_shas[sha]}"
+        path = os.path.join(this_dir, fixture)
+        with open(path) as fixture:
+            return json.load(fixture)
+
+    @staticmethod
+    def _parse_commits(commits):
+        return [commit["sha"] for commit in commits]
+
+    @staticmethod
+    def _parse_commit(commit):
+        author = commit.get("author")
+        commit_author = commit["commit"]["author"]
+        return {
+            "parent": commit["parents"][0]["sha"],
+            "date": dateutil.parser.isoparse(commit_author["date"]),
+            "message": commit["commit"]["message"].split("\n")[0],
+            "author_name": commit_author["name"],
+            "author_login": author["login"] if author else None,
+            "author_avatar": author["avatar_url"] if author else None,
+        }
+
+    def _get_response(self, url):
+        response = self.session.get(url) if self.session else requests.get(url)
+        if response.status_code != 200:
+            print(response.json())
+            return None
+        return response.json()

conbench/entities/summary.py

@@ -1,9 +1,7 @@
 import decimal
-import os
 
 import flask as f
 import marshmallow
-import requests
 import sqlalchemy as s
 from sqlalchemy import CheckConstraint as check
 from sqlalchemy.orm import relationship
@@ -19,7 +17,7 @@
 from ..entities._comparator import z_improvement, z_regression
 from ..entities.case import Case
 from ..entities.context import Context
-from ..entities.commit import Commit, parse_commit
+from ..entities.commit import Commit, get_github_commit
 from ..entities.data import Data
 from ..entities.distribution import update_distribution
 from ..entities.machine import Machine, MachineSchema
@@ -90,19 +88,7 @@ def create(data):
         sha, repository = data["github"]["commit"], data["github"]["repository"]
         commit = Commit.first(sha=sha)
         if not commit:
-            name = repository.split("github.com/")[1]
-            url = f"https://api.github.com/repos/{name}/commits/{sha}"
-
-            token, session = os.getenv("GITHUB_API_TOKEN"), None
-            if token:
-                session = requests.Session()
-                session.headers = {"Authorization": f"Bearer {token}"}
-
-            response = session.get(url) if session else requests.get(url)
-            if response.status_code != 200:
-                print(response.json())
-
-            github = parse_commit(response.json())
+            github = get_github_commit(repository, sha)
             commit = Commit.create(
                 {
                     "sha": sha,

conbench/tests/api/_fixtures.py

@@ -1,6 +1,7 @@
 CHILD = "02addad336ba19a654f9c857ede546331be7b631"
 PARENT = "4beb514d071c9beec69b8917b5265e77ade22fb3"
 GRANDPARENT = "6d703c4c7b15be630af48d5e9ef61628751674b2"
+ELDER = "81e9417eb68171e03a304097ae86e1fd83307130"
 
 
 RESULTS_UP = [[1, 2, 3], [2, 3, 4], [10, 20, 30]]