Fill in {commit} for Travis PR builds by querying GitHub

README.rst

@@ -318,12 +318,11 @@ Placeholder          Definition
                      Travis and Appveyor, this is the hash of an autogenerated
                      merge commit.
 ``{commit}``         The hash of the original commit that triggered the build
-                     or that was tagged for the release.  For GitHub Actions
-                     and all non-pull request builds, this is always the same
-                     as ``{build_commit}``.  For pull request builds on
-                     Appveyor, this is the head of the PR branch.  For pull
-                     request builds on Travis, this is ``UNK``, as the API does
-                     not expose the original commit.
+                     or that was tagged for the release.  For pull request
+                     builds, this is the head of the PR branch, or ``UNK`` if
+                     it cannot be determined.  For other builds (along with PR
+                     builds on GitHub Actions), this is always the same as
+                     ``{build_commit}``.
 ``{number}``         The run number of the workflow run (GitHub) or the build
                      number (Travis and Appveyor) [1]_
 ``{status}``         The success status of the workflow run (GitHub) or job
@@ -389,6 +388,10 @@ A Travis API access token can be acquired as follows:
 
 - Run ``travis token --com`` to retrieve the API access token.
 
+The Travis integration also requires a GitHub OAuth token in order to look up
+information on pull requests that the Travis API does not report; this token
+must be specified in the same way as for the GitHub integration.
+
 Appveyor
 ~~~~~~~~
 

src/tinuous/__main__.py

@@ -79,9 +79,9 @@ def fetch(cfg: Config, state: str, sanitize_secrets: bool) -> None:
     except FileNotFoundError:
         since_stamps = {}
     # Fetch tokens early in order to catch failures early:
-    tokens: Dict[str, str] = {}
+    tokens: Dict[str, Dict[str, str]] = {}
     for name, cicfg in cfg.ci.items():
-        tokens[name] = cicfg.get_auth_token()
+        tokens[name] = cicfg.get_auth_tokens()
     if cfg.datalad.enabled:
         try:
             from datalad.api import Dataset
@@ -100,7 +100,7 @@ def fetch(cfg: Config, state: str, sanitize_secrets: bool) -> None:
             since = datetime.fromisoformat(since_stamps[name])
         except KeyError:
             since = cfg.since
-        ci = cicfg.get_system(repo=cfg.repo, since=since, token=tokens[name])
+        ci = cicfg.get_system(repo=cfg.repo, since=since, tokens=tokens[name])
         for obj in ci.get_build_assets(cfg.types, artifacts=get_artifacts):
             if isinstance(obj, BuildLog):
                 path = obj.expand_path(cicfg.path, cfg.vars)

src/tinuous/appveyor.py

@@ -14,14 +14,14 @@ class Appveyor(CISystem):
     projectSlug: Optional[str]
 
     @staticmethod
-    def get_auth_token() -> str:
+    def get_auth_tokens() -> Dict[str, str]:
         token = os.environ.get("APPVEYOR_TOKEN")
         if not token:
             raise RuntimeError(
                 "Appveyor API key not set.  Set via APPVEYOR_TOKEN environment"
                 " variable."
             )
-        return token
+        return {"appveyor": token}
 
     @property
     def repo_slug(self) -> str:

src/tinuous/base.py

@@ -86,7 +86,7 @@ class CISystem(ABC, BaseModel):
 
     @staticmethod
     @abstractmethod
-    def get_auth_token() -> str:
+    def get_auth_tokens() -> Dict[str, str]:
         ...  # pragma: no cover
 
     @abstractmethod

src/tinuous/config.py

@@ -22,11 +22,13 @@ class CIConfig(NoExtraModel, ABC):
 
     @staticmethod
     @abstractmethod
-    def get_auth_token() -> str:
+    def get_auth_tokens() -> Dict[str, str]:
         ...  # pragma: no cover
 
     @abstractmethod
-    def get_system(self, repo: str, since: datetime, token: str) -> CISystem:
+    def get_system(
+        self, repo: str, since: datetime, tokens: Dict[str, str]
+    ) -> CISystem:
         ...  # pragma: no cover
 
 
@@ -36,28 +38,31 @@ class GitHubConfig(CIConfig):
     workflows: Optional[List[str]] = None
 
     @staticmethod
-    def get_auth_token() -> str:
-        return GitHubActions.get_auth_token()
+    def get_auth_tokens() -> Dict[str, str]:
+        return GitHubActions.get_auth_tokens()
 
-    def get_system(self, repo: str, since: datetime, token: str) -> GitHubActions:
+    def get_system(
+        self, repo: str, since: datetime, tokens: Dict[str, str]
+    ) -> GitHubActions:
         return GitHubActions(
             repo=repo,
             since=since,
-            token=token,
+            token=tokens["github"],
             workflows=self.workflows,
         )
 
 
 class TravisConfig(CIConfig):
     @staticmethod
-    def get_auth_token() -> str:
-        return Travis.get_auth_token()
+    def get_auth_tokens() -> Dict[str, str]:
+        return Travis.get_auth_tokens()
 
-    def get_system(self, repo: str, since: datetime, token: str) -> Travis:
+    def get_system(self, repo: str, since: datetime, tokens: Dict[str, str]) -> Travis:
         return Travis(
             repo=repo,
             since=since,
-            token=token,
+            token=tokens["travis"],
+            gh_token=tokens["github"],
         )
 
 
@@ -66,14 +71,16 @@ class AppveyorConfig(CIConfig):
     projectSlug: Optional[str] = None
 
     @staticmethod
-    def get_auth_token() -> str:
-        return Appveyor.get_auth_token()
+    def get_auth_tokens() -> Dict[str, str]:
+        return Appveyor.get_auth_tokens()
 
-    def get_system(self, repo: str, since: datetime, token: str) -> Appveyor:
+    def get_system(
+        self, repo: str, since: datetime, tokens: Dict[str, str]
+    ) -> Appveyor:
         return Appveyor(
             repo=repo,
             since=since,
-            token=token,
+            token=tokens["appveyor"],
             accountName=self.accountName,
             projectSlug=self.projectSlug,
         )

src/tinuous/github.py

@@ -5,7 +5,6 @@
 from pathlib import Path
 import re
 from shutil import rmtree
-import subprocess
 import tempfile
 from typing import Dict, Iterator, List, Optional, Tuple
 from zipfile import ZipFile
@@ -18,29 +17,23 @@
 import requests
 
 from .base import APIClient, Artifact, BuildAsset, BuildLog, CISystem, EventType
-from .util import ensure_aware, expand_template, iterfiles, log, stream_to_file
+from .util import (
+    ensure_aware,
+    expand_template,
+    get_github_token,
+    iterfiles,
+    log,
+    stream_to_file,
+)
 
 
 class GitHubActions(CISystem):
     workflows: Optional[List[str]] = None
     hash2pr: Dict[str, str] = Field(default_factory=dict)
 
     @staticmethod
-    def get_auth_token() -> str:
-        token = os.environ.get("GITHUB_TOKEN")
-        if not token:
-            r = subprocess.run(
-                ["git", "config", "hub.oauthtoken"],
-                stdout=subprocess.PIPE,
-                universal_newlines=True,
-            )
-            if r.returncode != 0 or not r.stdout.strip():
-                raise RuntimeError(
-                    "GitHub OAuth token not set.  Set via GITHUB_TOKEN"
-                    " environment variable or hub.oauthtoken Git config option."
-                )
-            token = r.stdout.strip()
-        return token
+    def get_auth_tokens() -> Dict[str, str]:
+        return {"github": get_github_token()}
 
     @cached_property
     def client(self) -> Github:

src/tinuous/travis.py

@@ -6,14 +6,18 @@
 from urllib.parse import quote
 
 from dateutil.parser import isoparse
+from github import Github
+from github.Repository import Repository
 
 from .base import APIClient, BuildAsset, BuildLog, CISystem, EventType
-from .util import log, removeprefix, stream_to_file
+from .util import get_github_token, log, removeprefix, stream_to_file
 
 
 class Travis(CISystem):
+    gh_token: str
+
     @staticmethod
-    def get_auth_token() -> str:
+    def get_auth_tokens() -> Dict[str, str]:
         token = os.environ.get("TRAVIS_TOKEN")
         if not token:
             try:
@@ -37,7 +41,7 @@ def get_auth_token() -> str:
                     " information."
                 )
             token = r.stdout.strip()
-        return token
+        return {"travis": token, "github": get_github_token()}
 
     @cached_property
     def client(self) -> APIClient:
@@ -49,6 +53,10 @@ def client(self) -> APIClient:
             },
         )
 
+    @cached_property
+    def ghrepo(self) -> Repository:
+        return Github(self.gh_token).get_repo(self.repo)
+
     def paginate(
         self, path: str, params: Optional[Dict[str, str]] = None
     ) -> Iterator[dict]:
@@ -69,7 +77,11 @@ def get_build_assets(
             f"/repo/{quote(self.repo, safe='')}/builds",
             params={"include": "build.jobs"},
         ):
-            run_event = EventType.from_travis_event(build["event_type"])
+            event_type = EventType.from_travis_event(build["event_type"])
+            if event_type is None:
+                raise ValueError(
+                    f"Build has unknown event type {build['event_type']!r}"
+                )
             if build["started_at"] is None:
                 ### TODO: If there are any builds with a higher number that
                 ### have already started and finished, this can lead to the
@@ -86,12 +98,32 @@ def get_build_assets(
             else:
                 log.info("Found build %s", build["number"])
                 self.register_build(ts, True)
-                if run_event in event_types:
+                if event_type in event_types:
+                    commit = self.get_commit(build, event_type)
                     for job in build["jobs"]:
-                        yield TravisJobLog.from_job(self.client, build, job)
+                        yield TravisJobLog.from_job(
+                            self.client, build, job, commit, event_type
+                        )
                 else:
                     log.info("Event type is %r; skipping", build["event_type"])
 
+    def get_commit(self, build: Dict[str, Any], event_type: EventType) -> Optional[str]:
+        if event_type in (EventType.CRON, EventType.PUSH):
+            commit = build["commit"]["sha"]
+            assert isinstance(commit, str)
+            return commit
+        elif event_type is EventType.PULL_REQUEST:
+            pr = self.ghrepo.get_pull(build["pull_request_number"])
+            if pr.merge_commit_sha == build["commit"]["sha"]:
+                return pr.head.sha
+            else:
+                log.info(
+                    "Could not determine PR head commit for build; setting to 'UNK'"
+                )
+                return None
+        else:
+            raise AssertionError(f"Unhandled EventType: {event_type!r}")
+
 
 class TravisJobLog(BuildLog):
     job: str
@@ -103,28 +135,23 @@ def from_job(
         client: APIClient,
         build: Dict[str, Any],
         job: Dict[str, Any],
+        commit: Optional[str],
+        event_type: EventType,
     ) -> "TravisJobLog":
         created_at = isoparse(build["started_at"])
-        event = EventType.from_travis_event(build["event_type"])
-        if event is None:
-            raise ValueError(f"Build has unknown event type {build['event_type']!r}")
         event_id: str
-        commit: Optional[str]
-        if event is EventType.CRON:
+        if event_type is EventType.CRON:
             event_id = created_at.strftime("%Y%m%dT%H%M%S")
-            commit = build["commit"]["sha"]
-        elif event is EventType.PUSH:
+        elif event_type is EventType.PUSH:
             event_id = build["branch"]["name"]
-            commit = build["commit"]["sha"]
-        elif event is EventType.PULL_REQUEST:
+        elif event_type is EventType.PULL_REQUEST:
             event_id = str(build["pull_request_number"])
-            commit = None
         else:
-            raise AssertionError(f"Unhandled EventType: {event!r}")
+            raise AssertionError(f"Unhandled EventType: {event_type!r}")
         return cls(
             client=client,
             created_at=created_at,
-            event_type=event,
+            event_type=event_type,
             event_id=event_id,
             build_commit=build["commit"]["sha"],
             commit=commit,

src/tinuous/util.py

@@ -1,7 +1,9 @@
 from collections import deque
 from datetime import datetime, timezone
 import logging
+import os
 from pathlib import Path
+import subprocess
 from typing import Dict, Iterator, cast
 
 import requests
@@ -54,3 +56,20 @@ def expand_template(
 
 def fstring(s: str, **kwargs: str) -> str:
     return cast(str, eval(f"f{s!r}", {}, kwargs))
+
+
+def get_github_token() -> str:
+    token = os.environ.get("GITHUB_TOKEN")
+    if not token:
+        r = subprocess.run(
+            ["git", "config", "hub.oauthtoken"],
+            stdout=subprocess.PIPE,
+            universal_newlines=True,
+        )
+        if r.returncode != 0 or not r.stdout.strip():
+            raise RuntimeError(
+                "GitHub OAuth token not set.  Set via GITHUB_TOKEN"
+                " environment variable or hub.oauthtoken Git config option."
+            )
+        token = r.stdout.strip()
+    return token