From 53752f3382cd663b5f2fd3a01c66eb4b680ecea7 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 8 Jun 2021 11:50:09 -0400 Subject: [PATCH] Add "until:" config setting --- README.rst | 4 ++++ src/tinuous/__main__.py | 4 +++- src/tinuous/appveyor.py | 6 +++++- src/tinuous/base.py | 1 + src/tinuous/config.py | 41 +++++++++++++++++++++++++++++++++-------- src/tinuous/github.py | 8 +++++++- src/tinuous/travis.py | 4 ++++ 7 files changed, 57 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 9cdf4d2..6d1df3f 100644 --- a/README.rst +++ b/README.rst @@ -197,6 +197,10 @@ keys: the latest such build is stored in the state file and used as the new ``since`` value for the respective CI system on subsequent runs. +``until`` + *(optional)* A timestamp (date, time, & timezone); only assets for builds + started before the given point in time will be retrieved + ``types`` A list of build trigger event types; only assets for builds triggered by one of the given events will be retrieved diff --git a/src/tinuous/__main__.py b/src/tinuous/__main__.py index 42c7ad7..e3fbd88 100644 --- a/src/tinuous/__main__.py +++ b/src/tinuous/__main__.py @@ -100,7 +100,9 @@ def fetch(cfg: Config, state: str, sanitize_secrets: bool) -> None: since = datetime.fromisoformat(since_stamps[name]) except KeyError: since = cfg.since - ci = cicfg.get_system(repo=cfg.repo, since=since, tokens=tokens[name]) + ci = cicfg.get_system( + repo=cfg.repo, since=since, until=cfg.until, tokens=tokens[name] + ) for obj in ci.get_build_assets(cfg.types, artifacts=get_artifacts): if isinstance(obj, BuildLog): path = obj.expand_path(cicfg.path, cfg.vars) diff --git a/src/tinuous/appveyor.py b/src/tinuous/appveyor.py index 4a6f99a..a854235 100644 --- a/src/tinuous/appveyor.py +++ b/src/tinuous/appveyor.py @@ -53,7 +53,9 @@ def get_builds(self) -> Iterator[dict]: def get_build_assets( self, event_types: List[EventType], artifacts: bool = False # noqa: U100 ) -> Iterator["BuildAsset"]: - log.info("Fetching runs newer than %s", self.since) + log.info("Fetching builds newer than %s", self.since) + if self.until is not None: + log.info("Skipping builds newer than %s", self.until) for build in self.get_builds(): if build.get("pullRequestId"): run_event = EventType.PULL_REQUEST @@ -62,6 +64,8 @@ def get_build_assets( ts = isoparse(build["created"]) if ts <= self.since: break + elif self.until is not None and ts > self.until: + log.info("Build %s is too new; skipping", build["buildNumber"]) elif build.get("finished") is None: log.info("Build %s not completed; skipping", build["buildNumber"]) self.register_build(ts, False) diff --git a/src/tinuous/base.py b/src/tinuous/base.py index 2e282ed..89c706c 100644 --- a/src/tinuous/base.py +++ b/src/tinuous/base.py @@ -82,6 +82,7 @@ class CISystem(ABC, BaseModel): repo: str token: str since: datetime + until: Optional[datetime] fetched: List[Tuple[datetime, bool]] = Field(default_factory=list) @staticmethod diff --git a/src/tinuous/config.py b/src/tinuous/config.py index c39563e..3d1d010 100644 --- a/src/tinuous/config.py +++ b/src/tinuous/config.py @@ -4,6 +4,7 @@ from typing import Dict, Iterator, List, Optional, Pattern, Tuple from pydantic import BaseModel, Field, validator +from pydantic.fields import ModelField from .appveyor import Appveyor from .base import CISystem, EventType @@ -27,7 +28,11 @@ def get_auth_tokens() -> Dict[str, str]: @abstractmethod def get_system( - self, repo: str, since: datetime, tokens: Dict[str, str] + self, + repo: str, + since: datetime, + until: Optional[datetime], + tokens: Dict[str, str], ) -> CISystem: ... # pragma: no cover @@ -42,11 +47,16 @@ def get_auth_tokens() -> Dict[str, str]: return GitHubActions.get_auth_tokens() def get_system( - self, repo: str, since: datetime, tokens: Dict[str, str] + self, + repo: str, + since: datetime, + until: Optional[datetime], + tokens: Dict[str, str], ) -> GitHubActions: return GitHubActions( repo=repo, since=since, + until=until, token=tokens["github"], workflows=self.workflows, ) @@ -57,10 +67,17 @@ class TravisConfig(CIConfig): def get_auth_tokens() -> Dict[str, str]: return Travis.get_auth_tokens() - def get_system(self, repo: str, since: datetime, tokens: Dict[str, str]) -> Travis: + def get_system( + self, + repo: str, + since: datetime, + until: Optional[datetime], + tokens: Dict[str, str], + ) -> Travis: return Travis( repo=repo, since=since, + until=until, token=tokens["travis"], gh_token=tokens["github"], ) @@ -75,11 +92,16 @@ def get_auth_tokens() -> Dict[str, str]: return Appveyor.get_auth_tokens() def get_system( - self, repo: str, since: datetime, tokens: Dict[str, str] + self, + repo: str, + since: datetime, + until: Optional[datetime], + tokens: Dict[str, str], ) -> Appveyor: return Appveyor( repo=repo, since=since, + until=until, token=tokens["appveyor"], accountName=self.accountName, projectSlug=self.projectSlug, @@ -110,6 +132,7 @@ class Config(NoExtraModel): vars: Dict[str, str] = Field(default_factory=dict) ci: CIConfigDict since: datetime + until: Optional[datetime] = None types: List[EventType] secrets: Dict[str, Pattern] = Field(default_factory=dict) allow_secrets_regex: Optional[Pattern] = Field(None, alias="allow-secrets-regex") @@ -121,8 +144,10 @@ def _validate_repo(cls, v: str) -> str: # noqa: B902, U100 raise ValueError("Repo must be in the form 'OWNER/NAME'") return v - @validator("since") - def _validate_since(cls, v: datetime) -> datetime: # noqa: B902, U100 - if v.tzinfo is None: - raise ValueError("'since' timestamp must include timezone offset") + @validator("since", "until") + def _validate_datetimes( + cls, v: Optional[datetime], field: ModelField # noqa: B902, U100 + ) -> Optional[datetime]: + if v is not None and v.tzinfo is None: + raise ValueError(f"{field.name!r} timestamp must include timezone offset") return v diff --git a/src/tinuous/github.py b/src/tinuous/github.py index 21ddd4c..faba76c 100644 --- a/src/tinuous/github.py +++ b/src/tinuous/github.py @@ -65,6 +65,8 @@ def get_build_assets( self, event_types: List[EventType], artifacts: bool = False ) -> Iterator["BuildAsset"]: log.info("Fetching runs newer than %s", self.since) + if self.until is not None: + log.info("Skipping runs newer than %s", self.until) for wf in self.get_workflows(): log.info("Fetching runs for workflow %s (%s)", wf.path, wf.name) for run in wf.get_runs(): @@ -72,6 +74,8 @@ def get_build_assets( ts = ensure_aware(run.created_at) if ts <= self.since: break + elif self.until is not None and ts > self.until: + log.info("Run %s is too new; skipping", run.run_number) elif run.status != "completed": log.info("Run %s not completed; skipping", run.run_number) self.register_build(ts, False) @@ -146,6 +150,8 @@ def get_artifacts(self, run: WorkflowRun) -> Iterator[Tuple[str, str]]: def get_release_assets(self) -> Iterator["GHReleaseAsset"]: log.info("Fetching releases newer than %s", self.since) + if self.until is not None: + log.info("Skipping releases newer than %s", self.until) for rel in self.ghrepo.get_releases(): if rel.draft: log.info("Release %s is draft; skipping", rel.tag_name) @@ -154,7 +160,7 @@ def get_release_assets(self) -> Iterator["GHReleaseAsset"]: log.info("Release %s is prerelease; skipping", rel.tag_name) continue ts = ensure_aware(rel.published_at) - if ts <= self.since: + if ts <= self.since or (self.until is not None and ts > self.until): continue self.register_build(ts, True) # TODO: Set to False for drafts? log.info("Found release %s", rel.tag_name) diff --git a/src/tinuous/travis.py b/src/tinuous/travis.py index 9a47368..c3d90d5 100644 --- a/src/tinuous/travis.py +++ b/src/tinuous/travis.py @@ -74,6 +74,8 @@ def get_build_assets( self, event_types: List[EventType], artifacts: bool = False # noqa: U100 ) -> Iterator["BuildAsset"]: log.info("Fetching builds newer than %s", self.since) + if self.until is not None: + log.info("Skipping builds newer than %s", self.until) for build in self.paginate( f"/repo/{quote(self.repo, safe='')}/builds", params={"include": "build.jobs"}, @@ -93,6 +95,8 @@ def get_build_assets( ts = isoparse(build["started_at"]) if ts <= self.since: break + elif self.until is not None and ts > self.until: + log.info("Build %s is too new; skipping", build["number"]) elif build["finished_at"] is None: log.info("Build %s not completed; skipping", build["number"]) self.register_build(ts, False)