From f49ec0d66074e64ff37db34ff4b3785ec803805a Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Thu, 16 Jun 2022 15:58:06 -0600 Subject: [PATCH 1/9] Add models for check suite and check run payloads These are the webhook event payloads for check suites and check runs, and corresponding models that can be obtained with GitHub's API. These are checks with examples from GitHub's webhook docs. --- src/timessquare/domain/githubapi.py | 113 ++++++ src/timessquare/domain/githubwebhook.py | 46 ++- .../github_webhooks/check_run_created.json | 321 ++++++++++++++++++ .../check_suite_completed.json | 226 ++++++++++++ tests/domain/githubwebhook_test.py | 43 +++ 5 files changed, 748 insertions(+), 1 deletion(-) create mode 100644 tests/data/github_webhooks/check_run_created.json create mode 100644 tests/data/github_webhooks/check_suite_completed.json diff --git a/src/timessquare/domain/githubapi.py b/src/timessquare/domain/githubapi.py index 52dc90e..e7dd584 100644 --- a/src/timessquare/domain/githubapi.py +++ b/src/timessquare/domain/githubapi.py @@ -3,6 +3,8 @@ from __future__ import annotations from base64 import b64decode +from enum import Enum +from typing import Optional from pydantic import BaseModel, Field, HttpUrl @@ -144,3 +146,114 @@ def decode(self) -> str: f"GitHub blob content encoding {self.encoding} " f"is unknown by GitHubBlobModel for url {self.url}" ) + + +class GitHubCheckSuiteStatus(str, Enum): + + queued = "queued" + in_progress = "in_progress" + completed = "completed" + + +class GitHubCheckSuiteConclusion(str, Enum): + + success = "success" + failure = "failure" + neutral = "neutral" + cancelled = "cancelled" + timed_out = "timed_out" + action_required = "action_required" + stale = "stale" + + +class GitHubCheckSuiteModel(BaseModel): + """A Pydantic model for the "check_suite" field in a check_suite webhook + (`GitHubCheckSuiteRequestModel`). + """ + + id: str = Field(description="Identifier for this check run") + + head_branch: str = Field( + title="Head branch", + description="Name of the branch the changes are on.", + ) + + head_sha: str = Field( + title="Head sha", + description="The SHA of the most recent commit for this check suite.", + ) + + url: HttpUrl = Field( + description="GitHub API URL for the check suite resource." + ) + + status: GitHubCheckSuiteStatus + + conclusion: Optional[GitHubCheckSuiteConclusion] + + +class GitHubCheckRunStatus(str, Enum): + """The check run status.""" + + queued = "queued" + in_progress = "in_progress" + completed = "completed" + + +class GitHubCheckRunConclusion(str, Enum): + """The check run conclusion state.""" + + success = "success" + failure = "failure" + neutral = "neutral" + cancelled = "cancelled" + timed_out = "timed_out" + action_required = "action_required" + stale = "stale" + + +class GitHubCheckRunAnnotationLevel(str, Enum): + """The level of a check run output annotation.""" + + notice = "notice" + warning = "warning" + failure = "failure" + + +class GitHubCheckSuiteId(BaseModel): + """Brief information about a check suite in the `GitHubCheckRunModel`.""" + + id: str = Field(description="Check suite ID") + + +class GitHubCheckRunModel(BaseModel): + """A Pydantic model for the "check_run" field in a check_run webhook + payload (`GitHubCheckRunPayloadModel`). + """ + + id: str = Field(description="Identifier for this check run") + + external_id: Optional[str] = Field( + description="Identifier set by the check runner." + ) + + head_sha: str = Field( + title="Head sha", + description="The SHA of the most recent commit for this check suite.", + ) + + status: GitHubCheckRunStatus = Field( + description="Status of the check run." + ) + + conclusion: Optional[GitHubCheckRunConclusion] = Field( + description="Conclusion status, if completed." + ) + + name: str = Field(description="Name of the check run.") + + url: HttpUrl = Field(description="URL of the check run API resource.") + + html_url: HttpUrl = Field(description="URL of the check run webpage.") + + check_suite: GitHubCheckSuiteId diff --git a/src/timessquare/domain/githubwebhook.py b/src/timessquare/domain/githubwebhook.py index 5397e4f..79e3813 100644 --- a/src/timessquare/domain/githubwebhook.py +++ b/src/timessquare/domain/githubwebhook.py @@ -10,7 +10,12 @@ from pydantic import BaseModel, Field -from .githubapi import GitHubPullRequestModel, GitHubRepositoryModel +from .githubapi import ( + GitHubCheckRunModel, + GitHubCheckSuiteModel, + GitHubPullRequestModel, + GitHubRepositoryModel, +) class GitHubAppInstallationModel(BaseModel): @@ -120,3 +125,42 @@ class GitHubPullRequestEventModel(BaseModel): number: int = Field(title="Pull request number") pull_request: GitHubPullRequestModel + + +class GitHubCheckSuiteEventModel(BaseModel): + """A Pydantic model for the "check_suite" webhook payload. + + https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#check_suite + """ + + action: str = Field( + title="Action performed", + description="Either requested or rerequested.", + ) + + check_suite: GitHubCheckSuiteModel + + repository: GitHubRepositoryModel + + installation: GitHubAppInstallationModel + + +class GitHubCheckRunEventModel(BaseModel): + """A Pydantic model for the "check_run" webhook payload. + + https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#check_run + """ + + action: str = Field( + title="The action that was performed.", + description=( + "Many event types are possible: created, completed, rerequested, " + "rerequested_action" + ), + ) + + repository: GitHubRepositoryModel + + installation: GitHubAppInstallationModel + + check_run: GitHubCheckRunModel diff --git a/tests/data/github_webhooks/check_run_created.json b/tests/data/github_webhooks/check_run_created.json new file mode 100644 index 0000000..8667adf --- /dev/null +++ b/tests/data/github_webhooks/check_run_created.json @@ -0,0 +1,321 @@ +{ + "action": "created", + "check_run": { + "id": 128620228, + "node_id": "MDg6Q2hlY2tSdW4xMjg2MjAyMjg=", + "head_sha": "ec26c3e57ca3a959ca5aad62de7213c562f8c821", + "external_id": "", + "url": "https://api.github.com/repos/Codertocat/Hello-World/check-runs/128620228", + "html_url": "https://github.com/Codertocat/Hello-World/runs/128620228", + "details_url": "https://octocoders.github.io", + "status": "queued", + "conclusion": null, + "started_at": "2019-05-15T15:21:12Z", + "completed_at": null, + "output": { + "title": null, + "summary": null, + "text": null, + "annotations_count": 0, + "annotations_url": "https://api.github.com/repos/Codertocat/Hello-World/check-runs/128620228/annotations" + }, + "name": "Octocoders-linter", + "check_suite": { + "id": 118578147, + "node_id": "MDEwOkNoZWNrU3VpdGUxMTg1NzgxNDc=", + "head_branch": "changes", + "head_sha": "ec26c3e57ca3a959ca5aad62de7213c562f8c821", + "status": "queued", + "conclusion": null, + "url": "https://api.github.com/repos/Codertocat/Hello-World/check-suites/118578147", + "before": "6113728f27ae82c7b1a177c8d03f9e96e0adf246", + "after": "ec26c3e57ca3a959ca5aad62de7213c562f8c821", + "pull_requests": [ + { + "url": "https://api.github.com/repos/Codertocat/Hello-World/pulls/2", + "id": 279147437, + "number": 2, + "head": { + "ref": "changes", + "sha": "ec26c3e57ca3a959ca5aad62de7213c562f8c821", + "repo": { + "id": 186853002, + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "name": "Hello-World" + } + }, + "base": { + "ref": "master", + "sha": "f95f852bd8fca8fcc58a9a2d6c842781e32a215e", + "repo": { + "id": 186853002, + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "name": "Hello-World" + } + } + } + ], + "app": { + "id": 29310, + "node_id": "MDM6QXBwMjkzMTA=", + "owner": { + "login": "Octocoders", + "id": 38302899, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", + "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Octocoders", + "html_url": "https://github.com/Octocoders", + "followers_url": "https://api.github.com/users/Octocoders/followers", + "following_url": "https://api.github.com/users/Octocoders/following{/other_user}", + "gists_url": "https://api.github.com/users/Octocoders/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Octocoders/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Octocoders/subscriptions", + "organizations_url": "https://api.github.com/users/Octocoders/orgs", + "repos_url": "https://api.github.com/users/Octocoders/repos", + "events_url": "https://api.github.com/users/Octocoders/events{/privacy}", + "received_events_url": "https://api.github.com/users/Octocoders/received_events", + "type": "Organization", + "site_admin": false + }, + "name": "octocoders-linter", + "description": "", + "external_url": "https://octocoders.github.io", + "html_url": "https://github.com/apps/octocoders-linter", + "created_at": "2019-04-19T19:36:24Z", + "updated_at": "2019-04-19T19:36:56Z", + "permissions": { + "administration": "write", + "checks": "write", + "contents": "write", + "deployments": "write", + "issues": "write", + "members": "write", + "metadata": "read", + "organization_administration": "write", + "organization_hooks": "write", + "organization_plan": "read", + "organization_projects": "write", + "organization_user_blocking": "write", + "pages": "write", + "pull_requests": "write", + "repository_hooks": "write", + "repository_projects": "write", + "statuses": "write", + "team_discussions": "write", + "vulnerability_alerts": "read" + }, + "events": [] + }, + "created_at": "2019-05-15T15:20:31Z", + "updated_at": "2019-05-15T15:20:31Z" + }, + "app": { + "id": 29310, + "node_id": "MDM6QXBwMjkzMTA=", + "owner": { + "login": "Octocoders", + "id": 38302899, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", + "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Octocoders", + "html_url": "https://github.com/Octocoders", + "followers_url": "https://api.github.com/users/Octocoders/followers", + "following_url": "https://api.github.com/users/Octocoders/following{/other_user}", + "gists_url": "https://api.github.com/users/Octocoders/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Octocoders/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Octocoders/subscriptions", + "organizations_url": "https://api.github.com/users/Octocoders/orgs", + "repos_url": "https://api.github.com/users/Octocoders/repos", + "events_url": "https://api.github.com/users/Octocoders/events{/privacy}", + "received_events_url": "https://api.github.com/users/Octocoders/received_events", + "type": "Organization", + "site_admin": false + }, + "name": "octocoders-linter", + "description": "", + "external_url": "https://octocoders.github.io", + "html_url": "https://github.com/apps/octocoders-linter", + "created_at": "2019-04-19T19:36:24Z", + "updated_at": "2019-04-19T19:36:56Z", + "permissions": { + "administration": "write", + "checks": "write", + "contents": "write", + "deployments": "write", + "issues": "write", + "members": "write", + "metadata": "read", + "organization_administration": "write", + "organization_hooks": "write", + "organization_plan": "read", + "organization_projects": "write", + "organization_user_blocking": "write", + "pages": "write", + "pull_requests": "write", + "repository_hooks": "write", + "repository_projects": "write", + "statuses": "write", + "team_discussions": "write", + "vulnerability_alerts": "read" + }, + "events": [] + }, + "pull_requests": [ + { + "url": "https://api.github.com/repos/Codertocat/Hello-World/pulls/2", + "id": 279147437, + "number": 2, + "head": { + "ref": "changes", + "sha": "ec26c3e57ca3a959ca5aad62de7213c562f8c821", + "repo": { + "id": 186853002, + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "name": "Hello-World" + } + }, + "base": { + "ref": "master", + "sha": "f95f852bd8fca8fcc58a9a2d6c842781e32a215e", + "repo": { + "id": 186853002, + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "name": "Hello-World" + } + } + } + ], + "deployment": { + "url": "https://api.github.com/repos/Codertocat/Hello-World/deployments/326191728", + "id": 326191728, + "node_id": "MDEwOkRlcGxveW1lbnQzMjYxOTE3Mjg=", + "task": "deploy", + "original_environment": "lab", + "environment": "lab", + "description": null, + "created_at": "2021-02-18T08:22:48Z", + "updated_at": "2021-02-18T09:47:16Z", + "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments/326191728/statuses", + "repository_url": "https://api.github.com/repos/Codertocat/Hello-World" + } + }, + "repository": { + "id": 186853002, + "node_id": "MDEwOlJlcG9zaXRvcnkxODY4NTMwMDI=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "private": false, + "owner": { + "login": "Codertocat", + "id": 21031067, + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/Codertocat/Hello-World", + "description": null, + "fork": false, + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", + "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", + "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", + "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", + "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", + "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", + "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", + "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", + "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", + "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", + "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", + "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", + "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", + "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", + "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", + "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", + "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", + "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", + "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", + "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", + "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", + "created_at": "2019-05-15T15:19:25Z", + "updated_at": "2019-05-15T15:21:03Z", + "pushed_at": "2019-05-15T15:20:57Z", + "git_url": "git://github.com/Codertocat/Hello-World.git", + "ssh_url": "git@github.com:Codertocat/Hello-World.git", + "clone_url": "https://github.com/Codertocat/Hello-World.git", + "svn_url": "https://github.com/Codertocat/Hello-World", + "homepage": null, + "size": 0, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Ruby", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 1, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 2, + "license": null, + "forks": 1, + "open_issues": 2, + "watchers": 0, + "default_branch": "master" + }, + "installation": { + "id": 957387 + }, + "sender": { + "login": "Codertocat", + "id": 21031067, + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": false + } +} diff --git a/tests/data/github_webhooks/check_suite_completed.json b/tests/data/github_webhooks/check_suite_completed.json new file mode 100644 index 0000000..7020f3f --- /dev/null +++ b/tests/data/github_webhooks/check_suite_completed.json @@ -0,0 +1,226 @@ +{ + "action": "completed", + "check_suite": { + "id": 118578147, + "node_id": "MDEwOkNoZWNrU3VpdGUxMTg1NzgxNDc=", + "head_branch": "changes", + "head_sha": "ec26c3e57ca3a959ca5aad62de7213c562f8c821", + "status": "completed", + "conclusion": "success", + "url": "https://api.github.com/repos/Codertocat/Hello-World/check-suites/118578147", + "before": "6113728f27ae82c7b1a177c8d03f9e96e0adf246", + "after": "ec26c3e57ca3a959ca5aad62de7213c562f8c821", + "pull_requests": [ + { + "url": "https://api.github.com/repos/Codertocat/Hello-World/pulls/2", + "id": 279147437, + "number": 2, + "head": { + "ref": "changes", + "sha": "ec26c3e57ca3a959ca5aad62de7213c562f8c821", + "repo": { + "id": 186853002, + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "name": "Hello-World" + } + }, + "base": { + "ref": "master", + "sha": "f95f852bd8fca8fcc58a9a2d6c842781e32a215e", + "repo": { + "id": 186853002, + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "name": "Hello-World" + } + } + } + ], + "app": { + "id": 29310, + "node_id": "MDM6QXBwMjkzMTA=", + "owner": { + "login": "Octocoders", + "id": 38302899, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", + "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Octocoders", + "html_url": "https://github.com/Octocoders", + "followers_url": "https://api.github.com/users/Octocoders/followers", + "following_url": "https://api.github.com/users/Octocoders/following{/other_user}", + "gists_url": "https://api.github.com/users/Octocoders/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Octocoders/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Octocoders/subscriptions", + "organizations_url": "https://api.github.com/users/Octocoders/orgs", + "repos_url": "https://api.github.com/users/Octocoders/repos", + "events_url": "https://api.github.com/users/Octocoders/events{/privacy}", + "received_events_url": "https://api.github.com/users/Octocoders/received_events", + "type": "Organization", + "site_admin": false + }, + "name": "octocoders-linter", + "description": "", + "external_url": "https://octocoders.github.io", + "html_url": "https://github.com/apps/octocoders-linter", + "created_at": "2019-04-19T19:36:24Z", + "updated_at": "2019-04-19T19:36:56Z", + "permissions": { + "administration": "write", + "checks": "write", + "contents": "write", + "deployments": "write", + "issues": "write", + "members": "write", + "metadata": "read", + "organization_administration": "write", + "organization_hooks": "write", + "organization_plan": "read", + "organization_projects": "write", + "organization_user_blocking": "write", + "pages": "write", + "pull_requests": "write", + "repository_hooks": "write", + "repository_projects": "write", + "statuses": "write", + "team_discussions": "write", + "vulnerability_alerts": "read" + }, + "events": [] + }, + "created_at": "2019-05-15T15:20:31Z", + "updated_at": "2019-05-15T15:21:14Z", + "latest_check_runs_count": 1, + "check_runs_url": "https://api.github.com/repos/Codertocat/Hello-World/check-suites/118578147/check-runs", + "head_commit": { + "id": "ec26c3e57ca3a959ca5aad62de7213c562f8c821", + "tree_id": "31b122c26a97cf9af023e9ddab94a82c6e77b0ea", + "message": "Update README.md", + "timestamp": "2019-05-15T15:20:30Z", + "author": { + "name": "Codertocat", + "email": "21031067+Codertocat@users.noreply.github.com" + }, + "committer": { + "name": "Codertocat", + "email": "21031067+Codertocat@users.noreply.github.com" + } + } + }, + "installation": { + "id": 957387 + }, + "repository": { + "id": 186853002, + "node_id": "MDEwOlJlcG9zaXRvcnkxODY4NTMwMDI=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "private": false, + "owner": { + "login": "Codertocat", + "id": 21031067, + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/Codertocat/Hello-World", + "description": null, + "fork": false, + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", + "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", + "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", + "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", + "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", + "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", + "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", + "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", + "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", + "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", + "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", + "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", + "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", + "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", + "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", + "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", + "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", + "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", + "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", + "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", + "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", + "created_at": "2019-05-15T15:19:25Z", + "updated_at": "2019-05-15T15:21:14Z", + "pushed_at": "2019-05-15T15:20:57Z", + "git_url": "git://github.com/Codertocat/Hello-World.git", + "ssh_url": "git@github.com:Codertocat/Hello-World.git", + "clone_url": "https://github.com/Codertocat/Hello-World.git", + "svn_url": "https://github.com/Codertocat/Hello-World", + "homepage": null, + "size": 0, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Ruby", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 2, + "license": null, + "forks": 0, + "open_issues": 2, + "watchers": 0, + "default_branch": "master" + }, + "sender": { + "login": "Codertocat", + "id": 21031067, + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": false + } +} diff --git a/tests/domain/githubwebhook_test.py b/tests/domain/githubwebhook_test.py index d6dbdaa..182cb7a 100644 --- a/tests/domain/githubwebhook_test.py +++ b/tests/domain/githubwebhook_test.py @@ -4,9 +4,16 @@ from pathlib import Path +from timessquare.domain.githubapi import ( + GitHubCheckRunStatus, + GitHubCheckSuiteConclusion, + GitHubCheckSuiteStatus, +) from timessquare.domain.githubwebhook import ( GitHubAppInstallationEventModel, GitHubAppInstallationRepositoriesEventModel, + GitHubCheckRunEventModel, + GitHubCheckSuiteEventModel, GitHubPullRequestEventModel, GitHubPushEventModel, ) @@ -56,3 +63,39 @@ def test_pull_request_event() -> None: assert data.action == "opened" assert data.pull_request.number == 2 assert data.pull_request.title == "Update the README with new information." + + +def test_check_suite_completed_event() -> None: + data_path = Path(__file__).parent.joinpath( + "../data/github_webhooks/check_suite_completed.json" + ) + data = GitHubCheckSuiteEventModel.parse_raw(data_path.read_text()) + + assert data.action == "completed" + assert data.check_suite.id == "118578147" + assert data.check_suite.head_branch == "changes" + assert data.check_suite.head_sha == ( + "ec26c3e57ca3a959ca5aad62de7213c562f8c821" + ) + assert data.check_suite.status == GitHubCheckSuiteStatus.completed + assert data.check_suite.conclusion == GitHubCheckSuiteConclusion.success + + +def test_check_run_created_event() -> None: + data_path = Path(__file__).parent.joinpath( + "../data/github_webhooks/check_run_created.json" + ) + data = GitHubCheckRunEventModel.parse_raw(data_path.read_text()) + + assert data.action == "created" + assert data.check_run.id == "128620228" + assert data.check_run.external_id == "" + assert data.check_run.url == ( + "https://api.github.com/repos/Codertocat/Hello-World" + "/check-runs/128620228" + ) + assert data.check_run.html_url == ( + "https://github.com/Codertocat/Hello-World/runs/128620228" + ) + assert data.check_run.status == GitHubCheckRunStatus.queued + assert data.check_run.check_suite.id == "118578147" From 075dfdfc0dc8e2aa23589e3d3a069c2221499b70 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Mon, 20 Jun 2022 15:11:14 -0400 Subject: [PATCH 2/9] Add services for GitHub check runs This catches check suite/run related GitHub webhook events, fires off async tasks and, creates + computes check runs in the GitHubRepoService. --- src/timessquare/services/github/repo.py | 60 +++++++++ src/timessquare/services/github/webhooks.py | 117 ++++++++++++++++++ src/timessquare/worker/functions/__init__.py | 6 + .../worker/functions/compute_check_run.py | 33 +++++ .../worker/functions/create_check_run.py | 33 +++++ .../functions/create_rerequested_check_run.py | 35 ++++++ src/timessquare/worker/main.py | 14 ++- 7 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 src/timessquare/worker/functions/compute_check_run.py create mode 100644 src/timessquare/worker/functions/create_check_run.py create mode 100644 src/timessquare/worker/functions/create_rerequested_check_run.py diff --git a/src/timessquare/services/github/repo.py b/src/timessquare/services/github/repo.py index 63a368b..bf0ea4d 100644 --- a/src/timessquare/services/github/repo.py +++ b/src/timessquare/services/github/repo.py @@ -6,6 +6,7 @@ from __future__ import annotations +import asyncio from pathlib import PurePosixPath from typing import List @@ -15,6 +16,8 @@ from timessquare.domain.githubapi import ( GitHubBlobModel, GitHubBranchModel, + GitHubCheckRunConclusion, + GitHubCheckRunStatus, GitHubRepositoryModel, ) from timessquare.domain.githubcheckout import ( @@ -23,6 +26,8 @@ RepositorySettingsFile, ) from timessquare.domain.githubwebhook import ( + GitHubCheckRunEventModel, + GitHubCheckSuiteEventModel, GitHubPullRequestModel, GitHubPushEventModel, ) @@ -285,3 +290,58 @@ async def update_page( page.repository_sidecar_sha = notebook.sidecar_git_tree_sha await self._page_service.update_page(page) + + async def create_check_run( + self, *, payload: GitHubCheckSuiteEventModel + ) -> None: + """Create a new GitHub check run suite, given a new Check Suite. + + NOTE: currently we're assuming that check suites are automatically + created when created a check run. See + https://docs.github.com/en/rest/checks/runs#create-a-check-run + """ + await self._create_yaml_config_check_run( + owner=payload.repository.owner.login, + repo=payload.repository.name, + head_sha=payload.check_suite.head_sha, + ) + + async def create_rerequested_check_run( + self, *, payload: GitHubCheckRunEventModel + ) -> None: + """Run a GitHub check run that was rerequested.""" + await self._create_yaml_config_check_run( + owner=payload.repository.owner.login, + repo=payload.repository.name, + head_sha=payload.check_run.head_sha, + ) + + async def _create_yaml_config_check_run( + self, *, owner: str, repo: str, head_sha: str + ) -> None: + await self._github_client.post( + "repos/{owner}/{repo}/check-runs", + url_vars={"owner": owner, "repo": repo}, + data={"name": "YAML configurations", "head_sha": head_sha}, + ) + + async def compute_check_run( + self, *, payload: GitHubCheckRunEventModel + ) -> None: + """Compute a GitHub check run.""" + # Set the check run to in-progress + await self._github_client.patch( + payload.check_run.url, + data={"status": GitHubCheckRunStatus.in_progress}, + ) + + await asyncio.sleep(30) + + # Set the check run to complete + await self._github_client.patch( + payload.check_run.url, + data={ + "status": GitHubCheckRunStatus.completed, + "conclusion": GitHubCheckRunConclusion.success, + }, + ) diff --git a/src/timessquare/services/github/webhooks.py b/src/timessquare/services/github/webhooks.py index e4a007a..31b0130 100644 --- a/src/timessquare/services/github/webhooks.py +++ b/src/timessquare/services/github/webhooks.py @@ -9,9 +9,12 @@ from safir.arq import ArqQueue from structlog.stdlib import BoundLogger +from timessquare.config import config from timessquare.domain.githubwebhook import ( GitHubAppInstallationEventModel, GitHubAppInstallationRepositoriesEventModel, + GitHubCheckRunEventModel, + GitHubCheckSuiteEventModel, GitHubPullRequestEventModel, GitHubPushEventModel, ) @@ -294,6 +297,120 @@ async def handle_pr_sync( ) +@router.register("check_suite") +async def handle_check_suite_request( + event: Event, + logger: BoundLogger, + arq_queue: ArqQueue, +) -> None: + """Handle the ``check_suite`` (requested and rerequested) webhook event + from GitHub. + + This handler is responsible for creating a check run. Once GitHub creates + the check run, GitHub sends a webhook handled by + `handle_check_run_created`. + + The rerequested action occurs when a re-runs the entire check suite from + the PR UI. Both "requested" and "rerequested" actions require Times Square + to create a new check run. + + https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#check_suite + + Parameters + ---------- + event : `gidgethub.sansio.Event` + The parsed event payload. + logger + The logger instance + arq_queue : `safir.arq.ArqQueue` + An arq queue client. + + """ + if event.data["action"] in ("requested", "rerequested"): + logger.info( + "GitHub check suite request event", + repo=event.data["repository"]["full_name"], + ) + payload = GitHubCheckSuiteEventModel.parse_obj(event.data) + logger.debug("GitHub check suite request payload", payload=payload) + + # Note that architecturally it might be possible to run this as part + # of the webhook handler or a BackgroundTask; but for now it's + # implemented as a queued task for uniformity with the other tasks + await arq_queue.enqueue( + "create_check_run", + payload=payload, + ) + + +@router.register("check_run", action="created") +async def handle_check_run_created( + event: Event, + logger: BoundLogger, + arq_queue: ArqQueue, +) -> None: + """Handle the ``check_run`` (created) webhook event from GitHub. + + Parameters + ---------- + event : `gidgethub.sansio.Event` + The parsed event payload. + logger + The logger instance + arq_queue : `safir.arq.ArqQueue` + An arq queue client. + """ + # Note that GitHub sends this webhook to any app with permissions to watch + # this event; Times Square needs to operate only on its own check run + # created events. + if event.data["check_run"]["app"]["id"] == config.github_app_id: + logger.info( + "GitHub check run created event", + repo=event.data["repository"]["full_name"], + ) + payload = GitHubCheckRunEventModel.parse_obj(event.data) + logger.debug("GitHub check run request payload", payload=payload) + + await arq_queue.enqueue( + "compute_check_run", + payload=payload, + ) + + +@router.register("check_run", action="rerequested") +async def handle_check_run_rerequested( + event: Event, + logger: BoundLogger, + arq_queue: ArqQueue, +) -> None: + """Handle the ``check_run`` (rerequested) webhook event from GitHub. + + Parameters + ---------- + event : `gidgethub.sansio.Event` + The parsed event payload. + logger + The logger instance + arq_queue : `safir.arq.ArqQueue` + An arq queue client. + """ + # Note that GitHub sends this webhook to any app with permissions to watch + # this event; Times Square needs to operate only on its own check run + # created events. + if event.data["check_run"]["app"]["id"] == config.github_app_id: + logger.info( + "GitHub check run rerequested event", + repo=event.data["repository"]["full_name"], + ) + payload = GitHubCheckRunEventModel.parse_obj(event.data) + logger.debug("GitHub check run request payload", payload=payload) + + await arq_queue.enqueue( + "create_reqrequested_check_run", + payload=payload, + ) + + @router.register("ping") async def handle_ping( event: Event, diff --git a/src/timessquare/worker/functions/__init__.py b/src/timessquare/worker/functions/__init__.py index 859229f..d5f23f3 100644 --- a/src/timessquare/worker/functions/__init__.py +++ b/src/timessquare/worker/functions/__init__.py @@ -1,3 +1,6 @@ +from .compute_check_run import compute_check_run +from .create_check_run import create_check_run +from .create_rerequested_check_run import create_rerequested_check_run from .ping import ping from .pull_request_sync import pull_request_sync from .repo_added import repo_added @@ -10,4 +13,7 @@ "repo_added", "repo_removed", "pull_request_sync", + "compute_check_run", + "create_check_run", + "create_rerequested_check_run", ] diff --git a/src/timessquare/worker/functions/compute_check_run.py b/src/timessquare/worker/functions/compute_check_run.py new file mode 100644 index 0000000..868f2c9 --- /dev/null +++ b/src/timessquare/worker/functions/compute_check_run.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from typing import Any, Dict + +from safir.dependencies.db_session import db_session_dependency + +from timessquare.domain.githubwebhook import GitHubCheckRunEventModel +from timessquare.worker.servicefactory import create_github_repo_service + + +async def compute_check_run( + ctx: Dict[Any, Any], *, payload: GitHubCheckRunEventModel +) -> str: + """Process compute queue tasks, triggered by check_run requested + events on GitHub repositories. + """ + logger = ctx["logger"].bind( + task="compute_check_run", + github_owner=payload.repository.owner.login, + github_repo=payload.repository.name, + ) + logger.info("Running compute_check_run") + + async for db_session in db_session_dependency(): + github_repo_service = await create_github_repo_service( + http_client=ctx["http_client"], + logger=logger, + installation_id=payload.installation.id, + db_session=db_session, + ) + async with db_session.begin(): + await github_repo_service.compute_check_run(payload=payload) + return "done" diff --git a/src/timessquare/worker/functions/create_check_run.py b/src/timessquare/worker/functions/create_check_run.py new file mode 100644 index 0000000..0d3c3db --- /dev/null +++ b/src/timessquare/worker/functions/create_check_run.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from typing import Any, Dict + +from safir.dependencies.db_session import db_session_dependency + +from timessquare.domain.githubwebhook import GitHubCheckSuiteEventModel +from timessquare.worker.servicefactory import create_github_repo_service + + +async def create_check_run( + ctx: Dict[Any, Any], *, payload: GitHubCheckSuiteEventModel +) -> str: + """Process create_check_run queue tasks, triggered by check_suite created + events on GitHub repositories. + """ + logger = ctx["logger"].bind( + task="create_check_run", + github_owner=payload.repository.owner.login, + github_repo=payload.repository.name, + ) + logger.info("Running create_check_run") + + async for db_session in db_session_dependency(): + github_repo_service = await create_github_repo_service( + http_client=ctx["http_client"], + logger=logger, + installation_id=payload.installation.id, + db_session=db_session, + ) + async with db_session.begin(): + await github_repo_service.create_check_run(payload=payload) + return "done" diff --git a/src/timessquare/worker/functions/create_rerequested_check_run.py b/src/timessquare/worker/functions/create_rerequested_check_run.py new file mode 100644 index 0000000..9c10778 --- /dev/null +++ b/src/timessquare/worker/functions/create_rerequested_check_run.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import Any, Dict + +from safir.dependencies.db_session import db_session_dependency + +from timessquare.domain.githubwebhook import GitHubCheckRunEventModel +from timessquare.worker.servicefactory import create_github_repo_service + + +async def create_rerequested_check_run( + ctx: Dict[Any, Any], *, payload: GitHubCheckRunEventModel +) -> str: + """Process create_rerequested_check_run queue tasks, triggered by + check_run rerequested events on GitHub repositories. + """ + logger = ctx["logger"].bind( + task="create_rerequested_check_run", + github_owner=payload.repository.owner.login, + github_repo=payload.repository.name, + ) + logger.info("Running create_rerequested_check_run") + + async for db_session in db_session_dependency(): + github_repo_service = await create_github_repo_service( + http_client=ctx["http_client"], + logger=logger, + installation_id=payload.installation.id, + db_session=db_session, + ) + async with db_session.begin(): + await github_repo_service.create_rerequested_check_run( + payload=payload + ) + return "done" diff --git a/src/timessquare/worker/main.py b/src/timessquare/worker/main.py index f7113d3..cf6f5fd 100644 --- a/src/timessquare/worker/main.py +++ b/src/timessquare/worker/main.py @@ -14,6 +14,9 @@ from timessquare.dependencies.redis import redis_dependency from .functions import ( + compute_check_run, + create_check_run, + create_rerequested_check_run, ping, pull_request_sync, repo_added, @@ -75,7 +78,16 @@ class WorkerSettings: See `arq.worker.Worker` for details on these attributes. """ - functions = [ping, repo_push, repo_added, repo_removed, pull_request_sync] + functions = [ + ping, + repo_push, + repo_added, + repo_removed, + pull_request_sync, + compute_check_run, + create_check_run, + create_rerequested_check_run, + ] redis_settings = config.arq_redis_settings From ee1c2bd188bc72b4970fbe7032a1ef8c8bc2b8b4 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Mon, 20 Jun 2022 17:07:53 -0400 Subject: [PATCH 3/9] Automatically run check_run for check_suite Rather than wait for the check_run webhook, we can actually run the check run immediately given the result from the check run creation endpoint. --- src/timessquare/services/github/repo.py | 12 +++++++++--- .../worker/functions/compute_check_run.py | 15 ++------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/timessquare/services/github/repo.py b/src/timessquare/services/github/repo.py index bf0ea4d..dcd4533 100644 --- a/src/timessquare/services/github/repo.py +++ b/src/timessquare/services/github/repo.py @@ -17,6 +17,7 @@ GitHubBlobModel, GitHubBranchModel, GitHubCheckRunConclusion, + GitHubCheckRunModel, GitHubCheckRunStatus, GitHubRepositoryModel, ) @@ -319,19 +320,24 @@ async def create_rerequested_check_run( async def _create_yaml_config_check_run( self, *, owner: str, repo: str, head_sha: str ) -> None: - await self._github_client.post( + data = await self._github_client.post( "repos/{owner}/{repo}/check-runs", url_vars={"owner": owner, "repo": repo}, data={"name": "YAML configurations", "head_sha": head_sha}, ) + check_run = GitHubCheckRunModel.parse_obj(data) + await self._compute_check_run(check_run) async def compute_check_run( self, *, payload: GitHubCheckRunEventModel ) -> None: """Compute a GitHub check run.""" + await self._compute_check_run(payload.check_run) + + async def _compute_check_run(self, check_run: GitHubCheckRunModel) -> None: # Set the check run to in-progress await self._github_client.patch( - payload.check_run.url, + check_run.url, data={"status": GitHubCheckRunStatus.in_progress}, ) @@ -339,7 +345,7 @@ async def compute_check_run( # Set the check run to complete await self._github_client.patch( - payload.check_run.url, + check_run.url, data={ "status": GitHubCheckRunStatus.completed, "conclusion": GitHubCheckRunConclusion.success, diff --git a/src/timessquare/worker/functions/compute_check_run.py b/src/timessquare/worker/functions/compute_check_run.py index 868f2c9..18af145 100644 --- a/src/timessquare/worker/functions/compute_check_run.py +++ b/src/timessquare/worker/functions/compute_check_run.py @@ -2,10 +2,7 @@ from typing import Any, Dict -from safir.dependencies.db_session import db_session_dependency - from timessquare.domain.githubwebhook import GitHubCheckRunEventModel -from timessquare.worker.servicefactory import create_github_repo_service async def compute_check_run( @@ -19,15 +16,7 @@ async def compute_check_run( github_owner=payload.repository.owner.login, github_repo=payload.repository.name, ) - logger.info("Running compute_check_run") + logger.info("Running compute_check_run", payload=payload.dict()) + logger.info("Skipping compute_check_run (not configured)") - async for db_session in db_session_dependency(): - github_repo_service = await create_github_repo_service( - http_client=ctx["http_client"], - logger=logger, - installation_id=payload.installation.id, - db_session=db_session, - ) - async with db_session.begin(): - await github_repo_service.compute_check_run(payload=payload) return "done" From a5ac198ecb3cd0fc5b887058a40b1b690595fb79 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Tue, 21 Jun 2022 14:08:56 -0400 Subject: [PATCH 4/9] Include payload in webhook task function logging --- src/timessquare/worker/functions/create_check_run.py | 2 +- .../worker/functions/create_rerequested_check_run.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/timessquare/worker/functions/create_check_run.py b/src/timessquare/worker/functions/create_check_run.py index 0d3c3db..3a5d5d4 100644 --- a/src/timessquare/worker/functions/create_check_run.py +++ b/src/timessquare/worker/functions/create_check_run.py @@ -19,7 +19,7 @@ async def create_check_run( github_owner=payload.repository.owner.login, github_repo=payload.repository.name, ) - logger.info("Running create_check_run") + logger.info("Running create_check_run", payload=payload.dict()) async for db_session in db_session_dependency(): github_repo_service = await create_github_repo_service( diff --git a/src/timessquare/worker/functions/create_rerequested_check_run.py b/src/timessquare/worker/functions/create_rerequested_check_run.py index 9c10778..93e2f47 100644 --- a/src/timessquare/worker/functions/create_rerequested_check_run.py +++ b/src/timessquare/worker/functions/create_rerequested_check_run.py @@ -19,7 +19,7 @@ async def create_rerequested_check_run( github_owner=payload.repository.owner.login, github_repo=payload.repository.name, ) - logger.info("Running create_rerequested_check_run") + logger.info("Running create_rerequested_check_run", payload=payload.dict()) async for db_session in db_session_dependency(): github_repo_service = await create_github_repo_service( From 1ef6e713c4984e046866b32b26846b7bcdea5272 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Wed, 22 Jun 2022 19:42:53 -0400 Subject: [PATCH 5/9] Make git_ref optional in checkout model It's convenient to know the git_ref corresponding to a checkout, but in truth only the head_sha is required. We're making it "optional" here to allow users to still set this metadata if relevant. --- src/timessquare/domain/githubcheckout.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/timessquare/domain/githubcheckout.py b/src/timessquare/domain/githubcheckout.py index 0ca7a6f..4464847 100644 --- a/src/timessquare/domain/githubcheckout.py +++ b/src/timessquare/domain/githubcheckout.py @@ -43,8 +43,9 @@ class GitHubRepositoryCheckout: settings: RepositorySettingsFile """Repository settings, read from times-square.yaml.""" - git_ref: str - """The "checked-out" full git ref. + git_ref: Optional[str] + """The "checked-out" full git ref, or `None` if a checkout of a bare + commit. Examples: From 239f6d3bb8fb2e8cb5e02cdedf26acf1f9c0a4ad Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Wed, 22 Jun 2022 19:48:39 -0400 Subject: [PATCH 6/9] Add GitHubRepositoryCheckout.create This classmethod makes it possible for a GitHubRepositoryCheckout to create itself from a github client, repository, and sha. This eliminates need for the "create_checkout" method on the RepositoryService. Overall this means it's easier to create checkouts from different parts of the code, which will be useful when writing the check run validation code. --- src/timessquare/domain/githubcheckout.py | 28 +++++++++++++++++++++++- src/timessquare/services/github/repo.py | 6 +++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/timessquare/domain/githubcheckout.py b/src/timessquare/domain/githubcheckout.py index 4464847..2116e1a 100644 --- a/src/timessquare/domain/githubcheckout.py +++ b/src/timessquare/domain/githubcheckout.py @@ -13,7 +13,7 @@ from gidgethub.httpx import GitHubAPI from pydantic import BaseModel, EmailStr, Field, HttpUrl, root_validator -from .githubapi import GitHubBlobModel +from .githubapi import GitHubBlobModel, GitHubRepositoryModel from .page import PageParameterSchema, PersonModel @@ -69,6 +69,32 @@ class GitHubRepositoryCheckout: URL variable is ``sha``. """ + @classmethod + async def create( + cls, + *, + github_client: GitHubAPI, + repo: GitHubRepositoryModel, + head_sha: str, + git_ref: Optional[str] = None, + ) -> GitHubRepositoryCheckout: + uri = repo.contents_url + "{?ref}" + data = await github_client.getitem( + uri, url_vars={"path": "times-square.yaml", "ref": head_sha} + ) + content_data = GitHubBlobModel.parse_obj(data) + file_content = content_data.decode() + settings = RepositorySettingsFile.parse_yaml(file_content) + return cls( + owner_name=repo.owner.login, + name=repo.name, + settings=settings, + git_ref=git_ref, + head_sha=head_sha, + trees_url=repo.trees_url, + blobs_url=repo.blobs_url, + ) + @property def full_name(self) -> str: """The full repository name (owner/repo format).""" diff --git a/src/timessquare/services/github/repo.py b/src/timessquare/services/github/repo.py index dcd4533..63e31a9 100644 --- a/src/timessquare/services/github/repo.py +++ b/src/timessquare/services/github/repo.py @@ -73,7 +73,8 @@ async def sync_from_repo_installation( branch = await self.request_github_branch( url_template=repo.branches_url, branch=repo.default_branch ) - checkout = await self.create_checkout( + checkout = await GitHubRepositoryCheckout.create( + github_client=self._github_client, repo=repo, git_ref=f"refs/heads/{branch.name}", head_sha=branch.commit.sha, @@ -85,7 +86,8 @@ async def sync_from_push( push_payload: GitHubPushEventModel, ) -> None: """Synchronize based on a GitHub push event.""" - checkout = await self.create_checkout( + checkout = await GitHubRepositoryCheckout.create( + github_client=self._github_client, repo=push_payload.repository, git_ref=push_payload.ref, head_sha=push_payload.after, From af64f58cbd6bdd143f085564e5bb8651fce2f62c Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Wed, 22 Jun 2022 19:50:49 -0400 Subject: [PATCH 7/9] Implementation a check run for YAML configs This is an initial attempt at implementing a check run, GitHubConfigsCheck, that reports Pydantic validation errors. --- src/timessquare/domain/githubcheckrun.py | 224 +++++++++++++++++++++++ src/timessquare/services/github/repo.py | 38 ++-- 2 files changed, 249 insertions(+), 13 deletions(-) create mode 100644 src/timessquare/domain/githubcheckrun.py diff --git a/src/timessquare/domain/githubcheckrun.py b/src/timessquare/domain/githubcheckrun.py new file mode 100644 index 0000000..ed7648a --- /dev/null +++ b/src/timessquare/domain/githubcheckrun.py @@ -0,0 +1,224 @@ +"""Domain models for a GitHub Check Runs computations.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Sequence, Union + +from gidgethub.httpx import GitHubAPI +from pydantic import ValidationError + +from .githubapi import ( + GitHubBlobModel, + GitHubCheckRunAnnotationLevel, + GitHubCheckRunConclusion, + GitHubRepositoryModel, +) +from .githubcheckout import ( + GitHubRepositoryCheckout, + NotebookSidecarFile, + RepositoryNotebookTreeRef, +) + + +@dataclass(kw_only=True) +class Annotation: + """Annotation of an issue in a file.""" + + path: str + + start_line: int + + message: str + + title: str + + annotation_level: GitHubCheckRunAnnotationLevel + + end_line: Optional[int] = None + + @classmethod + def from_validation_error( + cls, path: str, error: ValidationError + ) -> List[Annotation]: + annotations: List[Annotation] = [] + for item in error.errors(): + title = cls._format_title_for_pydantic_item(item["loc"]) + annotations.append( + Annotation( + path=path, + start_line=1, + message=item["msg"], + title=title, + annotation_level=GitHubCheckRunAnnotationLevel.failure, + ) + ) + return annotations + + @staticmethod + def _format_title_for_pydantic_item( + locations: Sequence[Union[str, int]] + ) -> str: + title_elements: List[str] = [] + for location in locations: + if isinstance(location, int): + title_elements.append(f"[{location}]") + else: + title_elements.append(f".{location}") + return "".join(title_elements).lstrip(".") + + def export(self) -> Dict[str, Any]: + """Export a GitHub check run output annotation object.""" + output = { + "path": self.path, + "start_line": self.start_line, + "message": self.message, + "title": self.title, + "annotation_level": self.annotation_level, + } + if self.end_line: + output["end_line"] = self.end_line + else: + output["end_line"] = self.start_line + return output + + +class GitHubConfigsCheck: + """A domain model for a YAML configuration GitHub Check run.""" + + def __init__(self) -> None: + self.annotations: List[Annotation] = [] + self.sidecar_files_checked: List[str] = [] + + @classmethod + async def validate_repo( + cls, + github_client: GitHubAPI, + repo: GitHubRepositoryModel, + head_sha: str, + ) -> GitHubConfigsCheck: + """Create a check run result model for a specific SHA of a GitHub + repository containing Times Square notebooks. + """ + check = cls() + + try: + checkout = await GitHubRepositoryCheckout.create( + github_client=github_client, + repo=repo, + head_sha=head_sha, + ) + except ValidationError as e: + annotations = Annotation.from_validation_error( + path="times-square.yaml", error=e + ) + check.annotations.extend(annotations) + return check + + # Validate each notebook yaml file + tree = await checkout.get_git_tree(github_client) + for notebook_ref in tree.find_notebooks(checkout.settings): + await check.validate_sidecar( + github_client=github_client, + repo=repo, + notebook_ref=notebook_ref, + ) + + return check + + async def validate_sidecar( + self, + *, + github_client: GitHubAPI, + repo: GitHubRepositoryModel, + notebook_ref: RepositoryNotebookTreeRef, + ) -> None: + """Validate the sidecar file for a notebook, adding its results + to the check. + """ + data = await github_client.getitem( + repo.blobs_url, + url_vars={"sha": notebook_ref.sidecar_git_tree_sha}, + ) + sidecar_blob = GitHubBlobModel.parse_obj(data) + try: + NotebookSidecarFile.parse_yaml(sidecar_blob.decode()) + except ValidationError as e: + annotations = Annotation.from_validation_error( + path=notebook_ref.sidecar_path, error=e + ) + self.annotations.extend(annotations) + self.sidecar_files_checked.append(notebook_ref.sidecar_path) + + @property + def conclusion(self) -> GitHubCheckRunConclusion: + """Synthesize a conclusion based on the annotations.""" + for annotation in self.annotations: + if ( + annotation.annotation_level + == GitHubCheckRunAnnotationLevel.failure + ): + return GitHubCheckRunConclusion.failure + + return GitHubCheckRunConclusion.success + + @property + def title(self) -> str: + return "YAML config validation" + + @property + def summary(self) -> str: + sidecar_count = len(self.sidecar_files_checked) + + if self.conclusion == GitHubCheckRunConclusion.success: + text = "Everything looks good ✅" + else: + text = "There are some issues 🧐" + + if sidecar_count == 1: + text = ( + f"{text} (checked times-square.yaml and 1 notebook sidecar " + "file)" + ) + else: + text = ( + f"{text} (checked times-square.yaml and " + f"{sidecar_count} notebook sidecar files)" + ) + + return text + + @property + def text(self) -> str: + text = "| File | Status |\n | --- | :-: |\n" + + if self._is_file_ok("times-square.yaml"): + text = f"{text}| times-square.yaml | ✅ |\n" + else: + text = f"{text}| times-square.yaml | ❌ |\n" + + sidecar_files = list(set(self.sidecar_files_checked)) + sidecar_files.sort() + for sidecar_path in sidecar_files: + if self._is_file_ok(sidecar_path): + text = f"{text}| {sidecar_path} | ✅ |\n" + else: + text = f"{text}| {sidecar_path} | ❌ |\n" + + return text + + def _is_file_ok(self, path: str) -> bool: + for annotation in self.annotations: + if annotation.path == path: + return False + return True + + def export_truncated_annotations(self) -> List[Dict[str, Any]]: + """Export the first 50 annotations to objects serializable to + GitHub. + + Sending more than 50 annotations requires multiple HTTP requests, + which we haven't implemented yet. See + https://docs.github.com/en/rest/checks/runs#update-a-check-run + """ + return [a.export() for a in self.annotations[:50]] diff --git a/src/timessquare/services/github/repo.py b/src/timessquare/services/github/repo.py index 63e31a9..4c673ef 100644 --- a/src/timessquare/services/github/repo.py +++ b/src/timessquare/services/github/repo.py @@ -6,7 +6,6 @@ from __future__ import annotations -import asyncio from pathlib import PurePosixPath from typing import List @@ -16,7 +15,6 @@ from timessquare.domain.githubapi import ( GitHubBlobModel, GitHubBranchModel, - GitHubCheckRunConclusion, GitHubCheckRunModel, GitHubCheckRunStatus, GitHubRepositoryModel, @@ -26,6 +24,7 @@ RepositoryNotebookModel, RepositorySettingsFile, ) +from timessquare.domain.githubcheckrun import GitHubConfigsCheck from timessquare.domain.githubwebhook import ( GitHubCheckRunEventModel, GitHubCheckSuiteEventModel, @@ -304,8 +303,7 @@ async def create_check_run( https://docs.github.com/en/rest/checks/runs#create-a-check-run """ await self._create_yaml_config_check_run( - owner=payload.repository.owner.login, - repo=payload.repository.name, + repo=payload.repository, head_sha=payload.check_suite.head_sha, ) @@ -314,42 +312,56 @@ async def create_rerequested_check_run( ) -> None: """Run a GitHub check run that was rerequested.""" await self._create_yaml_config_check_run( - owner=payload.repository.owner.login, - repo=payload.repository.name, + repo=payload.repository, head_sha=payload.check_run.head_sha, ) async def _create_yaml_config_check_run( - self, *, owner: str, repo: str, head_sha: str + self, *, repo: GitHubRepositoryModel, head_sha: str ) -> None: data = await self._github_client.post( "repos/{owner}/{repo}/check-runs", - url_vars={"owner": owner, "repo": repo}, + url_vars={"owner": repo.owner.login, "repo": repo.name}, data={"name": "YAML configurations", "head_sha": head_sha}, ) check_run = GitHubCheckRunModel.parse_obj(data) - await self._compute_check_run(check_run) + await self._compute_check_run(check_run=check_run, repo=repo) async def compute_check_run( self, *, payload: GitHubCheckRunEventModel ) -> None: """Compute a GitHub check run.""" - await self._compute_check_run(payload.check_run) + await self._compute_check_run( + repo=payload.repository, check_run=payload.check_run + ) - async def _compute_check_run(self, check_run: GitHubCheckRunModel) -> None: + async def _compute_check_run( + self, *, repo: GitHubRepositoryModel, check_run: GitHubCheckRunModel + ) -> None: + """Compute the YAML validation check run.""" # Set the check run to in-progress await self._github_client.patch( check_run.url, data={"status": GitHubCheckRunStatus.in_progress}, ) - await asyncio.sleep(30) + config_check = await GitHubConfigsCheck.validate_repo( + github_client=self._github_client, + repo=repo, + head_sha=check_run.head_sha, + ) # Set the check run to complete await self._github_client.patch( check_run.url, data={ "status": GitHubCheckRunStatus.completed, - "conclusion": GitHubCheckRunConclusion.success, + "conclusion": config_check.conclusion, + "output": { + "title": config_check.title, + "summary": config_check.summary, + "text": config_check.text, + "annotations": config_check.export_truncated_annotations(), + }, }, ) From 8d2b0b9687839295f0e0aa1a33c3489f05eda758 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Thu, 23 Jun 2022 12:58:15 -0400 Subject: [PATCH 8/9] Fix handling re-requested check run - There was a typo in the function name - No need to validate the app id for rerequests (and in fact, we were probably checking the wrong thing? --- src/timessquare/services/github/webhooks.py | 31 +++++++++++---------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/timessquare/services/github/webhooks.py b/src/timessquare/services/github/webhooks.py index 31b0130..3a670f2 100644 --- a/src/timessquare/services/github/webhooks.py +++ b/src/timessquare/services/github/webhooks.py @@ -363,7 +363,10 @@ async def handle_check_run_created( # Note that GitHub sends this webhook to any app with permissions to watch # this event; Times Square needs to operate only on its own check run # created events. - if event.data["check_run"]["app"]["id"] == config.github_app_id: + if ( + event.data["check_run"]["check_suite"]["app"]["id"] + == config.github_app_id + ): logger.info( "GitHub check run created event", repo=event.data["repository"]["full_name"], @@ -394,21 +397,19 @@ async def handle_check_run_rerequested( arq_queue : `safir.arq.ArqQueue` An arq queue client. """ - # Note that GitHub sends this webhook to any app with permissions to watch - # this event; Times Square needs to operate only on its own check run - # created events. - if event.data["check_run"]["app"]["id"] == config.github_app_id: - logger.info( - "GitHub check run rerequested event", - repo=event.data["repository"]["full_name"], - ) - payload = GitHubCheckRunEventModel.parse_obj(event.data) - logger.debug("GitHub check run request payload", payload=payload) + # Note that GitHub only sends this webhook to the app that's being + # re-requested. + logger.info( + "GitHub check run rerequested event", + repo=event.data["repository"]["full_name"], + ) + payload = GitHubCheckRunEventModel.parse_obj(event.data) + logger.debug("GitHub check run request payload", payload=payload) - await arq_queue.enqueue( - "create_reqrequested_check_run", - payload=payload, - ) + await arq_queue.enqueue( + "create_rerequested_check_run", + payload=payload, + ) @router.register("ping") From 0adca9e4f73efb001e1c8c94fa4920528458d25d Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Thu, 23 Jun 2022 18:22:30 -0400 Subject: [PATCH 9/9] Update change log --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 11fe3e5..773cbd6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Change log ========== +Unreleased +---------- + +Times Square now implements GitHub Checks for pull requests for notebook repositories. +Initially, Times Square validates the structure of YAML configuration files, specifically the ``times-square.yaml`` repository settings as well as the YAML sidecar files that describe each notebook. + 0.4.0 (2022-05-14) ------------------