Backport #48468 to 23.4: Implement status comment

tests/ci/ast_fuzzer_check.py

@@ -9,19 +9,21 @@
 
 from build_download_helper import get_build_name_for_check, read_build_urls
 from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
-from commit_status_helper import format_description, post_commit_status
+from commit_status_helper import (
+    RerunHelper,
+    format_description,
+    get_commit,
+    post_commit_status,
+)
 from docker_pull_helper import get_image_with_version
 from env_helper import (
-    GITHUB_REPOSITORY,
     GITHUB_RUN_URL,
     REPORTS_PATH,
-    REPO_COPY,
     TEMP_PATH,
 )
 from get_robot_token import get_best_robot_token
 from pr_info import PRInfo
 from report import TestResult
-from rerun_helper import RerunHelper
 from s3_helper import S3Helper
 from stopwatch import Stopwatch
 
@@ -41,19 +43,12 @@ def get_run_command(pr_number, sha, download_url, workspace_path, image):
     )
 
 
-def get_commit(gh, commit_sha):
-    repo = gh.get_repo(GITHUB_REPOSITORY)
-    commit = repo.get_commit(commit_sha)
-    return commit
-
-
-if __name__ == "__main__":
+def main():
     logging.basicConfig(level=logging.INFO)
 
     stopwatch = Stopwatch()
 
     temp_path = TEMP_PATH
-    repo_path = REPO_COPY
     reports_path = REPORTS_PATH
 
     check_name = sys.argv[1]
@@ -64,8 +59,9 @@ def get_commit(gh, commit_sha):
     pr_info = PRInfo()
 
     gh = Github(get_best_robot_token(), per_page=100)
+    commit = get_commit(gh, pr_info.sha)
 
-    rerun_helper = RerunHelper(gh, pr_info, check_name)
+    rerun_helper = RerunHelper(commit, check_name)
     if rerun_helper.is_already_finished_by_status():
         logging.info("Check is already finished according to github status, exiting")
         sys.exit(0)
@@ -172,4 +168,8 @@ def get_commit(gh, commit_sha):
 
     logging.info("Result: '%s', '%s', '%s'", status, description, report_url)
     print(f"::notice ::Report url: {report_url}")
-    post_commit_status(gh, pr_info.sha, check_name, description, status, report_url)
+    post_commit_status(commit, status, report_url, description, check_name, pr_info)
+
+
+if __name__ == "__main__":
+    main()

tests/ci/bugfix_validate_check.py

@@ -8,7 +8,7 @@
 
 from github import Github
 
-from commit_status_helper import post_commit_status
+from commit_status_helper import get_commit, post_commit_status
 from get_robot_token import get_best_robot_token
 from pr_info import PRInfo
 from report import TestResults, TestResult
@@ -81,13 +81,14 @@ def main(args):
     )
 
     gh = Github(get_best_robot_token(), per_page=100)
+    commit = get_commit(gh, pr_info.sha)
     post_commit_status(
-        gh,
-        pr_info.sha,
-        check_name_with_group,
-        "" if is_ok else "Changed tests don't reproduce the bug",
+        commit,
         "success" if is_ok else "error",
         report_url,
+        "" if is_ok else "Changed tests don't reproduce the bug",
+        check_name_with_group,
+        pr_info,
     )
 
 

tests/ci/build_check.py

@@ -9,7 +9,12 @@
 from typing import List, Tuple
 
 from ci_config import CI_CONFIG, BuildConfig
-from commit_status_helper import get_commit_filtered_statuses, get_commit
+from commit_status_helper import (
+    NotSet,
+    get_commit_filtered_statuses,
+    get_commit,
+    post_commit_status,
+)
 from docker_pull_helper import get_image_with_version
 from env_helper import (
     GITHUB_JOB,
@@ -232,10 +237,10 @@ def upload_master_static_binaries(
     print(f"::notice ::Binary static URL: {url}")
 
 
-def mark_failed_reports_pending(build_name: str, sha: str) -> None:
+def mark_failed_reports_pending(build_name: str, pr_info: PRInfo) -> None:
     try:
         gh = GitHub(get_best_robot_token())
-        commit = get_commit(gh, sha)
+        commit = get_commit(gh, pr_info.sha)
         statuses = get_commit_filtered_statuses(commit)
         report_status = [
             name
@@ -248,8 +253,13 @@ def mark_failed_reports_pending(build_name: str, sha: str) -> None:
                     "Commit already have failed status for '%s', setting it to 'pending'",
                     report_status,
                 )
-                commit.create_status(
-                    "pending", status.url, "Set to pending on rerun", report_status
+                post_commit_status(
+                    commit,
+                    "pending",
+                    status.target_url or NotSet,
+                    "Set to pending on rerun",
+                    report_status,
+                    pr_info,
                 )
     except:  # we do not care about any exception here
         logging.info("Failed to get or mark the reports status as pending, continue")
@@ -285,7 +295,7 @@ def main():
     check_for_success_run(s3_helper, s3_path_prefix, build_name, build_config)
 
     # If it's a latter running, we need to mark possible failed status
-    mark_failed_reports_pending(build_name, pr_info.sha)
+    mark_failed_reports_pending(build_name, pr_info)
 
     docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME)
     image_version = docker_image.version

tests/ci/build_report_check.py

@@ -22,11 +22,12 @@
 from get_robot_token import get_best_robot_token
 from pr_info import NeedsDataType, PRInfo
 from commit_status_helper import (
+    RerunHelper,
     get_commit,
+    post_commit_status,
     update_mergeable_check,
 )
 from ci_config import CI_CONFIG
-from rerun_helper import RerunHelper
 
 
 NEEDS_DATA_PATH = os.getenv("NEEDS_DATA_PATH", "")
@@ -136,10 +137,11 @@ def main():
 
     gh = Github(get_best_robot_token(), per_page=100)
     pr_info = PRInfo()
+    commit = get_commit(gh, pr_info.sha)
 
     atexit.register(update_mergeable_check, gh, pr_info, build_check_name)
 
-    rerun_helper = RerunHelper(gh, pr_info, build_check_name)
+    rerun_helper = RerunHelper(commit, build_check_name)
     if rerun_helper.is_already_finished_by_status():
         logging.info("Check is already finished according to github status, exiting")
         sys.exit(0)
@@ -274,12 +276,8 @@ def main():
 
     description = f"{ok_groups}/{total_groups} artifact groups are OK {addition}"
 
-    commit = get_commit(gh, pr_info.sha)
-    commit.create_status(
-        context=build_check_name,
-        description=description,
-        state=summary_status,
-        target_url=url,
+    post_commit_status(
+        commit, summary_status, url, description, build_check_name, pr_info
     )
 
     if summary_status == "error":

tests/ci/ci_config.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-from typing import Dict, TypeVar
+from dataclasses import dataclass
+from typing import Callable, Dict, TypeVar
 
 ConfValue = TypeVar("ConfValue", str, bool)
 BuildConfig = Dict[str, ConfValue]
@@ -399,3 +400,161 @@
     "Unit tests (tsan)",
     "Unit tests (ubsan)",
 ]
+
+
+@dataclass
+class CheckDescription:
+    name: str
+    description: str  # the check descriptions, will be put into the status table
+    match_func: Callable[[str], bool]  # the function to check vs the commit status
+
+    def __hash__(self) -> int:
+        return hash(self.name + self.description)
+
+
+CHECK_DESCRIPTIONS = [
+    CheckDescription(
+        "AST fuzzer",
+        "Runs randomly generated queries to catch program errors. "
+        "The build type is optionally given in parenthesis. "
+        "If it fails, ask a maintainer for help.",
+        lambda x: x.startswith("AST fuzzer"),
+    ),
+    CheckDescription(
+        "Bugfix validate check",
+        "Checks that either a new test (functional or integration) or there "
+        "some changed tests that fail with the binary built on master branch",
+        lambda x: x == "Bugfix validate check",
+    ),
+    CheckDescription(
+        "CI running",
+        "A meta-check that indicates the running CI. Normally, it's in <b>success</b> or "
+        "<b>pending</b> state. The failed status indicates some problems with the PR",
+        lambda x: x == "CI running",
+    ),
+    CheckDescription(
+        "ClickHouse build check",
+        "Builds ClickHouse in various configurations for use in further steps. "
+        "You have to fix the builds that fail. Build logs often has enough "
+        "information to fix the error, but you might have to reproduce the failure "
+        "locally. The <b>cmake</b> options can be found in the build log, grepping for "
+        '<b>cmake</b>. Use these options and follow the <a href="'
+        'https://clickhouse.com/docs/en/development/build">general build process</a>.',
+        lambda x: x.startswith("ClickHouse") and x.endswith("build check"),
+    ),
+    CheckDescription(
+        "Compatibility check",
+        "Checks that <b>clickhouse</b> binary runs on distributions with old libc "
+        "versions. If it fails, ask a maintainer for help.",
+        lambda x: x.startswith("Compatibility check"),
+    ),
+    CheckDescription(
+        "Docker image for servers",
+        "The check to build and optionally push the mentioned image to docker hub",
+        lambda x: x.startswith("Docker image")
+        and (x.endswith("building check") or x.endswith("build and push")),
+    ),
+    CheckDescription(
+        "Docs Check", "Builds and tests the documentation", lambda x: x == "Docs Check"
+    ),
+    CheckDescription(
+        "Fast test",
+        "Normally this is the first check that is ran for a PR. It builds ClickHouse "
+        'and runs most of <a href="https://clickhouse.com/docs/en/development/tests'
+        '#functional-tests">stateless functional tests</a>, '
+        "omitting some. If it fails, further checks are not started until it is fixed. "
+        "Look at the report to see which tests fail, then reproduce the failure "
+        'locally as described <a href="https://clickhouse.com/docs/en/development/'
+        'tests#functional-test-locally">here</a>.',
+        lambda x: x == "Fast test",
+    ),
+    CheckDescription(
+        "Flaky tests",
+        "Runs a flaky tests from master multiple times to identify if they are stable.",
+        lambda x: "tests flaky check" in x,
+    ),
+    CheckDescription(
+        "Install packages",
+        "Checks that the built packages are installable in a clear environment",
+        lambda x: x.startswith("Install packages ("),
+    ),
+    CheckDescription(
+        "Integration tests",
+        "The integration tests report. In parenthesis the package type is given, "
+        "and in square brackets are the optional part/total tests",
+        lambda x: x.startswith("Integration tests ("),
+    ),
+    CheckDescription(
+        "Mergeable Check",
+        "Checks if all other necessary checks are successful",
+        lambda x: x == "Mergeable Check",
+    ),
+    CheckDescription(
+        "Performance Comparison",
+        "Measure changes in query performance. The performance test report is "
+        'described in detail <a href="https://github.com/ClickHouse/ClickHouse/tree'
+        '/master/docker/test/performance-comparison#how-to-read-the-report">here</a>. '
+        "In square brackets are the optional part/total tests",
+        lambda x: x.startswith("Performance Comparison"),
+    ),
+    CheckDescription(
+        "Push to Dockerhub",
+        "The check for building and pushing the CI related docker images to docker hub",
+        lambda x: x.startswith("Push") and "to Dockerhub" in x,
+    ),
+    CheckDescription(
+        "Sqllogic",
+        "Run clickhouse on the "
+        '<a href="https://www.sqlite.org/sqllogictest">sqllogic</a> '
+        "test set against sqlite and checks that all statements are passed.",
+        lambda x: x.startswith("Sqllogic test"),
+    ),
+    CheckDescription(
+        "SQLancer",
+        "Fuzzing tests that detect logical bugs with "
+        '<a href="https://github.com/sqlancer/sqlancer">SQLancer</a> tool.',
+        lambda x: x.startswith("SQLancer"),
+    ),
+    CheckDescription(
+        "Stateful tests",
+        "Runs stateful functional tests for ClickHouse binaries built in various "
+        "configurations -- release, debug, with sanitizers, etc.",
+        lambda x: x.startswith("Stateful tests ("),
+    ),
+    CheckDescription(
+        "Stateless tests",
+        "Runs stateless functional tests for ClickHouse binaries built in various "
+        "configurations -- release, debug, with sanitizers, etc.",
+        lambda x: x.startswith("Stateless tests ("),
+    ),
+    CheckDescription(
+        "Stress test",
+        "Runs stateless functional tests concurrently from several clients to detect "
+        "concurrency-related errors.",
+        lambda x: x.startswith("Stress test ("),
+    ),
+    CheckDescription(
+        "Style Check",
+        "Runs a set of checks to keep the code style clean. If some of tests failed, "
+        "see the related log from the report.",
+        lambda x: x == "Style Check",
+    ),
+    CheckDescription(
+        "Unit tests",
+        "Runs the unit tests for different release types",
+        lambda x: x.startswith("Unit tests ("),
+    ),
+    CheckDescription(
+        "Upgrade check",
+        "Runs stress tests on server version from last release and then tries to "
+        "upgrade it to the version from the PR. It checks if the new server can "
+        "successfully startup without any errors, crashes or sanitizer asserts.",
+        lambda x: x.startswith("Upgrade check ("),
+    ),
+    CheckDescription(
+        "Falback for unknown",
+        "There's no description for the check yet, please add it to "
+        "tests/ci/ci_config.py:CHECK_DESCRIPTIONS",
+        lambda x: True,
+    ),
+]

tests/ci/codebrowser_check.py

@@ -7,7 +7,7 @@
 
 from github import Github
 
-from commit_status_helper import post_commit_status
+from commit_status_helper import get_commit, post_commit_status
 from docker_pull_helper import get_image_with_version
 from env_helper import (
     IMAGES_PATH,
@@ -43,6 +43,7 @@ def get_run_command(repo_path, output_path, image):
 
     gh = Github(get_best_robot_token(), per_page=100)
     pr_info = PRInfo()
+    commit = get_commit(gh, pr_info.sha)
 
     if not os.path.exists(TEMP_PATH):
         os.makedirs(TEMP_PATH)
@@ -87,4 +88,4 @@ def get_run_command(repo_path, output_path, image):
 
     print(f"::notice ::Report url: {report_url}")
 
-    post_commit_status(gh, pr_info.sha, NAME, "Report built", "success", report_url)
+    post_commit_status(commit, "success", report_url, "Report built", NAME, pr_info)