Merge pull request #54362 from ClickHouse/backport/23.3/52948

Backport #52948 to 23.3: Refactor CI_CONFIG

tests/ci/build_check.py

@@ -10,12 +10,6 @@
 import time
 
 from ci_config import CI_CONFIG, BuildConfig
-from commit_status_helper import (
-    NotSet,
-    get_commit_filtered_statuses,
-    get_commit,
-    post_commit_status,
-)
 from docker_pull_helper import get_image_with_version
 from env_helper import (
     GITHUB_JOB,
@@ -25,9 +19,7 @@
     S3_DOWNLOAD,
     TEMP_PATH,
 )
-from get_robot_token import get_best_robot_token
 from git_helper import Git, git_runner
-from github_helper import GitHub
 from pr_info import PRInfo
 from s3_helper import S3Helper
 from tee_popen import TeePopen
@@ -48,11 +40,11 @@
 
 
 def _can_export_binaries(build_config: BuildConfig) -> bool:
-    if build_config["package_type"] != "deb":
+    if build_config.package_type != "deb":
         return False
-    if build_config["sanitizer"] != "":
+    if build_config.sanitizer != "":
         return True
-    if build_config["build_type"] != "":
+    if build_config.debug_build:
         return True
     return False
 
@@ -65,26 +57,26 @@ def get_packager_cmd(
     image_version: str,
     official: bool,
 ) -> str:
-    package_type = build_config["package_type"]
-    comp = build_config["compiler"]
+    package_type = build_config.package_type
+    comp = build_config.compiler
     cmake_flags = "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=1"
     cmd = (
         f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager "
         f"--output-dir={output_path} --package-type={package_type} --compiler={comp}"
     )
 
-    if build_config["build_type"]:
-        cmd += f" --build-type={build_config['build_type']}"
-    if build_config["sanitizer"]:
-        cmd += f" --sanitizer={build_config['sanitizer']}"
-    if build_config["tidy"] == "enable":
+    if build_config.debug_build:
+        cmd += " --build-type=debug"
+    if build_config.sanitizer:
+        cmd += f" --sanitizer={build_config.sanitizer}"
+    if build_config.tidy:
         cmd += " --clang-tidy"
 
     cmd += " --cache=sccache"
     cmd += " --s3-rw-access"
     cmd += f" --s3-bucket={S3_BUILDS_BUCKET}"
 
-    if "additional_pkgs" in build_config and build_config["additional_pkgs"]:
+    if build_config.additional_pkgs:
         cmd += " --additional-pkgs"
 
     cmd += f" --docker-image-version={image_version}"
@@ -191,7 +183,7 @@ def create_json_artifact(
     result = {
         "log_url": log_url,
         "build_urls": build_urls,
-        "build_config": build_config,
+        "build_config": build_config.__dict__,
         "elapsed_seconds": elapsed,
         "status": success,
         "job_name": GITHUB_JOB,
@@ -231,7 +223,7 @@ def upload_master_static_binaries(
     build_output_path: Path,
 ) -> None:
     """Upload binary artifacts to a static S3 links"""
-    static_binary_name = build_config.get("static_binary_name", False)
+    static_binary_name = build_config.static_binary_name
     if pr_info.number != 0:
         return
     elif not static_binary_name:
@@ -245,41 +237,13 @@ def upload_master_static_binaries(
     print(f"::notice ::Binary static URL: {url}")
 
 
-def mark_failed_reports_pending(build_name: str, pr_info: PRInfo) -> None:
-    try:
-        gh = GitHub(get_best_robot_token())
-        commit = get_commit(gh, pr_info.sha)
-        statuses = get_commit_filtered_statuses(commit)
-        report_status = [
-            name
-            for name, builds in CI_CONFIG["builds_report_config"].items()
-            if build_name in builds
-        ][0]
-        for status in statuses:
-            if status.context == report_status and status.state in ["failure", "error"]:
-                logging.info(
-                    "Commit already have failed status for '%s', setting it to 'pending'",
-                    report_status,
-                )
-                post_commit_status(
-                    commit,
-                    "pending",
-                    status.target_url or NotSet,
-                    "Set to pending on rerun",
-                    report_status,
-                    pr_info,
-                )
-    except:  # we do not care about any exception here
-        logging.info("Failed to get or mark the reports status as pending, continue")
-
-
 def main():
     logging.basicConfig(level=logging.INFO)
 
     stopwatch = Stopwatch()
     build_name = sys.argv[1]
 
-    build_config = CI_CONFIG["build_config"][build_name]
+    build_config = CI_CONFIG.build_config[build_name]
 
     temp_path = Path(TEMP_PATH)
     os.makedirs(temp_path, exist_ok=True)
@@ -303,17 +267,12 @@ def main():
     # put them as github actions artifact (result)
     check_for_success_run(s3_helper, s3_path_prefix, build_name, build_config)
 
-    # If it's a latter running, we need to mark possible failed status
-    mark_failed_reports_pending(build_name, pr_info)
-
     docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME)
     image_version = docker_image.version
 
     logging.info("Got version from repo %s", version.string)
 
     official_flag = pr_info.number == 0
-    if "official" in build_config:
-        official_flag = build_config["official"]
 
     version_type = "testing"
     if "release" in pr_info.labels or "release-lts" in pr_info.labels:

tests/ci/build_download_helper.py

@@ -91,7 +91,7 @@ def set_auth_header():
 
 
 def get_build_name_for_check(check_name: str) -> str:
-    return CI_CONFIG["tests_config"][check_name]["required_build"]  # type: ignore
+    return CI_CONFIG.test_configs[check_name].required_build
 
 
 def read_build_urls(build_name: str, reports_path: str) -> List[str]:

tests/ci/build_report_check.py

@@ -23,6 +23,7 @@
 from pr_info import NeedsDataType, PRInfo
 from commit_status_helper import (
     RerunHelper,
+    format_description,
     get_commit,
     post_commit_status,
     update_mergeable_check,
@@ -69,10 +70,11 @@ def get_failed_report(
     message = f"{job_name} failed"
     build_result = BuildResult(
         compiler="unknown",
-        build_type="unknown",
+        debug_build=False,
         sanitizer="unknown",
         status=message,
         elapsed_seconds=0,
+        comment="",
     )
     return [build_result], [[""]], [GITHUB_RUN_URL]
 
@@ -83,10 +85,11 @@ def process_report(
     build_config = build_report["build_config"]
     build_result = BuildResult(
         compiler=build_config["compiler"],
-        build_type=build_config["build_type"],
+        debug_build=build_config["debug_build"],
         sanitizer=build_config["sanitizer"],
         status="success" if build_report["status"] else "failure",
         elapsed_seconds=build_report["elapsed_seconds"],
+        comment=build_config["comment"],
     )
     build_results = []
     build_urls = []
@@ -146,7 +149,7 @@ def main():
         logging.info("Check is already finished according to github status, exiting")
         sys.exit(0)
 
-    builds_for_check = CI_CONFIG["builds_report_config"][build_check_name]
+    builds_for_check = CI_CONFIG.builds_report_config[build_check_name]
     required_builds = required_builds or len(builds_for_check)
 
     # Collect reports from json artifacts
@@ -267,14 +270,20 @@ def main():
         if build_result.status == "success":
             ok_groups += 1
 
-    if ok_groups == 0 or some_builds_are_missing:
-        summary_status = "error"
+    # Check if there are no builds at all, do not override bad status
+    if summary_status == "success":
+        if some_builds_are_missing:
+            summary_status = "pending"
+        elif ok_groups == 0:
+            summary_status = "error"
 
     addition = ""
     if some_builds_are_missing:
-        addition = f"({len(build_reports)} of {required_builds} builds are OK)"
+        addition = f" ({len(build_reports)} of {required_builds} builds are OK)"
 
-    description = f"{ok_groups}/{total_groups} artifact groups are OK {addition}"
+    description = format_description(
+        f"{ok_groups}/{total_groups} artifact groups are OK{addition}"
+    )
 
     post_commit_status(
         commit, summary_status, url, description, build_check_name, pr_info