Backport #47945 to 23.2: Artifacts s3 prefix

docker/test/performance-comparison/download.sh

@@ -3,7 +3,9 @@ set -ex
 set -o pipefail
 trap "exit" INT TERM
 trap 'kill $(jobs -pr) ||:' EXIT
+S3_URL=${S3_URL:="https://clickhouse-builds.s3.amazonaws.com"}
 BUILD_NAME=${BUILD_NAME:-package_release}
+export S3_URL BUILD_NAME
 
 mkdir db0 ||:
 mkdir left ||:
@@ -28,8 +30,9 @@ function download
     # Historically there were various paths for the performance test package.
     # Test all of them.
     declare -a urls_to_try=(
-        "https://s3.amazonaws.com/clickhouse-builds/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
-        "https://s3.amazonaws.com/clickhouse-builds/$left_pr/$left_sha/$BUILD_NAME/performance.tgz"
+        "$S3_URL/PRs/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
+        "$S3_URL/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
+        "$S3_URL/$left_pr/$left_sha/$BUILD_NAME/performance.tgz"
     )
 
     for path in "${urls_to_try[@]}"

docker/test/performance-comparison/entrypoint.sh

@@ -6,11 +6,7 @@ export CHPC_CHECK_START_TIMESTAMP
 
 S3_URL=${S3_URL:="https://clickhouse-builds.s3.amazonaws.com"}
 BUILD_NAME=${BUILD_NAME:-package_release}
-
-COMMON_BUILD_PREFIX="/clickhouse_build_check"
-if [[ $S3_URL == *"s3.amazonaws.com"* ]]; then
-    COMMON_BUILD_PREFIX=""
-fi
+export S3_URL BUILD_NAME
 
 # Sometimes AWS responde with DNS error and it's impossible to retry it with
 # current curl version options.
@@ -66,8 +62,9 @@ function find_reference_sha
         # test all of them.
         unset found
         declare -a urls_to_try=(
-            "https://s3.amazonaws.com/clickhouse-builds/0/$REF_SHA/$BUILD_NAME/performance.tar.zst"
-            "https://s3.amazonaws.com/clickhouse-builds/0/$REF_SHA/$BUILD_NAME/performance.tgz"
+            "$S3_URL/PRs/0/$REF_SHA/$BUILD_NAME/performance.tar.zst"
+            "$S3_URL/0/$REF_SHA/$BUILD_NAME/performance.tar.zst"
+            "$S3_URL/0/$REF_SHA/$BUILD_NAME/performance.tgz"
         )
         for path in "${urls_to_try[@]}"
         do
@@ -92,10 +89,15 @@ chmod 777 workspace output
 cd workspace
 
 # Download the package for the version we are going to test.
-if curl_with_retry "$S3_URL/$PR_TO_TEST/$SHA_TO_TEST$COMMON_BUILD_PREFIX/$BUILD_NAME/performance.tar.zst"
-then
-    right_path="$S3_URL/$PR_TO_TEST/$SHA_TO_TEST$COMMON_BUILD_PREFIX/$BUILD_NAME/performance.tar.zst"
-fi
+# A temporary solution for migrating into PRs directory
+for prefix in "$S3_URL/PRs" "$S3_URL";
+do
+    if curl_with_retry "$prefix/$PR_TO_TEST/$SHA_TO_TEST/$BUILD_NAME/performance.tar.zst"
+    then
+        right_path="$prefix/$PR_TO_TEST/$SHA_TO_TEST/$BUILD_NAME/performance.tar.zst"
+        break
+    fi
+done
 
 mkdir right
 wget -nv -nd -c "$right_path" -O- | tar -C right --no-same-owner --strip-components=1 --zstd --extract --verbose

tests/ci/build_check.py

@@ -194,19 +194,21 @@ def create_json_artifact(
 
 
 def get_release_or_pr(pr_info: PRInfo, version: ClickHouseVersion) -> Tuple[str, str]:
+    "Return prefixes for S3 artifacts paths"
     # FIXME performance
     # performance builds are havily relies on a fixed path for artifacts, that's why
     # we need to preserve 0 for anything but PR number
     # It should be fixed in performance-comparison image eventually
-    performance_pr = "0"
+    # For performance tests we always set PRs prefix
+    performance_pr = "PRs/0"
     if "release" in pr_info.labels or "release-lts" in pr_info.labels:
         # for release pull requests we use branch names prefixes, not pr numbers
         return pr_info.head_ref, performance_pr
-    elif pr_info.number == 0:
+    if pr_info.number == 0:
         # for pushes to master - major version
         return f"{version.major}.{version.minor}", performance_pr
     # PR number for anything else
-    pr_number = str(pr_info.number)
+    pr_number = f"PRs/{pr_info.number}"
     return pr_number, pr_number
 
 

tests/ci/build_report_check.py

@@ -247,7 +247,7 @@ def main():
         str(pr_info.number) + "/" + pr_info.sha + "/" + context_name_for_path
     )
 
-    url = s3_helper.upload_build_file_to_s3(
+    url = s3_helper.upload_test_report_to_s3(
         report_path, s3_path_prefix + "/report.html"
     )
     logging.info("Report url %s", url)