diff --git a/.github/actionlint.yml b/.github/actionlint.yml index cf5f575e3c74..a7a76bd423dd 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -4,6 +4,10 @@ self-hosted-runner: - func-tester - func-tester-aarch64 - fuzzer-unit-tester + - altinity-on-demand + - altinity-type-cpx51 + - altinity-in-ash + - altinity-image-x86-system-ubuntu-22.04 - style-checker - style-checker-aarch64 - release-maker diff --git a/.github/actions/common_setup/action.yml b/.github/actions/common_setup/action.yml index b9299c64e72a..de7215d2103a 100644 --- a/.github/actions/common_setup/action.yml +++ b/.github/actions/common_setup/action.yml @@ -28,6 +28,16 @@ runs: run: | # to remove every leftovers sudo rm -fr "$TEMP_PATH" && mkdir -p "$TEMP_PATH" + - name: Setup zram + shell: bash + run: | + sudo modprobe zram + MemTotal=$(grep -Po "(?<=MemTotal:)\s+\d+" /proc/meminfo) # KiB + Percent=200 + ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB + .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0 + sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0 + sudo sysctl vm.swappiness=200 - name: Tune vm.mmap_rnd_bits for sanitizers shell: bash run: | diff --git a/.github/actions/docker_setup/action.yml b/.github/actions/docker_setup/action.yml new file mode 100644 index 000000000000..06388e4175c1 --- /dev/null +++ b/.github/actions/docker_setup/action.yml @@ -0,0 +1,29 @@ +name: Docker setup +description: Setup docker +inputs: + nested_job: + description: the fuse for unintended use inside of the reusable callable jobs + default: true + type: boolean +runs: + using: "composite" + steps: + - name: Docker IPv6 configuration + shell: bash + run: | + # make sure docker uses proper IPv6 config + sudo touch /etc/docker/daemon.json + sudo chown ubuntu:ubuntu /etc/docker/daemon.json + sudo cat < /etc/docker/daemon.json + { + "ipv6": true, + "fixed-cidr-v6": "2001:3984:3989::/64" + } + EOT + sudo chown root:root /etc/docker/daemon.json + sudo systemctl restart docker + sudo systemctl status docker + - name: Docker info + shell: bash + run: | + docker info diff --git a/.github/retry.sh b/.github/retry.sh new file mode 100755 index 000000000000..566c2cf11315 --- /dev/null +++ b/.github/retry.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Execute command until exitcode is 0 or +# maximum number of retries is reached +# Example: +# ./retry +retries=$1 +delay=$2 +command="${@:3}" +exitcode=0 +try=0 +until [ "$try" -ge $retries ] +do + echo "$command" + eval "$command" + exitcode=$? + if [ $exitcode -eq 0 ]; then + break + fi + try=$((try+1)) + sleep $2 +done +exit $exitcode diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 15e071420254..ad2e805f477a 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -22,7 +22,7 @@ jobs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version @@ -164,7 +164,7 @@ jobs: runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f - name: Download reports run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds @@ -194,7 +194,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Install packages (aarch64) - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce data: ${{ needs.RunConfig.outputs.data }} run_command: | python3 install_check.py "$CHECK_NAME" @@ -229,7 +229,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Stress test (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 data: ${{ needs.RunConfig.outputs.data }} ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -240,7 +240,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Integration tests (asan, old analyzer) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsTsan: needs: [RunConfig, BuilderDebTsan] @@ -248,7 +248,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Integration tests (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 data: ${{ needs.RunConfig.outputs.data }} FinishCheck: if: ${{ !cancelled() }} @@ -265,7 +265,7 @@ jobs: runs-on: [self-hosted, style-checker] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true - name: Finish label diff --git a/.github/workflows/cherry_pick.yml b/.github/workflows/cherry_pick.yml index 315673d4abcc..8e5191eb33cc 100644 --- a/.github/workflows/cherry_pick.yml +++ b/.github/workflows/cherry_pick.yml @@ -28,7 +28,7 @@ jobs: REPO_TEAM=core EOF - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}} diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 8b7ba6e62f6c..84182ec03aff 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -34,7 +34,7 @@ jobs: runs-on: [self-hosted, release-maker] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}} fetch-depth: 0 diff --git a/.github/workflows/docker_test_images.yml b/.github/workflows/docker_test_images.yml index 2138420f378e..be54cdaf3ddf 100644 --- a/.github/workflows/docker_test_images.yml +++ b/.github/workflows/docker_test_images.yml @@ -10,16 +10,31 @@ name: Build docker images description: set latest tag for resulting multiarch manifest required: false type: boolean - default: false + default: false + secrets: + secret_envs: + description: if given, it's passed to the environments + required: false + AWS_SECRET_ACCESS_KEY: + description: the access key to the aws param store. + required: true + AWS_ACCESS_KEY_ID: + description: the access key id to the aws param store. + required: true + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} jobs: DockerBuildAarch64: - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none] if: | !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_aarch64) != '[]' steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: ref: ${{ fromJson(inputs.data).git_ref }} - name: Build images @@ -29,12 +44,12 @@ jobs: --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \ --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_aarch64) }}' DockerBuildAmd64: - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] if: | !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_amd64) != '[]' steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: ref: ${{ fromJson(inputs.data).git_ref }} - name: Build images @@ -45,12 +60,12 @@ jobs: --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_amd64) }}' DockerMultiArchManifest: needs: [DockerBuildAmd64, DockerBuildAarch64] - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] if: | !failure() && !cancelled() && (toJson(fromJson(inputs.data).docker_data.missing_multi) != '[]' || inputs.set_latest) steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: ref: ${{ fromJson(inputs.data).git_ref }} - name: Build images diff --git a/.github/workflows/jepsen.yml b/.github/workflows/jepsen.yml index 92e4ce10ade5..1ef44d5aa902 100644 --- a/.github/workflows/jepsen.yml +++ b/.github/workflows/jepsen.yml @@ -16,7 +16,7 @@ jobs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version @@ -43,7 +43,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: ClickHouse Keeper Jepsen - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce data: ${{ needs.RunConfig.outputs.data }} run_command: | python3 jepsen_check.py keeper @@ -53,7 +53,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: ClickHouse Server Jepsen - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce data: ${{ needs.RunConfig.outputs.data }} run_command: | python3 jepsen_check.py server @@ -63,7 +63,7 @@ jobs: runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f - name: Check Workflow results if: ${{ !cancelled() }} run: | diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index b76bbbbbdbe6..09a6c2dbc747 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -16,7 +16,7 @@ jobs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version @@ -118,7 +118,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Builds - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce data: ${{ needs.RunConfig.outputs.data }} FinishCheck: @@ -127,7 +127,7 @@ jobs: runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f - name: Finish label run: | cd "$GITHUB_WORKSPACE/tests/ci" diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml index 45ce81c2cafd..376c153faa33 100644 --- a/.github/workflows/merge_queue.yml +++ b/.github/workflows/merge_queue.yml @@ -15,7 +15,7 @@ jobs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get a version @@ -56,7 +56,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Style check - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce run_command: | python3 style_check.py data: ${{ needs.RunConfig.outputs.data }} @@ -99,7 +99,7 @@ jobs: runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f - name: Check and set merge status if: ${{ needs.StyleCheck.result == 'success' }} run: | diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1cea94e75002..670019aac796 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -16,7 +16,7 @@ jobs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version @@ -50,7 +50,7 @@ jobs: runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f - name: Check Workflow results if: ${{ !cancelled() }} run: | diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index acd392978b63..aaec2037261c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -21,12 +21,12 @@ concurrency: jobs: RunConfig: - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce] outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get a version @@ -77,22 +77,17 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Style check - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce run_command: | - python3 style_check.py + python3 style_check.py --no-push data: ${{ needs.RunConfig.outputs.data }} - secrets: - secret_envs: | - ROBOT_CLICKHOUSE_SSH_KEY<> ${GITHUB_OUTPUT} + env: + input: ${{ inputs.runner_type }} + + Common: + strategy: + fail-fast: false + matrix: + SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, kafka, kerberos, key_value, lightweight_delete, memory, part_moves_between_shards, rbac, selects, session_timezone, ssl_server, tiered_storage, window_functions] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=${{ matrix.SUITE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} + + Alter: + strategy: + fail-fast: false + matrix: + ONLY: [replace, attach, move] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=alter + STORAGE=/${{ matrix.ONLY }}_partition + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u alter/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --only "/alter/${{ matrix.ONLY }} partition/*" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: alter-${{ matrix.ONLY }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} + + Benchmark: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=ontime_benchmark + STORAGE=/${{ matrix.STORAGE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/benchmark.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --storage ${{ matrix.STORAGE }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: benchmark-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + ClickHouseKeeperSSL: + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=clickhouse_keeper + STORAGE=/ssl + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --ssl + --clickhouse-binary-path ${{ env.clickhouse_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ inputs.arch }}-ssl-artifacts + path: ${{ env.artifact_paths }} + + LDAP: + strategy: + fail-fast: false + matrix: + SUITE: [authentication, external_user_directory, role_mapping] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=ldap/${{ matrix.SUITE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ldap-${{ matrix.SUITE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + Parquet: + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=parquet + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + ParquetS3: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=parquet + STORAGE=${{ matrix.STORAGE}} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --storage ${{ matrix.STORAGE }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ env.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + S3: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=s3 + STORAGE=/${{ matrix.STORAGE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --storage ${{ matrix.STORAGE }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} + + TieredStorage: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, s3amazon, s3gcs] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=tiered_storage + STORAGE=/${{ matrix.STORAGE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/ + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --with-${{ matrix.STORAGE }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 0a84d0931975..b606fdac826f 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -4,32 +4,42 @@ name: ReleaseBranchCI env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} on: # yamllint disable-line rule:truthy + pull_request: + types: + - synchronize + - reopened + - opened + branches: + # Anything/24.8 (e.g customizations/24.8.x) + - '**/24.8*' + release: + types: + - published + - prereleased push: branches: - # 22.1 and 22.10 - - '2[1-9].[1-9][0-9]' - - '2[1-9].[1-9]' + - 'releases/24.8**' + workflow_dispatch: jobs: RunConfig: - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax11, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none] outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 - name: Debug Info uses: ./.github/actions/debug - - name: Labels check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 run_check.py - name: Python unit tests run: | cd "$GITHUB_WORKSPACE/tests/ci" @@ -56,23 +66,26 @@ jobs: needs: [RunConfig] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/docker_test_images.yml + secrets: inherit with: data: ${{ needs.RunConfig.outputs.data }} CompatibilityCheckX86: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Compatibility check (release) - runner_type: style-checker + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} CompatibilityCheckAarch64: needs: [RunConfig, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Compatibility check (aarch64) - runner_type: style-checker + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} ######################################################################################### #################################### ORDINARY BUILDS #################################### @@ -81,6 +94,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_release checkout_depth: 0 @@ -91,6 +105,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_aarch64 checkout_depth: 0 @@ -101,6 +116,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_asan data: ${{ needs.RunConfig.outputs.data }} @@ -108,6 +124,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_ubsan data: ${{ needs.RunConfig.outputs.data }} @@ -115,6 +132,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_tsan data: ${{ needs.RunConfig.outputs.data }} @@ -122,6 +140,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_msan data: ${{ needs.RunConfig.outputs.data }} @@ -129,30 +148,11 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_debug data: ${{ needs.RunConfig.outputs.data }} force: true - BuilderBinDarwin: - needs: [RunConfig, BuildDockers] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/reusable_build.yml - with: - build_name: binary_darwin - checkout_depth: 0 - data: ${{ needs.RunConfig.outputs.data }} - # always rebuild on release branches to be able to publish from any commit - force: true - BuilderBinDarwinAarch64: - needs: [RunConfig, BuildDockers] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/reusable_build.yml - with: - build_name: binary_darwin_aarch64 - checkout_depth: 0 - data: ${{ needs.RunConfig.outputs.data }} - # always rebuild on release branches to be able to publish from any commit - force: true ############################################################################################ ##################################### Docker images ####################################### ############################################################################################ @@ -160,17 +160,19 @@ jobs: needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Docker server image - runner_type: style-checker + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} DockerKeeperImage: needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Docker keeper image - runner_type: style-checker + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} ############################################################################################ ##################################### BUILD REPORTER ####################################### @@ -178,29 +180,29 @@ jobs: Builds_Report: # run report check for failed builds to indicate the CI error if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }} - needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64, BuilderDebAsan, BuilderDebUBsan, BuilderDebMsan, BuilderDebTsan, BuilderDebDebug, BuilderBinDarwin, BuilderBinDarwinAarch64] - runs-on: [self-hosted, style-checker-aarch64] + needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64, BuilderDebAsan, BuilderDebUBsan, BuilderDebMsan, BuilderDebTsan, BuilderDebDebug] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax11, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 - name: Download reports run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds - name: Builds report run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_msan package_ubsan package_tsan package_debug binary_darwin binary_darwin_aarch64 + python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_msan package_ubsan package_tsan package_debug - name: Set status + # NOTE(vnemkov): generate and upload the report even if previous step failed + if: success() || failure() run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --post --job-name Builds MarkReleaseReady: if: ${{ !failure() && !cancelled() }} needs: - - BuilderBinDarwin - - BuilderBinDarwinAarch64 - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax11, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none] steps: - name: Debug run: | @@ -218,7 +220,7 @@ jobs: run: exit 1 - name: Check out repository code if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 - name: Mark Commit Release Ready if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} run: | @@ -231,9 +233,10 @@ jobs: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Install packages (release) - runner_type: style-checker + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} run_command: | python3 install_check.py "$CHECK_NAME" @@ -241,9 +244,10 @@ jobs: needs: [RunConfig, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Install packages (aarch64) - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} run_command: | python3 install_check.py "$CHECK_NAME" @@ -254,57 +258,64 @@ jobs: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (release) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash,altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestAarch64: needs: [RunConfig, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (aarch64) - runner_type: func-tester-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestAsan: needs: [RunConfig, BuilderDebAsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (asan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestTsan: needs: [RunConfig, BuilderDebTsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestMsan: needs: [RunConfig, BuilderDebMsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (msan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestUBsan: needs: [RunConfig, BuilderDebUBsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (ubsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestDebug: needs: [RunConfig, BuilderDebDebug] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (debug) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### @@ -313,57 +324,64 @@ jobs: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (release) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestAarch64: needs: [RunConfig, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (aarch64) - runner_type: func-tester-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestAsan: needs: [RunConfig, BuilderDebAsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (asan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestTsan: needs: [RunConfig, BuilderDebTsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestMsan: needs: [RunConfig, BuilderDebMsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (msan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestUBsan: needs: [RunConfig, BuilderDebUBsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (ubsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestDebug: needs: [RunConfig, BuilderDebDebug] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (debug) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} ############################################################################################## ######################################### STRESS TESTS ####################################### @@ -372,41 +390,46 @@ jobs: needs: [RunConfig, BuilderDebAsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stress test (asan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} StressTestTsan: needs: [RunConfig, BuilderDebTsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stress test (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} StressTestMsan: needs: [RunConfig, BuilderDebMsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stress test (msan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} StressTestUBsan: needs: [RunConfig, BuilderDebUBsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stress test (ubsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} StressTestDebug: needs: [RunConfig, BuilderDebDebug] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stress test (debug) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -415,33 +438,80 @@ jobs: needs: [RunConfig, BuilderDebAsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Integration tests (asan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsAnalyzerAsan: needs: [RunConfig, BuilderDebAsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Integration tests (asan, old analyzer) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsTsan: needs: [RunConfig, BuilderDebTsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Integration tests (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsRelease: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Integration tests (release) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + data: ${{ needs.RunConfig.outputs.data }} +############################################################################################# +##################################### REGRESSION TESTS ###################################### +############################################################################################# + RegressionTestsRelease: + needs: [BuilderDebRelease] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/regression.yml + secrets: inherit + with: + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression + commit: 634042960dd157489199ca0f9d028c7fd1e6adb0 + arch: release + build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout_minutes: 300 + RegressionTestsAarch64: + needs: [BuilderDebAarch64] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/regression.yml + secrets: inherit + with: + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression + commit: 634042960dd157489199ca0f9d028c7fd1e6adb0 + arch: aarch64 + build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout_minutes: 300 + SignRelease: + needs: [RunConfig, BuilderDebRelease] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_sign.yml + secrets: inherit + with: + test_name: Sign release + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + data: ${{ needs.RunConfig.outputs.data }} + SignAarch64: + needs: [RunConfig, BuilderDebAarch64] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_sign.yml + secrets: inherit + with: + test_name: Sign aarch64 + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none data: ${{ needs.RunConfig.outputs.data }} FinishCheck: if: ${{ !cancelled() }} @@ -475,10 +545,13 @@ jobs: - IntegrationTestsRelease - CompatibilityCheckX86 - CompatibilityCheckAarch64 - runs-on: [self-hosted, style-checker] + - RegressionTestsRelease + - RegressionTestsAarch64 + - SignRelease + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax11, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: clear-repository: true - name: Finish label diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index 5e254d785ecc..c64d0aaec500 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -4,6 +4,9 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} name: Build ClickHouse 'on': @@ -37,6 +40,12 @@ name: Build ClickHouse secret_envs: description: if given, it's passed to the environments required: false + AWS_SECRET_ACCESS_KEY: + description: the access key to the aws param store. + required: true + AWS_ACCESS_KEY_ID: + description: the access key id to the aws param store. + required: true jobs: Build: @@ -44,10 +53,10 @@ jobs: if: ${{ contains(fromJson(inputs.data).jobs_data.jobs_to_do, inputs.build_name) || inputs.force }} env: GITHUB_JOB_OVERRIDDEN: Build-${{inputs.build_name}} - runs-on: [self-hosted, '${{inputs.runner_type}}'] + runs-on: [self-hosted, altinity-type-ccx53, altinity-on-demand, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: clear-repository: true ref: ${{ fromJson(inputs.data).git_ref }} @@ -98,6 +107,11 @@ jobs: if: ${{ !cancelled() }} run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.build_name}}' + - name: Upload json report + uses: actions/upload-artifact@v4 + with: + path: ${{ env.TEMP_PATH }}/build_report_*.json + name: build_report_${{inputs.build_name}} - name: Clean if: always() uses: ./.github/actions/clean diff --git a/.github/workflows/reusable_sign.yml b/.github/workflows/reusable_sign.yml new file mode 100644 index 000000000000..2bd8ae430fa5 --- /dev/null +++ b/.github/workflows/reusable_sign.yml @@ -0,0 +1,164 @@ +### For the pure soul wishes to move it to another place +# https://github.com/orgs/community/discussions/9050 + +name: Testing workflow +'on': + workflow_call: + inputs: + test_name: + description: the value of test type from tests/ci/ci_config.py, ends up as $CHECK_NAME ENV + required: true + type: string + runner_type: + description: the label of runner to use + required: true + type: string + run_command: + description: the command to launch the check + default: "" + required: false + type: string + checkout_depth: + description: the value of the git shallow checkout + required: false + type: number + default: 1 + submodules: + description: if the submodules should be checked out + required: false + type: boolean + default: false + additional_envs: + description: additional ENV variables to setup the job + type: string + data: + description: ci data + type: string + required: true + working-directory: + description: sets custom working directory + type: string + default: "$GITHUB_WORKSPACE/tests/ci" + secrets: + secret_envs: + description: if given, it's passed to the environments + required: false + AWS_SECRET_ACCESS_KEY: + description: the access key to the aws param store. + required: true + AWS_ACCESS_KEY_ID: + description: the access key id to the aws param store. + required: true + GPG_BINARY_SIGNING_KEY: + description: gpg signing key for packages. + required: true + GPG_BINARY_SIGNING_PASSPHRASE: + description: gpg signing key passphrase. + required: true + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + CHECK_NAME: ${{inputs.test_name}} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + +jobs: + runner_labels_setup: + name: Compute proper runner labels for the rest of the jobs + runs-on: ubuntu-latest + outputs: + runner_labels: ${{ steps.setVariables.outputs.runner_labels }} + steps: + - id: setVariables + name: Prepare runner_labels variables for the later steps + run: | + + # Prepend self-hosted + input="self-hosted, ${input}" + + # Remove all whitespace + input="$(echo ${input} | tr -d [:space:])" + # Make something like a JSON array from comma-separated list + input="[ '${input//\,/\'\, \'}' ]" + + echo "runner_labels=$input" >> ${GITHUB_OUTPUT} + env: + input: ${{ inputs.runner_type }} + + Test: + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + name: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} + env: + GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} + strategy: + fail-fast: false # we always wait for entire matrix + matrix: + batch: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].batches }} + steps: + - name: Check out repository code + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 + with: + clear-repository: true + ref: ${{ fromJson(inputs.data).git_ref }} + submodules: ${{inputs.submodules}} + fetch-depth: ${{inputs.checkout_depth}} + filter: tree:0 + - name: Set build envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + CHECK_NAME=${{ inputs.test_name }} + ${{inputs.additional_envs}} + ${{secrets.secret_envs}} + DOCKER_TAG< 1 }} + run: | + cat >> "$GITHUB_ENV" << 'EOF' + RUN_BY_HASH_NUM=${{matrix.batch}} + RUN_BY_HASH_TOTAL=${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches }} + EOF + - name: Pre run + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --pre --job-name '${{inputs.test_name}}' + - name: Sign release + env: + GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }} + GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }} + run: | + cd "${{ inputs.working-directory }}" + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" \ + --infile ${{ toJson(inputs.data) }} \ + --job-name '${{inputs.test_name}}' \ + --run \ + --force \ + --run-command '''python3 sign_release.py''' + - name: Post run + if: ${{ !cancelled() }} + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.test_name}}' + - name: Mark as done + if: ${{ !cancelled() }} + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.test_name}}' --batch ${{matrix.batch}} + - name: Upload signed hashes + uses: actions/upload-artifact@v4 + with: + name: ${{inputs.test_name}} signed-hashes + path: ${{ env.TEMP_PATH }}/*.gpg + - name: Clean + if: always() + uses: ./.github/actions/clean diff --git a/.github/workflows/reusable_simple_job.yml b/.github/workflows/reusable_simple_job.yml index 7df98d96f790..9ecb28109450 100644 --- a/.github/workflows/reusable_simple_job.yml +++ b/.github/workflows/reusable_simple_job.yml @@ -63,7 +63,7 @@ jobs: GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: clear-repository: true ref: ${{ inputs.git_ref }} diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index ca6df5df14e0..dc81ebd2c3f3 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -43,17 +43,47 @@ name: Testing workflow secret_envs: description: if given, it's passed to the environments required: false - + AWS_SECRET_ACCESS_KEY: + description: the access key to the aws param store. + required: true + AWS_ACCESS_KEY_ID: + description: the access key id to the aws param store. + required: true env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 CHECK_NAME: ${{inputs.test_name}} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} jobs: + runner_labels_setup: + name: Compute proper runner labels for the rest of the jobs + runs-on: ubuntu-latest + outputs: + runner_labels: ${{ steps.setVariables.outputs.runner_labels }} + steps: + - id: setVariables + name: Prepare runner_labels variables for the later steps + run: | + + # Prepend self-hosted + input="self-hosted, ${input}" + + # Remove all whitespace + input="$(echo ${input} | tr -d [:space:])" + # Make something like a JSON array from comma-separated list + input="[ '${input//\,/\'\, \'}' ]" + + echo "runner_labels=$input" >> ${GITHUB_OUTPUT} + env: + input: ${{ inputs.runner_type }} + Test: - runs-on: [self-hosted, '${{inputs.runner_type}}'] - if: ${{ !failure() && !cancelled() && contains(fromJson(inputs.data).jobs_data.jobs_to_do, inputs.test_name) }} + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} name: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} env: GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} @@ -63,7 +93,7 @@ jobs: batch: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].batches }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: clear-repository: true ref: ${{ fromJson(inputs.data).git_ref }} @@ -84,6 +114,8 @@ jobs: uses: ./.github/actions/common_setup with: job_type: test + - name: Docker setup + uses: ./.github/actions/docker_setup - name: Setup batch if: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 }} run: | @@ -101,6 +133,7 @@ jobs: --infile ${{ toJson(inputs.data) }} \ --job-name '${{inputs.test_name}}' \ --run \ + --force \ --run-command '''${{inputs.run_command}}''' # shellcheck disable=SC2319 echo "JOB_EXIT_CODE=$?" >> "$GITHUB_ENV" diff --git a/README.md b/README.md index 2120a4d1211a..5f41b67e5ed8 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,60 @@
-[![Website](https://img.shields.io/website?up_message=AVAILABLE&down_message=DOWN&url=https%3A%2F%2Fclickhouse.com&style=for-the-badge)](https://clickhouse.com) +[![Website](https://img.shields.io/website?up_message=AVAILABLE&down_message=DOWN&url=https://docs.altinity.com/altinitystablebuilds&style=for-the-badge)](https://docs.altinity.com/altinitystablebuilds/) [![Apache 2.0 License](https://img.shields.io/badge/license-Apache%202.0-blueviolet?style=for-the-badge)](https://www.apache.org/licenses/LICENSE-2.0) - - - The ClickHouse company logo. + + + Altinity company logo -

ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time.

+

Altinity Stable Builds®

-## How To Install (Linux, macOS, FreeBSD) -``` -curl https://clickhouse.com/ | sh -``` +**Altinity Stable Builds** are releases of ClickHouse® that undergo rigorous testing to verify they are secure and ready for production use. Among other things, they are: -## Useful Links +* Supported for three years +* Validated against client libraries and visualization tools +* Tested for cloud use, including Kubernetes +* 100% open source and 100% compatible with ClickHouse upstream builds +* Available in FIPS-compatible versions + +**We encourage you to use Altinity Stable Builds whether you're an Altinity Support customer or not.** + +## Acknowledgement + +We at Altinity, Inc. are thoroughly grateful to the worldwide ClickHouse community, including the core committers who make ClickHouse the world's best analytics database. -* [Official website](https://clickhouse.com/) has a quick high-level overview of ClickHouse on the main page. -* [ClickHouse Cloud](https://clickhouse.cloud) ClickHouse as a service, built by the creators and maintainers. -* [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query a small ClickHouse cluster. -* [Documentation](https://clickhouse.com/docs/en/) provides more in-depth information. -* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format. -* [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time. -* [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events. -* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlighting, powered by github.dev. -* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any. +## What should I do if I find a bug in an Altinity Stable Build? -## Monthly Release & Community Call +ClickHouse’s thousands of core features are all well-tested and stable. To maintain that stability, Altinity Stable Builds are built with a CI system that runs tens of thousands of tests against every commit. But of course, things can always go wrong. If that happens, let us know! **We stand behind our work.** -Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know. +### If you're an Altinity customer: -* [v24.8 Community Call](https://clickhouse.com/company/events/v24-8-community-release-call) - August 29 +1. [Contact Altinity support](https://docs.altinity.com/support/) to file an issue. -## Upcoming Events +### If you're not an Altinity customer: + +1. Try to upgrade to the latest bugfix release. If you’re using v23.8.8 but you know that v23.8.11.29 exists, start by upgrading to the bugfix. Upgrades to the latest maintenance releases are smooth and safe. +2. Look for similar issues in the [Altinity/ClickHouse](https://github.com/Altinity/ClickHouse/issues) or [ClickHouse/ClickHouse](https://github.com/ClickHouse/ClickHouse/issues) repos; it's possible the problem has been logged and a fix is on the way. +3. If you can reproduce the bug, try to isolate it. For example, remove pieces of a failing query one by one, creating the simplest scenario where the error still occurs. Creating [a minimal reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) is a huge step towards a solution. +4. [File an issue](https://github.com/Altinity/ClickHouse/issues/new/choose) in the Altinity/ClickHouse repo. + +## Useful Links -Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `` clickhouse `` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc. +* [Release notes](https://docs.altinity.com/releasenotes/altinity-stable-release-notes/) - Complete details on the changes and fixes in each Altinity Stable Build release +* [Builds page](https://builds.altinity.cloud/) - Download and installation instructions for Altinity Stable Builds +* [Dockerhub page](https://hub.docker.com/r/altinity/clickhouse-server) - Home of the Altinity Stable Build container images +* [Knowledge base](https://kb.altinity.com/) - Insight, knowledge, and advice from the Altinity Engineering and Support teams +* [Documentation](https://docs.altinity.com/altinitystablebuilds/) - A list of current releases and their lifecycles +* [Altinity support for ClickHouse](https://altinity.com/clickhouse-support/) - The best ClickHouse support in the industry, delivered by the most knowledgeable ClickHouse team in the industry +* [Altinity administrator training for ClickHouse](https://altinity.com/clickhouse-training/) - Understand how ClickHouse works, not just how to use it +* [Altinity blog](https://altinity.com/blog/) - The latest news on Altinity's ClickHouse-related projects, release notes, tutorials, and more +* [Altinity YouTube channel](https://www.youtube.com/@AltinityB) - ClickHouse tutorials, webinars, conference presentations, and other useful things +* [Altinity Slack channel](https://altinitydbworkspace.slack.com/join/shared_invite/zt-1togw9b4g-N0ZOXQyEyPCBh_7IEHUjdw#/shared-invite/email) - The Altinity Slack channel -* MORE COMING SOON! +
-## Recent Recordings -* **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments" -* **Recording available**: [**v24.4 Release Call**](https://www.youtube.com/watch?v=dtUqgcfOGmE) All the features of 24.4, one convenient video! Watch it now! - - ## Interested in joining ClickHouse and making it your full-time job? - -We are a globally diverse and distributed team, united behind a common goal of creating industry-leading, real-time analytics. Here, you will have an opportunity to solve some of the most cutting-edge technical challenges and have direct ownership of your work and vision. If you are a contributor by nature, a thinker and a doer - we’ll definitely click! - -Check out our **current openings** here: https://clickhouse.com/company/careers - -Can't find what you are looking for, but want to let us know you are interested in joining ClickHouse? Email careers@clickhouse.com! +*©2024 Altinity Inc. All rights reserved. Altinity®, Altinity.Cloud®, and Altinity Stable Builds® are registered trademarks of Altinity, Inc. ClickHouse® is a registered trademark of ClickHouse, Inc.* diff --git a/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h b/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h index acc79ec92b23..6e44d9f45b7b 100644 --- a/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h +++ b/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h @@ -188,8 +188,9 @@ namespace Crypto pFile = fopen(keyFile.c_str(), "r"); if (pFile) { - pem_password_cb * pCB = pass.empty() ? (pem_password_cb *)0 : &passCB; - void * pPassword = pass.empty() ? (void *)0 : (void *)pass.c_str(); + pem_password_cb * pCB = &passCB; + static constexpr char * no_password = ""; + void * pPassword = pass.empty() ? (void *)no_password : (void *)pass.c_str(); if (readFunc(pFile, &pKey, pCB, pPassword)) { fclose(pFile); @@ -225,6 +226,13 @@ namespace Crypto error: if (pFile) fclose(pFile); + if (*ppKey) + { + if constexpr (std::is_same_v) + EVP_PKEY_free(*ppKey); + else + EC_KEY_free(*ppKey); + } throw OpenSSLException("EVPKey::loadKey(string)"); } @@ -286,6 +294,13 @@ namespace Crypto error: if (pBIO) BIO_free(pBIO); + if (*ppKey) + { + if constexpr (std::is_same_v) + EVP_PKEY_free(*ppKey); + else + EC_KEY_free(*ppKey); + } throw OpenSSLException("EVPKey::loadKey(stream)"); } diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h index c19eecf5c737..2c56875835e7 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h @@ -248,6 +248,9 @@ namespace Net SSL_CTX * sslContext() const; /// Returns the underlying OpenSSL SSL Context object. + SSL_CTX * takeSslContext(); + /// Takes ownership of the underlying OpenSSL SSL Context object. + Usage usage() const; /// Returns whether the context is for use by a client or by a server /// and whether TLSv1 is required. @@ -401,6 +404,13 @@ namespace Net return _pSSLContext; } + inline SSL_CTX * Context::takeSslContext() + { + auto * result = _pSSLContext; + _pSSLContext = nullptr; + return result; + } + inline bool Context::extendedCertificateVerificationEnabled() const { diff --git a/base/poco/NetSSL_OpenSSL/src/Context.cpp b/base/poco/NetSSL_OpenSSL/src/Context.cpp index da1c121286b6..69c88eef63ac 100644 --- a/base/poco/NetSSL_OpenSSL/src/Context.cpp +++ b/base/poco/NetSSL_OpenSSL/src/Context.cpp @@ -106,6 +106,11 @@ Context::Context( Context::~Context() { + if (_pSSLContext == nullptr) + { + return; + } + try { SSL_CTX_free(_pSSLContext); diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index efb178591748..46ab7b8bf48d 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -7,6 +7,13 @@ SET(VERSION_MAJOR 24) SET(VERSION_MINOR 8) SET(VERSION_PATCH 8) SET(VERSION_GITHASH e28553d4f2ba78643f9ef47b698954a2c54e6bcc) -SET(VERSION_DESCRIBE v24.8.8.1-lts) -SET(VERSION_STRING 24.8.8.1) + +#1000 for altinitystable candidates +#2000 for altinityedge candidates +SET(VERSION_TWEAK 181000) +SET(VERSION_FLAVOUR altinitystable) + +SET(VERSION_DESCRIBE v24.8.8.181000.altinitystable) +SET(VERSION_STRING 24.8.8.181000.altinitystable) + # end of autochange diff --git a/cmake/version.cmake b/cmake/version.cmake index 9ca21556f4d4..b008c989c0b0 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -3,9 +3,10 @@ include(${PROJECT_SOURCE_DIR}/cmake/autogenerated_versions.txt) set(VERSION_EXTRA "" CACHE STRING "") set(VERSION_TWEAK "" CACHE STRING "") -if (VERSION_TWEAK) - string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_TWEAK}) -endif () +# NOTE(vnemkov): we rely on VERSION_TWEAK portion to be already present in VERSION_STRING +# if (VERSION_TWEAK) +# string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_TWEAK}) +# endif () if (VERSION_EXTRA) string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_EXTRA}) @@ -19,5 +20,5 @@ set (VERSION_STRING_SHORT "${VERSION_MAJOR}.${VERSION_MINOR}") math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000") if(CLICKHOUSE_OFFICIAL_BUILD) - set(VERSION_OFFICIAL " (official build)") + set(VERSION_OFFICIAL " (altinity build)") endif() diff --git a/contrib/numactl b/contrib/numactl index 8d13d63a05f0..ff32c618d63c 160000 --- a/contrib/numactl +++ b/contrib/numactl @@ -1 +1 @@ -Subproject commit 8d13d63a05f0c3cd88bf777cbb61541202b7da08 +Subproject commit ff32c618d63ca7ac48cce366c5a04bb3563683a0 diff --git a/docker/docs/builder/Dockerfile b/docker/docs/builder/Dockerfile index b7b706a8a5ce..a61b44b60798 100644 --- a/docker/docs/builder/Dockerfile +++ b/docker/docs/builder/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/docs-builder . +# docker build -t altinityinfra/docs-builder . # nodejs 17 prefers ipv6 and is broken in our environment FROM node:16-alpine diff --git a/docker/images.json b/docker/images.json index 716b76ee217d..8985368b3b07 100644 --- a/docker/images.json +++ b/docker/images.json @@ -1,125 +1,128 @@ { "docker/packager/binary-builder": { - "name": "clickhouse/binary-builder", + "name": "altinityinfra/binary-builder", "dependent": [] }, "docker/packager/cctools": { - "name": "clickhouse/cctools", + "name": "altinityinfra/cctools", "dependent": [] }, "docker/test/compatibility/centos": { - "name": "clickhouse/test-old-centos", + "name": "altinityinfra/test-old-centos", "dependent": [] }, "docker/test/compatibility/ubuntu": { - "name": "clickhouse/test-old-ubuntu", + "name": "altinityinfra/test-old-ubuntu", "dependent": [] }, "docker/test/integration/base": { - "name": "clickhouse/integration-test", + "only_amd64": true, + "name": "altinityinfra/integration-test", "dependent": [] }, "docker/test/fuzzer": { - "name": "clickhouse/fuzzer", + "name": "altinityinfra/fuzzer", "dependent": [] }, "docker/test/libfuzzer": { - "name": "clickhouse/libfuzzer", + "name": "altinityinfra/libfuzzer", "dependent": [] }, "docker/test/performance-comparison": { - "name": "clickhouse/performance-comparison", + "name": "altinityinfra/performance-comparison", "dependent": [] }, "docker/test/util": { - "name": "clickhouse/test-util", + "name": "altinityinfra/test-util", "dependent": [ "docker/test/base", "docker/test/fasttest" ] }, "docker/test/stateless": { - "name": "clickhouse/stateless-test", + "name": "altinityinfra/stateless-test", "dependent": [ "docker/test/stateful" ] }, "docker/test/stateful": { - "name": "clickhouse/stateful-test", + "name": "altinityinfra/stateful-test", "dependent": [ "docker/test/stress", "docker/test/upgrade" ] }, "docker/test/unit": { - "name": "clickhouse/unit-test", + "name": "altinityinfra/unit-test", "dependent": [] }, "docker/test/stress": { - "name": "clickhouse/stress-test", + "name": "altinityinfra/stress-test", "dependent": [] }, "docker/test/upgrade": { - "name": "clickhouse/upgrade-check", + "name": "altinityinfra/upgrade-check", "dependent": [] }, "docker/test/integration/runner": { - "name": "clickhouse/integration-tests-runner", + "only_amd64": true, + "name": "altinityinfra/integration-tests-runner", "dependent": [] }, "docker/test/fasttest": { - "name": "clickhouse/fasttest", + "name": "altinityinfra/fasttest", "dependent": [ "docker/packager/binary-builder" ] }, "docker/test/style": { - "name": "clickhouse/style-test", + "name": "altinityinfra/style-test", "dependent": [] }, "docker/test/integration/s3_proxy": { - "name": "clickhouse/s3-proxy", + "name": "altinityinfra/s3-proxy", "dependent": [] }, "docker/test/integration/resolver": { - "name": "clickhouse/python-bottle", + "name": "altinityinfra/python-bottle", "dependent": [] }, "docker/test/integration/helper_container": { - "name": "clickhouse/integration-helper", + "only_amd64": true, + "name": "altinityinfra/integration-helper", "dependent": [] }, "docker/test/integration/mysql_golang_client": { - "name": "clickhouse/mysql-golang-client", + "name": "altinityinfra/mysql-golang-client", "dependent": [] }, "docker/test/integration/dotnet_client": { - "name": "clickhouse/dotnet-client", + "name": "altinityinfra/dotnet-client", "dependent": [] }, "docker/test/integration/mysql_java_client": { - "name": "clickhouse/mysql-java-client", + "name": "altinityinfra/mysql-java-client", "dependent": [] }, "docker/test/integration/mysql_js_client": { - "name": "clickhouse/mysql-js-client", + "name": "altinityinfra/mysql-js-client", "dependent": [] }, "docker/test/integration/mysql_php_client": { - "name": "clickhouse/mysql-php-client", + "name": "altinityinfra/mysql-php-client", "dependent": [] }, "docker/test/integration/postgresql_java_client": { - "name": "clickhouse/postgresql-java-client", + "name": "altinityinfra/postgresql-java-client", "dependent": [] }, "docker/test/integration/kerberos_kdc": { "only_amd64": true, - "name": "clickhouse/kerberos-kdc", + "name": "altinityinfra/kerberos-kdc", "dependent": [] }, "docker/test/base": { - "name": "clickhouse/test-base", + "name": "altinityinfra/test-base", "dependent": [ "docker/test/clickbench", "docker/test/fuzzer", @@ -133,49 +136,44 @@ "docker/test/unit" ] }, - "docker/test/integration/kerberized_hadoop": { - "only_amd64": true, - "name": "clickhouse/kerberized-hadoop", - "dependent": [] - }, "docker/test/sqlancer": { - "name": "clickhouse/sqlancer-test", + "name": "altinityinfra/sqlancer-test", "dependent": [] }, "docker/test/keeper-jepsen": { - "name": "clickhouse/keeper-jepsen-test", + "name": "altinityinfra/keeper-jepsen-test", "dependent": [] }, "docker/test/server-jepsen": { - "name": "clickhouse/server-jepsen-test", + "name": "altinityinfra/server-jepsen-test", "dependent": [] }, "docker/test/clickbench": { - "name": "clickhouse/clickbench", + "name": "altinityinfra/clickbench", "dependent": [] }, "docker/test/install/deb": { - "name": "clickhouse/install-deb-test", + "name": "altinityinfra/install-deb-test", "dependent": [] }, "docker/test/install/rpm": { - "name": "clickhouse/install-rpm-test", + "name": "altinityinfra/install-rpm-test", "dependent": [] }, "docker/docs/builder": { - "name": "clickhouse/docs-builder", + "name": "altinityinfra/docs-builder", "dependent": [] }, "docker/test/sqllogic": { - "name": "clickhouse/sqllogic-test", + "name": "altinityinfra/sqllogic-test", "dependent": [] }, "docker/test/sqltest": { - "name": "clickhouse/sqltest", + "name": "altinityinfra/sqltest", "dependent": [] }, "docker/test/integration/nginx_dav": { - "name": "clickhouse/nginx-dav", + "name": "altinityinfra/nginx-dav", "dependent": [] } } diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index a44664259fbc..646ef74164ba 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -12,7 +12,9 @@ RUN arch=${TARGETARCH:-amd64} \ && ln -s "${rarch}-linux-gnu" /lib/linux-gnu -FROM alpine +# As of Nov 3rd 2024, alpine:3.20.3 has 1 "unknown"-type CVE: +# * https://security.alpinelinux.org/vuln/CVE-2024-9143 +FROM alpine:3.20.3 ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ diff --git a/docker/packager/binary-builder/Dockerfile b/docker/packager/binary-builder/Dockerfile index 7d6acdcd856c..2f6778ba2112 100644 --- a/docker/packager/binary-builder/Dockerfile +++ b/docker/packager/binary-builder/Dockerfile @@ -1,11 +1,12 @@ -# docker build -t clickhouse/binary-builder . +# docker build -t altinityinfra/binary-builder . ARG FROM_TAG=latest -FROM clickhouse/fasttest:$FROM_TAG +FROM altinityinfra/fasttest:$FROM_TAG ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} # If the cctools is updated, then first build it in the CI, then update here in a different commit -COPY --from=clickhouse/cctools:d9e3596e706b /cctools /cctools +# NOTE: a9e509c9bbc0 created from commit 1bc0e6ee583d49a472fd01f0e9960a11bf240c6c +COPY --from=altinityinfra/cctools:a9e509c9bbc0 /cctools /cctools # Rust toolchain and libraries ENV RUSTUP_HOME=/rust/rustup diff --git a/docker/packager/cctools/Dockerfile b/docker/packager/cctools/Dockerfile index 570a42d42d51..3555bf7c428b 100644 --- a/docker/packager/cctools/Dockerfile +++ b/docker/packager/cctools/Dockerfile @@ -1,10 +1,10 @@ -# docker build -t clickhouse/cctools . +# docker build -t altinityinfra/cctools . -# This is a hack to significantly reduce the build time of the clickhouse/binary-builder +# This is a hack to significantly reduce the build time of the altinityinfra/binary-builder # It's based on the assumption that we don't care of the cctools version so much -# It event does not depend on the clickhouse/fasttest in the `docker/images.json` +# It event does not depend on the altinityinfra/fasttest in the `docker/images.json` ARG FROM_TAG=latest -FROM clickhouse/fasttest:$FROM_TAG as builder +FROM altinityinfra/fasttest:$FROM_TAG as builder ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} diff --git a/docker/packager/packager b/docker/packager/packager index da4af7fc1bee..7975b6a03aae 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -10,7 +10,7 @@ from typing import List, Optional SCRIPT_PATH = Path(__file__).absolute() IMAGE_TYPE = "binary-builder" -IMAGE_NAME = f"clickhouse/{IMAGE_TYPE}" +IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}" class BuildException(Exception): @@ -130,9 +130,11 @@ def parse_env_variables( sanitizer: str, package_type: str, cache: str, + s3_access_key_id: str, s3_bucket: str, s3_directory: str, s3_rw_access: bool, + s3_secret_access_key: str, clang_tidy: bool, version: str, official: bool, @@ -321,6 +323,10 @@ def parse_env_variables( result.append(f"SCCACHE_S3_KEY_PREFIX={sccache_dir}") if not s3_rw_access: result.append("SCCACHE_S3_NO_CREDENTIALS=true") + if s3_access_key_id: + result.append(f"AWS_ACCESS_KEY_ID={s3_access_key_id}") + if s3_secret_access_key: + result.append(f"AWS_SECRET_ACCESS_KEY={s3_secret_access_key}") if clang_tidy: # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above. @@ -447,6 +453,14 @@ def parse_args() -> argparse.Namespace: type=dir_name, help="a directory with ccache", ) + parser.add_argument( + "--s3-access-key-id", + help="an S3 access key id used for sscache bucket", + ) + parser.add_argument( + "--s3-secret-access-key", + help="an S3 secret access key used for sscache bucket", + ) parser.add_argument( "--s3-bucket", help="an S3 bucket used for sscache and clang-tidy-cache", @@ -530,9 +544,11 @@ def main() -> None: args.sanitizer, args.package_type, args.cache, + args.s3_access_key_id, args.s3_bucket, args.s3_directory, args.s3_rw_access, + args.s3_secret_access_key, args.clang_tidy, args.version, args.official, diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 6ff18fb301c6..9e22f9154967 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 # see https://github.com/moby/moby/issues/4032#issuecomment-192327844 # It could be removed after we move on a version 23:04+ diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 848da4ef5b0e..1fc5a1f5f424 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/test-base . +# docker build -t altinityinfra/test-base . ARG FROM_TAG=latest -FROM clickhouse/test-util:$FROM_TAG +FROM altinityinfra/test-util:$FROM_TAG RUN apt-get update \ && apt-get install \ diff --git a/docker/test/clickbench/Dockerfile b/docker/test/clickbench/Dockerfile index 0b6b1736e031..214191a8b488 100644 --- a/docker/test/clickbench/Dockerfile +++ b/docker/test/clickbench/Dockerfile @@ -1,5 +1,5 @@ ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG ENV TZ=Europe/Amsterdam RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone diff --git a/docker/test/compatibility/centos/Dockerfile b/docker/test/compatibility/centos/Dockerfile index 628609e374f6..1edb42422b1f 100644 --- a/docker/test/compatibility/centos/Dockerfile +++ b/docker/test/compatibility/centos/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/test-old-centos . +# docker build -t altinityinfra/test-old-centos . FROM centos:5 CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \ diff --git a/docker/test/compatibility/ubuntu/Dockerfile b/docker/test/compatibility/ubuntu/Dockerfile index ddd0a76bd446..0eb283ff3daf 100644 --- a/docker/test/compatibility/ubuntu/Dockerfile +++ b/docker/test/compatibility/ubuntu/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/test-old-ubuntu . +# docker build -t altinityinfra/test-old-ubuntu . FROM ubuntu:12.04 CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \ diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 5d311c673a42..c4fae94d4abe 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -1,6 +1,6 @@ -# docker build -t clickhouse/fasttest . +# docker build -t altinityinfra/fasttest . ARG FROM_TAG=latest -FROM clickhouse/test-util:$FROM_TAG +FROM altinityinfra/test-util:$FROM_TAG RUN apt-get update \ && apt-get install \ diff --git a/docker/test/fasttest/requirements.txt b/docker/test/fasttest/requirements.txt index 993ea22e5ae4..bc5afd26b6c7 100644 --- a/docker/test/fasttest/requirements.txt +++ b/docker/test/fasttest/requirements.txt @@ -1,41 +1,41 @@ -Jinja2==3.1.3 -MarkupSafe==2.1.5 -PyJWT==2.3.0 -PyYAML==6.0.1 -Pygments==2.11.2 -SecretStorage==3.3.1 -blinker==1.4 -certifi==2020.6.20 -chardet==4.0.0 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -idna==3.3 -importlib-metadata==4.6.4 -jeepney==0.7.1 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -lxml==4.8.0 -more-itertools==8.10.0 -numpy==1.26.3 -oauthlib==3.2.0 -packaging==24.1 -pandas==1.5.3 -pip==24.1.1 -pipdeptree==2.23.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -python-dateutil==2.9.0.post0 -pytz==2024.1 -requests==2.32.3 -scipy==1.12.0 -setuptools==59.6.0 -six==1.16.0 -termcolor==1.1.0 -urllib3==1.26.5 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +Jinja2~=3.1.3 +MarkupSafe~=2.1.5 +PyJWT~=2.3.0 +PyYAML~=6.0.1 +Pygments~=2.11.2 +SecretStorage~=3.3.1 +blinker~=1.4 +certifi +chardet~=4.0.0 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +idna~=3.3 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +lxml~=4.8.0 +more-itertools~=8.10.0 +numpy~=1.26.3 +oauthlib~=3.2.0 +packaging~=24.1 +pandas~=1.5.3 +pip~=24.1.1 +pipdeptree~=2.23.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +python-dateutil~=2.9.0 +pytz +requests~=2.32.3 +scipy~=1.12.0 +setuptools~=59.6.0 +six~=1.16.0 +termcolor~=1.1.0 +urllib3~=1.26.5 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/fuzzer/Dockerfile b/docker/test/fuzzer/Dockerfile index e1fb09b8ed57..f79fa706e72e 100644 --- a/docker/test/fuzzer/Dockerfile +++ b/docker/test/fuzzer/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/fuzzer . +# docker build -t altinityinfra/fuzzer . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" @@ -41,5 +41,5 @@ CMD set -o pipefail \ && cd /workspace \ && timeout -s 9 1h /run-fuzzer.sh 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log -# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/fuzzer +# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> altinityinfra/fuzzer diff --git a/docker/test/fuzzer/requirements.txt b/docker/test/fuzzer/requirements.txt index 3dce93e023b2..fa70d89e7ac3 100644 --- a/docker/test/fuzzer/requirements.txt +++ b/docker/test/fuzzer/requirements.txt @@ -1,27 +1,27 @@ -blinker==1.4 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -Jinja2==3.1.4 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -MarkupSafe==2.1.5 -more-itertools==8.10.0 -oauthlib==3.2.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +Jinja2~=3.1.4 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +MarkupSafe~=2.1.5 +more-itertools~=8.10.0 +oauthlib~=3.2.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index dc4d470a2623..8af06623deaf 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/integration-test . +# docker build -t altinityinfra/integration-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG SHELL ["/bin/bash", "-c"] @@ -29,6 +29,7 @@ RUN apt-get update \ python3-pip \ libcurl4-openssl-dev \ libssl-dev \ + iptables \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* diff --git a/docker/test/integration/base/requirements.txt b/docker/test/integration/base/requirements.txt index d195d8deaf60..98df4d94bf58 100644 --- a/docker/test/integration/base/requirements.txt +++ b/docker/test/integration/base/requirements.txt @@ -1,26 +1,26 @@ -blinker==1.4 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -more-itertools==8.10.0 -oauthlib==3.2.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -pycurl==7.45.3 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +more-itertools~=8.10.0 +oauthlib~=3.2.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +pycurl~=7.45.3 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/integration/helper_container/Dockerfile b/docker/test/integration/helper_container/Dockerfile index 49a3d3cd84b8..a453533ce852 100644 --- a/docker/test/integration/helper_container/Dockerfile +++ b/docker/test/integration/helper_container/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/integration-helper . +# docker build -t altinityinfra/integration-helper . # Helper docker container to run iptables without sudo FROM alpine:3.18 diff --git a/docker/test/integration/kerberized_hadoop/Dockerfile b/docker/test/integration/kerberized_hadoop/Dockerfile index 62dcd2ca9973..ddc0850b3c5a 100644 --- a/docker/test/integration/kerberized_hadoop/Dockerfile +++ b/docker/test/integration/kerberized_hadoop/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/kerberized-hadoop . +# docker build -t altinityinfra/kerberized-hadoop . FROM sequenceiq/hadoop-docker:2.7.0 diff --git a/docker/test/integration/kerberos_kdc/Dockerfile b/docker/test/integration/kerberos_kdc/Dockerfile index a203c33a3313..a7f989bf4a56 100644 --- a/docker/test/integration/kerberos_kdc/Dockerfile +++ b/docker/test/integration/kerberos_kdc/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/kerberos-kdc . +# docker build -t altinityinfra/kerberos-kdc . FROM centos:6 RUN sed -i '/^mirrorlist/s/^/#/;/^#baseurl/{s/#//;s/mirror.centos.org\/centos\/$releasever/vault.centos.org\/6.10/}' /etc/yum.repos.d/*B* diff --git a/docker/test/integration/mysql_golang_client/Dockerfile b/docker/test/integration/mysql_golang_client/Dockerfile index 5281f786ae2d..52be68126e47 100644 --- a/docker/test/integration/mysql_golang_client/Dockerfile +++ b/docker/test/integration/mysql_golang_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/mysql-golang-client . +# docker build -t altinityinfra/mysql-golang-client . # MySQL golang client docker container FROM golang:1.17 diff --git a/docker/test/integration/mysql_java_client/Dockerfile b/docker/test/integration/mysql_java_client/Dockerfile index 38fefac070e7..5826ee77d501 100644 --- a/docker/test/integration/mysql_java_client/Dockerfile +++ b/docker/test/integration/mysql_java_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/mysql-java-client . +# docker build -t altinityinfra/mysql-java-client . # MySQL Java client docker container FROM openjdk:8-jdk-alpine diff --git a/docker/test/integration/mysql_js_client/Dockerfile b/docker/test/integration/mysql_js_client/Dockerfile index 4c9df10ace1c..2b821f243234 100644 --- a/docker/test/integration/mysql_js_client/Dockerfile +++ b/docker/test/integration/mysql_js_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/mysql-js-client . +# docker build -t altinityinfra/mysql-js-client . # MySQL JavaScript client docker container FROM node:16.14.2 diff --git a/docker/test/integration/mysql_php_client/Dockerfile b/docker/test/integration/mysql_php_client/Dockerfile index 0e11ae023e63..b060e93f70a3 100644 --- a/docker/test/integration/mysql_php_client/Dockerfile +++ b/docker/test/integration/mysql_php_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/mysql-php-client . +# docker build -t altinityinfra/mysql-php-client . # MySQL PHP client docker container FROM php:8-cli-alpine diff --git a/docker/test/integration/postgresql_java_client/Dockerfile b/docker/test/integration/postgresql_java_client/Dockerfile index c5583085ef37..5a7458cc1d2f 100644 --- a/docker/test/integration/postgresql_java_client/Dockerfile +++ b/docker/test/integration/postgresql_java_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/postgresql-java-client . +# docker build -t altinityinfra/postgresql-java-client . # PostgreSQL Java client docker container FROM ubuntu:18.04 diff --git a/docker/test/integration/resolver/Dockerfile b/docker/test/integration/resolver/Dockerfile index b35a7262651b..a7fec32620e1 100644 --- a/docker/test/integration/resolver/Dockerfile +++ b/docker/test/integration/resolver/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/python-bottle . +# docker build -t altinityinfra/python-bottle . # Helper docker container to run python bottle apps FROM python:3 diff --git a/docker/test/integration/resolver/requirements.txt b/docker/test/integration/resolver/requirements.txt index fbf852953296..314b112319b3 100644 --- a/docker/test/integration/resolver/requirements.txt +++ b/docker/test/integration/resolver/requirements.txt @@ -1,6 +1,6 @@ -bottle==0.12.25 -packaging==24.1 -pip==23.2.1 -pipdeptree==2.23.0 -setuptools==69.0.3 -wheel==0.42.0 +bottle~=0.13 +packaging~=24.1 +pip~=23.2.1 +pipdeptree~=2.23.0 +setuptools~=69.0.3 +wheel~=0.42.0 diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index d62009f1be38..f5784428e5fd 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/integration-tests-runner . +# docker build -t altinityinfra/integration-tests-runner . FROM ubuntu:22.04 # ARG for quick switch to a given ubuntu mirror diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh index 8882daa38ea3..84baa6b1f342 100755 --- a/docker/test/integration/runner/dockerd-entrypoint.sh +++ b/docker/test/integration/runner/dockerd-entrypoint.sh @@ -4,12 +4,12 @@ set -e mkdir -p /etc/docker/ echo '{ "ipv6": true, - "fixed-cidr-v6": "fd00::/8", + "fixed-cidr-v6": "2001:db8:1::/64", "ip-forward": true, "log-level": "debug", "storage-driver": "overlay2", - "insecure-registries" : ["dockerhub-proxy.dockerhub-proxy-zone:5000"], - "registry-mirrors" : ["http://dockerhub-proxy.dockerhub-proxy-zone:5000"] + "insecure-registries" : ["65.108.242.32:5000"], + "registry-mirrors" : ["http://65.108.242.32:5000"] }' | dd of=/etc/docker/daemon.json 2>/dev/null if [ -f /sys/fs/cgroup/cgroup.controllers ]; then diff --git a/docker/test/integration/runner/requirements.txt b/docker/test/integration/runner/requirements.txt index 2c9df73ccca3..f1ab83e89ec4 100644 --- a/docker/test/integration/runner/requirements.txt +++ b/docker/test/integration/runner/requirements.txt @@ -1,116 +1,116 @@ -PyHDFS==0.3.1 -PyJWT==2.3.0 -PyMySQL==1.1.0 -PyNaCl==1.5.0 -PyYAML==5.3.1 -SecretStorage==3.3.1 -argon2-cffi-bindings==21.2.0 -argon2-cffi==23.1.0 -async-timeout==4.0.3 -asyncio==3.4.3 -attrs==23.2.0 -avro==1.10.2 -azure-core==1.30.1 -azure-storage-blob==12.19.0 -bcrypt==4.1.3 -beautifulsoup4==4.12.3 -blinker==1.4 -boto3==1.34.24 -botocore==1.34.101 -bs4==0.0.2 -cassandra-driver==3.29.0 -certifi==2024.2.2 -cffi==1.16.0 -charset-normalizer==3.3.2 -click==8.1.7 -confluent-kafka==2.3.0 -cryptography==3.4.8 -dbus-python==1.2.18 -decorator==5.1.1 -delta-spark==2.3.0 -dict2xml==1.7.4 -dicttoxml==1.7.16 -distro-info==1.1+ubuntu0.2 -distro==1.7.0 -docker-compose==1.29.2 -docker==6.1.3 -dockerpty==0.4.1 -docopt==0.6.2 -exceptiongroup==1.2.1 -execnet==2.1.1 -geomet==0.2.1.post1 -grpcio-tools==1.60.0 -grpcio==1.60.0 -gssapi==1.8.3 -httplib2==0.20.2 -idna==3.7 -importlib-metadata==4.6.4 -iniconfig==2.0.0 -isodate==0.6.1 -jeepney==0.7.1 -jmespath==1.0.1 -jsonschema==3.2.0 -jwcrypto==1.5.6 -kafka-python==2.0.2 -kazoo==2.9.0 -keyring==23.5.0 -krb5==0.5.1 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -lxml==5.1.0 -lz4==4.3.3 -minio==7.2.3 -more-itertools==8.10.0 -nats-py==2.6.0 -oauthlib==3.2.0 -packaging==24.0 -paramiko==3.4.0 -pika==1.2.0 -pip==24.1.1 -pipdeptree==2.23.0 -pluggy==1.5.0 -protobuf==4.25.2 -psycopg2-binary==2.9.6 -py4j==0.10.9.5 -py==1.11.0 -pyarrow==17.0.0 -pycparser==2.22 -pycryptodome==3.20.0 -pymongo==3.11.0 -pyparsing==2.4.7 -pyrsistent==0.20.0 -pyspark==3.3.2 -pyspnego==0.10.2 -pytest-order==1.0.0 -pytest-random==0.2 -pytest-repeat==0.9.3 -pytest-reportlog==0.4.0 -pytest-timeout==2.2.0 -pytest-xdist==3.5.0 -pytest==7.4.4 -python-apt==2.4.0+ubuntu3 -python-dateutil==2.9.0.post0 -python-dotenv==0.21.1 -pytz==2023.3.post1 -redis==5.0.1 -requests-kerberos==0.14.0 -requests==2.31.0 -retry==0.9.2 -s3transfer==0.10.1 -setuptools==59.6.0 -simplejson==3.19.2 -six==1.16.0 -soupsieve==2.5 -texttable==1.7.0 -tomli==2.0.1 -typing_extensions==4.11.0 -tzlocal==2.1 -unattended-upgrades==0.1 -urllib3==2.0.7 -wadllib==1.3.6 -websocket-client==0.59.0 -wheel==0.37.1 -zipp==1.0.0 +PyHDFS~=0.3.1 +PyJWT~=2.3.0 +PyMySQL~=1.1.0 +PyNaCl~=1.5.0 +PyYAML~=5.3.1 +SecretStorage~=3.3.1 +argon2-cffi-bindings~=21.2.0 +argon2-cffi~=23.1.0 +async-timeout~=4.0.3 +asyncio~=3.4.3 +attrs~=23.2.0 +avro~=1.10.2 +azure-core~=1.30.1 +azure-storage-blob~=12.19.0 +bcrypt~=4.1.3 +beautifulsoup4~=4.12.3 +blinker~=1.4 +boto3~=1.34.24 +botocore~=1.34.101 +bs4~=0.0.2 +cassandra-driver~=3.29.0 +certifi +cffi~=1.16.0 +charset-normalizer~=3.3.2 +click~=8.1.7 +confluent-kafka~=2.3.0 +cryptography~=3.4.8 +dbus-python~=1.2.18 +decorator~=5.1.1 +delta-spark~=2.3.0 +dict2xml~=1.7.4 +dicttoxml~=1.7.16 +distro-info~=1.1 +distro~=1.7.0 +docker-compose~=1.29.2 +docker~=6.1.3 +dockerpty~=0.4.1 +docopt~=0.6.2 +exceptiongroup~=1.2.1 +execnet~=2.1.1 +geomet~=0.2.1 +grpcio-tools~=1.60.0 +grpcio~=1.60.0 +gssapi~=1.8.3 +httplib2~=0.20.2 +idna~=3.7 +importlib-metadata~=4.6.4 +iniconfig~=2.0.0 +isodate~=0.6.1 +jeepney~=0.7.1 +jmespath~=1.0.1 +jsonschema~=3.2.0 +jwcrypto~=1.5.6 +kafka-python~=2.0.2 +kazoo~=2.9.0 +keyring~=23.5.0 +krb5~=0.5.1 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +lxml~=5.1.0 +lz4~=4.3.3 +minio~=7.2.3 +more-itertools~=8.10.0 +nats-py~=2.6.0 +oauthlib~=3.2.0 +packaging~=24.0 +paramiko~=3.4.0 +pika~=1.2.0 +pip~=24.1.1 +pipdeptree~=2.23.0 +pluggy~=1.5.0 +protobuf~=4.25.2 +psycopg2-binary~=2.9.6 +py4j~=0.10.9.5 +py~=1.11.0 +pyarrow~=17.0.0 +pycparser~=2.22 +pycryptodome~=3.20.0 +pymongo~=3.11.0 +pyparsing~=2.4.7 +pyrsistent~=0.20.0 +pyspark~=3.3.2 +pyspnego~=0.10.2 +pytest-order~=1.0.0 +pytest-random~=0.2 +pytest-repeat~=0.9.3 +pytest-reportlog~=0.4.0 +pytest-timeout~=2.2.0 +pytest-xdist~=3.5.0 +pytest~=7.4.4 +python-apt~=2.4.0 +python-dateutil~=2.9.0 +python-dotenv~=0.21.1 +pytz +redis~=5.0.1 +requests-kerberos~=0.14.0 +requests~=2.31.0 +retry~=0.9.2 +s3transfer~=0.10.1 +setuptools~=59.6.0 +simplejson~=3.19.2 +six~=1.16.0 +soupsieve~=2.5 +texttable~=1.7.0 +tomli~=2.0.1 +typing_extensions~=4.11.0 +tzlocal~=2.1 +unattended-upgrades~=0.1 +urllib3~=2.0.7 +wadllib~=1.3.6 +websocket-client~=0.59.0 +wheel~=0.37.1 +zipp~=1.0.0 deltalake==0.16.0 diff --git a/docker/test/integration/s3_proxy/Dockerfile b/docker/test/integration/s3_proxy/Dockerfile index 5858218e4e4c..df8d8f00f216 100644 --- a/docker/test/integration/s3_proxy/Dockerfile +++ b/docker/test/integration/s3_proxy/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/s3-proxy . +# docker build -t altinityinfra/s3-proxy . FROM nginx:alpine COPY run.sh /run.sh diff --git a/docker/test/keeper-jepsen/Dockerfile b/docker/test/keeper-jepsen/Dockerfile index 3c5d0a6ecb42..d3080a526711 100644 --- a/docker/test/keeper-jepsen/Dockerfile +++ b/docker/test/keeper-jepsen/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/keeper-jepsen-test . +# docker build -t altinityinfra/keeper-jepsen-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/libfuzzer/Dockerfile b/docker/test/libfuzzer/Dockerfile index 3ffae0cd921f..6f3f3aeb6eb7 100644 --- a/docker/test/libfuzzer/Dockerfile +++ b/docker/test/libfuzzer/Dockerfile @@ -1,6 +1,6 @@ -# docker build -t clickhouse/libfuzzer . +# docker build -t altinityinfra/libfuzzer . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" @@ -37,5 +37,5 @@ ENV FUZZER_ARGS="-max_total_time=60" SHELL ["/bin/bash", "-c"] -# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer +# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> altinityinfra/libfuzzer diff --git a/docker/test/libfuzzer/requirements.txt b/docker/test/libfuzzer/requirements.txt index 3dce93e023b2..fa70d89e7ac3 100644 --- a/docker/test/libfuzzer/requirements.txt +++ b/docker/test/libfuzzer/requirements.txt @@ -1,27 +1,27 @@ -blinker==1.4 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -Jinja2==3.1.4 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -MarkupSafe==2.1.5 -more-itertools==8.10.0 -oauthlib==3.2.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +Jinja2~=3.1.4 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +MarkupSafe~=2.1.5 +more-itertools~=8.10.0 +oauthlib~=3.2.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index f71392752826..ad4866ed742a 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -1,7 +1,7 @@ -# docker build -t clickhouse/performance-comparison . +# docker build -t altinityinfra/performance-comparison . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ @@ -46,4 +46,4 @@ ENV PATH="/opt/gdb/bin:${PATH}" CMD ["bash", "/run.sh"] -# docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/performance-comparison +# docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> altinityinfra/performance-comparison diff --git a/docker/test/performance-comparison/requirements.txt b/docker/test/performance-comparison/requirements.txt index 932527cc0226..eb927584d593 100644 --- a/docker/test/performance-comparison/requirements.txt +++ b/docker/test/performance-comparison/requirements.txt @@ -1,32 +1,32 @@ -blinker==1.4 -clickhouse-driver==0.2.7 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -more-itertools==8.10.0 -numpy==1.26.3 -oauthlib==3.2.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -Pygments==2.11.2 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -pytz==2023.4 -PyYAML==6.0.1 -scipy==1.12.0 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -tzlocal==2.1 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +clickhouse-driver~=0.2.7 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +more-itertools~=8.10.0 +numpy~=1.26.3 +oauthlib~=3.2.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +Pygments~=2.11.2 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4 +pytz +PyYAML~=6.0.1 +scipy~=1.12.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +tzlocal~=2.1 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/server-jepsen/Dockerfile b/docker/test/server-jepsen/Dockerfile index fd70fc457020..5207f31b953f 100644 --- a/docker/test/server-jepsen/Dockerfile +++ b/docker/test/server-jepsen/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/server-jepsen-test . +# docker build -t altinityinfra/server-jepsen-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/sqlancer/Dockerfile b/docker/test/sqlancer/Dockerfile index 3b919ffb3e39..b77066bcbac4 100644 --- a/docker/test/sqlancer/Dockerfile +++ b/docker/test/sqlancer/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/sqlancer-test . +# docker build -t altinityinfra/sqlancer-test . FROM ubuntu:22.04 # ARG for quick switch to a given ubuntu mirror diff --git a/docker/test/sqllogic/Dockerfile b/docker/test/sqllogic/Dockerfile index 0d21a2da44ee..767e5eecfa84 100644 --- a/docker/test/sqllogic/Dockerfile +++ b/docker/test/sqllogic/Dockerfile @@ -1,6 +1,6 @@ -# docker build -t clickhouse/sqllogic-test . +# docker build -t altinityinfra/sqllogic-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG RUN apt-get update --yes \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/sqllogic/requirements.txt b/docker/test/sqllogic/requirements.txt index abc0a3686590..3a9ec5f1c556 100644 --- a/docker/test/sqllogic/requirements.txt +++ b/docker/test/sqllogic/requirements.txt @@ -1,30 +1,30 @@ -blinker==1.4 -cryptography==3.4.8 -dbus-python==1.2.18 -deepdiff==7.0.1 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -more-itertools==8.10.0 -numpy==1.26.4 -oauthlib==3.2.0 -ordered-set==4.1.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -PyJWT==2.3.0 -pyodbc==5.1.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -sqlglot==23.16.0 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +cryptography~=3.4.8 +dbus-python~=1.2.18 +deepdiff~=7.0.1 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +more-itertools~=8.10.0 +numpy~=1.26.4 +oauthlib~=3.2.0 +ordered-set~=4.1.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +PyJWT~=2.3.0 +pyodbc~=5.1.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +sqlglot~=23.16.0 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/sqltest/Dockerfile b/docker/test/sqltest/Dockerfile index b805bb03c2b0..e21cb2d7febb 100644 --- a/docker/test/sqltest/Dockerfile +++ b/docker/test/sqltest/Dockerfile @@ -1,6 +1,6 @@ -# docker build -t clickhouse/sqltest . +# docker build -t altinityinfra/sqltest . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG RUN apt-get update --yes \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/sqltest/requirements.txt b/docker/test/sqltest/requirements.txt index 4a0ae3edbacd..4676ffd9ae8f 100644 --- a/docker/test/sqltest/requirements.txt +++ b/docker/test/sqltest/requirements.txt @@ -1,29 +1,29 @@ -blinker==1.4 -clickhouse-driver==0.2.7 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -more-itertools==8.10.0 -oauthlib==3.2.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -pytz==2024.1 -PyYAML==6.0.1 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -tzlocal==5.2 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +clickhouse-driver~=0.2.7 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +more-itertools~=8.10.0 +oauthlib~=3.2.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +pytz +PyYAML~=6.0.1 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +tzlocal~=5.2 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 0daf88cad7ee..e0f77526e242 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #47031 -# docker build -t clickhouse/stateful-test . +# docker build -t altinityinfra/stateful-test . ARG FROM_TAG=latest -FROM clickhouse/stateless-test:$FROM_TAG +FROM altinityinfra/stateless-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index c072eeb0fa8b..af578423b043 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -22,8 +22,7 @@ source /utils.lib # install test configs /usr/share/clickhouse-test/config/install.sh -azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence & - +azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & ./setup_minio.sh stateful ./mc admin trace clickminio > /test_output/minio.log & MC_ADMIN_PID=$! diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index b0c4914a4e88..7e57ad900a49 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/stateless-test . +# docker build -t altinityinfra/stateless-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz" @@ -87,7 +87,7 @@ ENV MINIO_ROOT_PASSWORD="clickhouse" ENV EXPORT_S3_STORAGE_POLICIES=1 ENV CLICKHOUSE_GRPC_CLIENT="/usr/share/clickhouse-utils/grpc-client/clickhouse-grpc-client.py" -RUN npm install -g azurite@3.30.0 \ +RUN npm install -g azurite@^3.33.0 \ && npm install -g tslib && npm install -g node COPY run.sh / diff --git a/docker/test/stateless/requirements.txt b/docker/test/stateless/requirements.txt index 74860d5fec3a..1537cd261900 100644 --- a/docker/test/stateless/requirements.txt +++ b/docker/test/stateless/requirements.txt @@ -1,53 +1,53 @@ -awscli==1.22.34 -blinker==1.4 -botocore==1.23.34 -certifi==2020.6.20 -chardet==4.0.0 -colorama==0.4.4 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -docutils==0.17.1 -grpcio==1.47.0 -gyp==0.1 -httplib2==0.20.2 -idna==3.3 -importlib-metadata==4.6.4 -jeepney==0.7.1 -Jinja2==3.1.3 -jmespath==0.10.0 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -lxml==4.8.0 -MarkupSafe==2.1.5 -more-itertools==8.10.0 -numpy==1.26.3 -oauthlib==3.2.0 -packaging==24.1 -pandas==1.5.3 -pip==24.1.1 -pipdeptree==2.23.0 -protobuf==4.25.3 -pyarrow==15.0.0 -pyasn1==0.4.8 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -python-dateutil==2.8.1 -pytz==2024.1 -PyYAML==6.0.1 -requests==2.32.3 -roman==3.3 -rsa==4.8 -s3transfer==0.5.0 -scipy==1.12.0 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -termcolor==1.1.0 -urllib3==1.26.5 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +awscli~=1.22.34 +blinker~=1.4 +botocore~=1.23.34 +certifi +chardet~=4.0.0 +colorama~=0.4.4 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +docutils~=0.17.1 +grpcio~=1.47.0 +gyp~=0.1 +httplib2~=0.20.2 +idna~=3.3 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +Jinja2~=3.1.3 +jmespath~=0.10.0 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +lxml~=4.8.0 +MarkupSafe~=2.1.5 +more-itertools~=8.10.0 +numpy~=1.26.3 +oauthlib~=3.2.0 +packaging~=24.1 +pandas~=1.5.3 +pip~=24.1.1 +pipdeptree~=2.23.0 +protobuf~=4.25.3 +pyarrow~=15.0.0 +pyasn1~=0.4.8 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +python-dateutil~=2.8.1 +pytz +PyYAML~=6.0.1 +requests~=2.32.3 +roman~=3.3 +rsa~=4.8 +s3transfer~=0.5.0 +scipy~=1.12.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +termcolor~=1.1.0 +urllib3~=1.26.5 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index ad0cd321cc55..13d79243713b 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -50,6 +50,12 @@ source /utils.lib # install test configs /usr/share/clickhouse-test/config/install.sh +if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then + echo "Azure is disabled" +else + azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & +fi + ./setup_minio.sh stateless ./setup_hdfs_minicluster.sh diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index 0f81a1cd07fb..507bf90bc2dd 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/stress-test . +# docker build -t altinityinfra/stress-test . ARG FROM_TAG=latest -FROM clickhouse/stateful-test:$FROM_TAG +FROM altinityinfra/stateful-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index b21114e456f4..48348aa131ca 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -55,6 +55,7 @@ export ZOOKEEPER_FAULT_INJECTION=1 # available for dump via clickhouse-local configure +azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & ./setup_minio.sh stateless # to have a proper environment config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index cdc1d1fa095d..88894e444c63 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/style-test . +# docker build -t altinityinfra/style-test . FROM ubuntu:22.04 ARG ACT_VERSION=0.2.33 ARG ACTIONLINT_VERSION=1.6.22 diff --git a/docker/test/style/requirements.txt b/docker/test/style/requirements.txt index ed73d0d3636c..b3f62d122efb 100644 --- a/docker/test/style/requirements.txt +++ b/docker/test/style/requirements.txt @@ -1,58 +1,58 @@ -aiohttp==3.9.5 -aiosignal==1.3.1 -astroid==3.1.0 -async-timeout==4.0.3 -attrs==23.2.0 -black==24.4.2 -boto3==1.34.131 -botocore==1.34.131 -certifi==2024.6.2 -cffi==1.16.0 -charset-normalizer==3.3.2 -click==8.1.7 -codespell==2.2.1 -cryptography==42.0.8 -Deprecated==1.2.14 -dill==0.3.8 -flake8==4.0.1 -frozenlist==1.4.1 -idna==3.7 -isort==5.13.2 -jmespath==1.0.1 -jwt==1.3.1 -mccabe==0.6.1 -multidict==6.0.5 -mypy==1.8.0 -mypy-extensions==1.0.0 -packaging==24.1 -pathspec==0.9.0 -pip==24.1.1 -pipdeptree==2.23.0 -platformdirs==4.2.2 -pycodestyle==2.8.0 -pycparser==2.22 -pyflakes==2.4.0 -PyGithub==2.3.0 -PyJWT==2.8.0 -pylint==3.1.0 -PyNaCl==1.5.0 -python-dateutil==2.9.0.post0 -python-magic==0.4.24 -PyYAML==6.0.1 -rapidfuzz==3.9.3 -requests==2.32.3 -s3transfer==0.10.1 -setuptools==59.6.0 -six==1.16.0 -thefuzz==0.22.1 -tomli==2.0.1 -tomlkit==0.12.5 -tqdm==4.66.4 -types-requests==2.32.0.20240622 -typing_extensions==4.12.2 -unidiff==0.7.5 -urllib3==2.2.2 -wheel==0.37.1 -wrapt==1.16.0 -yamllint==1.26.3 -yarl==1.9.4 +aiohttp~=3.9.5 +aiosignal~=1.3.1 +astroid~=3.1.0 +async-timeout~=4.0.3 +attrs~=23.2.0 +black~=24.4.2 +boto3~=1.34.131 +botocore~=1.34.131 +certifi +cffi~=1.16.0 +charset-normalizer~=3.3.2 +click~=8.1.7 +codespell~=2.2.1 +cryptography~=42.0.8 +Deprecated~=1.2.14 +dill~=0.3.8 +flake8~=4.0.1 +frozenlist~=1.4.1 +idna~=3.7 +isort~=5.13.2 +jmespath~=1.0.1 +jwt~=1.3.1 +mccabe~=0.6.1 +multidict~=6.0.5 +mypy~=1.8.0 +mypy-extensions~=1.0.0 +packaging~=24.1 +pathspec~=0.9.0 +pip~=24.1.1 +pipdeptree~=2.23.0 +platformdirs~=4.2.2 +pycodestyle~=2.8.0 +pycparser~=2.22 +pyflakes~=2.4.0 +PyGithub~=2.3.0 +PyJWT~=2.8.0 +pylint~=3.1.0 +PyNaCl~=1.5.0 +python-dateutil~=2.9.0 +python-magic~=0.4.24 +PyYAML~=6.0.1 +rapidfuzz~=3.9.3 +requests~=2.32.3 +s3transfer~=0.10.1 +setuptools~=59.6.0 +six~=1.16.0 +thefuzz~=0.22.1 +tomli~=2.0.1 +tomlkit~=0.12.5 +tqdm~=4.66.4 +types-requests~=2.32.0 +typing_extensions~=4.12.2 +unidiff~=0.7.5 +urllib3~=2.2.2 +wheel~=0.37.1 +wrapt~=1.16.0 +yamllint~=1.26.3 +yarl~=1.9.4 diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index 9f4b86aa0ca7..adc72011bd1d 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/unit-test . +# docker build -t altinityinfra/unit-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG COPY run.sh / RUN chmod +x run.sh diff --git a/docker/test/upgrade/Dockerfile b/docker/test/upgrade/Dockerfile index 78d912fd0312..c66868c2a046 100644 --- a/docker/test/upgrade/Dockerfile +++ b/docker/test/upgrade/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/upgrade-check . +# docker build -t altinityinfra/upgrade-check . ARG FROM_TAG=latest -FROM clickhouse/stateful-test:$FROM_TAG +FROM altinityinfra/stateful-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index 8b949ed95dbd..9c4d75ffd2f4 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/test-util . +# docker build -t altinityinfra/test-util . FROM ubuntu:22.04 # ARG for quick switch to a given ubuntu mirror diff --git a/docs/logo_horizontal_blue_black.png b/docs/logo_horizontal_blue_black.png new file mode 100644 index 000000000000..b67e8ed45385 Binary files /dev/null and b/docs/logo_horizontal_blue_black.png differ diff --git a/docs/logo_horizontal_blue_white.png b/docs/logo_horizontal_blue_white.png new file mode 100644 index 000000000000..44ed6cca6d75 Binary files /dev/null and b/docs/logo_horizontal_blue_white.png differ diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index d40e320e741c..0e9e0dbb7ac4 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -51,12 +51,12 @@ AsyncLoader::Pool::Pool(const AsyncLoader::PoolInitializer & init) , priority(init.priority) , max_threads(init.max_threads > 0 ? init.max_threads : getNumberOfPhysicalCPUCores()) , thread_pool(std::make_unique( - init.metric_threads, - init.metric_active_threads, - init.metric_scheduled_threads, - /* max_threads = */ std::numeric_limits::max(), // Unlimited number of threads, we do worker management ourselves - /* max_free_threads = */ 0, // We do not require free threads - /* queue_size = */0)) // Unlimited queue to avoid blocking during worker spawning + init.metric_threads, + init.metric_active_threads, + init.metric_scheduled_threads, + /* max_threads = */ ThreadPool::MAX_THEORETICAL_THREAD_COUNT, // Unlimited number of threads, we do worker management ourselves + /* max_free_threads = */ 0, // We do not require free threads + /* queue_size = */ 0)) // Unlimited queue to avoid blocking during worker spawning {} AsyncLoader::Pool::Pool(Pool&& o) noexcept diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 23ef44f3b323..adc074738786 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -75,9 +75,9 @@ M(GlobalThread, "Number of threads in global thread pool.") \ M(GlobalThreadActive, "Number of threads in global thread pool running a task.") \ M(GlobalThreadScheduled, "Number of queued or active jobs in global thread pool.") \ - M(LocalThread, "Number of threads in local thread pools. The threads in local thread pools are taken from the global thread pool.") \ - M(LocalThreadActive, "Number of threads in local thread pools running a task.") \ - M(LocalThreadScheduled, "Number of queued or active jobs in local thread pools.") \ + M(LocalThread, "Obsolete. Number of threads in local thread pools. The threads in local thread pools are taken from the global thread pool.") \ + M(LocalThreadActive, "Obsolete. Number of threads in local thread pools running a task.") \ + M(LocalThreadScheduled, "Obsolete. Number of queued or active jobs in local thread pools.") \ M(MergeTreeDataSelectExecutorThreads, "Number of threads in the MergeTreeDataSelectExecutor thread pool.") \ M(MergeTreeDataSelectExecutorThreadsActive, "Number of threads in the MergeTreeDataSelectExecutor thread pool running a task.") \ M(MergeTreeDataSelectExecutorThreadsScheduled, "Number of queued or active jobs in the MergeTreeDataSelectExecutor thread pool.") \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index d43d9fdcea8e..044f787aee98 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -86,6 +86,20 @@ M(NetworkReceiveBytes, "Total number of bytes received from network. Only ClickHouse-related network interaction is included, not by 3rd party libraries.") \ M(NetworkSendBytes, "Total number of bytes send to network. Only ClickHouse-related network interaction is included, not by 3rd party libraries.") \ \ + M(GlobalThreadPoolExpansions, "Counts the total number of times new threads have been added to the global thread pool. This metric indicates the frequency of expansions in the global thread pool to accommodate increased processing demands.") \ + M(GlobalThreadPoolShrinks, "Counts the total number of times the global thread pool has shrunk by removing threads. This occurs when the number of idle threads exceeds max_thread_pool_free_size, indicating adjustments in the global thread pool size in response to decreased thread utilization.") \ + M(GlobalThreadPoolThreadCreationMicroseconds, "Total time spent waiting for new threads to start.") \ + M(GlobalThreadPoolLockWaitMicroseconds, "Total time threads have spent waiting for locks in the global thread pool.") \ + M(GlobalThreadPoolJobs, "Counts the number of jobs that have been pushed to the global thread pool.") \ + M(GlobalThreadPoolJobWaitTimeMicroseconds, "Measures the elapsed time from when a job is scheduled in the thread pool to when it is picked up for execution by a worker thread. This metric helps identify delays in job processing, indicating the responsiveness of the thread pool to new tasks.") \ + M(LocalThreadPoolExpansions, "Counts the total number of times threads have been borrowed from the global thread pool to expand local thread pools.") \ + M(LocalThreadPoolShrinks, "Counts the total number of times threads have been returned to the global thread pool from local thread pools.") \ + M(LocalThreadPoolThreadCreationMicroseconds, "Total time local thread pools have spent waiting to borrow a thread from the global pool.") \ + M(LocalThreadPoolLockWaitMicroseconds, "Total time threads have spent waiting for locks in the local thread pools.") \ + M(LocalThreadPoolJobs, "Counts the number of jobs that have been pushed to the local thread pools.") \ + M(LocalThreadPoolBusyMicroseconds, "Total time threads have spent executing the actual work.") \ + M(LocalThreadPoolJobWaitTimeMicroseconds, "Measures the elapsed time from when a job is scheduled in the thread pool to when it is picked up for execution by a worker thread. This metric helps identify delays in job processing, indicating the responsiveness of the thread pool to new tasks.") \ + \ M(DiskS3GetRequestThrottlerCount, "Number of DiskS3 GET and SELECT requests passed through throttler.") \ M(DiskS3GetRequestThrottlerSleepMicroseconds, "Total time a query was sleeping to conform DiskS3 GET and SELECT request throttling.") \ M(DiskS3PutRequestThrottlerCount, "Number of DiskS3 PUT, COPY, POST and LIST requests passed through throttler.") \ diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index c8f1ae999698..5dfefcbfccf1 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -27,6 +28,66 @@ namespace CurrentMetrics extern const Metric GlobalThreadScheduled; } +namespace ProfileEvents +{ + extern const Event GlobalThreadPoolExpansions; + extern const Event GlobalThreadPoolShrinks; + extern const Event GlobalThreadPoolThreadCreationMicroseconds; + extern const Event GlobalThreadPoolLockWaitMicroseconds; + extern const Event GlobalThreadPoolJobs; + extern const Event GlobalThreadPoolJobWaitTimeMicroseconds; + + extern const Event LocalThreadPoolExpansions; + extern const Event LocalThreadPoolShrinks; + extern const Event LocalThreadPoolThreadCreationMicroseconds; + extern const Event LocalThreadPoolLockWaitMicroseconds; + extern const Event LocalThreadPoolJobs; + extern const Event LocalThreadPoolBusyMicroseconds; + extern const Event LocalThreadPoolJobWaitTimeMicroseconds; + +} + +namespace +{ + struct ScopedDecrement + { + std::optional>> atomic_var; + + // Deleted copy constructor and copy assignment operator + ScopedDecrement(const ScopedDecrement&) = delete; + ScopedDecrement& operator=(const ScopedDecrement&) = delete; + + // Move constructor + ScopedDecrement(ScopedDecrement&& other) noexcept + : atomic_var(std::move(other.atomic_var)) + { + other.atomic_var.reset(); + } + + // Move assignment operator + ScopedDecrement& operator=(ScopedDecrement&& other) noexcept + { + if (this != &other) + { + atomic_var.swap(other.atomic_var); + } + return *this; + } + + explicit ScopedDecrement(std::atomic& var) + : atomic_var(var) + { + atomic_var->get().fetch_sub(1, std::memory_order_relaxed); + } + + ~ScopedDecrement() + { + if (atomic_var) + atomic_var->get().fetch_add(1, std::memory_order_relaxed); + } + }; +} + class JobWithPriority { public: @@ -35,17 +96,29 @@ class JobWithPriority Job job; Priority priority; CurrentMetrics::Increment metric_increment; + ScopedDecrement available_threads_decrement; + DB::OpenTelemetry::TracingContextOnThread thread_trace_context; /// Call stacks of all jobs' schedulings leading to this one std::vector frame_pointers; bool enable_job_stack_trace = false; + Stopwatch job_create_time; + + // Deleted copy constructor and copy assignment operator + JobWithPriority(const JobWithPriority&) = delete; + JobWithPriority& operator=(const JobWithPriority&) = delete; + + // Move constructor and move assignment operator + JobWithPriority(JobWithPriority&&) noexcept = default; + JobWithPriority& operator=(JobWithPriority&&) noexcept = default; JobWithPriority( Job job_, Priority priority_, CurrentMetrics::Metric metric, const DB::OpenTelemetry::TracingContextOnThread & thread_trace_context_, - bool capture_frame_pointers) + bool capture_frame_pointers, ScopedDecrement available_threads_decrement_) : job(job_), priority(priority_), metric_increment(metric), + available_threads_decrement(std::move(available_threads_decrement_)), thread_trace_context(thread_trace_context_), enable_job_stack_trace(capture_frame_pointers) { if (!capture_frame_pointers) @@ -59,6 +132,11 @@ class JobWithPriority { return priority > rhs.priority; // Reversed for `priority_queue` max-heap to yield minimum value (i.e. highest priority) first } + + UInt64 elapsedMicroseconds() const + { + return job_create_time.elapsedMicroseconds(); + } }; static constexpr auto DEFAULT_THREAD_NAME = "ThreadPool"; @@ -97,12 +175,19 @@ ThreadPoolImpl::ThreadPoolImpl( , queue_size(queue_size_ ? std::max(queue_size_, max_threads) : 0 /* zero means the queue is unlimited */) , shutdown_on_exception(shutdown_on_exception_) { + max_threads = std::min(max_threads, static_cast(MAX_THEORETICAL_THREAD_COUNT)); + max_free_threads = std::min(max_free_threads, static_cast(MAX_THEORETICAL_THREAD_COUNT)); + remaining_pool_capacity.store(max_threads, std::memory_order_relaxed); + available_threads.store(0, std::memory_order_relaxed); } template void ThreadPoolImpl::setMaxThreads(size_t value) { + value = std::min(value, static_cast(MAX_THEORETICAL_THREAD_COUNT)); std::lock_guard lock(mutex); + remaining_pool_capacity.fetch_add(value - max_threads, std::memory_order_relaxed); + bool need_start_threads = (value > max_threads); bool need_finish_free_threads = (value < max_free_threads); @@ -135,6 +220,7 @@ size_t ThreadPoolImpl::getMaxThreads() const template void ThreadPoolImpl::setMaxFreeThreads(size_t value) { + value = std::min(value, static_cast(MAX_THEORETICAL_THREAD_COUNT)); std::lock_guard lock(mutex); bool need_finish_free_threads = (value < max_free_threads); @@ -156,7 +242,6 @@ void ThreadPoolImpl::setQueueSize(size_t value) jobs.reserve(queue_size); } - template template ReturnType ThreadPoolImpl::scheduleImpl(Job job, Priority priority, std::optional wait_microseconds, bool propagate_opentelemetry_tracing_context) @@ -179,15 +264,52 @@ ReturnType ThreadPoolImpl::scheduleImpl(Job job, Priority priority, std: return false; }; + // Decrement available_threads, scoped to the job lifecycle. + // This ensures that available_threads decreases when a new job starts + // and automatically increments when the job completes or goes out of scope. + ScopedDecrement available_threads_decrement(available_threads); + + std::unique_ptr new_thread; + + // Load the current capacity + int64_t capacity = remaining_pool_capacity.load(std::memory_order_relaxed); + int64_t currently_available_threads = available_threads.load(std::memory_order_relaxed); + + while (currently_available_threads <= 0 && capacity > 0) { + if (remaining_pool_capacity.compare_exchange_weak(capacity, capacity - 1, std::memory_order_relaxed)) + { + try + { + new_thread = std::make_unique(*this); + break; // Exit the loop once a thread is successfully created. + } + catch (...) + { + // Failed to create the thread, restore capacity + remaining_pool_capacity.fetch_add(1, std::memory_order_relaxed); + std::lock_guard lock(mutex); // needed to change first_exception. + return on_error("failed to start the thread"); + } + } + // capacity gets reloaded by (unsuccessful) compare_exchange_weak + currently_available_threads = available_threads.load(std::memory_order_relaxed); + } + + { + Stopwatch watch; std::unique_lock lock(mutex); + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolLockWaitMicroseconds : ProfileEvents::LocalThreadPoolLockWaitMicroseconds, + watch.elapsedMicroseconds()); if (CannotAllocateThreadFaultInjector::injectFault()) return on_error("fault injected"); auto pred = [this] { return !queue_size || scheduled_jobs < queue_size || shutdown; }; - if (wait_microseconds) /// Check for optional. Condition is true if the optional is set and the value is zero. + /// Wait for available threads or timeout + if (wait_microseconds) /// Check for optional. Condition is true if the optional is set. Even if the value is zero. { if (!job_finished.wait_for(lock, std::chrono::microseconds(*wait_microseconds), pred)) return on_error(fmt::format("no free thread (timeout={})", *wait_microseconds)); @@ -198,47 +320,97 @@ ReturnType ThreadPoolImpl::scheduleImpl(Job job, Priority priority, std: if (shutdown) return on_error("shutdown"); - /// We must not to allocate any memory after we emplaced a job in a queue. - /// Because if an exception would be thrown, we won't notify a thread about job occurrence. - - /// Check if there are enough threads to process job. - if (threads.size() < std::min(max_threads, scheduled_jobs + 1)) + /// We must not allocate memory or perform operations that could throw exceptions after adding a job to the queue, + /// because if an exception occurs, it may leave the job in the queue without notifying any threads. + typename ThreadFromThreadPool::ThreadList::iterator thread_slot; + + /// The decision to start a new thread is made outside the locked section. + /// However, thread load and demand can change dynamically, and decisions based on + /// atomic variables outside the critical section might become outdated by the time we acquire the lock. + /// This can lead to two possible scenarios: + /// + /// 1) Relatively common: A new thread was started outside the lock, but by the time we acquire the lock, + /// demand for threads has decreased (e.g., other threads have finished their jobs and are now idle). + /// In this case, even though there are now enough threads, we still attempt to add the new thread + /// to the pool, provided it does not exceed the `max_threads` or `max_free_threads` limits. Keeping + /// an extra thread in the pool may help accommodate a sudden increase in demand without the need + /// to wait for thread creation. + /// + /// 2) Very unlikely (but possible): Outside the lock, it appeared there were enough threads + /// to handle the workload. However, after acquiring the lock, it turns out the new thread + /// is needed (possibly because one of the existing threads was removed or became unavailable). + /// In this case, we create the thread inside the critical section, even though this may introduce + /// a small delay. + + /// Check if we can add the thread created outside the critical section to the pool. + bool adding_new_thread = new_thread && threads.size() < std::min(max_threads, 1 /* current job */ + scheduled_jobs + max_free_threads); + + // If we didn't create a new thread initially but realize we actually need one (unlikely scenario). + if (unlikely(!adding_new_thread && threads.size() < std::min(max_threads, scheduled_jobs + 1))) { try { - threads.emplace_front(); + remaining_pool_capacity.fetch_sub(1, std::memory_order_relaxed); + new_thread = std::make_unique(*this); } catch (...) { - /// Most likely this is a std::bad_alloc exception - return on_error("cannot allocate thread slot"); + // If thread creation fails, restore the pool capacity and return an error. + remaining_pool_capacity.fetch_add(1, std::memory_order_relaxed); + return on_error("failed to start the thread"); } + adding_new_thread = true; + } + if (adding_new_thread) + { try { - threads.front() = Thread([this, it = threads.begin()] { worker(it); }); + threads.emplace_front(std::move(new_thread)); + thread_slot = threads.begin(); } catch (...) { - threads.pop_front(); - return on_error("cannot allocate thread"); + /// Most likely this is a std::bad_alloc exception + return on_error("cannot emplace the thread in the pool"); } } + else // we have a thread but there is no space for that in the pool. + { + new_thread.reset(); + } - jobs.emplace(std::move(job), - priority, - metric_scheduled_jobs, - /// Tracing context on this thread is used as parent context for the sub-thread that runs the job - propagate_opentelemetry_tracing_context ? DB::OpenTelemetry::CurrentContext() : DB::OpenTelemetry::TracingContextOnThread(), - /// capture_frame_pointers - DB::Exception::enable_job_stack_trace); + try + { + jobs.emplace(std::move(job), + priority, + metric_scheduled_jobs, + /// Tracing context on this thread is used as parent context for the sub-thread that runs the job + propagate_opentelemetry_tracing_context ? DB::OpenTelemetry::CurrentContext() : DB::OpenTelemetry::TracingContextOnThread(), + /// capture_frame_pointers + DB::Exception::enable_job_stack_trace, + std::move(available_threads_decrement)); - ++scheduled_jobs; + ++scheduled_jobs; + + if (adding_new_thread) + (*thread_slot)->start(thread_slot); + + } + catch (...) + { + if (adding_new_thread) + threads.pop_front(); + + return on_error("cannot start the job or thread"); + } } /// Wake up a free thread to run the new job. new_job_or_shutdown.notify_one(); + ProfileEvents::increment(std::is_same_v ? ProfileEvents::GlobalThreadPoolJobs : ProfileEvents::LocalThreadPoolJobs); + return static_cast(true); } @@ -251,23 +423,51 @@ void ThreadPoolImpl::startNewThreadsNoLock() /// Start new threads while there are more scheduled jobs in the queue and the limit `max_threads` is not reached. while (threads.size() < std::min(scheduled_jobs, max_threads)) { + std::unique_ptr new_thread; + + int64_t capacity = remaining_pool_capacity.load(std::memory_order_relaxed); + + while (capacity > 0) + { + if (remaining_pool_capacity.compare_exchange_weak(capacity, capacity - 1, std::memory_order_relaxed)) + { + try + { + // Successfully decremented, attempt to create a new thread + new_thread = std::make_unique(*this); + } + catch (...) + { + // Failed to create the thread, restore capacity + remaining_pool_capacity.fetch_add(1, std::memory_order_relaxed); + } + break; // Exit loop whether thread creation succeeded or not + } + } + + if (!new_thread) + break; /// failed to start more threads + + typename ThreadFromThreadPool::ThreadList::iterator thread_slot; + try { - threads.emplace_front(); + threads.emplace_front(std::move(new_thread)); + thread_slot = threads.begin(); } catch (...) { - break; /// failed to start more threads + break; } try { - threads.front() = Thread([this, it = threads.begin()] { worker(it); }); + (*thread_slot)->start(thread_slot); } catch (...) { threads.pop_front(); - break; /// failed to start more threads + break; } } } @@ -293,7 +493,11 @@ void ThreadPoolImpl::scheduleOrThrow(Job job, Priority priority, uint64_ template void ThreadPoolImpl::wait() { + Stopwatch watch; std::unique_lock lock(mutex); + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolLockWaitMicroseconds : ProfileEvents::LocalThreadPoolLockWaitMicroseconds, + watch.elapsedMicroseconds()); /// Signal here just in case. /// If threads are waiting on condition variables, but there are some jobs in the queue /// then it will prevent us from deadlock. @@ -325,17 +529,29 @@ void ThreadPoolImpl::finalize() { std::lock_guard lock(mutex); shutdown = true; - /// We don't want threads to remove themselves from `threads` anymore, otherwise `thread.join()` will go wrong below in this function. + + /// scheduleImpl doesn't check for shutdown outside the critical section, + /// so we set remaining_pool_capacity to a large negative value + /// (e.g., -MAX_THEORETICAL_THREAD_COUNT) to signal that no new threads are needed. + /// This effectively prevents any new threads from being started during shutdown. + remaining_pool_capacity.store(-MAX_THEORETICAL_THREAD_COUNT, std::memory_order_relaxed); + + /// Disable thread self-removal from `threads`. Otherwise, if threads remove themselves, + /// the thread.join() operation will fail later in this function. threads_remove_themselves = false; } - /// Wake up threads so they can finish themselves. + /// Notify all threads to wake them up, so they can complete their work and exit gracefully. new_job_or_shutdown.notify_all(); - /// Wait for all currently running jobs to finish (we don't wait for all scheduled jobs here like the function wait() does). - for (auto & thread : threads) - thread.join(); + /// Join all threads before clearing the list + for (auto& thread_ptr : threads) + { + if (thread_ptr) + thread_ptr->join(); + } + // now it's safe to clear the threads threads.clear(); } @@ -371,11 +587,88 @@ bool ThreadPoolImpl::finished() const return shutdown; } + +template +ThreadPoolImpl::ThreadFromThreadPool::ThreadFromThreadPool(ThreadPoolImpl& parent_pool_) + : parent_pool(parent_pool_) + , thread_state(ThreadState::Preparing) // Initial state is Preparing +{ + Stopwatch watch2; + + thread = Thread(&ThreadFromThreadPool::worker, this); + + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolThreadCreationMicroseconds : ProfileEvents::LocalThreadPoolThreadCreationMicroseconds, + watch2.elapsedMicroseconds()); + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolExpansions : ProfileEvents::LocalThreadPoolExpansions); + + parent_pool.available_threads.fetch_add(1, std::memory_order_relaxed); +} + + +template +void ThreadPoolImpl::ThreadFromThreadPool::start(typename ThreadList::iterator & it) +{ + /// the thread which created ThreadFromThreadPool should start it after adding it to the pool, or destroy it. + /// no parallelism is expected here. So the only valid transition for the start method is Preparing to Running. + chassert(thread_state.load(std::memory_order_relaxed) == ThreadState::Preparing); + thread_it = it; + thread_state.store(ThreadState::Running, std::memory_order_relaxed); /// now worker can start executing the main loop +} + +template +void ThreadPoolImpl::ThreadFromThreadPool::join() +{ + // Ensure the thread is joined before destruction if still joinable + if (thread.joinable()) + thread.join(); +} + template -void ThreadPoolImpl::worker(typename std::list::iterator thread_it) +void ThreadPoolImpl::ThreadFromThreadPool::removeSelfFromPoolNoPoolLock() +{ + if (thread.joinable()) + thread.detach(); + + parent_pool.threads.erase(thread_it); +} + +template +ThreadPoolImpl::ThreadFromThreadPool::~ThreadFromThreadPool() +{ + parent_pool.available_threads.fetch_sub(1, std::memory_order_relaxed); + + // The thread is being destructed, so the remaining pool capacity increases + parent_pool.remaining_pool_capacity.fetch_add(1, std::memory_order_relaxed); + + // If the worker was still waiting in the loop for thread initialization, + // signal it to terminate and be destroyed now. + thread_state.store(ThreadState::Destructing, std::memory_order_relaxed); + + join(); + + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolShrinks : ProfileEvents::LocalThreadPoolShrinks); +} + + +template +void ThreadPoolImpl::ThreadFromThreadPool::worker() { DENY_ALLOCATIONS_IN_SCOPE; - CurrentMetrics::Increment metric_pool_threads(metric_threads); + + // wait until the thread will be started + while (thread_state.load(std::memory_order_relaxed) == ThreadState::Preparing) + { + std::this_thread::yield(); // let's try to yield to avoid consuming too much CPU in the busy-loop + } + + // If the thread transitions to Destructing, exit + if (thread_state.load(std::memory_order_relaxed) == ThreadState::Destructing) + return; + + CurrentMetrics::Increment metric_pool_threads(parent_pool.metric_threads); bool job_is_done = false; std::exception_ptr exception_from_job; @@ -391,7 +684,11 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ std::optional job_data; { - std::unique_lock lock(mutex); + Stopwatch watch; + std::unique_lock lock(parent_pool.mutex); + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolLockWaitMicroseconds : ProfileEvents::LocalThreadPoolLockWaitMicroseconds, + watch.elapsedMicroseconds()); // Finish with previous job if any if (job_is_done) @@ -399,42 +696,55 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ job_is_done = false; if (exception_from_job) { - if (!first_exception) - first_exception = exception_from_job; - if (shutdown_on_exception) - shutdown = true; + if (!parent_pool.first_exception) + parent_pool.first_exception = exception_from_job; + if (parent_pool.shutdown_on_exception) + { + parent_pool.shutdown = true; + + // Prevent new thread creation, as explained in finalize. + parent_pool.remaining_pool_capacity.store(-MAX_THEORETICAL_THREAD_COUNT, std::memory_order_relaxed); + } exception_from_job = {}; } - --scheduled_jobs; + --parent_pool.scheduled_jobs; - job_finished.notify_all(); - if (shutdown) - new_job_or_shutdown.notify_all(); /// `shutdown` was set, wake up other threads so they can finish themselves. + parent_pool.job_finished.notify_all(); + if (parent_pool.shutdown) + parent_pool.new_job_or_shutdown.notify_all(); /// `shutdown` was set, wake up other threads so they can finish themselves. } - new_job_or_shutdown.wait(lock, [&] { return !jobs.empty() || shutdown || threads.size() > std::min(max_threads, scheduled_jobs + max_free_threads); }); + parent_pool.new_job_or_shutdown.wait(lock, [this] { + return !parent_pool.jobs.empty() + || parent_pool.shutdown + || parent_pool.threads.size() > std::min(parent_pool.max_threads, parent_pool.scheduled_jobs + parent_pool.max_free_threads); + }); - if (jobs.empty() || threads.size() > std::min(max_threads, scheduled_jobs + max_free_threads)) + + if (parent_pool.jobs.empty() || parent_pool.threads.size() > std::min(parent_pool.max_threads, parent_pool.scheduled_jobs + parent_pool.max_free_threads)) { // We enter here if: // - either this thread is not needed anymore due to max_free_threads excess; // - or shutdown happened AND all jobs are already handled. - if (threads_remove_themselves) - { - thread_it->detach(); - threads.erase(thread_it); - } + + if (parent_pool.threads_remove_themselves) + removeSelfFromPoolNoPoolLock(); // Detach and remove itself from the pool + return; } /// boost::priority_queue does not provide interface for getting non-const reference to an element /// to prevent us from modifying its priority. We have to use const_cast to force move semantics on JobWithPriority. - job_data = std::move(const_cast(jobs.top())); - jobs.pop(); + job_data = std::move(const_cast(parent_pool.jobs.top())); + parent_pool.jobs.pop(); + + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolJobWaitTimeMicroseconds : ProfileEvents::LocalThreadPoolJobWaitTimeMicroseconds, + job_data->elapsedMicroseconds()); /// We don't run jobs after `shutdown` is set, but we have to properly dequeue all jobs and finish them. - if (shutdown) + if (parent_pool.shutdown) { { ALLOW_ALLOCATIONS_IN_SCOPE; @@ -457,9 +767,24 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ if (DB::Exception::enable_job_stack_trace) DB::Exception::setThreadFramePointers(std::move(job_data->frame_pointers)); - CurrentMetrics::Increment metric_active_pool_threads(metric_active_threads); + CurrentMetrics::Increment metric_active_pool_threads(parent_pool.metric_active_threads); + + if constexpr (!std::is_same_v) + { + Stopwatch watch; + job_data->job(); + // This metric is less relevant for the global thread pool, as it would show large values (time while + // a thread was used by local pools) and increment only when local pools are destroyed. + // + // In cases where global pool threads are used directly (without a local thread pool), distinguishing + // them is difficult. + ProfileEvents::increment(ProfileEvents::LocalThreadPoolBusyMicroseconds, watch.elapsedMicroseconds()); + } + else + { + job_data->job(); + } - job_data->job(); if (thread_trace_context.root_span.isTraceEnabled()) { @@ -495,7 +820,6 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ } } - template class ThreadPoolImpl; template class ThreadPoolImpl>; template class ThreadPoolImpl>; diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index a31d793264e9..7e497245acc3 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -32,7 +32,7 @@ class JobWithPriority; * * This thread pool can be used as a task queue. * For example, you can create a thread pool with 10 threads (and queue of size 10) and schedule 1000 tasks - * - in this case you will be blocked to keep 10 tasks in fly. + * - in this case you will be blocked to keep 10 tasks in flight. * * Thread: std::thread or something with identical interface. */ @@ -40,9 +40,57 @@ template class ThreadPoolImpl { public: + // used as 'unlimited' thread pool size + // on linux you can not have more threads even if the RAM is unlimited + // see https://docs.kernel.org/admin-guide/sysctl/kernel.html#threads-max + static constexpr int MAX_THEORETICAL_THREAD_COUNT = 0x3fffffff; // ~1 billion + using Job = std::function; using Metric = CurrentMetrics::Metric; + // Subclass that encapsulates the thread and has the ability to remove itself from the pool. + class ThreadFromThreadPool + { + public: + using ThreadList = std::list>; + + /// Constructor to initialize and start the thread (but not associate it with the pool) + explicit ThreadFromThreadPool(ThreadPoolImpl& parent_pool); + + // Shift the thread state from Preparing to Running to allow the worker to start. + void start(ThreadList::iterator& it); + + void join(); + + // Destructor to join the thread if needed (shift the state to Destructing if it was not running) + ~ThreadFromThreadPool(); + + private: + ThreadPoolImpl& parent_pool; + Thread thread; + + enum class ThreadState + { + Preparing, + Running, + Destructing + }; + + // Atomic state to track the thread's state + std::atomic thread_state; + + // Stores the position of the thread in the parent thread pool list + typename std::list>::iterator thread_it; + + // Remove itself from the parent pool + void removeSelfFromPoolNoPoolLock(); + + // Worker does a busy loop (with yield) while the state is Preparing. + // After that, immediately returns if the state changed to Destructing, + // or starts the main working loop if the state is Running. + void worker(); + }; + /// Maximum number of threads is based on the number of physical cores. ThreadPoolImpl(Metric metric_threads_, Metric metric_active_threads_, Metric metric_scheduled_jobs_); @@ -63,14 +111,14 @@ class ThreadPoolImpl size_t queue_size_, bool shutdown_on_exception_ = true); - /// Add new job. Locks until number of scheduled jobs is less than maximum or exception in one of threads was thrown. - /// If any thread was throw an exception, first exception will be rethrown from this method, - /// and exception will be cleared. + /// Add new job. Locks until the number of scheduled jobs is less than the maximum or an exception in one of the threads was thrown. + /// If any thread has thrown an exception, the first exception will be rethrown from this method, + /// and the exception will be cleared. /// Also throws an exception if cannot create thread. /// Priority: lower is higher. - /// NOTE: Probably you should call wait() if exception was thrown. If some previously scheduled jobs are using some objects, - /// located on stack of current thread, the stack must not be unwinded until all jobs finished. However, - /// if ThreadPool is a local object, it will wait for all scheduled jobs in own destructor. + /// NOTE: Probably you should call wait() if an exception was thrown. If some previously scheduled jobs are using some objects, + /// located on the stack of the current thread, the stack must not be unwound until all jobs are finished. However, + /// if ThreadPool is a local object, it will wait for all scheduled jobs in its own destructor. void scheduleOrThrowOnError(Job job, Priority priority = {}); /// Similar to scheduleOrThrowOnError(...). Wait for specified amount of time and schedule a job or return false. @@ -81,12 +129,12 @@ class ThreadPoolImpl /// Wait for all currently active jobs to be done. /// You may call schedule and wait many times in arbitrary order. - /// If any thread was throw an exception, first exception will be rethrown from this method, - /// and exception will be cleared. + /// If any thread has thrown an exception, the first exception will be rethrown from this method, + /// and the exception will be cleared. void wait(); /// Waits for all threads. Doesn't rethrow exceptions (use 'wait' method to rethrow exceptions). - /// You should not destroy object while calling schedule or wait methods from another threads. + /// You should not destroy the object while calling schedule or wait methods from other threads. ~ThreadPoolImpl(); /// Returns number of running and scheduled jobs. @@ -127,28 +175,40 @@ class ThreadPoolImpl size_t queue_size; size_t scheduled_jobs = 0; + + // Originally equals to max_threads, but changes dynamically. + // Decrements with every new thread started, increments when it finishes. + // If positive, then more threads can be started. + // When it comes to zero, it means that max_threads threads have already been started. + // it can be below zero when the threadpool is shutting down + std::atomic remaining_pool_capacity; + + // Increments every time a new thread joins the thread pool or a job finishes. + // Decrements every time a task is scheduled. + // If positive, it means that there are more threads than jobs (and some are idle). + // If zero, it means that every thread has a job. + // If negative, it means that we have more jobs than threads. + std::atomic available_threads; + bool shutdown = false; bool threads_remove_themselves = true; const bool shutdown_on_exception = true; - boost::heap::priority_queue jobs; - std::list threads; + boost::heap::priority_queue> jobs; + ThreadFromThreadPool::ThreadList threads; std::exception_ptr first_exception; std::stack on_destroy_callbacks; template ReturnType scheduleImpl(Job job, Priority priority, std::optional wait_microseconds, bool propagate_opentelemetry_tracing_context = true); - void worker(typename std::list::iterator thread_it); - - /// Tries to start new threads if there are scheduled jobs and the limit `max_threads` is not reached. Must be called with `mutex` locked. + /// Tries to start new threads if there are scheduled jobs and the limit `max_threads` is not reached. Must be called with the mutex locked. void startNewThreadsNoLock(); void finalize(); void onDestroy(); }; - /// ThreadPool with std::thread for threads. using FreeThreadPool = ThreadPoolImpl; diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index f09ea56391a6..0521245c55c9 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -28,6 +28,16 @@ #include #include +#if USE_SSL +# include +# include +# include +# include +# include +# include +# include +#endif + #include #include #include @@ -48,6 +58,7 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; extern const int LOGICAL_ERROR; extern const int INVALID_CONFIG_PARAMETER; + extern const int BAD_ARGUMENTS; } using namespace std::chrono_literals; @@ -56,6 +67,16 @@ namespace { #if USE_SSL + +int callSetCertificate(SSL * ssl, void * arg) +{ + if (!arg) + return -1; + + const CertificateReloader::Data * data = reinterpret_cast(arg); + return setCertificateCallback(ssl, data, getLogger("SSLContext")); +} + void setSSLParams(nuraft::asio_service::options & asio_opts) { const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config(); @@ -69,18 +90,55 @@ void setSSLParams(nuraft::asio_service::options & asio_opts) if (!config.has(private_key_file_property)) throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server private key file is not set."); - asio_opts.enable_ssl_ = true; - asio_opts.server_cert_file_ = config.getString(certificate_file_property); - asio_opts.server_key_file_ = config.getString(private_key_file_property); + Poco::Net::Context::Params params; + params.certificateFile = config.getString(certificate_file_property); + if (params.certificateFile.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server certificate file in config '{}' is empty", certificate_file_property); + + params.privateKeyFile = config.getString(private_key_file_property); + if (params.privateKeyFile.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server key file in config '{}' is empty", private_key_file_property); + + auto pass_phrase = config.getString("openSSL.server.privateKeyPassphraseHandler.options.password", ""); + auto certificate_data = std::make_shared(params.certificateFile, params.privateKeyFile, pass_phrase); if (config.has(root_ca_file_property)) - asio_opts.root_cert_file_ = config.getString(root_ca_file_property); + params.caLocation = config.getString(root_ca_file_property); - if (config.getBool("openSSL.server.loadDefaultCAFile", false)) - asio_opts.load_default_ca_file_ = true; + params.loadDefaultCAs = config.getBool("openSSL.server.loadDefaultCAFile", false); + params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString("openSSL.server.verificationMode", "none")); - if (config.getString("openSSL.server.verificationMode", "none") == "none") - asio_opts.skip_verification_ = true; + std::string disabled_protocols_list = config.getString("openSSL.server.disableProtocols", ""); + Poco::StringTokenizer dp_tok(disabled_protocols_list, ";,", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY); + int disabled_protocols = 0; + for (const auto & token : dp_tok) + { + if (token == "sslv2") + disabled_protocols |= Poco::Net::Context::PROTO_SSLV2; + else if (token == "sslv3") + disabled_protocols |= Poco::Net::Context::PROTO_SSLV3; + else if (token == "tlsv1") + disabled_protocols |= Poco::Net::Context::PROTO_TLSV1; + else if (token == "tlsv1_1") + disabled_protocols |= Poco::Net::Context::PROTO_TLSV1_1; + else if (token == "tlsv1_2") + disabled_protocols |= Poco::Net::Context::PROTO_TLSV1_2; + } + + asio_opts.ssl_context_provider_server_ = [params, certificate_data, disabled_protocols] + { + Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_SERVER_USE, params); + context.disableProtocols(disabled_protocols); + SSL_CTX * ssl_ctx = context.takeSslContext(); + SSL_CTX_set_cert_cb(ssl_ctx, callSetCertificate, reinterpret_cast(certificate_data.get())); + return ssl_ctx; + }; + + asio_opts.ssl_context_provider_client_ = [ctx_params = std::move(params)] + { + Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_CLIENT_USE, ctx_params); + return context.takeSslContext(); + }; } #endif diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 4f9abb306274..3edb6ff6c1b8 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -1078,6 +1078,7 @@ class IColumn; M(Bool, input_format_parquet_preserve_order, false, "Avoid reordering rows when reading from Parquet files. Usually makes it much slower.", 0) \ M(Bool, input_format_parquet_filter_push_down, true, "When reading Parquet files, skip whole row groups based on the WHERE/PREWHERE expressions and min/max statistics in the Parquet metadata.", 0) \ M(Bool, input_format_parquet_use_native_reader, false, "When reading Parquet files, to use native reader instead of arrow reader.", 0) \ + M(Bool, input_format_parquet_bloom_filter_push_down, true, "When reading Parquet files, skip whole row groups based on the WHERE/PREWHERE expressions and bloom filter in the Parquet metadata.", 0) \ M(Bool, input_format_allow_seeks, true, "Allow seeks while reading in ORC/Parquet/Arrow input formats", 0) \ M(Bool, input_format_orc_allow_missing_columns, true, "Allow missing columns while reading ORC input formats", 0) \ M(Bool, input_format_orc_use_fast_decoder, true, "Use a faster ORC decoder implementation.", 0) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index b3e7a59c5e2f..ee2cce99ff2b 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -75,6 +75,7 @@ static std::initializer_list skip_row_groups = {}; bool output_string_as_string = false; bool output_fixed_string_as_fixed_byte_array = true; diff --git a/src/Interpreters/Set.h b/src/Interpreters/Set.h index 8b44f36b278d..45c570a9bed7 100644 --- a/src/Interpreters/Set.h +++ b/src/Interpreters/Set.h @@ -239,6 +239,8 @@ class MergeTreeSetIndex const Columns & getOrderedSet() const { return ordered_set; } + const std::vector & getIndexesMapping() const { return indexes_mapping; } + private: // If all arguments in tuple are key columns, we can optimize NOT IN when there is only one element. bool has_all_keys; diff --git a/src/Processors/Formats/Impl/ArrowFieldIndexUtil.h b/src/Processors/Formats/Impl/ArrowFieldIndexUtil.h index 24ffdc105811..fca954b6cfb4 100644 --- a/src/Processors/Formats/Impl/ArrowFieldIndexUtil.h +++ b/src/Processors/Formats/Impl/ArrowFieldIndexUtil.h @@ -15,6 +15,7 @@ #include #include #include +#include namespace arrow @@ -65,11 +66,22 @@ class ArrowFieldIndexUtil return result; } + // For a parquet schema {x: {i: int, j: int}}, this should be populated as follows + // clickhouse_index = 0, parquet_indexes = {0, 1} + struct ClickHouseIndexToParquetIndex + { + std::size_t clickhouse_index; + std::vector parquet_indexes; + }; + /// Only collect the required fields' indices. Eg. when just read a field of a struct, /// don't need to collect the whole indices in this struct. - std::vector findRequiredIndices(const Block & header, const arrow::Schema & schema) + std::vector findRequiredIndices( + const Block & header, + const arrow::Schema & schema, + const parquet::FileMetaData & file) { - std::vector required_indices; + std::vector required_indices; std::unordered_set added_indices; /// Flat all named fields' index information into a map. auto fields_indices = calculateFieldIndices(schema); @@ -79,7 +91,7 @@ class ArrowFieldIndexUtil std::string col_name = named_col.name; if (ignore_case) boost::to_lower(col_name); - findRequiredIndices(col_name, named_col.type, fields_indices, added_indices, required_indices); + findRequiredIndices(col_name, i, named_col.type, fields_indices, added_indices, required_indices, file); } return required_indices; } @@ -169,10 +181,12 @@ class ArrowFieldIndexUtil void findRequiredIndices( const String & name, + std::size_t header_index, DataTypePtr data_type, const std::unordered_map> & field_indices, std::unordered_set & added_indices, - std::vector & required_indices) + std::vector & required_indices, + const parquet::FileMetaData & file) { auto nested_type = removeNullable(data_type); if (const DB::DataTypeTuple * type_tuple = typeid_cast(nested_type.get())) @@ -187,20 +201,20 @@ class ArrowFieldIndexUtil if (ignore_case) boost::to_lower(field_name); const auto & field_type = field_types[i]; - findRequiredIndices(Nested::concatenateName(name, field_name), field_type, field_indices, added_indices, required_indices); + findRequiredIndices(Nested::concatenateName(name, field_name), header_index, field_type, field_indices, added_indices, required_indices, file); } return; } } else if (const auto * type_array = typeid_cast(nested_type.get())) { - findRequiredIndices(name, type_array->getNestedType(), field_indices, added_indices, required_indices); + findRequiredIndices(name, header_index, type_array->getNestedType(), field_indices, added_indices, required_indices, file); return; } else if (const auto * type_map = typeid_cast(nested_type.get())) { - findRequiredIndices(name, type_map->getKeyType(), field_indices, added_indices, required_indices); - findRequiredIndices(name, type_map->getValueType(), field_indices, added_indices, required_indices); + findRequiredIndices(name, header_index, type_map->getKeyType(), field_indices, added_indices, required_indices, file); + findRequiredIndices(name, header_index, type_map->getValueType(), field_indices, added_indices, required_indices, file); return; } auto it = field_indices.find(name); @@ -211,14 +225,18 @@ class ArrowFieldIndexUtil } else { + ClickHouseIndexToParquetIndex index_mapping; + index_mapping.clickhouse_index = header_index; for (int j = 0; j < it->second.second; ++j) { auto index = it->second.first + j; if (added_indices.insert(index).second) { - required_indices.emplace_back(index); + index_mapping.parquet_indexes.emplace_back(index); } } + + required_indices.emplace_back(index_mapping); } } }; diff --git a/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.cpp b/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.cpp new file mode 100644 index 000000000000..75eeb15a5190 --- /dev/null +++ b/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.cpp @@ -0,0 +1,525 @@ +#include +#include + +#if USE_PARQUET + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace +{ + +bool isParquetStringTypeSupportedForBloomFilters( + const std::shared_ptr & logical_type, + parquet::ConvertedType::type converted_type) +{ + if (logical_type && + !logical_type->is_none() + && !(logical_type->is_string() || logical_type->is_BSON() || logical_type->is_JSON())) + { + return false; + } + + if (parquet::ConvertedType::type::NONE != converted_type && + !(converted_type == parquet::ConvertedType::JSON || converted_type == parquet::ConvertedType::UTF8 + || converted_type == parquet::ConvertedType::BSON)) + { + return false; + } + + return true; +} + +bool isParquetIntegerTypeSupportedForBloomFilters(const std::shared_ptr & logical_type, parquet::ConvertedType::type converted_type) +{ + if (logical_type && !logical_type->is_none() && !logical_type->is_int()) + { + return false; + } + + if (parquet::ConvertedType::type::NONE != converted_type && !(converted_type == parquet::ConvertedType::INT_8 || converted_type == parquet::ConvertedType::INT_16 + || converted_type == parquet::ConvertedType::INT_32 || converted_type == parquet::ConvertedType::INT_64 + || converted_type == parquet::ConvertedType::UINT_8 || converted_type == parquet::ConvertedType::UINT_16 + || converted_type == parquet::ConvertedType::UINT_32 || converted_type == parquet::ConvertedType::UINT_64)) + { + return false; + } + + return true; +} + +template +uint64_t hashSpecialFLBATypes(const Field & field) +{ + const T & value = field.safeGet(); + + parquet::FLBA flba(reinterpret_cast(&value)); + + parquet::XxHasher hasher; + + return hasher.Hash(&flba, sizeof(T)); +}; + +std::optional tryHashStringWithoutCompatibilityCheck(const Field & field) +{ + const auto field_type = field.getType(); + + if (field_type != Field::Types::Which::String) + { + return std::nullopt; + } + + parquet::XxHasher hasher; + parquet::ByteArray ba { field.safeGet() }; + + return hasher.Hash(&ba); +} + +std::optional tryHashString( + const Field & field, + const std::shared_ptr & logical_type, + parquet::ConvertedType::type converted_type) +{ + if (!isParquetStringTypeSupportedForBloomFilters(logical_type, converted_type)) + { + return std::nullopt; + } + + return tryHashStringWithoutCompatibilityCheck(field); +} + +std::optional tryHashFLBA( + const Field & field, + const std::shared_ptr & logical_type, + parquet::ConvertedType::type converted_type, + std::size_t parquet_column_length) +{ + if (!isParquetStringTypeSupportedForBloomFilters(logical_type, converted_type)) + { + return std::nullopt; + } + + const auto field_type = field.getType(); + + if (field_type == Field::Types::Which::IPv6 && parquet_column_length == sizeof(IPv6)) + { + return hashSpecialFLBATypes(field); + } + + return tryHashStringWithoutCompatibilityCheck(field); +} + +template +std::optional tryHashInt(const Field & field, const std::shared_ptr & logical_type, parquet::ConvertedType::type converted_type) +{ + if (!isParquetIntegerTypeSupportedForBloomFilters(logical_type, converted_type)) + { + return std::nullopt; + } + + parquet::XxHasher hasher; + + if (field.getType() == Field::Types::Which::Int64) + { + return hasher.Hash(static_cast(field.safeGet())); + } + else if (field.getType() == Field::Types::Which::UInt64) + { + return hasher.Hash(static_cast(field.safeGet())); + } + else if (field.getType() == Field::Types::IPv4) + { + /* + * In theory, we could accept IPv4 over 64 bits variables. It would only be a problem in case it was hashed using the byte array api + * with a zero-ed buffer that had a 32 bits variable copied into it. + * + * To be on the safe side, accept only in case physical type is 32 bits. + * */ + if constexpr (std::is_same_v) + { + return hasher.Hash(static_cast(field.safeGet())); + } + } + + return std::nullopt; +} + +std::optional tryHash(const Field & field, const parquet::ColumnDescriptor * parquet_column_descriptor) +{ + const auto physical_type = parquet_column_descriptor->physical_type(); + const auto & logical_type = parquet_column_descriptor->logical_type(); + const auto converted_type = parquet_column_descriptor->converted_type(); + + switch (physical_type) + { + case parquet::Type::type::INT32: + return tryHashInt(field, logical_type, converted_type); + case parquet::Type::type::INT64: + return tryHashInt(field, logical_type, converted_type); + case parquet::Type::type::BYTE_ARRAY: + return tryHashString(field, logical_type, converted_type); + case parquet::Type::type::FIXED_LEN_BYTE_ARRAY: + return tryHashFLBA(field, logical_type, converted_type, parquet_column_descriptor->type_length()); + default: + return std::nullopt; + } +} + +std::optional> hash(const IColumn * data_column, const parquet::ColumnDescriptor * parquet_column_descriptor) +{ + std::vector hashes; + + for (size_t i = 0u; i < data_column->size(); i++) + { + Field f; + data_column->get(i, f); + + auto hashed_value = tryHash(f, parquet_column_descriptor); + + if (!hashed_value) + { + return std::nullopt; + } + + hashes.emplace_back(*hashed_value); + } + + return hashes; +} + +bool maybeTrueOnBloomFilter(const std::vector & hashes, const std::unique_ptr & bloom_filter) +{ + for (const auto hash : hashes) + { + if (bloom_filter->FindHash(hash)) + { + return true; + } + } + + return false; +} + +const parquet::ColumnDescriptor * getColumnDescriptorIfBloomFilterIsPresent( + const std::unique_ptr & parquet_rg_metadata, + const std::vector & clickhouse_column_index_to_parquet_index, + std::size_t clickhouse_column_index) +{ + if (clickhouse_column_index_to_parquet_index.size() <= clickhouse_column_index) + { + return nullptr; + } + + const auto & parquet_indexes = clickhouse_column_index_to_parquet_index[clickhouse_column_index].parquet_indexes; + + // complex types like structs, tuples and maps will have more than one index. + // we don't support those for now + if (parquet_indexes.size() > 1) + { + return nullptr; + } + + if (parquet_indexes.empty()) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Something bad happened, raise an issue and try the query with `input_format_parquet_bloom_filter_push_down=false`"); + } + + auto parquet_column_index = parquet_indexes[0]; + + const auto * parquet_column_descriptor = parquet_rg_metadata->schema()->Column(parquet_column_index); + + bool column_has_bloom_filter = parquet_rg_metadata->ColumnChunk(parquet_column_index)->bloom_filter_offset().has_value(); + if (!column_has_bloom_filter) + { + return nullptr; + } + + return parquet_column_descriptor; +} + +} + +ParquetBloomFilterCondition::ParquetBloomFilterCondition(const std::vector & condition_, const Block & header_) + : condition(condition_), header(header_) +{ +} + +bool ParquetBloomFilterCondition::mayBeTrueOnRowGroup(const ColumnIndexToBF & column_index_to_column_bf) const +{ + using Function = ConditionElement::Function; + std::vector rpn_stack; + + for (const auto & element : condition) + { + if (element.function == Function::FUNCTION_IN + || element.function == Function::FUNCTION_NOT_IN) + { + bool maybe_true = true; + for (auto column_index = 0u; column_index < element.hashes_per_column.size(); column_index++) + { + // in case bloom filter is not present for this row group + // https://github.com/ClickHouse/ClickHouse/pull/62966#discussion_r1722361237 + if (!column_index_to_column_bf.contains(element.key_columns[column_index])) + { + rpn_stack.emplace_back(true, true); + continue; + } + + bool column_maybe_contains = maybeTrueOnBloomFilter( + element.hashes_per_column[column_index], + column_index_to_column_bf.at(element.key_columns[column_index])); + + if (!column_maybe_contains) + { + maybe_true = false; + break; + } + } + + rpn_stack.emplace_back(maybe_true, true); + if (element.function == Function::FUNCTION_NOT_IN) + rpn_stack.back() = !rpn_stack.back(); + } + else if (element.function == Function::FUNCTION_NOT) + { + rpn_stack.back() = !rpn_stack.back(); + } + else if (element.function == Function::FUNCTION_OR) + { + auto arg1 = rpn_stack.back(); + rpn_stack.pop_back(); + auto arg2 = rpn_stack.back(); + rpn_stack.back() = arg1 | arg2; + } + else if (element.function == Function::FUNCTION_AND) + { + auto arg1 = rpn_stack.back(); + rpn_stack.pop_back(); + auto arg2 = rpn_stack.back(); + rpn_stack.back() = arg1 & arg2; + } + else if (element.function == Function::ALWAYS_TRUE) + { + rpn_stack.emplace_back(true, false); + } + else if (element.function == Function::ALWAYS_FALSE) + { + rpn_stack.emplace_back(false, true); + } + else + { + rpn_stack.emplace_back(true, true); + } + } + + if (rpn_stack.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected stack size in KeyCondition::mayBeTrueOnRowGroup"); + + return rpn_stack[0].can_be_true; +} + +std::unordered_set ParquetBloomFilterCondition::getFilteringColumnKeys() const +{ + std::unordered_set column_keys; + + for (const auto & element : condition) + { + for (const auto index : element.key_columns) + { + column_keys.insert(index); + } + } + + return column_keys; +} + +/* + * `KeyCondition::rpn` is overly complex for bloom filters, some operations are not even supported. Not only that, but to avoid hashing each time + * we loop over a rpn element, we need to store hashes instead of where predicate values. To address this, we loop over `KeyCondition::rpn` + * and build a simplified RPN that holds hashes instead of values. + * + * `KeyCondition::RPNElement::FUNCTION_IN_RANGE` becomes: + * `FUNCTION_IN` + * `FUNCTION_UNKNOWN` when range limits are different + * `KeyCondition::RPNElement::FUNCTION_IN_SET` becomes + * `FUNCTION_IN` + * + * Complex types and structs are not supported. + * There are two sources of data types being analyzed, and they need to be compatible: DB::Field type and parquet type. + * This is determined by the `isColumnSupported` method. + * + * Some interesting examples: + * 1. file(..., 'str_column UInt64') where str_column = 50; Field.type == UInt64. Parquet type string. Not supported. + * 2. file(...) where str_column = 50; Field.type == String (conversion already taken care by `KeyCondition`). Parquet type string. + * 3. file(...) where uint32_column = toIPv4(5). Field.type == IPv4. Incompatible column types, resolved by `KeyCondition` itself. + * 4. file(...) where toIPv4(uint32_column) = toIPv4(5). Field.type == IPv4. We know it is safe to hash it using an int32 API. + * */ +std::vector keyConditionRPNToParquetBloomFilterCondition( + const std::vector & rpn, + const std::vector & clickhouse_column_index_to_parquet_index, + const std::unique_ptr & parquet_rg_metadata) +{ + std::vector condition_elements; + + using RPNElement = KeyCondition::RPNElement; + using Function = ParquetBloomFilterCondition::ConditionElement::Function; + + for (const auto & rpn_element : rpn) + { + // this would be a problem for `where negate(x) = -58`. + // It would perform a bf search on `-58`, and possibly miss row groups containing this data. + if (!rpn_element.monotonic_functions_chain.empty()) + { + condition_elements.emplace_back(Function::FUNCTION_UNKNOWN); + continue; + } + + ParquetBloomFilterCondition::ConditionElement::HashesForColumns hashes; + + if (rpn_element.function == RPNElement::FUNCTION_IN_RANGE + || rpn_element.function == RPNElement::FUNCTION_NOT_IN_RANGE) + { + // Only FUNCTION_EQUALS is supported and for that extremes need to be the same + if (rpn_element.range.left != rpn_element.range.right) + { + condition_elements.emplace_back(Function::FUNCTION_UNKNOWN); + continue; + } + + const auto * parquet_column_descriptor = + getColumnDescriptorIfBloomFilterIsPresent(parquet_rg_metadata, clickhouse_column_index_to_parquet_index, rpn_element.key_column); + + if (!parquet_column_descriptor) + { + condition_elements.emplace_back(Function::FUNCTION_UNKNOWN); + continue; + } + + auto hashed_value = tryHash(rpn_element.range.left, parquet_column_descriptor); + + if (!hashed_value) + { + condition_elements.emplace_back(Function::FUNCTION_UNKNOWN); + continue; + } + + std::vector hashes_for_column; + hashes_for_column.emplace_back(*hashed_value); + + hashes.emplace_back(std::move(hashes_for_column)); + + auto function = rpn_element.function == RPNElement::FUNCTION_IN_RANGE + ? ParquetBloomFilterCondition::ConditionElement::Function::FUNCTION_IN + : ParquetBloomFilterCondition::ConditionElement::Function::FUNCTION_NOT_IN; + + std::vector key_columns; + key_columns.emplace_back(rpn_element.key_column); + + condition_elements.emplace_back(function, std::move(hashes), std::move(key_columns)); + } + else if (rpn_element.function == RPNElement::FUNCTION_IN_SET + || rpn_element.function == RPNElement::FUNCTION_NOT_IN_SET) + { + const auto & set_index = rpn_element.set_index; + const auto & ordered_set = set_index->getOrderedSet(); + const auto & indexes_mapping = set_index->getIndexesMapping(); + bool found_empty_column = false; + + std::vector key_columns; + + for (auto i = 0u; i < ordered_set.size(); i++) + { + const auto & set_column = ordered_set[i]; + + const auto * parquet_column_descriptor = getColumnDescriptorIfBloomFilterIsPresent( + parquet_rg_metadata, + clickhouse_column_index_to_parquet_index, + indexes_mapping[i].key_index); + + if (!parquet_column_descriptor) + { + continue; + } + + auto column = set_column; + + if (column->empty()) + { + found_empty_column = true; + break; + } + + if (const auto & nullable_column = checkAndGetColumn(set_column.get())) + { + column = nullable_column->getNestedColumnPtr(); + } + + auto hashes_for_column_opt = hash(column.get(), parquet_column_descriptor); + + if (!hashes_for_column_opt) + { + continue; + } + + auto & hashes_for_column = *hashes_for_column_opt; + + if (hashes_for_column.empty()) + { + continue; + } + + hashes.emplace_back(hashes_for_column); + + key_columns.push_back(indexes_mapping[i].key_index); + } + + if (found_empty_column) + { + condition_elements.emplace_back(Function::ALWAYS_FALSE); + continue; + } + + if (hashes.empty()) + { + condition_elements.emplace_back(Function::FUNCTION_UNKNOWN); + continue; + } + + auto function = RPNElement::FUNCTION_IN_SET == rpn_element.function ? Function::FUNCTION_IN : Function::FUNCTION_NOT_IN; + + condition_elements.emplace_back(function, hashes, key_columns); + } + else if (rpn_element.function == RPNElement::FUNCTION_NOT) + { + condition_elements.emplace_back(Function::FUNCTION_NOT); + } + else if (rpn_element.function == RPNElement::FUNCTION_OR) + { + condition_elements.emplace_back(Function::FUNCTION_OR); + } + else if (rpn_element.function == RPNElement::FUNCTION_AND) + { + condition_elements.emplace_back(Function::FUNCTION_AND); + } + else + { + condition_elements.emplace_back(Function::ALWAYS_TRUE); + } + } + + return condition_elements; +} + +} + +#endif diff --git a/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.h b/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.h new file mode 100644 index 000000000000..6de6030b23cb --- /dev/null +++ b/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.h @@ -0,0 +1,73 @@ +#pragma once + +#include + +#if USE_PARQUET + +#include +#include +#include + +namespace parquet +{ +class BloomFilter; +} + +namespace DB +{ + +class ParquetBloomFilterCondition +{ +public: + + struct ConditionElement + { + enum Function + { + /// Atoms of a Boolean expression. + FUNCTION_IN, + FUNCTION_NOT_IN, + /// Can take any value. + FUNCTION_UNKNOWN, + /// Operators of the logical expression. + FUNCTION_NOT, + FUNCTION_AND, + FUNCTION_OR, + /// Constants + ALWAYS_FALSE, + ALWAYS_TRUE, + }; + + using ColumnPtr = IColumn::Ptr; + using HashesForColumns = std::vector>; + using KeyColumns = std::vector; + + Function function; + // each entry represents a list of hashes per column + // suppose there are three columns with 2 rows each + // hashes_per_column.size() == 3 and hashes_per_column[0].size() == 2 + HashesForColumns hashes_per_column; + KeyColumns key_columns; + }; + + using RPNElement = KeyCondition::RPNElement; + using ColumnIndexToBF = std::unordered_map>; + + explicit ParquetBloomFilterCondition(const std::vector & condition_, const Block & header_); + + bool mayBeTrueOnRowGroup(const ColumnIndexToBF & column_index_to_column_bf) const; + std::unordered_set getFilteringColumnKeys() const; + +private: + std::vector condition; + Block header; +}; + +std::vector keyConditionRPNToParquetBloomFilterCondition( + const std::vector & rpn, + const std::vector & clickhouse_column_index_to_parquet_index, + const std::unique_ptr & parquet_rg_metadata); + +} + +#endif diff --git a/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.cpp b/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.cpp index b8e4db8700cc..b471989076b4 100644 --- a/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.cpp +++ b/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.cpp @@ -296,6 +296,31 @@ void ParquetPlainValuesReader::readBatch( ); } +template +void ParquetBitPlainReader::readBatch( + MutableColumnPtr & col_ptr, LazyNullMap & null_map, UInt32 num_values) +{ + auto cursor = col_ptr->size(); + auto * column_data = getResizedPrimitiveData(*assert_cast(col_ptr.get()), cursor + num_values); + + def_level_reader->visitNullableValues( + cursor, + num_values, + max_def_level, + null_map, + /* individual_visitor */ [&](size_t nest_cursor) + { + uint8_t byte; + bit_reader->GetValue(1, &byte); + column_data[nest_cursor] = byte; + }, + /* repeated_visitor */ [&](size_t nest_cursor, UInt32 count) + { + bit_reader->GetBatch(1, &column_data[nest_cursor], count); + } + ); +} + template <> void ParquetPlainValuesReader, ParquetReaderTypes::TimestampInt96>::readBatch( @@ -561,6 +586,9 @@ template class ParquetPlainValuesReader>; template class ParquetPlainValuesReader>; template class ParquetPlainValuesReader>; template class ParquetPlainValuesReader; +template class ParquetPlainValuesReader; + +template class ParquetBitPlainReader; template class ParquetFixedLenPlainReader>; template class ParquetFixedLenPlainReader>; @@ -569,6 +597,7 @@ template class ParquetRleLCReader; template class ParquetRleLCReader; template class ParquetRleLCReader; +template class ParquetRleDictReader; template class ParquetRleDictReader; template class ParquetRleDictReader; template class ParquetRleDictReader; diff --git a/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.h b/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.h index fbccb612b3c0..db55f7e2d6ab 100644 --- a/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.h +++ b/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.h @@ -172,6 +172,27 @@ class ParquetPlainValuesReader : public ParquetDataValuesReader ParquetDataBuffer plain_data_buffer; }; +template +class ParquetBitPlainReader : public ParquetDataValuesReader +{ +public: + ParquetBitPlainReader( + Int32 max_def_level_, + std::unique_ptr def_level_reader_, + std::unique_ptr bit_reader_) + : max_def_level(max_def_level_) + , def_level_reader(std::move(def_level_reader_)) + , bit_reader(std::move(bit_reader_)) + {} + + void readBatch(MutableColumnPtr & col_ptr, LazyNullMap & null_map, UInt32 num_values) override; + +private: + Int32 max_def_level; + std::unique_ptr def_level_reader; + std::unique_ptr bit_reader; +}; + /** * The data and definition level encoding are same as ParquetPlainValuesReader. * But the element size is const and bigger than primitive data type. diff --git a/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.cpp b/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.cpp index 9e1cae9bb657..c3ced4a47b0f 100644 --- a/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.cpp +++ b/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.cpp @@ -370,6 +370,7 @@ template void ParquetLeafColReader::readPage() { // refer to: ColumnReaderImplBase::ReadNewPage in column_reader.cc + // this is where decompression happens auto cur_page = parquet_page_reader->NextPage(); switch (cur_page->type()) { @@ -409,46 +410,13 @@ void ParquetLeafColReader::readPage() } template -void ParquetLeafColReader::readPageV1(const parquet::DataPageV1 & page) +void ParquetLeafColReader::initDataReader( + parquet::Encoding::type enconding_type, + const uint8_t * buffer, + std::size_t max_size, + std::unique_ptr && def_level_reader) { - static parquet::LevelDecoder repetition_level_decoder; - - cur_page_values = page.num_values(); - - // refer to: VectorizedColumnReader::readPageV1 in Spark and LevelDecoder::SetData in column_reader.cc - if (page.definition_level_encoding() != parquet::Encoding::RLE && col_descriptor.max_definition_level() != 0) - { - throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unsupported encoding: {}", page.definition_level_encoding()); - } - const auto * buffer = page.data(); - auto max_size = page.size(); - - if (col_descriptor.max_repetition_level() > 0) - { - auto rep_levels_bytes = repetition_level_decoder.SetData( - page.repetition_level_encoding(), col_descriptor.max_repetition_level(), 0, buffer, max_size); - buffer += rep_levels_bytes; - max_size -= rep_levels_bytes; - } - - assert(col_descriptor.max_definition_level() >= 0); - std::unique_ptr def_level_reader; - if (col_descriptor.max_definition_level() > 0) - { - auto bit_width = arrow::bit_util::Log2(col_descriptor.max_definition_level() + 1); - auto num_bytes = ::arrow::util::SafeLoadAs(buffer); - auto bit_reader = std::make_unique(buffer + 4, num_bytes); - num_bytes += 4; - buffer += num_bytes; - max_size -= num_bytes; - def_level_reader = std::make_unique(std::move(bit_reader), bit_width); - } - else - { - def_level_reader = std::make_unique(page.num_values()); - } - - switch (page.encoding()) + switch (enconding_type) { case parquet::Encoding::PLAIN: { @@ -458,16 +426,29 @@ void ParquetLeafColReader::readPageV1(const parquet::DataPageV1 & page) degradeDictionary(); } - ParquetDataBuffer parquet_buffer = [&]() + if (col_descriptor.physical_type() == parquet::Type::BOOLEAN) { - if constexpr (!std::is_same_v, TColumn>) - return ParquetDataBuffer(buffer, max_size); - - auto scale = assert_cast(*base_data_type).getScale(); - return ParquetDataBuffer(buffer, max_size, scale); - }(); - data_values_reader = createPlainReader( - col_descriptor, std::move(def_level_reader), std::move(parquet_buffer)); + if constexpr (std::is_same_v) + { + auto bit_reader = std::make_unique(buffer, max_size); + data_values_reader = std::make_unique>(col_descriptor.max_definition_level(), + std::move(def_level_reader), + std::move(bit_reader)); + } + } + else + { + ParquetDataBuffer parquet_buffer = [&]() + { + if constexpr (!std::is_same_v, TColumn>) + return ParquetDataBuffer(buffer, max_size); + + auto scale = assert_cast(*base_data_type).getScale(); + return ParquetDataBuffer(buffer, max_size, scale); + }(); + data_values_reader = createPlainReader( + col_descriptor, std::move(def_level_reader), std::move(parquet_buffer)); + } break; } case parquet::Encoding::RLE_DICTIONARY: @@ -489,17 +470,144 @@ void ParquetLeafColReader::readPageV1(const parquet::DataPageV1 & page) case parquet::Encoding::DELTA_BINARY_PACKED: case parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: case parquet::Encoding::DELTA_BYTE_ARRAY: - throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unsupported encoding: {}", page.encoding()); + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unsupported encoding: {}", enconding_type); default: - throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unknown encoding type: {}", page.encoding()); + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unknown encoding type: {}", enconding_type); } } template -void ParquetLeafColReader::readPageV2(const parquet::DataPageV2 & /*page*/) +void ParquetLeafColReader::readPageV1(const parquet::DataPageV1 & page) { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "read page V2 is not implemented yet"); + cur_page_values = page.num_values(); + + // refer to: VectorizedColumnReader::readPageV1 in Spark and LevelDecoder::SetData in column_reader.cc + if (page.definition_level_encoding() != parquet::Encoding::RLE && col_descriptor.max_definition_level() != 0) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unsupported encoding: {}", page.definition_level_encoding()); + } + + const auto * buffer = page.data(); + auto max_size = static_cast(page.size()); + + if (col_descriptor.max_repetition_level() > 0) + { + if (max_size < sizeof(int32_t)) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Not enough bytes in parquet page buffer, corrupt?"); + } + + auto num_bytes = ::arrow::util::SafeLoadAs(buffer); + + if (num_bytes < 0) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Number of bytes for dl is negative, corrupt?"); + } + + if (num_bytes + 4u > max_size) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Not enough bytes in parquet page buffer, corrupt?"); + } + + // not constructing level reader because we are not using it atm + num_bytes += 4; + buffer += num_bytes; + max_size -= num_bytes; + } + + assert(col_descriptor.max_definition_level() >= 0); + std::unique_ptr def_level_reader; + if (col_descriptor.max_definition_level() > 0) + { + auto bit_width = arrow::bit_util::Log2(col_descriptor.max_definition_level() + 1); + + if (max_size < sizeof(int32_t)) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Not enough bytes in parquet page buffer, corrupt?"); + } + + auto num_bytes = ::arrow::util::SafeLoadAs(buffer); + + if (num_bytes < 0) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Number of bytes for dl is negative, corrupt?"); + } + + if (num_bytes + 4u > max_size) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Not enough bytes in parquet page buffer, corrupt?"); + } + + auto bit_reader = std::make_unique(buffer + 4, num_bytes); + num_bytes += 4; + buffer += num_bytes; + max_size -= num_bytes; + def_level_reader = std::make_unique(std::move(bit_reader), bit_width); + } + else + { + def_level_reader = std::make_unique(page.num_values()); + } + + initDataReader(page.encoding(), buffer, max_size, std::move(def_level_reader)); +} + +/* + * As far as I understand, the difference between page v1 and page v2 lies primarily on the below: + * 1. repetition and definition levels are not compressed; + * 2. size of repetition and definition levels is present in the header; + * 3. the encoding is always RLE + * + * Therefore, this method leverages the existing `parquet::LevelDecoder::SetDataV2` method to build the repetition level decoder. + * The data buffer is "offset-ed" by rl bytes length and then dl decoder is built using RLE decoder. Since dl bytes length was present in the header, + * there is no need to read it and apply an offset like in page v1. + * */ +template +void ParquetLeafColReader::readPageV2(const parquet::DataPageV2 & page) +{ + cur_page_values = page.num_values(); + + const auto * buffer = page.data(); + + if (page.repetition_levels_byte_length() < 0 || page.definition_levels_byte_length() < 0) + { + throw Exception( + ErrorCodes::PARQUET_EXCEPTION, "Either RL or DL is negative, this should not happen. Most likely corrupt file or parsing issue"); + } + + const int64_t total_levels_length = + static_cast(page.repetition_levels_byte_length()) + + page.definition_levels_byte_length(); + + if (total_levels_length > page.size()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Data page too small for levels (corrupt header?)"); + } + + // ARROW-17453: Even if max_rep_level_ is 0, there may still be + // repetition level bytes written and/or reported in the header by + // some writers (e.g. Athena) + buffer += page.repetition_levels_byte_length(); + + assert(col_descriptor.max_definition_level() >= 0); + std::unique_ptr def_level_reader; + if (col_descriptor.max_definition_level() > 0) + { + auto bit_width = arrow::bit_util::Log2(col_descriptor.max_definition_level() + 1); + auto num_bytes = page.definition_levels_byte_length(); + auto bit_reader = std::make_unique(buffer, num_bytes); + def_level_reader = std::make_unique(std::move(bit_reader), bit_width); + } + else + { + def_level_reader = std::make_unique(page.num_values()); + } + + buffer += page.definition_levels_byte_length(); + + initDataReader(page.encoding(), buffer, page.size() - total_levels_length, std::move(def_level_reader)); } template @@ -518,6 +626,12 @@ std::unique_ptr ParquetLeafColReader::createDi }); return res; } + + if (col_descriptor.physical_type() == parquet::Type::type::BOOLEAN) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Dictionary encoding for booleans is not supported"); + } + return std::make_unique>( col_descriptor.max_definition_level(), std::move(def_level_reader), @@ -526,6 +640,7 @@ std::unique_ptr ParquetLeafColReader::createDi } +template class ParquetLeafColReader; template class ParquetLeafColReader; template class ParquetLeafColReader; template class ParquetLeafColReader; diff --git a/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.h b/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.h index c5b14132f176..e1eb7702def1 100644 --- a/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.h +++ b/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.h @@ -54,6 +54,10 @@ class ParquetLeafColReader : public ParquetColumnReader void readPage(); void readPageV1(const parquet::DataPageV1 & page); void readPageV2(const parquet::DataPageV2 & page); + void initDataReader(parquet::Encoding::type enconding_type, + const uint8_t * buffer, + std::size_t max_size, + std::unique_ptr && def_level_reader); std::unique_ptr createDictReader( std::unique_ptr def_level_reader, std::unique_ptr rle_data_reader); diff --git a/src/Processors/Formats/Impl/Parquet/ParquetRecordReader.cpp b/src/Processors/Formats/Impl/Parquet/ParquetRecordReader.cpp index 18a0db7484ec..74107df4e26b 100644 --- a/src/Processors/Formats/Impl/Parquet/ParquetRecordReader.cpp +++ b/src/Processors/Formats/Impl/Parquet/ParquetRecordReader.cpp @@ -265,7 +265,7 @@ std::unique_ptr ColReaderFactory::makeReader() switch (col_descriptor.physical_type()) { case parquet::Type::BOOLEAN: - break; + return makeLeafReader(); case parquet::Type::INT32: return fromInt32(); case parquet::Type::INT64: diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index 4231ef3c7ab5..bfab82148d32 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include "ArrowBufferedStreams.h" @@ -25,6 +27,7 @@ #include #include #include +#include #include namespace CurrentMetrics @@ -264,6 +267,50 @@ static Field decodePlainParquetValueSlow(const std::string & data, parquet::Type return field; } +static ParquetBloomFilterCondition::ColumnIndexToBF buildColumnIndexToBF( + parquet::BloomFilterReader & bf_reader, + int row_group, + const std::vector & clickhouse_column_index_to_parquet_index, + const std::unordered_set & filtering_columns +) +{ + auto rg_bf = bf_reader.RowGroup(row_group); + + if (!rg_bf) + { + return {}; + } + + ParquetBloomFilterCondition::ColumnIndexToBF index_to_column_bf; + + for (const auto & [clickhouse_index, parquet_indexes] : clickhouse_column_index_to_parquet_index) + { + if (!filtering_columns.contains(clickhouse_index)) + { + continue; + } + + // Complex / nested types contain more than one index. We don't support those. + if (parquet_indexes.size() > 1) + { + continue; + } + + auto parquet_index = parquet_indexes[0]; + + auto bf = rg_bf->GetColumnBloomFilter(parquet_index); + + if (!bf) + { + continue; + } + + index_to_column_bf[clickhouse_index] = std::move(bf); + } + + return index_to_column_bf; +} + /// Range of values for each column, based on statistics in the Parquet metadata. /// This is lower/upper bounds, not necessarily exact min and max, e.g. the min/max can be just /// missing in the metadata. @@ -471,9 +518,27 @@ void ParquetBlockInputFormat::initializeIfNeeded() ArrowFieldIndexUtil field_util( format_settings.parquet.case_insensitive_column_matching, format_settings.parquet.allow_missing_columns); - column_indices = field_util.findRequiredIndices(getPort().getHeader(), *schema); + + auto index_mapping = field_util.findRequiredIndices(getPort().getHeader(), *schema, *metadata); + + for (const auto & [clickhouse_header_index, parquet_indexes] : index_mapping) + { + for (auto parquet_index : parquet_indexes) + { + column_indices.push_back(parquet_index); + } + } int num_row_groups = metadata->num_row_groups(); + + if (num_row_groups == 0) + { + return; + } + + const auto bf_reader_properties = parquet::default_reader_properties(); + std::unique_ptr bf_reader; + row_group_batches.reserve(num_row_groups); auto adaptive_chunk_size = [&](int row_group_idx) -> size_t @@ -494,11 +559,38 @@ void ParquetBlockInputFormat::initializeIfNeeded() return std::min(std::max(preferred_num_rows, MIN_ROW_NUM), static_cast(format_settings.parquet.max_block_size)); }; + std::unique_ptr parquet_bloom_filter_condition; + + std::unordered_set filtering_columns; + + if (format_settings.parquet.bloom_filter_push_down && key_condition) + { + bf_reader = parquet::BloomFilterReader::Make(arrow_file, metadata, bf_reader_properties, nullptr); + + const auto parquet_conditions = keyConditionRPNToParquetBloomFilterCondition( + key_condition->getRPN(), + index_mapping, + metadata->RowGroup(0)); + parquet_bloom_filter_condition = std::make_unique(parquet_conditions, getPort().getHeader()); + + filtering_columns = parquet_bloom_filter_condition->getFilteringColumnKeys(); + } + for (int row_group = 0; row_group < num_row_groups; ++row_group) { if (skip_row_groups.contains(row_group)) continue; + if (parquet_bloom_filter_condition) + { + const auto column_index_to_bf = buildColumnIndexToBF(*bf_reader, row_group, index_mapping, filtering_columns); + + if (!parquet_bloom_filter_condition->mayBeTrueOnRowGroup(column_index_to_bf)) + { + continue; + } + } + if (format_settings.parquet.filter_push_down && key_condition && !key_condition ->checkInHyperrectangle( diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index df7b6e7fbd7d..5b981fc7a875 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -34,8 +34,12 @@ int CertificateReloader::setCertificate(SSL * ssl, const CertificateReloader::Mu auto current = pdata->data.get(); if (!current) return -1; + return setCertificateCallback(ssl, current.get(), log); +} - if (current->certs_chain.empty()) +int setCertificateCallback(SSL * ssl, const CertificateReloader::Data * current_data, LoggerPtr log) +{ + if (current_data->certs_chain.empty()) return -1; if (auto err = SSL_clear_chain_certs(ssl); err != 1) @@ -43,12 +47,12 @@ int CertificateReloader::setCertificate(SSL * ssl, const CertificateReloader::Mu LOG_ERROR(log, "Clear certificates {}", Poco::Net::Utility::getLastError()); return -1; } - if (auto err = SSL_use_certificate(ssl, const_cast(current->certs_chain[0].certificate())); err != 1) + if (auto err = SSL_use_certificate(ssl, const_cast(current_data->certs_chain[0].certificate())); err != 1) { LOG_ERROR(log, "Use certificate {}", Poco::Net::Utility::getLastError()); return -1; } - for (auto cert = current->certs_chain.begin() + 1; cert != current->certs_chain.end(); cert++) + for (auto cert = current_data->certs_chain.begin() + 1; cert != current_data->certs_chain.end(); cert++) { if (auto err = SSL_add1_chain_cert(ssl, const_cast(cert->certificate())); err != 1) { @@ -56,7 +60,7 @@ int CertificateReloader::setCertificate(SSL * ssl, const CertificateReloader::Mu return -1; } } - if (auto err = SSL_use_PrivateKey(ssl, const_cast(static_cast(current->key))); err != 1) + if (auto err = SSL_use_PrivateKey(ssl, const_cast(static_cast(current_data->key))); err != 1) { LOG_ERROR(log, "Use private key {}", Poco::Net::Utility::getLastError()); return -1; diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 7472d2f6baa3..28737988fddc 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -104,6 +104,9 @@ class CertificateReloader mutable std::mutex data_mutex; }; +/// A callback for OpenSSL +int setCertificateCallback(SSL * ssl, const CertificateReloader::Data * current_data, LoggerPtr log); + } #endif diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index caaaf53b9d46..e54d5237ae75 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -800,7 +800,7 @@ void KeyCondition::getAllSpaceFillingCurves() KeyCondition::KeyCondition( const ActionsDAG * filter_dag, ContextPtr context, - const Names & key_column_names, + const Names & key_column_names_, const ExpressionActionsPtr & key_expr_, bool single_point_) : key_expr(key_expr_) @@ -808,7 +808,7 @@ KeyCondition::KeyCondition( , single_point(single_point_) { size_t key_index = 0; - for (const auto & name : key_column_names) + for (const auto & name : key_column_names_) { if (!key_columns.contains(name)) { diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py index 8bc0f51dfc73..697c094b5525 100644 --- a/tests/ci/ast_fuzzer_check.py +++ b/tests/ci/ast_fuzzer_check.py @@ -16,7 +16,7 @@ from tee_popen import TeePopen from ci_config import CI -IMAGE_NAME = "clickhouse/fuzzer" +IMAGE_NAME = "altinityinfra/fuzzer" def get_run_command( diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 77d91c8400b0..384435311069 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -10,19 +10,20 @@ import docker_images_helper from ci_config import CI -from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH +from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY from git_helper import Git -from pr_info import PRInfo +from pr_info import PRInfo, EventType from report import FAILURE, SUCCESS, JobReport, StatusType from stopwatch import Stopwatch from tee_popen import TeePopen from version_helper import ( ClickHouseVersion, + VersionType, get_version_from_repo, update_version_local, ) -IMAGE_NAME = "clickhouse/binary-builder" +IMAGE_NAME = "altinityinfra/binary-builder" BUILD_LOG_NAME = "build_log.log" @@ -64,6 +65,8 @@ def get_packager_cmd( cmd += " --cache=sccache" cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" + cmd += f" --s3-access-key-id={S3_ACCESS_KEY_ID}" + cmd += f" --s3-secret-access-key={S3_SECRET_ACCESS_KEY}" if build_config.additional_pkgs: cmd += " --additional-pkgs" @@ -162,16 +165,31 @@ def main(): version = get_version_from_repo(git=Git(True)) logging.info("Got version from repo %s", version.string) - official_flag = pr_info.number == 0 + # official_flag = pr_info.number == 0 - version_type = "testing" - if is_release_pr(pr_info): - version_type = "stable" - official_flag = True + # version_type = "testing" + # if is_release_pr(pr_info): + # version_type = "stable" + # official_flag = True + + # NOTE(vnemkov): For Altinity Stable builds, version flavor + # (last part of version, like 'altinitystable') is obtained from tag. + # If there is no tag, then version is considered to be 'testing' + version_type = version._flavour = VersionType.TESTING + official_flag = True + + if pr_info.event_type == EventType.PUSH \ + and pr_info.ref.startswith('/ref/tags/'): + tag_name = pr_info.ref.removeprefix('/ref/tags/') + version_type = tag_name.split('.')[-1] + version._flavour = version_type + logging.info("Using version from tag: %s => %s", tag_name, version) + + # TODO(vnemkov): make sure tweak part is incremented by 1 each time we merge a PR update_version_local(version, version_type) - logging.info("Updated local files with version") + logging.info("Updated local files with version %s", version) logging.info("Build short name %s", build_name) diff --git a/tests/ci/changelog.py b/tests/ci/changelog.py index 39e426945d3d..9bac4de4ee74 100755 --- a/tests/ci/changelog.py +++ b/tests/ci/changelog.py @@ -161,7 +161,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--repo", - default="ClickHouse/ClickHouse", + default="Altinity/ClickHouse", help="a repository to query for pull-requests from GitHub", ) parser.add_argument( diff --git a/tests/ci/ci.py b/tests/ci/ci.py index ea1fe872439a..465820a7488f 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -388,7 +388,7 @@ def _pre_action(s3, job_name, batch, indata, pr_info): _get_ext_check_name(job_name), ) ClickHouseHelper().insert_events_into( - db="default", table="checks", events=prepared_events + db="gh-data", table="checks", events=prepared_events ) print(f"Pre action done. Report files [{reports_files}] have been downloaded") @@ -966,7 +966,7 @@ def _add_build_to_version_history( print(f"::notice ::Log Adding record to versions history: {data}") - ch_helper.insert_event_into(db="default", table="version_history", event=data) + ch_helper.insert_event_into(db="gh-data", table="version_history", event=data) def _run_test(job_name: str, run_command: str) -> int: @@ -1302,7 +1302,7 @@ def main() -> int: job_report.check_name or _get_ext_check_name(args.job_name), ) ch_helper.insert_events_into( - db="default", table="checks", events=prepared_events + db="gh-data", table="checks", events=prepared_events ) if "DockerServerImage" in args.job_name and indata is not None: @@ -1376,7 +1376,7 @@ def main() -> int: _get_ext_check_name(args.job_name), ) ClickHouseHelper().insert_events_into( - db="default", table="checks", events=prepared_events + db="gh-data", table="checks", events=prepared_events ) ### POST action: end diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py index 164af72f4be7..ae64e2c44c21 100644 --- a/tests/ci/ci_buddy.py +++ b/tests/ci/ci_buddy.py @@ -104,7 +104,9 @@ def _get_webhooks(): return json_string def post(self, message: str, channels: List[str]) -> None: - print(f"Posting slack message, dry_run [{self.dry_run}]") + print(f"Would've posted slack message, dry_run [{self.dry_run}], message: {message}") + # NOTE(vnemkov): we don't use slack for CI/CD no need to post messages + return if self.dry_run: urls = [self.channels[Channels.DRY_RUN]] else: diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 29c74a9f6ccf..425552398dab 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -365,15 +365,12 @@ class CI: ), JobNames.STRESS_TEST_ASAN: CommonJobConfigs.STRESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_ASAN], - random_bucket="stress_with_sanitizer", ), JobNames.STRESS_TEST_UBSAN: CommonJobConfigs.STRESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_UBSAN], - random_bucket="stress_with_sanitizer", ), JobNames.STRESS_TEST_MSAN: CommonJobConfigs.STRESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_MSAN], - random_bucket="stress_with_sanitizer", ), JobNames.STRESS_TEST_AZURE_TSAN: CommonJobConfigs.STRESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_TSAN], release_only=True @@ -400,7 +397,7 @@ class CI: required_builds=[BuildNames.PACKAGE_DEBUG], pr_only=True ), JobNames.INTEGRATION_TEST_ASAN: CommonJobConfigs.INTEGRATION_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_ASAN], release_only=True, num_batches=4 + required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4 ), JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_ASAN], @@ -419,7 +416,7 @@ class CI: JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=4, - release_only=True, + #release_only=True, ), JobNames.INTEGRATION_TEST_FLAKY: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_ASAN], @@ -530,7 +527,7 @@ class CI: JobNames.DOCS_CHECK: JobConfig( digest=DigestConfig( include_paths=["**/*.md", "./docs", "tests/ci/docs_check.py"], - docker=["clickhouse/docs-builder"], + docker=["altinityinfra/docs-builder"], ), run_command="docs_check.py", runner_type=Runners.FUNC_TESTER, @@ -540,7 +537,7 @@ class CI: digest=DigestConfig( include_paths=["./tests/queries/0_stateless/"], exclude_files=[".md"], - docker=["clickhouse/fasttest"], + docker=["altinityinfra/fasttest"], ), timeout=2400, runner_type=Runners.BUILDER, @@ -555,6 +552,14 @@ class CI: timeout=900, runner_type=Runners.STYLE_CHECKER, ), + JobNames.SIGN_RELEASE: JobConfig( + required_builds=[BuildNames.PACKAGE_RELEASE], + runner_type=Runners.STYLE_CHECKER + ), + JobNames.SIGN_AARCH64: JobConfig( + required_builds=[BuildNames.PACKAGE_AARCH64], + runner_type=Runners.STYLE_CHECKER_ARM + ), } @classmethod diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index 70aba2bc16b0..c10aec14b06b 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -223,6 +223,9 @@ class JobNames(metaclass=WithIter): DOCS_CHECK = "Docs check" BUGFIX_VALIDATE = "Bugfix validation" + SIGN_RELEASE = "Sign release" + SIGN_AARCH64 = "Sign aarch64" + # hack to concatenate Build and non-build jobs under JobNames class for attr_name in dir(BuildNames): @@ -338,7 +341,7 @@ class JobConfig: # label that enables job in CI, if set digest isn't used run_by_label: str = "" # to run always regardless of the job digest or/and label - run_always: bool = False + run_always: bool = True # disables CI await for a given job disable_await: bool = False # if the job needs to be run on the release branch, including master (building packages, docker server). @@ -391,7 +394,7 @@ class CommonJobConfigs: job_name_keyword="compatibility", digest=DigestConfig( include_paths=["./tests/ci/compatibility_check.py"], - docker=["clickhouse/test-old-ubuntu", "clickhouse/test-old-centos"], + docker=["altinityinfra/test-old-ubuntu", "altinityinfra/test-old-centos"], ), run_command="compatibility_check.py", runner_type=Runners.STYLE_CHECKER, @@ -400,7 +403,7 @@ class CommonJobConfigs: job_name_keyword="install", digest=DigestConfig( include_paths=["./tests/ci/install_check.py"], - docker=["clickhouse/install-deb-test", "clickhouse/install-rpm-test"], + docker=["altinityinfra/install-deb-test", "altinityinfra/install-rpm-test"], ), run_command='install_check.py "$CHECK_NAME"', runner_type=Runners.STYLE_CHECKER, @@ -417,7 +420,7 @@ class CommonJobConfigs: "./tests/*.txt", ], exclude_files=[".md"], - docker=["clickhouse/stateless-test"], + docker=["altinityinfra/stateless-test"], ), run_command='functional_test_check.py "$CHECK_NAME"', runner_type=Runners.FUNC_TESTER, @@ -433,7 +436,7 @@ class CommonJobConfigs: "./tests/*.txt", ], exclude_files=[".md"], - docker=["clickhouse/stateful-test"], + docker=["altinityinfra/stateful-test"], ), run_command='functional_test_check.py "$CHECK_NAME"', runner_type=Runners.FUNC_TESTER, @@ -450,7 +453,7 @@ class CommonJobConfigs: "./tests/*.txt", ], exclude_files=[".md"], - docker=["clickhouse/stress-test"], + docker=["altinityinfra/stress-test"], ), run_command="stress_check.py", runner_type=Runners.STRESS_TESTER, @@ -461,7 +464,7 @@ class CommonJobConfigs: digest=DigestConfig( include_paths=["./tests/ci/upgrade_check.py"], exclude_files=[".md"], - docker=["clickhouse/upgrade-check"], + docker=["altinityinfra/upgrade-check"], ), run_command="upgrade_check.py", runner_type=Runners.STRESS_TESTER, @@ -487,7 +490,7 @@ class CommonJobConfigs: include_paths=[ "./tests/ci/ast_fuzzer_check.py", ], - docker=["clickhouse/fuzzer"], + docker=["altinityinfra/fuzzer"], ), run_command="ast_fuzzer_check.py", run_always=True, @@ -498,7 +501,7 @@ class CommonJobConfigs: digest=DigestConfig( include_paths=["./tests/ci/unit_tests_check.py"], exclude_files=[".md"], - docker=["clickhouse/unit-test"], + docker=["altinityinfra/unit-test"], ), run_command="unit_tests_check.py", runner_type=Runners.FUZZER_UNIT_TESTER, @@ -511,7 +514,7 @@ class CommonJobConfigs: "./tests/performance/", ], exclude_files=[".md"], - docker=["clickhouse/performance-comparison"], + docker=["altinityinfra/performance-comparison"], ), run_command="performance_comparison_check.py", runner_type=Runners.STRESS_TESTER, @@ -529,7 +532,7 @@ class CommonJobConfigs: digest=DigestConfig( include_paths=["./tests/ci/sqllogic_test.py"], exclude_files=[".md"], - docker=["clickhouse/sqllogic-test"], + docker=["altinityinfra/sqllogic-test"], ), run_command="sqllogic_test.py", timeout=10800, @@ -541,7 +544,7 @@ class CommonJobConfigs: digest=DigestConfig( include_paths=["./tests/ci/sqltest.py"], exclude_files=[".md"], - docker=["clickhouse/sqltest"], + docker=["altinityinfra/sqltest"], ), run_command="sqltest.py", timeout=10800, @@ -558,7 +561,7 @@ class CommonJobConfigs: DOCKER_SERVER = JobConfig( job_name_keyword="docker", required_on_release_branch=True, - run_command='docker_server.py --check-name "$CHECK_NAME" --tag-type head --allow-build-reuse', + run_command='docker_server.py --check-name "$CHECK_NAME" --tag-type head --allow-build-reuse --push', digest=DigestConfig( include_paths=[ "tests/ci/docker_server.py", @@ -574,7 +577,7 @@ class CommonJobConfigs: include_paths=[ "tests/ci/clickbench.py", ], - docker=["clickhouse/clickbench"], + docker=["altinityinfra/clickbench"], ), run_command='clickbench.py "$CHECK_NAME"', timeout=900, @@ -611,7 +614,7 @@ class CommonJobConfigs: "./tests/performance", ], exclude_files=[".md"], - docker=["clickhouse/binary-builder"], + docker=["altinityinfra/binary-builder"], git_submodules=True, ), run_command="build_check.py $BUILD_NAME", diff --git a/tests/ci/ci_settings.py b/tests/ci/ci_settings.py index 05929179e06e..64de92e289a5 100644 --- a/tests/ci/ci_settings.py +++ b/tests/ci/ci_settings.py @@ -48,7 +48,8 @@ def create_from_pr_message( res = CiSettings() pr_info = PRInfo() if ( - not pr_info.is_pr and not debug_message + (not pr_info.is_pr and not debug_message) + or pr_info.body is None ): # if commit_message is provided it's test/debug scenario - do not return # CI options can be configured in PRs only # if debug_message is provided - it's a test diff --git a/tests/ci/clickbench.py b/tests/ci/clickbench.py index a1988abb1f54..3d995289c530 100644 --- a/tests/ci/clickbench.py +++ b/tests/ci/clickbench.py @@ -22,7 +22,7 @@ def get_image_name() -> str: - return "clickhouse/clickbench" + return "altinityinfra/clickbench" def get_run_command( diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index ef93a07e68cd..029a3f539d6d 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -212,11 +212,11 @@ def prepare_tests_results_for_clickhouse( report_url: str, check_name: str, ) -> List[dict]: - base_ref = pr_info.base_ref - base_repo = pr_info.base_name - head_ref = pr_info.head_ref - head_repo = pr_info.head_name - pull_request_url = f"https://github.com/{GITHUB_REPOSITORY}/commits/{head_ref}" + pull_request_url = "https://github.com/Altinity/ClickHouse/commits/master" + base_ref = "master" + head_ref = "master" + base_repo = pr_info.repo_full_name + head_repo = pr_info.repo_full_name if pr_info.number != 0: pull_request_url = pr_info.pr_html_url diff --git a/tests/ci/compatibility_check.py b/tests/ci/compatibility_check.py index bb0c717160e2..3b161bdcc88e 100644 --- a/tests/ci/compatibility_check.py +++ b/tests/ci/compatibility_check.py @@ -16,8 +16,8 @@ from report import FAILURE, SUCCESS, JobReport, TestResult, TestResults from stopwatch import Stopwatch -IMAGE_UBUNTU = "clickhouse/test-old-ubuntu" -IMAGE_CENTOS = "clickhouse/test-old-centos" +IMAGE_UBUNTU = "altinityinfra/test-old-ubuntu" +IMAGE_CENTOS = "altinityinfra/test-old-centos" DOWNLOAD_RETRIES_COUNT = 5 diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 49d2b63b87ca..c401f218df56 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -275,7 +275,7 @@ def main(): NAME, ) ch_helper = ClickHouseHelper() - ch_helper.insert_events_into(db="default", table="checks", events=prepared_events) + ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) if status == FAILURE: sys.exit(1) diff --git a/tests/ci/docker_images_helper.py b/tests/ci/docker_images_helper.py index f0323145cfa1..9faefef12c24 100644 --- a/tests/ci/docker_images_helper.py +++ b/tests/ci/docker_images_helper.py @@ -20,9 +20,9 @@ def docker_login(relogin: bool = True) -> None: "docker system info | grep --quiet -E 'Username|Registry'" ): Shell.check( # pylint: disable=unexpected-keyword-arg - "docker login --username 'robotclickhouse' --password-stdin", + "docker login --username 'altinityinfra' --password-stdin", strict=True, - stdin_str=get_parameter_from_ssm("dockerhub_robot_password"), + stdin_str=get_parameter_from_ssm("dockerhub-password"), encoding="utf-8", ) diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index 772cbbf1b024..914e93935deb 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -212,7 +212,7 @@ def main(): NAME, ) ch_helper = ClickHouseHelper() - ch_helper.insert_events_into(db="default", table="checks", events=prepared_events) + ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) if status == FAILURE: sys.exit(1) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 3251ec5644e0..ca2732e0af78 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -58,7 +58,7 @@ def parse_args() -> argparse.Namespace: "--version", type=version_arg, default=get_version_from_repo(git=git).string, - help="a version to build, automaticaly got from version_helper, accepts either " + help="a version to build, automatically got from version_helper, accepts either " "tag ('refs/tags/' is removed automatically) or a normal 22.2.2.2 format", ) parser.add_argument( @@ -214,7 +214,7 @@ def build_and_push_image( init_args = ["docker", "buildx", "build"] if push: init_args.append("--push") - init_args.append("--output=type=image,push-by-digest=true") + init_args.append("--output=type=image") init_args.append(f"--tag={image.repo}") else: init_args.append("--output=type=docker") @@ -311,10 +311,10 @@ def main(): assert not args.image_path and not args.image_repo if "server image" in args.check_name: image_path = "docker/server" - image_repo = "clickhouse/clickhouse-server" + image_repo = "altinityinfra/clickhouse-server" elif "keeper image" in args.check_name: image_path = "docker/keeper" - image_repo = "clickhouse/clickhouse-keeper" + image_repo = "altinityinfra/clickhouse-keeper" else: assert False, "Invalid --check-name" else: @@ -332,6 +332,8 @@ def main(): image = DockerImageData(image_path, image_repo, False) tags = gen_tags(args.version, args.tag_type) + tags.append(f'{pr_info.number}-{args.version}') + repo_urls = {} direct_urls: Dict[str, List[str]] = {} diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py index 6bd4ef496755..b850c28ab2b1 100644 --- a/tests/ci/docs_check.py +++ b/tests/ci/docs_check.py @@ -61,7 +61,7 @@ def main(): elif args.force: logging.info("Check the docs because of force flag") - docker_image = pull_image(get_docker_image("clickhouse/docs-builder")) + docker_image = pull_image(get_docker_image("altinityinfra/docs-builder")) test_output = temp_path / "docs_check_log" test_output.mkdir(parents=True, exist_ok=True) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 5217e4035da1..cad781a17445 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -20,20 +20,24 @@ CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") GITHUB_JOB = os.getenv("GITHUB_JOB_OVERRIDDEN", "") or os.getenv("GITHUB_JOB", "local") -GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse") +GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "Altinity/ClickHouse") GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID", "0") GITHUB_SERVER_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") GITHUB_UPSTREAM_REPOSITORY = os.getenv( - "GITHUB_UPSTREAM_REPOSITORY", "ClickHouse/ClickHouse" + "GITHUB_UPSTREAM_REPOSITORY", "Altinity/ClickHouse" ) GITHUB_WORKSPACE = os.getenv("GITHUB_WORKSPACE", git_root) GITHUB_RUN_URL = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}" IMAGES_PATH = os.getenv("IMAGES_PATH", TEMP_PATH) REPO_COPY = os.getenv("REPO_COPY", GITHUB_WORKSPACE) RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp"))) -S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "clickhouse-builds") -S3_BUILDS_BUCKET_PUBLIC = "clickhouse-builds" -S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "clickhouse-test-reports") + +S3_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") +S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "altinity-build-artifacts") +S3_BUILDS_BUCKET_PUBLIC = "altinity-build-artifacts" +S3_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") +S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "altinity-build-artifacts") + S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") S3_DOWNLOAD = os.getenv("S3_DOWNLOAD", S3_URL) S3_ARTIFACT_DOWNLOAD_TEMPLATE = ( diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index ed727dd3659f..3e725d8c6a9b 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -80,7 +80,7 @@ def main(): pr_info = PRInfo() - docker_image = pull_image(get_docker_image("clickhouse/fasttest")) + docker_image = pull_image(get_docker_image("altinityinfra/fasttest")) workspace = temp_path / "fasttest-workspace" workspace.mkdir(parents=True, exist_ok=True) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index d08f98fa05fe..0fd8ba4dd638 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -16,7 +16,6 @@ from docker_images_helper import DockerImage, get_docker_image, pull_image from download_release_packages import download_last_release from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH -from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import ( ERROR, @@ -52,8 +51,9 @@ def get_additional_envs( check_name: str, run_by_hash_num: int, run_by_hash_total: int ) -> List[str]: result = [] - azure_connection_string = get_parameter_from_ssm("azure_connection_string") - result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") + # TODO(vnemkov): put proper Azure connection string into SSM and re-enable this one + # azure_connection_string = get_parameter_from_ssm("azure_connection_string") + # result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") if "DatabaseReplicated" in check_name: result.append("USE_DATABASE_REPLICATED=1") if "DatabaseOrdinary" in check_name: @@ -67,9 +67,6 @@ def get_additional_envs( result.append("RANDOMIZE_OBJECT_KEY_TYPE=1") if "analyzer" in check_name: result.append("USE_OLD_ANALYZER=1") - if "azure" in check_name: - assert "USE_S3_STORAGE_FOR_MERGE_TREE=1" not in result - result.append("USE_AZURE_STORAGE_FOR_MERGE_TREE=1") if run_by_hash_total != 0: result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}") @@ -80,9 +77,9 @@ def get_additional_envs( def get_image_name(check_name: str) -> str: if "stateless" in check_name.lower(): - return "clickhouse/stateless-test" + return "altinityinfra/stateless-test" if "stateful" in check_name.lower(): - return "clickhouse/stateful-test" + return "altinityinfra/stateful-test" raise ValueError(f"Cannot deduce image name based on check name {check_name}") diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index a4317d5caff5..a218202fd76e 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -59,8 +59,20 @@ def get_parameters_from_ssm( ROBOT_TOKEN = None # type: Optional[Token] +# NOTE(Arthur Passos): Original CI code uses the "_original" version of this method. Each robot token is rate limited +# and the original implementation selects the "best one". To make it simpler and iterate faster, +# we are using only one robot and keeping the method signature. In the future we might reconsider +# having multiple robot tokens +def get_best_robot_token(token_prefix_env_name="github_robot_token"): + # Re-use already fetched token (same as in get_best_robot_token_original) + # except here we assume it is always a string (since we use only one token and don't do token rotation) + global ROBOT_TOKEN + if ROBOT_TOKEN is not None: + return ROBOT_TOKEN + ROBOT_TOKEN = get_parameter_from_ssm(token_prefix_env_name) + return ROBOT_TOKEN -def get_best_robot_token(tokens_path: str = "/github-tokens") -> str: +def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: global ROBOT_TOKEN if ROBOT_TOKEN is not None: return ROBOT_TOKEN.value diff --git a/tests/ci/git_helper.py b/tests/ci/git_helper.py index 6b66bc44d100..ca924a378a45 100644 --- a/tests/ci/git_helper.py +++ b/tests/ci/git_helper.py @@ -11,13 +11,29 @@ logger = logging.getLogger(__name__) +class VersionType: + LTS = "lts" + NEW = "new" + PRESTABLE = "altinityedge" + STABLE = "altinitystable" + TESTING = "altinitytest" + + VALID = (NEW, TESTING, PRESTABLE, STABLE, LTS, + # NOTE (vnemkov): we don't use those directly, but it is used in unit-tests + "stable", + "prestable", + "testing", + ) + # ^ and $ match subline in `multiple\nlines` # \A and \Z match only start and end of the whole string +# NOTE (vnemkov): support both upstream tag style: v22.x.y.z-lts and Altinity tag style: v22.x.y.z.altinitystable +# Because at early release stages there could be no Altinity tag set on commit, only upstream one. RELEASE_BRANCH_REGEXP = r"\A\d+[.]\d+\Z" TAG_REGEXP = ( r"\Av\d{2}" # First two digits of major part r"([.][1-9]\d*){3}" # minor.patch.tweak parts - r"-(new|testing|prestable|stable|lts)\Z" # suffix with a version type + fr"[.-]({'|'.join(VersionType.VALID)})\Z" # suffix with a version type ) SHA_REGEXP = re.compile(r"\A([0-9]|[a-f]){40}\Z") @@ -26,9 +42,9 @@ with tempfile.NamedTemporaryFile("w", delete=False) as f: GIT_KNOWN_HOSTS_FILE = f.name - GIT_PREFIX = ( # All commits to remote are done as robot-clickhouse - "git -c user.email=robot-clickhouse@users.noreply.github.com " - "-c user.name=robot-clickhouse -c commit.gpgsign=false " + GIT_PREFIX = ( # All commits to remote are done as altinity-robot + "git -c user.email=altinity-robot@users.noreply.github.com " + "-c user.name=altinity-robot -c commit.gpgsign=false " "-c core.sshCommand=" f"'ssh -o UserKnownHostsFile={GIT_KNOWN_HOSTS_FILE} " "-o StrictHostKeyChecking=accept-new'" diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py index 6c33b1f20660..af556285ecb4 100644 --- a/tests/ci/install_check.py +++ b/tests/ci/install_check.py @@ -17,8 +17,8 @@ from stopwatch import Stopwatch from tee_popen import TeePopen -RPM_IMAGE = "clickhouse/install-rpm-test" -DEB_IMAGE = "clickhouse/install-deb-test" +RPM_IMAGE = "altinityinfra/install-rpm-test" +DEB_IMAGE = "altinityinfra/install-deb-test" TEMP_PATH = Path(TEMP) LOGS_PATH = TEMP_PATH / "tests_logs" diff --git a/tests/ci/integration_test_images.py b/tests/ci/integration_test_images.py index 8148ac611817..d51998cc1549 100644 --- a/tests/ci/integration_test_images.py +++ b/tests/ci/integration_test_images.py @@ -1,19 +1,19 @@ #!/usr/bin/env python3 IMAGES_ENV = { - "clickhouse/dotnet-client": "DOCKER_DOTNET_CLIENT_TAG", - "clickhouse/integration-helper": "DOCKER_HELPER_TAG", - "clickhouse/integration-test": "DOCKER_BASE_TAG", - "clickhouse/integration-tests-runner": "", - "clickhouse/kerberized-hadoop": "DOCKER_KERBERIZED_HADOOP_TAG", - "clickhouse/kerberos-kdc": "DOCKER_KERBEROS_KDC_TAG", - "clickhouse/mysql-golang-client": "DOCKER_MYSQL_GOLANG_CLIENT_TAG", - "clickhouse/mysql-java-client": "DOCKER_MYSQL_JAVA_CLIENT_TAG", - "clickhouse/mysql-js-client": "DOCKER_MYSQL_JS_CLIENT_TAG", - "clickhouse/mysql-php-client": "DOCKER_MYSQL_PHP_CLIENT_TAG", - "clickhouse/nginx-dav": "DOCKER_NGINX_DAV_TAG", - "clickhouse/postgresql-java-client": "DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", - "clickhouse/python-bottle": "DOCKER_PYTHON_BOTTLE_TAG", + "altinityinfra/dotnet-client": "DOCKER_DOTNET_CLIENT_TAG", + "altinityinfra/integration-helper": "DOCKER_HELPER_TAG", + "altinityinfra/integration-test": "DOCKER_BASE_TAG", + "altinityinfra/integration-tests-runner": "", + # "altinityinfra/kerberized-hadoop": "DOCKER_KERBERIZED_HADOOP_TAG", + "altinityinfra/kerberos-kdc": "DOCKER_KERBEROS_KDC_TAG", + "altinityinfra/mysql-golang-client": "DOCKER_MYSQL_GOLANG_CLIENT_TAG", + "altinityinfra/mysql-java-client": "DOCKER_MYSQL_JAVA_CLIENT_TAG", + "altinityinfra/mysql-js-client": "DOCKER_MYSQL_JS_CLIENT_TAG", + "altinityinfra/mysql-php-client": "DOCKER_MYSQL_PHP_CLIENT_TAG", + "altinityinfra/nginx-dav": "DOCKER_NGINX_DAV_TAG", + "altinityinfra/postgresql-java-client": "DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", + "altinityinfra/python-bottle": "DOCKER_PYTHON_BOTTLE_TAG", } IMAGES = list(IMAGES_ENV.keys()) diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py index 6023ad8f0762..476c7b149c20 100755 --- a/tests/ci/integration_tests_runner.py +++ b/tests/ci/integration_tests_runner.py @@ -26,8 +26,8 @@ from stopwatch import Stopwatch from tee_popen import TeePopen -MAX_RETRY = 1 -NUM_WORKERS = 5 +MAX_RETRY = 3 +NUM_WORKERS = 10 SLEEP_BETWEEN_RETRIES = 5 PARALLEL_GROUP_SIZE = 100 CLICKHOUSE_BINARY_PATH = "usr/bin/clickhouse" @@ -318,7 +318,7 @@ def _pre_pull_images(self, repo_path): cmd = ( f"cd {repo_path}/tests/integration && " - f"timeout --signal=KILL 1h ./runner {self._get_runner_opts()} {image_cmd} " + f"timeout --signal=KILL 2h ./runner {self._get_runner_opts()} {image_cmd} " "--pre-pull --command ' echo Pre Pull finished ' " ) @@ -428,7 +428,7 @@ def _get_all_tests(self, repo_path): out_file_full = os.path.join(self.result_path, "runner_get_all_tests.log") cmd = ( f"cd {repo_path}/tests/integration && " - f"timeout --signal=KILL 1h ./runner {runner_opts} {image_cmd} -- --setup-plan " + f"timeout --signal=KILL 2h ./runner {runner_opts} {image_cmd} -- --setup-plan " ) logging.info( @@ -515,7 +515,7 @@ def _get_runner_image_cmd(self, repo_path): "--docker-image-version", ): for img in IMAGES: - if img == "clickhouse/integration-tests-runner": + if img == "altinityinfra/integration-tests-runner": runner_version = self.get_image_version(img) logging.info( "Can run with custom docker image version %s", runner_version @@ -645,7 +645,8 @@ def run_test_group( info_path = os.path.join(repo_path, "tests/integration", info_basename) test_cmd = " ".join([shlex.quote(test) for test in sorted(test_names)]) - parallel_cmd = f" --parallel {num_workers} " if num_workers > 0 else "" + # run in parallel only the first time, re-runs are sequential to give chance to flappy tests to pass. + parallel_cmd = f" --parallel {num_workers} " if num_workers > 0 and i == 0 else "" repeat_cmd = f" --count {repeat_count} " if repeat_count > 0 else "" # -r -- show extra test summary: # -f -- (f)ailed diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py index 772467d42459..85f33374e8a4 100644 --- a/tests/ci/jepsen_check.py +++ b/tests/ci/jepsen_check.py @@ -28,10 +28,10 @@ KEEPER_DESIRED_INSTANCE_COUNT = 3 SERVER_DESIRED_INSTANCE_COUNT = 4 -KEEPER_IMAGE_NAME = "clickhouse/keeper-jepsen-test" +KEEPER_IMAGE_NAME = "altinityinfra/keeper-jepsen-test" KEEPER_CHECK_NAME = CI.JobNames.JEPSEN_KEEPER -SERVER_IMAGE_NAME = "clickhouse/server-jepsen-test" +SERVER_IMAGE_NAME = "altinityinfra/server-jepsen-test" SERVER_CHECK_NAME = CI.JobNames.JEPSEN_SERVER SUCCESSFUL_TESTS_ANCHOR = "# Successful tests" diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index 8f19dd7d0234..984534682224 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -109,7 +109,7 @@ def main(): run_by_hash_num = 0 run_by_hash_total = 0 - docker_image = pull_image(get_docker_image("clickhouse/libfuzzer")) + docker_image = pull_image(get_docker_image("altinityinfra/libfuzzer")) fuzzers_path = temp_path / "fuzzers" fuzzers_path.mkdir(parents=True, exist_ok=True) diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index b3488ac0af21..c677ac92bc75 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -31,7 +31,7 @@ from stopwatch import Stopwatch from tee_popen import TeePopen -IMAGE_NAME = "clickhouse/performance-comparison" +IMAGE_NAME = "altinityinfra/performance-comparison" def get_run_command( @@ -100,7 +100,7 @@ def main(): if pr_info.number == 0: pr_link = commit.html_url else: - pr_link = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_info.number}" + pr_link = f"https://github.com/Altinity/ClickHouse/pull/{pr_info.number}" docker_env += ( f' -e CHPC_ADD_REPORT_LINKS="' @@ -188,7 +188,7 @@ def main(): def too_many_slow(msg): match = re.search(r"(|.* )(\d+) slower.*", msg) # This threshold should be synchronized with the value in - # https://github.com/ClickHouse/ClickHouse/blob/master/docker/test/performance-comparison/report.py#L629 + # https://github.com/Altinity/ClickHouse/blob/master/docker/test/performance-comparison/report.py#L629 threshold = 5 return int(match.group(2).strip()) > threshold if match else False diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 09a8cb563a1f..f97ea05e14da 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -80,8 +80,12 @@ def get_pr_for_commit(sha, ref): ref, sha, ) - first_pr = our_prs[0] - return first_pr + if len(our_prs) != 0: + first_pr = our_prs[0] + return first_pr + else: + return None + except Exception as ex: logging.error( "Cannot fetch PR info from commit ref %s, sha %s, exception: %s", @@ -132,8 +136,9 @@ def __init__( ref = github_event.get("ref", "refs/heads/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] + self.ref = ref # type: str e.g. "refs/pull/509/merge" or "refs/tags/v24.3.12.76.altinitystable" # Default values - self.base_ref = "" # type: str + self.base_ref = github_event.get("base_ref","") # type: str self.base_name = "" # type: str self.head_ref = "" # type: str self.head_name = "" # type: str diff --git a/tests/ci/release.py b/tests/ci/release.py index b26d6205f3b2..20a4eff61300 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -603,7 +603,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--repo", - default="ClickHouse/ClickHouse", + default="Altinity/ClickHouse", help="repository to create the release", ) parser.add_argument( diff --git a/tests/ci/sign_release.py b/tests/ci/sign_release.py new file mode 100644 index 000000000000..8a5827097c8b --- /dev/null +++ b/tests/ci/sign_release.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +import sys +import os +import logging +from env_helper import TEMP_PATH, REPO_COPY, REPORT_PATH +from s3_helper import S3Helper +from pr_info import PRInfo +from build_download_helper import download_builds_filter +import hashlib +from pathlib import Path + +GPG_BINARY_SIGNING_KEY = os.getenv("GPG_BINARY_SIGNING_KEY") +GPG_BINARY_SIGNING_PASSPHRASE = os.getenv("GPG_BINARY_SIGNING_PASSPHRASE") + +CHECK_NAME = os.getenv("CHECK_NAME", "Sign release") + +def hash_file(file_path): + BLOCK_SIZE = 65536 # The size of each read from the file + + file_hash = hashlib.sha256() # Create the hash object, can use something other than `.sha256()` if you wish + with open(file_path, 'rb') as f: # Open the file to read it's bytes + fb = f.read(BLOCK_SIZE) # Read from the file. Take in the amount declared above + while len(fb) > 0: # While there is still data being read from the file + file_hash.update(fb) # Update the hash + fb = f.read(BLOCK_SIZE) # Read the next block from the file + + hash_file_path = file_path + '.sha256' + with open(hash_file_path, 'x') as f: + digest = file_hash.hexdigest() + f.write(digest) + print(f'Hashed {file_path}: {digest}') + + return hash_file_path + +def sign_file(file_path): + priv_key_file_path = 'priv.key' + with open(priv_key_file_path, 'x') as f: + f.write(GPG_BINARY_SIGNING_KEY) + + out_file_path = f'{file_path}.gpg' + + os.system(f'echo {GPG_BINARY_SIGNING_PASSPHRASE} | gpg --batch --import {priv_key_file_path}') + os.system(f'gpg -o {out_file_path} --pinentry-mode=loopback --batch --yes --passphrase {GPG_BINARY_SIGNING_PASSPHRASE} --sign {file_path}') + print(f"Signed {file_path}") + os.remove(priv_key_file_path) + + return out_file_path + +def main(): + reports_path = Path(REPORT_PATH) + + if not os.path.exists(TEMP_PATH): + os.makedirs(TEMP_PATH) + + pr_info = PRInfo() + + logging.info("Repo copy path %s", REPO_COPY) + + s3_helper = S3Helper() + + s3_path_prefix = Path(f"{pr_info.number}/{pr_info.sha}/" + CHECK_NAME.lower().replace( + " ", "_" + ).replace("(", "_").replace(")", "_").replace(",", "_")) + + # downloads `package_release` artifacts generated + download_builds_filter(CHECK_NAME, reports_path, Path(TEMP_PATH)) + + for f in os.listdir(TEMP_PATH): + full_path = os.path.join(TEMP_PATH, f) + if os.path.isdir(full_path): + continue + hashed_file_path = hash_file(full_path) + signed_file_path = sign_file(hashed_file_path) + s3_path = s3_path_prefix / os.path.basename(signed_file_path) + s3_helper.upload_build_file_to_s3(Path(signed_file_path), str(s3_path)) + print(f'Uploaded file {signed_file_path} to {s3_path}') + + # Signed hashes are: + # clickhouse-client_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-keeper_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-client-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-keeper-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-client_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-keeper-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg + # clickhouse-client-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-keeper-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg + # clickhouse-common-static_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-keeper-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-common-static-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-keeper-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-common-static_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-keeper.sha512.gpg + # clickhouse-common-static-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-library-bridge.sha512.gpg + # clickhouse-common-static-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-odbc-bridge.sha512.gpg + # clickhouse-common-static-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-server_22.3.15.2.altinitystable_amd64.deb.sha512.gpg + # clickhouse-common-static-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-server-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg + # clickhouse-common-static-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-server_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-keeper_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-server-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-keeper-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse.sha512.gpg + + sys.exit(0) + +if __name__ == "__main__": + main() diff --git a/tests/ci/sqlancer_check.py b/tests/ci/sqlancer_check.py index a68db8b97919..1c173eb55519 100644 --- a/tests/ci/sqlancer_check.py +++ b/tests/ci/sqlancer_check.py @@ -14,7 +14,7 @@ from tee_popen import TeePopen from ci_config import CI -IMAGE_NAME = "clickhouse/sqlancer-test" +IMAGE_NAME = "altinityinfra/sqlancer-test" def get_run_command(download_url: str, workspace_path: Path, image: DockerImage) -> str: diff --git a/tests/ci/sqllogic_test.py b/tests/ci/sqllogic_test.py index 7fe44c235c7b..27ab9441826b 100755 --- a/tests/ci/sqllogic_test.py +++ b/tests/ci/sqllogic_test.py @@ -26,7 +26,7 @@ from tee_popen import TeePopen NO_CHANGES_MSG = "Nothing to run" -IMAGE_NAME = "clickhouse/sqllogic-test" +IMAGE_NAME = "altinityinfra/sqllogic-test" def get_run_command( diff --git a/tests/ci/sqltest.py b/tests/ci/sqltest.py index 8e6ca6ff87fa..35fa6fbaa199 100644 --- a/tests/ci/sqltest.py +++ b/tests/ci/sqltest.py @@ -14,7 +14,7 @@ from stopwatch import Stopwatch from ci_config import CI -IMAGE_NAME = "clickhouse/sqltest" +IMAGE_NAME = "altinityinfra/sqltest" def get_run_command(pr_number, sha, download_url, workspace_path, image): diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 85da601e379c..48a4e2f87798 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -13,7 +13,6 @@ from clickhouse_helper import CiLogsCredentials from docker_images_helper import DockerImage, get_docker_image, pull_image from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH -from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import ERROR, JobReport, TestResults, read_test_results from stopwatch import Stopwatch @@ -34,14 +33,13 @@ def format(self, record): def get_additional_envs(check_name: str) -> List[str]: result = [] - azure_connection_string = get_parameter_from_ssm("azure_connection_string") - result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") + # TODO(vnemkov): put proper Azure connection string into SSM and re-enable this one + # azure_connection_string = get_parameter_from_ssm("azure_connection_string") + # result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") # some cloud-specific features require feature flags enabled # so we need this ENV to be able to disable the randomization # of feature flags result.append("RANDOMIZE_KEEPER_FEATURE_FLAGS=1") - if "azure" in check_name: - result.append("USE_AZURE_STORAGE_FOR_MERGE_TREE=1") if "s3" in check_name: result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") @@ -208,4 +206,4 @@ def run_stress_test(docker_image_name: str) -> None: if __name__ == "__main__": - run_stress_test("clickhouse/stress-test") + run_stress_test("altinityinfra/stress-test") diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 36620d44a2d0..ba5aef398a19 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -168,7 +168,7 @@ def main(): run_shell_check = any(is_shell(file) for file in pr_info.changed_files) run_python_check = any(is_python(file) for file in pr_info.changed_files) - IMAGE_NAME = "clickhouse/style-test" + IMAGE_NAME = "altinityinfra/style-test" image = pull_image(get_docker_image(IMAGE_NAME)) docker_command = ( f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE " diff --git a/tests/ci/test_git.py b/tests/ci/test_git.py index 60cd95b6869d..833582e01bd7 100644 --- a/tests/ci/test_git.py +++ b/tests/ci/test_git.py @@ -56,16 +56,16 @@ def setUp(self): self.git.commits_since_new = 0 def test_tags(self): - self.git.new_tag = "v21.12.333.22222-stable" - self.git.latest_tag = "v21.12.333.22222-stable" + self.git.new_tag = "v21.12.333.22222-altinitystable" + self.git.latest_tag = "v21.12.333.22222-altinitystable" for tag_attr in ("new_tag", "latest_tag"): - self.assertEqual(getattr(self.git, tag_attr), "v21.12.333.22222-stable") + self.assertEqual(getattr(self.git, tag_attr), "v21.12.333.22222-altinitystable") setattr(self.git, tag_attr, "") self.assertEqual(getattr(self.git, tag_attr), "") for tag in ( - "v21.12.333-stable", + "v21.12.333-altinitystable", "v21.12.333-prestable", - "21.12.333.22222-stable", + "21.12.333.22222-altinitystable", "v21.12.333.22222-production", ): with self.assertRaises(Exception): diff --git a/tests/ci/test_version.py b/tests/ci/test_version.py index c4f12091ec09..30f7dbc6a762 100644 --- a/tests/ci/test_version.py +++ b/tests/ci/test_version.py @@ -18,13 +18,14 @@ def test_version_arg(self): ("1.1.1.2", vh.get_version_from_string("1.1.1.2")), ("v11.1.1.2-lts", vh.get_version_from_string("11.1.1.2")), ("v01.1.1.2-prestable", vh.get_version_from_string("1.1.1.2")), - ("v21.1.1.2-stable", vh.get_version_from_string("21.1.1.2")), + ("v21.1.1.2-altinitystable", vh.get_version_from_string("21.1.1.2")), ("v31.1.1.2-testing", vh.get_version_from_string("31.1.1.2")), ("refs/tags/v31.1.1.2-testing", vh.get_version_from_string("31.1.1.2")), ) - for test_case in cases: - version = vh.version_arg(test_case[0]) - self.assertEqual(test_case[1], version) + for i, test_case in enumerate(cases): + with self.subTest(test_case, i=i): + version = vh.version_arg(test_case[0]) + self.assertEqual(test_case[1], version) error_cases = ( "0.0.0", "1.1.1.a", @@ -48,44 +49,52 @@ class TestCase: expected: CHV cases = ( + # TestCase( + # "v24.6.1.1-new", + # 15, + # "v24.4.1.2088-stable", + # 415, + # CHV(24, 5, 1, 54487, None, 415), + # ), + # TestCase( + # "v24.6.1.1-testing", + # 15, + # "v24.4.1.2088-stable", + # 415, + # CHV(24, 5, 1, 54487, None, 15), + # ), + # TestCase( + # "v24.6.1.1-stable", + # 15, + # "v24.4.1.2088-stable", + # 415, + # CHV(24, 5, 1, 54487, None, 15), + # ), + # TestCase( + # "v24.5.1.1-stable", + # 15, + # "v24.4.1.2088-stable", + # 415, + # CHV(24, 5, 1, 54487, None, 15), + # ), TestCase( - "v24.6.1.1-new", - 15, + "v24.5.1.100-stable", + 0, "v24.4.1.2088-stable", 415, - CHV(24, 5, 1, 54487, None, 415), - ), - TestCase( - "v24.6.1.1-testing", - 15, - "v24.4.1.2088-stable", - 415, - CHV(24, 5, 1, 54487, None, 16), - ), - TestCase( - "v24.6.1.1-stable", - 15, - "v24.4.1.2088-stable", - 415, - CHV(24, 5, 1, 54487, None, 15), - ), - TestCase( - "v24.5.1.1-stable", - 15, - "v24.4.1.2088-stable", - 415, - CHV(24, 5, 1, 54487, None, 15), + CHV(24, 5, 1, 54487, None, 100), ), ) git = Git(True) - for tc in cases: - git.latest_tag = tc.latest_tag - git.commits_since_latest = tc.commits_since_latest - git.new_tag = tc.new_tag - git.commits_since_new = tc.commits_since_new - self.assertEqual( - vh.get_version_from_repo( - Path("tests/ci/tests/autogenerated_versions.txt"), git - ), - tc.expected, - ) + for i, tc in enumerate(cases): + with self.subTest(tc, i=i): + git.latest_tag = tc.latest_tag + git.commits_since_latest = tc.commits_since_latest + git.new_tag = tc.new_tag + git.commits_since_new = tc.commits_since_new + self.assertEqual( + vh.get_version_from_repo( + Path("tests/ci/tests/autogenerated_versions.txt"), git + ), + tc.expected, + ) diff --git a/tests/ci/tests/autogenerated_versions.txt b/tests/ci/tests/autogenerated_versions.txt index 10028bf50c81..675fc161a2fc 100644 --- a/tests/ci/tests/autogenerated_versions.txt +++ b/tests/ci/tests/autogenerated_versions.txt @@ -6,6 +6,7 @@ SET(VERSION_REVISION 54487) SET(VERSION_MAJOR 24) SET(VERSION_MINOR 5) SET(VERSION_PATCH 1) +SET(VERSION_TWEAK 15) SET(VERSION_GITHASH 70a1d3a63d47f0be077d67b8deb907230fc7cfb0) SET(VERSION_DESCRIBE v24.5.1.1-testing) SET(VERSION_STRING 24.5.1.1) diff --git a/tests/ci/tests/docker_images_for_tests.json b/tests/ci/tests/docker_images_for_tests.json index 70db87605616..5faa3522e7ec 100644 --- a/tests/ci/tests/docker_images_for_tests.json +++ b/tests/ci/tests/docker_images_for_tests.json @@ -1,120 +1,120 @@ { "docker/packager/deb": { - "name": "clickhouse/deb-builder", + "name": "altinityinfra/deb-builder", "dependent": [] }, "docker/packager/binary": { - "name": "clickhouse/binary-builder", + "name": "altinityinfra/binary-builder", "dependent": [ "docker/test/codebrowser" ] }, "docker/test/compatibility/centos": { - "name": "clickhouse/test-old-centos", + "name": "altinityinfra/test-old-centos", "dependent": [] }, "docker/test/compatibility/ubuntu": { - "name": "clickhouse/test-old-ubuntu", + "name": "altinityinfra/test-old-ubuntu", "dependent": [] }, "docker/test/integration/base": { - "name": "clickhouse/integration-test", + "name": "altinityinfra/integration-test", "dependent": [] }, "docker/test/fuzzer": { - "name": "clickhouse/fuzzer", + "name": "altinityinfra/fuzzer", "dependent": [] }, "docker/test/performance-comparison": { - "name": "clickhouse/performance-comparison", + "name": "altinityinfra/performance-comparison", "dependent": [] }, "docker/test/util": { - "name": "clickhouse/test-util", + "name": "altinityinfra/test-util", "dependent": [ "docker/test/base", "docker/test/fasttest" ] }, "docker/test/stateless": { - "name": "clickhouse/stateless-test", + "name": "altinityinfra/stateless-test", "dependent": [ "docker/test/stateful", "docker/test/unit" ] }, "docker/test/stateful": { - "name": "clickhouse/stateful-test", + "name": "altinityinfra/stateful-test", "dependent": [ "docker/test/stress" ] }, "docker/test/unit": { - "name": "clickhouse/unit-test", + "name": "altinityinfra/unit-test", "dependent": [] }, "docker/test/stress": { - "name": "clickhouse/stress-test", + "name": "altinityinfra/stress-test", "dependent": [] }, "docker/test/codebrowser": { - "name": "clickhouse/codebrowser", + "name": "altinityinfra/codebrowser", "dependent": [] }, "docker/test/integration/runner": { - "name": "clickhouse/integration-tests-runner", + "name": "altinityinfra/integration-tests-runner", "dependent": [] }, "docker/test/fasttest": { - "name": "clickhouse/fasttest", + "name": "altinityinfra/fasttest", "dependent": [] }, "docker/test/style": { - "name": "clickhouse/style-test", + "name": "altinityinfra/style-test", "dependent": [] }, "docker/test/integration/s3_proxy": { - "name": "clickhouse/s3-proxy", + "name": "altinityinfra/s3-proxy", "dependent": [] }, "docker/test/integration/resolver": { - "name": "clickhouse/python-bottle", + "name": "altinityinfra/python-bottle", "dependent": [] }, "docker/test/integration/helper_container": { - "name": "clickhouse/integration-helper", + "name": "altinityinfra/integration-helper", "dependent": [] }, "docker/test/integration/mysql_golang_client": { - "name": "clickhouse/mysql-golang-client", + "name": "altinityinfra/mysql-golang-client", "dependent": [] }, "docker/test/integration/dotnet_client": { - "name": "clickhouse/dotnet-client", + "name": "altinityinfra/dotnet-client", "dependent": [] }, "docker/test/integration/mysql_java_client": { - "name": "clickhouse/mysql-java-client", + "name": "altinityinfra/mysql-java-client", "dependent": [] }, "docker/test/integration/mysql_js_client": { - "name": "clickhouse/mysql-js-client", + "name": "altinityinfra/mysql-js-client", "dependent": [] }, "docker/test/integration/mysql_php_client": { - "name": "clickhouse/mysql-php-client", + "name": "altinityinfra/mysql-php-client", "dependent": [] }, "docker/test/integration/postgresql_java_client": { - "name": "clickhouse/postgresql-java-client", + "name": "altinityinfra/postgresql-java-client", "dependent": [] }, "docker/test/integration/kerberos_kdc": { - "name": "clickhouse/kerberos-kdc", + "name": "altinityinfra/kerberos-kdc", "dependent": [] }, "docker/test/base": { - "name": "clickhouse/test-base", + "name": "altinityinfra/test-base", "dependent": [ "docker/test/stateless", "docker/test/integration/base", @@ -123,20 +123,16 @@ "docker/test/sqltest" ] }, - "docker/test/integration/kerberized_hadoop": { - "name": "clickhouse/kerberized-hadoop", - "dependent": [] - }, "docker/test/sqlancer": { - "name": "clickhouse/sqlancer-test", + "name": "altinityinfra/sqlancer-test", "dependent": [] }, "docker/test/keeper-jepsen": { - "name": "clickhouse/keeper-jepsen-test", + "name": "altinityinfra/keeper-jepsen-test", "dependent": [] }, "docker/docs/builder": { - "name": "clickhouse/docs-builder", + "name": "altinityinfra/docs-builder", "only_amd64": true, "dependent": [ "docker/docs/check", @@ -144,19 +140,19 @@ ] }, "docker/docs/check": { - "name": "clickhouse/docs-check", + "name": "altinityinfra/docs-check", "dependent": [] }, "docker/docs/release": { - "name": "clickhouse/docs-release", + "name": "altinityinfra/docs-release", "dependent": [] }, "docker/test/sqllogic": { - "name": "clickhouse/sqllogic-test", + "name": "altinityinfra/sqllogic-test", "dependent": [] }, "docker/test/sqltest": { - "name": "clickhouse/sqltest", + "name": "altinityinfra/sqltest", "dependent": [] } } diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py index 9cc8ec379bfe..7aabe009bc99 100644 --- a/tests/ci/unit_tests_check.py +++ b/tests/ci/unit_tests_check.py @@ -15,7 +15,7 @@ from stopwatch import Stopwatch from tee_popen import TeePopen -IMAGE_NAME = "clickhouse/unit-test" +IMAGE_NAME = "altinityinfra/unit-test" def get_test_name(line): diff --git a/tests/ci/upgrade_check.py b/tests/ci/upgrade_check.py index 83b6f9e299fd..f84451cad81d 100644 --- a/tests/ci/upgrade_check.py +++ b/tests/ci/upgrade_check.py @@ -1,4 +1,4 @@ import stress_check if __name__ == "__main__": - stress_check.run_stress_test("clickhouse/upgrade-check") + stress_check.run_stress_test("altinityinfra/upgrade-check") diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 07a7a9601c04..559af989540f 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Any, Dict, Iterable, List, Literal, Optional, Set, Tuple, Union -from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix +from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix, VersionType FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt" CHANGELOG_IN_PATH = "debian/changelog.in" @@ -29,6 +29,7 @@ SET(VERSION_MINOR {minor}) SET(VERSION_PATCH {patch}) SET(VERSION_GITHASH {githash}) +SET(VERSION_TWEAK {tweak}) SET(VERSION_DESCRIBE {describe}) SET(VERSION_STRING {string}) # end of autochange @@ -48,6 +49,7 @@ def __init__( revision: Union[int, str], git: Optional[Git], tweak: Optional[Union[int, str]] = None, + flavour: Optional[str] = None, ): self._major = int(major) self._minor = int(minor) @@ -61,6 +63,7 @@ def __init__( self._tweak = self._git.tweak self._describe = "" self._description = "" + self._flavour = flavour def update(self, part: PART_TYPE) -> "ClickHouseVersion": """If part is valid, returns a new version""" @@ -152,9 +155,12 @@ def description(self) -> str: @property def string(self): - return ".".join( + version_as_string = ".".join( (str(self.major), str(self.minor), str(self.patch), str(self.tweak)) ) + if self._flavour: + version_as_string = f"{version_as_string}.{self._flavour}" + return version_as_string @property def is_lts(self) -> bool: @@ -185,7 +191,10 @@ def with_description(self, version_type): if version_type not in VersionType.VALID: raise ValueError(f"version type {version_type} not in {VersionType.VALID}") self._description = version_type - self._describe = f"v{self.string}-{version_type}" + if version_type == self._flavour: + self._describe = f"v{self.string}" + else: + self._describe = f"v{self.string}-{version_type}" return self def copy(self) -> "ClickHouseVersion": @@ -242,21 +251,12 @@ def __repr__(self): ClickHouseVersions = List[ClickHouseVersion] - -class VersionType: - LTS = "lts" - NEW = "new" - PRESTABLE = "prestable" - STABLE = "stable" - TESTING = "testing" - VALID = (NEW, TESTING, PRESTABLE, STABLE, LTS) - - def validate_version(version: str) -> None: + # NOTE(vnemkov): minor but important fixes, so versions with 'flavour' are threated as valid (e.g. 22.8.8.4.altinitystable) parts = version.split(".") - if len(parts) != 4: + if len(parts) < 4: raise ValueError(f"{version} does not contain 4 parts") - for part in parts: + for part in parts[:4]: int(part) @@ -297,18 +297,36 @@ def get_version_from_repo( versions["patch"], versions["revision"], git, + # Explicitly use tweak value from version file + tweak=versions.get("tweak", versions["revision"]), + flavour=versions.get("flavour", None) ) - # Since 24.5 we have tags like v24.6.1.1-new, and we must check if the release - # branch already has it's own commit. It's necessary for a proper tweak version + + # if this commit is tagged, use tag's version instead of something stored in cmake if git is not None and git.latest_tag: version_from_tag = get_version_from_tag(git.latest_tag) - if ( - version_from_tag.description == VersionType.NEW - and cmake_version < version_from_tag - ): - # We are in a new release branch without existing release. - # We should change the tweak version to a `tweak_to_new` - cmake_version.tweak = git.tweak_to_new + logging.debug(f'Git latest tag: {git.latest_tag} ({git.commits_since_latest} commits ago)\n' + f'"new" tag: {git.new_tag} ({git.commits_since_new})\n' + f'current commit: {git.sha}\n' + f'current brach: {git.branch}' + ) + if git.commits_since_latest == 0: + # Tag has a priority over the version written in CMake. + # Version must match (except tweak, flavour, description, etc.) to avoid accidental mess. + if not (version_from_tag.major == cmake_version.major \ + and version_from_tag.minor == cmake_version.minor \ + and version_from_tag.patch == cmake_version.patch): + raise RuntimeError(f"Version generated from tag ({version_from_tag}) should have same major, minor, and patch values as version generated from cmake ({cmake_version})") + + # Don't need to reset version completely, mostly because revision part is not set in tag, but must be preserved + logging.debug(f"Resetting TWEAK and FLAVOUR of version from cmake {cmake_version} to values from tag: {version_from_tag.tweak}.{version_from_tag._flavour}") + cmake_version._flavour = version_from_tag._flavour + cmake_version.tweak = version_from_tag.tweak + else: + # We've had some number of commits since the latest tag. + logging.debug(f"Bumping the TWEAK of version from cmake {cmake_version} by {git.commits_since_latest}") + cmake_version.tweak = cmake_version.tweak + git.commits_since_latest + return cmake_version @@ -316,15 +334,31 @@ def get_version_from_string( version: str, git: Optional[Git] = None ) -> ClickHouseVersion: validate_version(version) - parts = version.split(".") - return ClickHouseVersion(parts[0], parts[1], parts[2], -1, git, parts[3]) + # dict for simple handling of missing parts with parts.get(index, default) + parts = dict(enumerate(version.split("."))) + return ClickHouseVersion( + parts[0], + parts[1], + parts[2], + -1, + git, + parts.get(3, None), + parts.get(4, None) + ) def get_version_from_tag(tag: str) -> ClickHouseVersion: Git.check_tag(tag) - tag, description = tag[1:].split("-", 1) - version = get_version_from_string(tag) - version.with_description(description) + tag = tag[1:] # strip initial 'v' + if '-' in tag: + # Upstream tags with dash + tag, description = tag.split("-", 1) + version = get_version_from_string(tag) + version.with_description(description) + else: + # Altinity's tags, with dots as separators between parts (handled properly down the road) + version = get_version_from_string(tag) + return version @@ -424,7 +458,7 @@ def update_contributors( get_abs_path(relative_contributors_path).write_text(content, encoding="utf-8") -def update_version_local(version, version_type="testing"): +def update_version_local(version : ClickHouseVersion, version_type="testing"): update_contributors() version.with_description(version_type) update_cmake_version(version) diff --git a/tests/config/config.d/azure_storage_conf.xml b/tests/config/config.d/azure_storage_conf.xml index f24b62b87b10..412d40111a70 100644 --- a/tests/config/config.d/azure_storage_conf.xml +++ b/tests/config/config.d/azure_storage_conf.xml @@ -4,10 +4,13 @@ object_storage azure + http://localhost:10000/devstoreaccount1 + cont false + + devstoreaccount1 + Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== 33554432 - openbucketforpublicci - cache diff --git a/tests/config/config.d/azure_storage_policy_by_default.xml b/tests/config/config.d/azure_storage_policy_by_default.xml deleted file mode 100644 index cab8a106f1b8..000000000000 --- a/tests/config/config.d/azure_storage_policy_by_default.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - azure_cache - - diff --git a/tests/integration/compose/docker_compose_clickhouse.yml b/tests/integration/compose/docker_compose_clickhouse.yml index fdd124ede91a..ff4523c5b0d7 100644 --- a/tests/integration/compose/docker_compose_clickhouse.yml +++ b/tests/integration/compose/docker_compose_clickhouse.yml @@ -2,4 +2,4 @@ version: '2.3' # Used to pre-pull images with docker-compose services: clickhouse1: - image: clickhouse/integration-test + image: altinityinfra/integration-test diff --git a/tests/integration/compose/docker_compose_dotnet_client.yml b/tests/integration/compose/docker_compose_dotnet_client.yml index b63dac51522c..e5746fa209fb 100644 --- a/tests/integration/compose/docker_compose_dotnet_client.yml +++ b/tests/integration/compose/docker_compose_dotnet_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: dotnet1: - image: clickhouse/dotnet-client:${DOCKER_DOTNET_CLIENT_TAG:-latest} + image: altinityinfra/dotnet-client:${DOCKER_DOTNET_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/tests/integration/compose/docker_compose_jdbc_bridge.yml b/tests/integration/compose/docker_compose_jdbc_bridge.yml index b3686adc21c4..3f8edfb88268 100644 --- a/tests/integration/compose/docker_compose_jdbc_bridge.yml +++ b/tests/integration/compose/docker_compose_jdbc_bridge.yml @@ -1,6 +1,7 @@ version: '2.3' services: bridge1: + # NOTE(vnemkov): not produced by CI/CD, so must not be replaced with altinityinfra/jdbc-bridge image: clickhouse/jdbc-bridge command: | /bin/bash -c 'cat << EOF > config/datasources/self.json @@ -24,4 +25,4 @@ services: volumes: - type: ${JDBC_BRIDGE_FS:-tmpfs} source: ${JDBC_BRIDGE_LOGS:-} - target: /app/logs \ No newline at end of file + target: /app/logs diff --git a/tests/integration/compose/docker_compose_keeper.yml b/tests/integration/compose/docker_compose_keeper.yml index 91010c4aa83d..fba5bc728f88 100644 --- a/tests/integration/compose/docker_compose_keeper.yml +++ b/tests/integration/compose/docker_compose_keeper.yml @@ -1,7 +1,7 @@ version: '2.3' services: zoo1: - image: ${image:-clickhouse/integration-test} + image: ${image:-altinityinfra/integration-test} restart: always user: ${user:-} volumes: @@ -37,7 +37,7 @@ services: - inet6 - rotate zoo2: - image: ${image:-clickhouse/integration-test} + image: ${image:-altinityinfra/integration-test} restart: always user: ${user:-} volumes: @@ -73,7 +73,7 @@ services: - inet6 - rotate zoo3: - image: ${image:-clickhouse/integration-test} + image: ${image:-altinityinfra/integration-test} restart: always user: ${user:-} volumes: diff --git a/tests/integration/compose/docker_compose_kerberized_hdfs.yml b/tests/integration/compose/docker_compose_kerberized_hdfs.yml index e955a14eb3df..58d321177c0d 100644 --- a/tests/integration/compose/docker_compose_kerberized_hdfs.yml +++ b/tests/integration/compose/docker_compose_kerberized_hdfs.yml @@ -4,7 +4,7 @@ services: kerberizedhdfs1: cap_add: - DAC_READ_SEARCH - image: clickhouse/kerberized-hadoop:${DOCKER_KERBERIZED_HADOOP_TAG:-latest} + image: altinityinfra/kerberized-hadoop:${DOCKER_KERBERIZED_HADOOP_TAG:-latest} hostname: kerberizedhdfs1 restart: always volumes: @@ -24,7 +24,7 @@ services: net.ipv4.ip_local_port_range: '55000 65535' hdfskerberos: - image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} + image: altinityinfra/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} hostname: hdfskerberos volumes: - ${KERBERIZED_HDFS_DIR}/secrets:/tmp/keytab diff --git a/tests/integration/compose/docker_compose_kerberized_kafka.yml b/tests/integration/compose/docker_compose_kerberized_kafka.yml index 49d4c1db90fe..7ae1011b1876 100644 --- a/tests/integration/compose/docker_compose_kerberized_kafka.yml +++ b/tests/integration/compose/docker_compose_kerberized_kafka.yml @@ -52,7 +52,7 @@ services: net.ipv4.ip_local_port_range: '55000 65535' kafka_kerberos: - image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} + image: altinityinfra/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} hostname: kafka_kerberos volumes: - ${KERBERIZED_KAFKA_DIR}/secrets:/tmp/keytab diff --git a/tests/integration/compose/docker_compose_kerberos_kdc.yml b/tests/integration/compose/docker_compose_kerberos_kdc.yml index 3ce9a6df1fb6..062bdace6e9c 100644 --- a/tests/integration/compose/docker_compose_kerberos_kdc.yml +++ b/tests/integration/compose/docker_compose_kerberos_kdc.yml @@ -2,7 +2,7 @@ version: '2.3' services: kerberoskdc: - image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} + image: altinityinfra/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} hostname: kerberoskdc volumes: - ${KERBEROS_KDC_DIR}/secrets:/tmp/keytab diff --git a/tests/integration/compose/docker_compose_minio.yml b/tests/integration/compose/docker_compose_minio.yml index 40098d05b04b..b709161296d9 100644 --- a/tests/integration/compose/docker_compose_minio.yml +++ b/tests/integration/compose/docker_compose_minio.yml @@ -19,14 +19,14 @@ services: # HTTP proxies for Minio. proxy1: - image: clickhouse/s3-proxy + image: altinityinfra/s3-proxy expose: - "8080" # Redirect proxy port - "80" # Reverse proxy port - "443" # Reverse proxy port (secure) proxy2: - image: clickhouse/s3-proxy + image: altinityinfra/s3-proxy expose: - "8080" - "80" @@ -34,7 +34,7 @@ services: # Empty container to run proxy resolver. resolver: - image: clickhouse/python-bottle:${DOCKER_PYTHON_BOTTLE_TAG:-latest} + image: altinityinfra/python-bottle:${DOCKER_PYTHON_BOTTLE_TAG:-latest} expose: - "8080" tty: true diff --git a/tests/integration/compose/docker_compose_mysql_golang_client.yml b/tests/integration/compose/docker_compose_mysql_golang_client.yml index 56cc04105740..09154b584244 100644 --- a/tests/integration/compose/docker_compose_mysql_golang_client.yml +++ b/tests/integration/compose/docker_compose_mysql_golang_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: golang1: - image: clickhouse/mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest} + image: altinityinfra/mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/tests/integration/compose/docker_compose_mysql_java_client.yml b/tests/integration/compose/docker_compose_mysql_java_client.yml index 529974dd4bfe..ffda321958cf 100644 --- a/tests/integration/compose/docker_compose_mysql_java_client.yml +++ b/tests/integration/compose/docker_compose_mysql_java_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: java1: - image: clickhouse/mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest} + image: altinityinfra/mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest} # to keep container running command: sleep 1d diff --git a/tests/integration/compose/docker_compose_mysql_js_client.yml b/tests/integration/compose/docker_compose_mysql_js_client.yml index 90939449c5f3..b46eb2706c47 100644 --- a/tests/integration/compose/docker_compose_mysql_js_client.yml +++ b/tests/integration/compose/docker_compose_mysql_js_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: mysqljs1: - image: clickhouse/mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG:-latest} + image: altinityinfra/mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/tests/integration/compose/docker_compose_mysql_php_client.yml b/tests/integration/compose/docker_compose_mysql_php_client.yml index 408b8ff089a9..662783a00a1f 100644 --- a/tests/integration/compose/docker_compose_mysql_php_client.yml +++ b/tests/integration/compose/docker_compose_mysql_php_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: php1: - image: clickhouse/mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG:-latest} + image: altinityinfra/mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/tests/integration/compose/docker_compose_nginx.yml b/tests/integration/compose/docker_compose_nginx.yml index 38d2a6d84c84..9d4403f283fb 100644 --- a/tests/integration/compose/docker_compose_nginx.yml +++ b/tests/integration/compose/docker_compose_nginx.yml @@ -5,7 +5,7 @@ services: # Files will be put into /usr/share/nginx/files. nginx: - image: clickhouse/nginx-dav:${DOCKER_NGINX_DAV_TAG:-latest} + image: altinityinfra/nginx-dav:${DOCKER_NGINX_DAV_TAG:-latest} restart: always ports: - 80:80 diff --git a/tests/integration/compose/docker_compose_postgresql_java_client.yml b/tests/integration/compose/docker_compose_postgresql_java_client.yml index 904bfffdfd5b..5c8673ae3eeb 100644 --- a/tests/integration/compose/docker_compose_postgresql_java_client.yml +++ b/tests/integration/compose/docker_compose_postgresql_java_client.yml @@ -1,6 +1,6 @@ version: '2.2' services: java: - image: clickhouse/postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:-latest} + image: altinityinfra/postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 215718463e85..7f142673cfdf 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -74,7 +74,8 @@ # Minimum version we use in integration tests to check compatibility with old releases # Keep in mind that we only support upgrading between releases that are at most 1 year different. # This means that this minimum need to be, at least, 1 year older than the current release -CLICKHOUSE_CI_MIN_TESTED_VERSION = "23.3" +# NOTE(vnemkov): this is a docker tag, make sure it doesn't include initial 'v' +CLICKHOUSE_CI_MIN_TESTED_VERSION = "23.3.19.33.altinitystable" # to create docker-compose env file @@ -1071,7 +1072,7 @@ def setup_keeper_cmd(self, instance, env_variables, docker_compose_yml_dir): env_variables["keeper_binary"] = binary_path env_variables["keeper_cmd_prefix"] = keeper_cmd_prefix - env_variables["image"] = "clickhouse/integration-test:" + self.docker_base_tag + env_variables["image"] = "altinityinfra/integration-test:" + self.docker_base_tag env_variables["user"] = str(os.getuid()) env_variables["keeper_fs"] = "bind" for i in range(1, 4): @@ -1721,7 +1722,7 @@ def add_instance( hostname=None, env_variables=None, instance_env_variables=False, - image="clickhouse/integration-test", + image="altinityinfra/integration-test", tag=None, stay_alive=False, ipv4_address=None, @@ -3422,7 +3423,7 @@ def __init__( hostname=None, env_variables=None, instance_env_variables=False, - image="clickhouse/integration-test", + image="altinityinfra/integration-test", tag="latest", stay_alive=False, ipv4_address=None, @@ -4063,7 +4064,7 @@ def contains_in_log( exclusion_substring="", ): if from_host: - # We check fist file exists but want to look for all rotated logs as well + # We check first file exists but want to look for all rotated logs as well result = subprocess_check_call( [ "bash", diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index e6e79dc79478..3689bb409d15 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -243,7 +243,7 @@ def __init__( def _ensure_container(self): if self._container is None or self._container_expire_time <= time.time(): - image_name = "clickhouse/integration-helper:" + os.getenv( + image_name = "altinityinfra/integration-helper:" + os.getenv( "DOCKER_HELPER_TAG", "latest" ) for i in range(5): diff --git a/tests/integration/runner b/tests/integration/runner index 0667541b196d..c8f23ba1258c 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -29,7 +29,7 @@ CONFIG_DIR_IN_REPO = "programs/server" INTEGRATION_DIR_IN_REPO = "tests/integration" UTILS_DIR_IN_REPO = "utils" -DIND_INTEGRATION_TESTS_IMAGE_NAME = "clickhouse/integration-tests-runner" +DIND_INTEGRATION_TESTS_IMAGE_NAME = "altinityinfra/integration-tests-runner" def check_args_and_update_paths(args): diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index 7de5f51921b7..953ae47c109d 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -6,7 +6,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py index 6b385bf84020..d794d53a8401 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py @@ -6,7 +6,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, ) diff --git a/tests/integration/test_backward_compatibility/test_aggregate_function_state.py b/tests/integration/test_backward_compatibility/test_aggregate_function_state.py index 9878c1ed70ea..7775bbcb6153 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_function_state.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_function_state.py @@ -6,7 +6,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, @@ -14,7 +14,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_convert_ordinary.py b/tests/integration/test_backward_compatibility/test_convert_ordinary.py index f5d0c0666007..7011a384bde8 100644 --- a/tests/integration/test_backward_compatibility/test_convert_ordinary.py +++ b/tests/integration/test_backward_compatibility/test_convert_ordinary.py @@ -4,7 +4,7 @@ cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_zookeeper=True, diff --git a/tests/integration/test_backward_compatibility/test_cte_distributed.py b/tests/integration/test_backward_compatibility/test_cte_distributed.py index e0be009e8744..73330ac04732 100644 --- a/tests/integration/test_backward_compatibility/test_cte_distributed.py +++ b/tests/integration/test_backward_compatibility/test_cte_distributed.py @@ -7,7 +7,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_functions.py b/tests/integration/test_backward_compatibility/test_functions.py index 202a741bfb59..af13c53ab8f2 100644 --- a/tests/integration/test_backward_compatibility/test_functions.py +++ b/tests/integration/test_backward_compatibility/test_functions.py @@ -12,7 +12,7 @@ upstream = cluster.add_instance("upstream", use_old_analyzer=True) backward = cluster.add_instance( "backward", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, ) diff --git a/tests/integration/test_backward_compatibility/test_insert_profile_events.py b/tests/integration/test_backward_compatibility/test_insert_profile_events.py index a90453d045be..d3955f5cf3c0 100644 --- a/tests/integration/test_backward_compatibility/test_insert_profile_events.py +++ b/tests/integration/test_backward_compatibility/test_insert_profile_events.py @@ -10,7 +10,7 @@ upstream_node = cluster.add_instance("upstream_node", use_old_analyzer=True) old_node = cluster.add_instance( "old_node", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, ) diff --git a/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py b/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py index 4752a589a44f..0b4178451986 100644 --- a/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py +++ b/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py @@ -6,7 +6,7 @@ # Version 21.6.3.14 has incompatible partition id for tables with UUID in partition key. node = cluster.add_instance( "node", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py index b13e6c975e80..c090b0f91be7 100644 --- a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py @@ -6,7 +6,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, @@ -14,7 +14,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py index 83be0e4c5a34..de8fcb3a940f 100644 --- a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py +++ b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py @@ -7,7 +7,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py index cbe147dc07bd..ba5d797982a3 100644 --- a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py +++ b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py @@ -7,7 +7,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py index ad573c7ffe06..8fe2ff6f13ce 100644 --- a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py @@ -7,7 +7,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag="24.1" if is_arm() else CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, @@ -15,7 +15,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag="24.1" if is_arm() else CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py index e36c3310e4ad..9401091821d8 100644 --- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py +++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py @@ -6,7 +6,7 @@ node_old = cluster.add_instance( "node1", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_disk_over_web_server/test.py b/tests/integration/test_disk_over_web_server/test.py index 891ee8f00f5f..9573eec6b5a6 100644 --- a/tests/integration/test_disk_over_web_server/test.py +++ b/tests/integration/test_disk_over_web_server/test.py @@ -37,7 +37,7 @@ def cluster(): with_nginx=True, stay_alive=True, with_installed_binary=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, ) cluster.add_instance( diff --git a/tests/integration/test_distributed_insert_backward_compatibility/test.py b/tests/integration/test_distributed_insert_backward_compatibility/test.py index 9e794555d49d..ed786eccc117 100644 --- a/tests/integration/test_distributed_insert_backward_compatibility/test.py +++ b/tests/integration/test_distributed_insert_backward_compatibility/test.py @@ -10,7 +10,7 @@ node_dist = cluster.add_instance( "node2", main_configs=["configs/remote_servers.xml"], - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index d74cf97e5c66..7f5c4c7841c8 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -17,6 +17,7 @@ def make_instance(name, *args, **kwargs): main_configs.append("configs/remote_servers.xml") user_configs = kwargs.pop("user_configs", []) user_configs.append("configs/users.xml") + return cluster.add_instance( name, with_zookeeper=True, diff --git a/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt new file mode 100644 index 000000000000..cabc53fa809e --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPDCCAiQCFBXNOvsLA+dqmX/TkYG9JXdD5m72MA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMCmNsaWNraG91c2UwIBcNMjIw +NDIxMTAzNDU1WhgPMjEyMjAzMjgxMDM0NTVaMFkxCzAJBgNVBAYTAkFVMRMwEQYD +VQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBM +dGQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAKaXz596N4NC2zZdIqdwZbSYAtNdBCsBVPt5YT9F640aF5zOogPZyxGP +ENyOZwABi/7HhwFbH657xyRvi8lTau8dZL+0tbakyoIn1Tw6j+/3GXTjLduJSy6C +mOf4OzsrFC8mYgU+7p5ijvWVlO9h5NMbLdAPSIB5WSHhmSORH5LgjoK6oMOYdRod +GmfHqSbwPVwy3Li5SXlniCQmJsM0zl64LFbJ/NU+13qETmhBiDgmh0Svi+wzSzqZ +q1PIX92T3k44IXNZbvF7lKbUOS9Xb3BoxA4cDqRcTx4x73xRDwodSmqiuQOC99HI +A0C/tZJ25VNAGjLKukPSHqYscq2PAsUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA +IDQwjf/ja3TfOXrz+Gn1eErSKnWS3asjRT9rYWQsy3tzVUkMIcszrG+FqTR16g5H +ZWyuEOi6KIRmda3SYKdLKmtQLrgx6/d/jvH5TQ0LTFZrp6vh0lo3pV+L6fLo1ZRD +V1i8jW/7HHNyqJamUXOjwA0DpPOMkdtwuyV+rJ+2bTG1ZSK33O4Ae2CY5+dad6zy +YI6b1c9flWfDznuNEMH7jDDjKgXwjZGeU53FiuuhHiNyRchsr/B9eIBsom8oykiD +kch4cnAxx2E+m3hLYzupkXHOVQ5CNpVk8PGUCIGcyqDxPt+fOj1WbDQ9laEcfhmV +kR+vHmzOqWZnHU4QeMqDig== +-----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key new file mode 100644 index 000000000000..1e29a4c8fa12 --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,4E14FF586022476CD22AAFB662BB0E40 + +dpJZKq5k+fMuC7XECfTSRjPeOEl9wNuVtZkcjEWaHN8ky4umND7ARyRyuU1Nk7cy +fpCFlFKOqDfCkT5zVK/fB6pF32wqAI7sqeSuYPfQY0+L77yRdwM6L46WslzVKZYE +lXD1AmqKT/LgF3+eBY5slkAAJo10zYDgKEwnoQVBp31YW2/+6oAGaY/O6x3p7aTG +dw9CP+SFc0o8lPl1lsSovdNXDUiVCftvClog7hwyDv8AhHyGgynw3UJXX8UlyWu+ +Zz5zpgrvB2gvDLeoZZ6qjMGvtpEwlYBh4de9ZOsvQUpXEEfkQFtJV0j613OCQune +LoTxKpYV1V/mZX4HPaJ1oC0OJ7XcliAOSS9K49YobTuz7Kg5Wg3bVGo9xRfYDjch +rVeMP4u5RXSGuHL23FPMfK6EqcldrFyRwLaY/IV1Yl6UNUMKAphn/WMtWVuT3TiT +QMCI2VRt7ItwZwwFn5RgyDweWdFf5v3AmN/lOhATDBqosahkPxDZ1VZ6OBPoJLPM +UrDWH/lqrByeEjtAOwr5UsWKwLuJ8qUxQ4TchHwFKOwy6VsrRwMQ3ZWi2govPF9I +W0sfLj5Ulfjx6zHdqnF48a1Elit4JH6inBEWFuj7bmlOotq+PHoeT61zAwW+gnrG +3JTo3XnaE2WwRDpqvKYHWLv/J218rq8PtIaq9gjr55odPfIt8lkJ1XzF4WQ21rIJ +GNWZ3xz4fxpvrKnQyAKGu0ZcdjA1nqs16oiVr+UnJoXmkM5yBCic4fZYwPTSQHYS +ZxwaTzEjfeGxrSeLrN9CgoweucvogOvUjJOBcW/py80du8vWz0YyzMhg3o0YeGME +C+Kts/YWxmyfw4DaWt8RtWCKl85hEmz8RODvkMLGtLzvVoSyLQWqp1NhGIlFtzXs +7sPLulUeyD2avTC/RB/Pu9Nk80c0368BxCoeYbiFWZpaN70SJmCUE5H59J2d0olw +5v2RVjLBi8wqnzoa0+2L8wnG7IQGadS97dj0eBR+JXNtoJhVrurS80RJ6B0bNxdu +gX8otfnJYsZyK5hbEhcQqLdnyGhDEE8YHe7Hv9stWwLAFOfOMzyzC06lFS1eNiw4 +FJyXJUhDieb8EqetouAC8dNVXz4Q1zOTlGuAbGoKm5v0U5IhCQap9GUSW5QiUgOQ +AEMs9aGfd91R+IcDf19mZptsQLYA6MGBN6fm+3O2iZImKIbF+ZZo0S6liFFmn6lm +M+diTzaoiqgEkiXOuRhdQUMaiGV8BMZxv8qUH6/vyC3gSueoTio0f9PfASDYfvXD +A3GuI87P6LF1it2UlN6ssFoXTZdfQQZwRmNuqOqw+BJOJHrR6trcXOCZOQ77Qnvd +M5a348gIzluVUkExAPGCsySQWMx4Of5NBF28jEC3+TAwkRqBV2ZHmfGLWnvwaB+A +YUeKtpWblfG1lsrDAdwL2dilU95oby+35sExX7M2dCrL9Y2P5oTCW3u12//ZSLeL +Yhi1Rzol6LAuesZCVF0Zv/YYDhzAckJfT/qXK5B5pz9saswxCUBEpiKlLpVsjOFJ +2bHm8NgOMD5b3cdh1kvts4wZe+giry7LHsn46f+9VqN+gA6XxeVsPyb4uO1KW3SN +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_keeper_internal_secure/configs/server.crt b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt similarity index 100% rename from tests/integration/test_keeper_internal_secure/configs/server.crt rename to tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt diff --git a/tests/integration/test_keeper_internal_secure/configs/server.key b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key similarity index 100% rename from tests/integration/test_keeper_internal_secure/configs/server.key rename to tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml index 986b503ebe35..dabf280bc363 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml @@ -1,5 +1,6 @@ + 0 9181 1 /var/lib/clickhouse/coordination/log diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml index 652b1992f463..21d8f1e0eb37 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml @@ -1,5 +1,6 @@ + 0 9181 2 /var/lib/clickhouse/coordination/log diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml index 6507f97473b8..5d8cfb8b3e7b 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml @@ -1,5 +1,6 @@ + 0 9181 3 /var/lib/clickhouse/coordination/log diff --git a/tests/integration/test_keeper_internal_secure/configs/ssl_conf.xml b/tests/integration/test_keeper_internal_secure/configs/ssl_conf.xml deleted file mode 100644 index 37eb2624b1b6..000000000000 --- a/tests/integration/test_keeper_internal_secure/configs/ssl_conf.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - /etc/clickhouse-server/config.d/server.crt - /etc/clickhouse-server/config.d/server.key - /etc/clickhouse-server/config.d/rootCA.pem - true - none - true - sslv2,sslv3 - true - - - diff --git a/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml b/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml new file mode 100644 index 000000000000..a444122b09df --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml @@ -0,0 +1,10 @@ +openSSL: + server: + certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt' + privateKeyFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.key' + caConfig: '/etc/clickhouse-server/config.d/rootCA.pem' + loadDefaultCAFile: true + verificationMode: 'none' + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true diff --git a/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml b/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml new file mode 100644 index 000000000000..51b65c5253af --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml @@ -0,0 +1,14 @@ +openSSL: + server: + certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt' + privateKeyFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.key' + privateKeyPassphraseHandler: + name: KeyFileHandler + options: + password: 'PASSWORD' + caConfig: '/etc/clickhouse-server/config.d/rootCA.pem' + loadDefaultCAFile: true + verificationMode: 'none' + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true diff --git a/tests/integration/test_keeper_internal_secure/test.py b/tests/integration/test_keeper_internal_secure/test.py index 2d45e95e4fff..8cab03b6e2df 100644 --- a/tests/integration/test_keeper_internal_secure/test.py +++ b/tests/integration/test_keeper_internal_secure/test.py @@ -2,44 +2,55 @@ import pytest from helpers.cluster import ClickHouseCluster -import random -import string +import helpers.keeper_utils as ku +from multiprocessing.dummy import Pool import os -import time +CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance( - "node1", - main_configs=[ - "configs/enable_secure_keeper1.xml", - "configs/ssl_conf.xml", - "configs/server.crt", - "configs/server.key", - "configs/rootCA.pem", - ], -) -node2 = cluster.add_instance( - "node2", - main_configs=[ - "configs/enable_secure_keeper2.xml", - "configs/ssl_conf.xml", - "configs/server.crt", - "configs/server.key", - "configs/rootCA.pem", - ], -) -node3 = cluster.add_instance( - "node3", - main_configs=[ - "configs/enable_secure_keeper3.xml", - "configs/ssl_conf.xml", - "configs/server.crt", - "configs/server.key", - "configs/rootCA.pem", - ], -) - -from kazoo.client import KazooClient, KazooState +nodes = [ + cluster.add_instance( + "node1", + main_configs=[ + "configs/enable_secure_keeper1.xml", + "configs/ssl_conf.yml", + "configs/WithoutPassPhrase.crt", + "configs/WithoutPassPhrase.key", + "configs/WithPassPhrase.crt", + "configs/WithPassPhrase.key", + "configs/rootCA.pem", + ], + stay_alive=True, + ), + cluster.add_instance( + "node2", + main_configs=[ + "configs/enable_secure_keeper2.xml", + "configs/ssl_conf.yml", + "configs/WithoutPassPhrase.crt", + "configs/WithoutPassPhrase.key", + "configs/WithPassPhrase.crt", + "configs/WithPassPhrase.key", + "configs/rootCA.pem", + ], + stay_alive=True, + ), + cluster.add_instance( + "node3", + main_configs=[ + "configs/enable_secure_keeper3.xml", + "configs/ssl_conf.yml", + "configs/WithoutPassPhrase.crt", + "configs/WithoutPassPhrase.key", + "configs/WithPassPhrase.crt", + "configs/WithPassPhrase.key", + "configs/rootCA.pem", + ], + stay_alive=True, + ), +] + +from kazoo.client import KazooClient @pytest.fixture(scope="module") @@ -61,23 +72,113 @@ def get_fake_zk(nodename, timeout=30.0): return _fake_zk_instance -def test_secure_raft_works(started_cluster): +def run_test(): + node_zks = [] try: - node1_zk = get_fake_zk("node1") - node2_zk = get_fake_zk("node2") - node3_zk = get_fake_zk("node3") + for node in nodes: + node_zks.append(get_fake_zk(node.name)) - node1_zk.create("/test_node", b"somedata1") - node2_zk.sync("/test_node") - node3_zk.sync("/test_node") + node_zks[0].create("/test_node", b"somedata1") + node_zks[1].sync("/test_node") + node_zks[2].sync("/test_node") - assert node1_zk.exists("/test_node") is not None - assert node2_zk.exists("/test_node") is not None - assert node3_zk.exists("/test_node") is not None + for node_zk in node_zks: + assert node_zk.exists("/test_node") is not None finally: try: - for zk_conn in [node1_zk, node2_zk, node3_zk]: + for zk_conn in node_zks: + if zk_conn is None: + continue zk_conn.stop() zk_conn.close() except: pass + + +def setupSsl(node, filename, password): + if password is None: + node.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "configs/ssl_conf.yml"), + "/etc/clickhouse-server/config.d/ssl_conf.yml", + ) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/ssl_conf.yml", + "WithoutPassPhrase", + filename, + ) + return + + node.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "configs/ssl_conf_password.yml"), + "/etc/clickhouse-server/config.d/ssl_conf.yml", + ) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/ssl_conf.yml", + "WithoutPassPhrase", + filename, + ) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/ssl_conf.yml", + "PASSWORD", + password, + ) + + +def stop_all_clickhouse(): + for node in nodes: + node.stop_clickhouse() + + for node in nodes: + node.exec_in_container(["rm", "-rf", "/var/lib/clickhouse/coordination"]) + + +def start_clickhouse(node): + node.start_clickhouse() + + +def start_all_clickhouse(): + p = Pool(3) + waiters = [] + + for node in nodes: + waiters.append(p.apply_async(start_clickhouse, args=(node,))) + + for waiter in waiters: + waiter.wait() + + for node in nodes: + ku.wait_until_connected(cluster, node) + + +def check_valid_configuration(filename, password): + stop_all_clickhouse() + for node in nodes: + setupSsl(node, filename, password) + start_all_clickhouse() + run_test() + + +def check_invalid_configuration(filename, password): + stop_all_clickhouse() + for node in nodes: + setupSsl(node, filename, password) + + nodes[0].start_clickhouse(expected_to_fail=True) + nodes[0].wait_for_log_line( + "OpenSSLException: EVPKey::loadKey.*error:0480006C:PEM routines::no start line", + ) + + +def test_secure_raft_works(started_cluster): + check_valid_configuration("WithoutPassPhrase", None) + + +def test_secure_raft_works_with_password(started_cluster): + check_valid_configuration("WithoutPassPhrase", "unusedpassword") + check_invalid_configuration("WithPassPhrase", "wrongpassword") + check_invalid_configuration("WithPassPhrase", "") + check_valid_configuration("WithPassPhrase", "test") + check_invalid_configuration("WithPassPhrase", None) diff --git a/tests/integration/test_old_versions/test.py b/tests/integration/test_old_versions/test.py index a5e62a380bfd..39ea7a78e815 100644 --- a/tests/integration/test_old_versions/test.py +++ b/tests/integration/test_old_versions/test.py @@ -6,7 +6,7 @@ cluster = ClickHouseCluster(__file__) node_oldest = cluster.add_instance( "node_oldest", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"], diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index 2b30170b203e..c3b8b14560c4 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -365,7 +365,7 @@ def test_different_part_types_on_replicas(start_cluster, table, part_type): "node7", user_configs=["configs_old/users.d/not_optimize_count.xml"], with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_replicated_merge_tree_compatibility/test.py b/tests/integration/test_replicated_merge_tree_compatibility/test.py index a70f3234c1eb..22de74d073e4 100644 --- a/tests/integration/test_replicated_merge_tree_compatibility/test.py +++ b/tests/integration/test_replicated_merge_tree_compatibility/test.py @@ -5,7 +5,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, @@ -13,7 +13,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml b/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml index 30a0b6276b76..e69df7d39f65 100644 --- a/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml +++ b/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml @@ -3,6 +3,7 @@ 1 1 + 1 default 100 0 diff --git a/tests/integration/test_replicating_constants/test.py b/tests/integration/test_replicating_constants/test.py index af8916dd625b..8da83038d146 100644 --- a/tests/integration/test_replicating_constants/test.py +++ b/tests/integration/test_replicating_constants/test.py @@ -8,8 +8,8 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, - image="clickhouse/clickhouse-server", - tag="23.3", + image="altinity/clickhouse-server", + tag="23.3.19.33.altinitystable", with_installed_binary=True, ) diff --git a/tests/integration/test_storage_s3_queue/configs/users_old.xml b/tests/integration/test_storage_s3_queue/configs/users_old.xml new file mode 100644 index 000000000000..227dbe9bde5b --- /dev/null +++ b/tests/integration/test_storage_s3_queue/configs/users_old.xml @@ -0,0 +1,11 @@ + + + + 1 + 1 + + + 1 + + + diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 66dad88ccbcc..09a6be1ec191 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -107,7 +107,9 @@ def started_cluster(): ) cluster.add_instance( "old_instance", + user_configs=["configs/users_old.xml"], with_zookeeper=True, + # NOTE (vnemkov) Can't use altinity/clickhouse here since 23.8 doesn't hve S3Queue (and associated settings yet) image="clickhouse/clickhouse-server", tag="23.12", stay_alive=True, diff --git a/tests/integration/test_trace_log_build_id/test.py b/tests/integration/test_trace_log_build_id/test.py index 8d654aef342d..19c26a3a5bf9 100644 --- a/tests/integration/test_trace_log_build_id/test.py +++ b/tests/integration/test_trace_log_build_id/test.py @@ -11,7 +11,7 @@ node = cluster.add_instance( "node", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 538322473ee3..21b5885ffb49 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -16,7 +16,7 @@ node4 = cluster.add_instance( "node4", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, @@ -28,7 +28,7 @@ node5 = cluster.add_instance( "node5", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, @@ -39,7 +39,7 @@ node6 = cluster.add_instance( "node6", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_version_update/test.py b/tests/integration/test_version_update/test.py index b386a79c932f..f42d791d1bda 100644 --- a/tests/integration/test_version_update/test.py +++ b/tests/integration/test_version_update/test.py @@ -10,7 +10,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, stay_alive=True, diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 9365498f89d4..73a01a51ce74 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -9,7 +9,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, stay_alive=True, @@ -20,7 +20,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, stay_alive=True, @@ -31,7 +31,7 @@ node3 = cluster.add_instance( "node3", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, stay_alive=True, diff --git a/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.reference b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.reference new file mode 100644 index 000000000000..99ede711976d --- /dev/null +++ b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.reference @@ -0,0 +1,347 @@ +1000 +bloom filter is off, all row groups should be read +expect rows_read = select count() +{ + "data": [ + { + "string": "AZSR", + "flba": "WNMM" + }, + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 2, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +bloom filter is on, some row groups should be skipped +expect rows_read much less than select count() +{ + "data": [ + { + "string": "AZSR", + "flba": "WNMM" + }, + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 2, + "statistics": { + "rows_read": 464, + "bytes_read": 21703 + } +} +bloom filter is on, but where predicate contains data from 2 row groups out of 3. +Rows read should be less than select count, but greater than previous selects +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + }, + { + "string": "ZHZK", + "flba": "HRWD" + } + ], + "rows": 2, + "statistics": { + "rows_read": 536, + "bytes_read": 25708 + } +} +bloom filter is on, but where predicate contains data from all row groups +expect rows_read = select count() +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + }, + { + "string": "OKAI", + "flba": "UXGT" + }, + { + "string": "ZHZK", + "flba": "HRWD" + } + ], + "rows": 3, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +IN check +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + }, + { + "string": "ZHZK", + "flba": "HRWD" + } + ], + "rows": 2, + "statistics": { + "rows_read": 536, + "bytes_read": 25708 + } +} +tuple in case, bf is off. +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +tuple in case, bf is on. +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 1, + "statistics": { + "rows_read": 464, + "bytes_read": 21703 + } +} +complex tuple in case, bf is off +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +complex tuple in case, bf is on +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 1, + "statistics": { + "rows_read": 464, + "bytes_read": 21703 + } +} +complex tuple in case, bf is on. Non existent +{ + "data": [], + "rows": 0, + "statistics": { + "rows_read": 0, + "bytes_read": 0 + } +} +Bloom filter for json column. BF is off +{ + "data": [ + { + "json": "{\"key\":38, \"value\":\"NXONM\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +Bloom filter for json column. BF is on +{ + "data": [ + { + "json": "{\"key\":38, \"value\":\"NXONM\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 72, + "bytes_read": 4005 + } +} +Bloom filter for ipv4 column. BF is off +{ + "data": [ + { + "json": "{\"key\":38, \"value\":\"NXONM\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +Bloom filter for ipv4 column. BF is on +{ + "data": [ + { + "json": "{\"key\":38, \"value\":\"NXONM\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 72, + "bytes_read": 4005 + } +} +Bloom filter for ipv4 column. BF is on. Specified in the schema +{ + "data": [ + { + "ipv4": "0.0.1.143" + } + ], + "rows": 1, + "statistics": { + "rows_read": 72, + "bytes_read": 4005 + } +} +Bloom filter on 64 bit column read as ipv4. We explicitly deny it, should read all rg +{ + "data": [ + { + "uint64_logical": "22.230.220.164" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +BF off for parquet uint64 logical type. Should read everything +{ + "data": [ + { + "json": "{\"key\":683, \"value\":\"YKCPD\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +BF on for parquet uint64 logical type. Uint64 is stored as a signed int 64, but with logical annotation. Make sure a value greater than int64 can be queried +{ + "data": [ + { + "json": "{\"key\":683, \"value\":\"YKCPD\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 464, + "bytes_read": 21711 + } +} +Uint16 is stored as physical type int32 with bidwidth = 16 and sign = false. Make sure a value greater than int16 can be queried. BF is on. +{ + "data": [ + { + "json": "{\"key\":874, \"value\":\"JENHW\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 464, + "bytes_read": 21703 + } +} +BF off for parquet int8 logical type. Should read everything +{ + "data": [ + { + "json": "{\"key\":89, \"value\":\"MFIYP\"}" + }, + { + "json": "{\"key\":321, \"value\":\"JNOIA\"}" + }, + { + "json": "{\"key\":938, \"value\":\"UBMLO\"}" + }, + { + "json": "{\"key\":252, \"value\":\"ZVLKF\"}" + } + ], + "rows": 4, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +BF on for parquet int8 logical type. Should skip row groups +{ + "data": [ + { + "json": "{\"key\":89, \"value\":\"MFIYP\"}" + }, + { + "json": "{\"key\":321, \"value\":\"JNOIA\"}" + }, + { + "json": "{\"key\":938, \"value\":\"UBMLO\"}" + }, + { + "json": "{\"key\":252, \"value\":\"ZVLKF\"}" + } + ], + "rows": 4, + "statistics": { + "rows_read": 536, + "bytes_read": 25716 + } +} +Invalid column conversion with in operation. String type can not be hashed against parquet int64 physical type. Should read everything +{ + "data": [], + "rows": 0, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +Transformations on key column shall not be allowed. Should read everything +{ + "data": [ + { + "uint64_logical": "7711695863945021976" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} diff --git a/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.sh b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.sh new file mode 100755 index 000000000000..b7d40a1be63a --- /dev/null +++ b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Tags: no-ubsan, no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +USER_FILES_PATH=$($CLICKHOUSE_CLIENT_BINARY --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +WORKING_DIR="${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +mkdir -p "${WORKING_DIR}" + +DATA_FILE="${CUR_DIR}/data_parquet/multi_column_bf.gz.parquet" + +DATA_FILE_USER_PATH="${WORKING_DIR}/multi_column_bf.gz.parquet" + +cp ${DATA_FILE} ${DATA_FILE_USER_PATH} + +${CLICKHOUSE_CLIENT} --query="select count(*) from file('${DATA_FILE_USER_PATH}', Parquet) SETTINGS use_cache_for_count_from_files=false;" + +echo "bloom filter is off, all row groups should be read" +echo "expect rows_read = select count()" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where string='PFJH' or flba='WNMM' order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false" | jq 'del(.meta,.statistics.elapsed)' + +echo "bloom filter is on, some row groups should be skipped" +echo "expect rows_read much less than select count()" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where string='PFJH' or flba='WNMM' order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false" | jq 'del(.meta,.statistics.elapsed)' + +echo "bloom filter is on, but where predicate contains data from 2 row groups out of 3." +echo "Rows read should be less than select count, but greater than previous selects" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where string='PFJH' or string='ZHZK' order by uint16_logical asc Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "bloom filter is on, but where predicate contains data from all row groups" +echo "expect rows_read = select count()" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where string='PFJH' or string='ZHZK' or uint64_logical=18441251162536403933 order by uint16_logical asc Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "IN check" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where string in ('PFJH', 'ZHZK') order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "tuple in case, bf is off." +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where (string, flba) in ('PFJH', 'GKJC') order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "tuple in case, bf is on." +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where (string, flba) in ('PFJH', 'GKJC') order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "complex tuple in case, bf is off" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where (string, flba) in (('NON1', 'NON1'), ('PFJH', 'GKJC'), ('NON2', 'NON2')) order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "complex tuple in case, bf is on" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where (string, flba) in (('NON1', 'NON1'), ('PFJH', 'GKJC'), ('NON2', 'NON2')) order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "complex tuple in case, bf is on. Non existent" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where (string, flba) in (('NON1', 'NON1'), ('NON2', 'NON2'), ('NON3', 'NON3')) order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter for json column. BF is off" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where json = '{\"key\":38, \"value\":\"NXONM\"}' order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter for json column. BF is on" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where json = '{\"key\":38, \"value\":\"NXONM\"}' order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter for ipv4 column. BF is off" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where ipv4 = IPv4StringToNum('0.0.1.143') order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter for ipv4 column. BF is on" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where ipv4 = IPv4StringToNum('0.0.1.143') order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter for ipv4 column. BF is on. Specified in the schema" +${CLICKHOUSE_CLIENT} --query="select ipv4 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv4 IPv4') where ipv4 = toIPv4('0.0.1.143') order by ipv4 asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter on 64 bit column read as ipv4. We explicitly deny it, should read all rg" +${CLICKHOUSE_CLIENT} --query="select uint64_logical from file ('${DATA_FILE_USER_PATH}', Parquet, 'uint64_logical IPv4') where uint64_logical = toIPv4(5552715629697883300) order by uint64_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "BF off for parquet uint64 logical type. Should read everything" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where uint64_logical=18441251162536403933 order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "BF on for parquet uint64 logical type. Uint64 is stored as a signed int 64, but with logical annotation. Make sure a value greater than int64 can be queried" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where uint64_logical=18441251162536403933 order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Uint16 is stored as physical type int32 with bidwidth = 16 and sign = false. Make sure a value greater than int16 can be queried. BF is on." +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where uint16_logical=65528 order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "BF off for parquet int8 logical type. Should read everything" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where int8_logical=-126 order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "BF on for parquet int8 logical type. Should skip row groups" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where int8_logical=-126 order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Invalid column conversion with in operation. String type can not be hashed against parquet int64 physical type. Should read everything" +${CLICKHOUSE_CLIENT} --query="select uint64_logical from file('${DATA_FILE_USER_PATH}', Parquet, 'uint64_logical String') where uint64_logical in ('5') order by uint64_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Transformations on key column shall not be allowed. Should read everything" +${CLICKHOUSE_CLIENT} --query="select uint64_logical from file('${DATA_FILE_USER_PATH}', Parquet) where negate(uint64_logical) = -7711695863945021976 order by uint64_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +rm -rf ${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME:?}/* diff --git a/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.reference b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.reference new file mode 100644 index 000000000000..acb66d986e5f --- /dev/null +++ b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.reference @@ -0,0 +1,76 @@ +bloom filter is off, row groups should be read +expect rows_read = select count() +{ + "data": [ + { + "ipv6": "7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995" + } + ], + "rows": 1, + "statistics": { + "rows_read": 5, + "bytes_read": 128 + } +} +bloom filter is on for ipv6, row groups should also be read since there is only one. Below queries just make sure the data is properly returned +{ + "data": [ + { + "ipv6": "7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995" + } + ], + "rows": 1, + "statistics": { + "rows_read": 5, + "bytes_read": 128 + } +} +{ + "data": [ + { + "ipv6": "7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995" + } + ], + "rows": 1, + "statistics": { + "rows_read": 5, + "bytes_read": 128 + } +} +{ + "data": [ + { + "toIPv6(ipv6)": "7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995" + } + ], + "rows": 1, + "statistics": { + "rows_read": 5, + "bytes_read": 128 + } +} +non existent ipv6, row group should be skipped +{ + "data": [], + "rows": 0, + "statistics": { + "rows_read": 0, + "bytes_read": 0 + } +} +{ + "data": [], + "rows": 0, + "statistics": { + "rows_read": 0, + "bytes_read": 0 + } +} +{ + "data": [], + "rows": 0, + "statistics": { + "rows_read": 5, + "bytes_read": 128 + } +} diff --git a/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.sh b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.sh new file mode 100755 index 000000000000..752e7ed38a58 --- /dev/null +++ b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Tags: no-ubsan, no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +USER_FILES_PATH=$($CLICKHOUSE_CLIENT_BINARY --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +WORKING_DIR="${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +mkdir -p "${WORKING_DIR}" + +DATA_FILE="${CUR_DIR}/data_parquet/ipv6_bloom_filter.gz.parquet" + +DATA_FILE_USER_PATH="${WORKING_DIR}/ipv6_bloom_filter.gz.parquet" + +cp ${DATA_FILE} ${DATA_FILE_USER_PATH} + +echo "bloom filter is off, row groups should be read" +echo "expect rows_read = select count()" +${CLICKHOUSE_CLIENT} --query="select ipv6 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv6 IPv6') where ipv6 = '7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995' Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "bloom filter is on for ipv6, row groups should also be read since there is only one. Below queries just make sure the data is properly returned" +${CLICKHOUSE_CLIENT} --query="select ipv6 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv6 IPv6') where ipv6 = '7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995' Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' +${CLICKHOUSE_CLIENT} --query="select ipv6 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv6 IPv6') where ipv6 = toIPv6('7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995') Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' +${CLICKHOUSE_CLIENT} --query="select toIPv6(ipv6) from file('${DATA_FILE_USER_PATH}', Parquet) where ipv6 = toIPv6('7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995') Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "non existent ipv6, row group should be skipped" +${CLICKHOUSE_CLIENT} --query="select ipv6 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv6 IPv6') where ipv6 = 'fafe:b9d4:e754:4e78:8783:37f5:b2ea:9995' Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' +${CLICKHOUSE_CLIENT} --query="select ipv6 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv6 IPv6') where ipv6 = toIPv6('fafe:b9d4:e754:4e78:8783:37f5:b2ea:9995') Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' +${CLICKHOUSE_CLIENT} --query="select toIPv6(ipv6) from file('${DATA_FILE_USER_PATH}', Parquet) where ipv6 = toIPv6('fafe:b9d4:e754:4e78:8783:37f5:b2ea:9995') Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' diff --git a/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.reference b/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.reference new file mode 100644 index 000000000000..3ae545099d6a --- /dev/null +++ b/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.reference @@ -0,0 +1,10 @@ +abc 2 +abc 2 +abc 3 +abc 4 +\N 5 +abc 2 +abc 2 +abc 3 +abc 4 +\N 5 diff --git a/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.sh b/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.sh new file mode 100755 index 000000000000..217491534e99 --- /dev/null +++ b/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Tags: no-ubsan, no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER_FILES_PATH=$($CLICKHOUSE_CLIENT_BINARY --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +WORKING_DIR="${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +mkdir -p "${WORKING_DIR}" + +DATA_FILE="${CUR_DIR}/data_parquet/datapage_v2.snappy.parquet" + +DATA_FILE_USER_PATH="${WORKING_DIR}/datapage_v2.snappy.parquet" + +cp ${DATA_FILE} ${DATA_FILE_USER_PATH} + +# Not reading all columns because some data types and encodings are not supported by native reader yet +# TODO read all columns once implemented +${CLICKHOUSE_CLIENT} --query="select a, c from file('${DATA_FILE_USER_PATH}', Parquet) order by c SETTINGS input_format_parquet_use_native_reader=false;" +${CLICKHOUSE_CLIENT} --query="select a, c from file('${DATA_FILE_USER_PATH}', Parquet) order by c SETTINGS input_format_parquet_use_native_reader=true;" diff --git a/tests/queries/0_stateless/03254_parquet_bool_native_reader.reference b/tests/queries/0_stateless/03254_parquet_bool_native_reader.reference new file mode 100644 index 000000000000..0c7e55ad2344 --- /dev/null +++ b/tests/queries/0_stateless/03254_parquet_bool_native_reader.reference @@ -0,0 +1,20 @@ +0 false +1 \N +2 false +3 \N +4 false +5 \N +6 false +7 \N +8 true +9 \N +0 false +1 \N +2 false +3 \N +4 false +5 \N +6 false +7 \N +8 true +9 \N diff --git a/tests/queries/0_stateless/03254_parquet_bool_native_reader.sh b/tests/queries/0_stateless/03254_parquet_bool_native_reader.sh new file mode 100755 index 000000000000..c28523b3c549 --- /dev/null +++ b/tests/queries/0_stateless/03254_parquet_bool_native_reader.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: no-ubsan, no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER_FILES_PATH=$($CLICKHOUSE_CLIENT_BINARY --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +WORKING_DIR="${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +mkdir -p "${WORKING_DIR}" + +DATA_FILE="${CUR_DIR}/data_parquet/nullbool.parquet" + +DATA_FILE_USER_PATH="${WORKING_DIR}/nullbool.parquet" + +cp ${DATA_FILE} ${DATA_FILE_USER_PATH} + +${CLICKHOUSE_CLIENT} --query="select id, bool from file('${DATA_FILE_USER_PATH}', Parquet) order by id SETTINGS input_format_parquet_use_native_reader=false;" +${CLICKHOUSE_CLIENT} --query="select id, bool from file('${DATA_FILE_USER_PATH}', Parquet) order by id SETTINGS input_format_parquet_use_native_reader=true;" diff --git a/tests/queries/0_stateless/data_parquet/ipv6_bloom_filter.gz.parquet b/tests/queries/0_stateless/data_parquet/ipv6_bloom_filter.gz.parquet new file mode 100644 index 000000000000..335fddc1f0ad Binary files /dev/null and b/tests/queries/0_stateless/data_parquet/ipv6_bloom_filter.gz.parquet differ diff --git a/tests/queries/0_stateless/data_parquet/multi_column_bf.gz.parquet b/tests/queries/0_stateless/data_parquet/multi_column_bf.gz.parquet new file mode 100644 index 000000000000..bfa0a39062a0 Binary files /dev/null and b/tests/queries/0_stateless/data_parquet/multi_column_bf.gz.parquet differ diff --git a/tests/queries/0_stateless/data_parquet/nullbool.parquet b/tests/queries/0_stateless/data_parquet/nullbool.parquet new file mode 100644 index 000000000000..d9b365bbe75b Binary files /dev/null and b/tests/queries/0_stateless/data_parquet/nullbool.parquet differ diff --git a/utils/clickhouse-docker b/utils/clickhouse-docker index cfe515f1de54..62d980f2f407 100755 --- a/utils/clickhouse-docker +++ b/utils/clickhouse-docker @@ -24,13 +24,13 @@ param="$1" if [ "${param}" = "list" ] then # https://stackoverflow.com/a/39454426/1555175 - wget -nv https://registry.hub.docker.com/v1/repositories/clickhouse/clickhouse-server/tags -O - | sed -e 's/[][]//g' -e 's/"//g' -e 's/ //g' | tr '}' '\n' | awk -F: '{print $3}' + wget -nv https://registry.hub.docker.com/v1/repositories/altinity/clickhouse-server/tags -O - | sed -e 's/[][]//g' -e 's/"//g' -e 's/ //g' | tr '}' '\n' | awk -F: '{print $3}' else - docker pull clickhouse/clickhouse-server:${param} + docker pull altinity/clickhouse-server:${param} tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX) # older version require /nonexistent folder to exist to run clickhouse client :D chmod 777 ${tmp_dir} set -e - containerid=`docker run -v${tmp_dir}:/nonexistent -d clickhouse/clickhouse-server:${param}` + containerid=`docker run -v${tmp_dir}:/nonexistent -d altinity/clickhouse-server:${param}` set +e while : do diff --git a/utils/test_history/requirements.txt b/utils/test_history/requirements.txt index 07970e9fab5f..220519696891 100644 --- a/utils/test_history/requirements.txt +++ b/utils/test_history/requirements.txt @@ -1,3 +1,3 @@ -pygithub==1.43.5 -tabulate==0.8.6 -termcolor==1.1.0 +pygithub~=1.43.5 +tabulate~=0.8.6 +termcolor~=1.1.0