Antalya 25.6.5 Add "Run without cache" option to workflows

.github/workflows/backport_branches.yml

@@ -4,14 +4,20 @@ name: BackportPR
 
 on:
   workflow_dispatch:
+    inputs:
+      no_cache:
+        description: Run without cache
+        required: false
+        type: boolean
+        default: false
   pull_request:
     branches: ['2[1-9].[1-9][0-9]', '2[1-9].[1-9]']
 
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
   DISABLE_CI_MERGE_COMMIT: ${{ vars.DISABLE_CI_MERGE_COMMIT || '0' }}
-  DISABLE_CI_CACHE: ${{ vars.DISABLE_CI_CACHE || '0' }}
+  DISABLE_CI_CACHE: ${{ github.event.inputs.no_cache || '0' }}
   CHECKOUT_REF: ${{ vars.DISABLE_CI_MERGE_COMMIT == '1' && github.event.pull_request.head.sha || '' }}
   DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
   CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}

.github/workflows/master.yml

@@ -4,12 +4,19 @@ name: MasterCI
 
 on:
   workflow_dispatch:
+    inputs:
+      no_cache:
+        description: Run without cache
+        required: false
+        type: boolean
+        default: false
   push:
     branches: ['antalya', 'releases/*', 'antalya-*']
 
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
+  DISABLE_CI_CACHE: ${{ github.event.inputs.no_cache || '0' }}
   CHECKOUT_REF: ""
   DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
   CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}

.github/workflows/merge_queue.yml

@@ -348,101 +348,3 @@ jobs:
           else
             python3 -m praktika run 'Finish Workflow' --workflow "MergeQueueCI" --ci |& tee ./ci/tmp/job.log
           fi
-
-##########################################################################################
-##################################### ALTINITY JOBS ######################################
-##########################################################################################
-  GrypeScanServer:
-    needs: [config_workflow, docker_server_image]
-    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIHNlcnZlciBpbWFnZQ==') }}
-    strategy:
-      fail-fast: false
-      matrix:
-        suffix: ['', '-alpine']
-    uses: ./.github/workflows/grype_scan.yml
-    secrets: inherit
-    with:
-      docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
-      tag-suffix: ${{ matrix.suffix }}
-  GrypeScanKeeper:
-      needs: [config_workflow, docker_keeper_image]
-      if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIGtlZXBlciBpbWFnZQ==') }}
-      uses: ./.github/workflows/grype_scan.yml
-      secrets: inherit
-      with:
-        docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
-
-  RegressionTestsRelease:
-    needs: [config_workflow, build_amd_release]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression')}}
-    uses: ./.github/workflows/regression.yml
-    secrets: inherit
-    with:
-      runner_type: altinity-on-demand, altinity-regression-tester
-      commit: eadf0647501a547d57c49b71ca256e10c6c304f6
-      arch: release
-      build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
-      workflow_config: ${{ needs.config_workflow.outputs.data }}
-  RegressionTestsAarch64:
-    needs: [config_workflow, build_arm_release]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression') && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_aarch64')}}
-    uses: ./.github/workflows/regression.yml
-    secrets: inherit
-    with:
-      runner_type: altinity-on-demand, altinity-regression-tester-aarch64
-      commit: eadf0647501a547d57c49b71ca256e10c6c304f6
-      arch: aarch64
-      build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
-      workflow_config: ${{ needs.config_workflow.outputs.data }}
-
-  SignRelease:
-    needs: [config_workflow, build_amd_release]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_sign.yml
-    secrets: inherit
-    with:
-      test_name: Sign release
-      runner_type: altinity-style-checker
-      data: ${{ needs.config_workflow.outputs.data }}
-  SignAarch64:
-    needs: [config_workflow, build_arm_release]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_sign.yml
-    secrets: inherit
-    with:
-      test_name: Sign aarch64
-      runner_type: altinity-style-checker-aarch64
-      data: ${{ needs.config_workflow.outputs.data }}
-
-  FinishCIReport:
-    if: ${{ !cancelled() }}
-    needs:
-      - config_workflow
-      - dockers_build_amd
-      - dockers_build_arm
-      - style_check
-      - fast_test
-      - build_amd_binary
-      - finish_workflow
-      - SignRelease
-      - SignAarch64
-      - RegressionTestsRelease
-      - RegressionTestsAarch64
-      - GrypeScanServer
-      - GrypeScanKeeper
-    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
-    steps:
-      - name: Check out repository code
-        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
-        with:
-          clear-repository: true
-      - name: Finalize workflow report
-        if: ${{ !cancelled() }}
-        uses: ./.github/actions/create_workflow_report
-        with:
-          workflow_config: ${{ needs.config_workflow.outputs.data }}
-          final: true

.github/workflows/nightly_fuzzers.yml

@@ -246,99 +246,3 @@ jobs:
           else
             python3 -m praktika run 'libFuzzer tests' --workflow "NightlyFuzzers" --ci |& tee ./ci/tmp/job.log
           fi
-
-##########################################################################################
-##################################### ALTINITY JOBS ######################################
-##########################################################################################
-  GrypeScanServer:
-    needs: [config_workflow, docker_server_image]
-    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIHNlcnZlciBpbWFnZQ==') }}
-    strategy:
-      fail-fast: false
-      matrix:
-        suffix: ['', '-alpine']
-    uses: ./.github/workflows/grype_scan.yml
-    secrets: inherit
-    with:
-      docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
-      tag-suffix: ${{ matrix.suffix }}
-  GrypeScanKeeper:
-      needs: [config_workflow, docker_keeper_image]
-      if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIGtlZXBlciBpbWFnZQ==') }}
-      uses: ./.github/workflows/grype_scan.yml
-      secrets: inherit
-      with:
-        docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
-
-  RegressionTestsRelease:
-    needs: [config_workflow, build_amd_release]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression')}}
-    uses: ./.github/workflows/regression.yml
-    secrets: inherit
-    with:
-      runner_type: altinity-on-demand, altinity-regression-tester
-      commit: eadf0647501a547d57c49b71ca256e10c6c304f6
-      arch: release
-      build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
-      workflow_config: ${{ needs.config_workflow.outputs.data }}
-  RegressionTestsAarch64:
-    needs: [config_workflow, build_arm_release]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression') && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_aarch64')}}
-    uses: ./.github/workflows/regression.yml
-    secrets: inherit
-    with:
-      runner_type: altinity-on-demand, altinity-regression-tester-aarch64
-      commit: eadf0647501a547d57c49b71ca256e10c6c304f6
-      arch: aarch64
-      build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
-      workflow_config: ${{ needs.config_workflow.outputs.data }}
-
-  SignRelease:
-    needs: [config_workflow, build_amd_release]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_sign.yml
-    secrets: inherit
-    with:
-      test_name: Sign release
-      runner_type: altinity-style-checker
-      data: ${{ needs.config_workflow.outputs.data }}
-  SignAarch64:
-    needs: [config_workflow, build_arm_release]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_sign.yml
-    secrets: inherit
-    with:
-      test_name: Sign aarch64
-      runner_type: altinity-style-checker-aarch64
-      data: ${{ needs.config_workflow.outputs.data }}
-
-  FinishCIReport:
-    if: ${{ !cancelled() }}
-    needs:
-      - config_workflow
-      - dockers_build_amd
-      - dockers_build_arm
-      - build_fuzzers
-      - libfuzzer_tests
-      - SignRelease
-      - SignAarch64
-      - RegressionTestsRelease
-      - RegressionTestsAarch64
-      - GrypeScanServer
-      - GrypeScanKeeper
-    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
-    steps:
-      - name: Check out repository code
-        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
-        with:
-          clear-repository: true
-      - name: Finalize workflow report
-        if: ${{ !cancelled() }}
-        uses: ./.github/actions/create_workflow_report
-        with:
-          workflow_config: ${{ needs.config_workflow.outputs.data }}
-          final: true

.github/workflows/nightly_jepsen.yml

@@ -246,99 +246,3 @@ jobs:
           else
             python3 -m praktika run 'ClickHouse Keeper Jepsen' --workflow "NightlyJepsen" --ci |& tee ./ci/tmp/job.log
           fi
-
-##########################################################################################
-##################################### ALTINITY JOBS ######################################
-##########################################################################################
-  GrypeScanServer:
-    needs: [config_workflow, docker_server_image]
-    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIHNlcnZlciBpbWFnZQ==') }}
-    strategy:
-      fail-fast: false
-      matrix:
-        suffix: ['', '-alpine']
-    uses: ./.github/workflows/grype_scan.yml
-    secrets: inherit
-    with:
-      docker_image: altinityinfra/clickhouse-server
-      version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
-      tag-suffix: ${{ matrix.suffix }}
-  GrypeScanKeeper:
-      needs: [config_workflow, docker_keeper_image]
-      if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'RG9ja2VyIGtlZXBlciBpbWFnZQ==') }}
-      uses: ./.github/workflows/grype_scan.yml
-      secrets: inherit
-      with:
-        docker_image: altinityinfra/clickhouse-keeper
-        version: ${{ fromJson(needs.config_workflow.outputs.data).custom_data.version.string }}
-
-  RegressionTestsRelease:
-    needs: [config_workflow, build_amd_release]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression')}}
-    uses: ./.github/workflows/regression.yml
-    secrets: inherit
-    with:
-      runner_type: altinity-on-demand, altinity-regression-tester
-      commit: eadf0647501a547d57c49b71ca256e10c6c304f6
-      arch: release
-      build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
-      workflow_config: ${{ needs.config_workflow.outputs.data }}
-  RegressionTestsAarch64:
-    needs: [config_workflow, build_arm_release]
-    if: ${{ !failure() && !cancelled() && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_regression') && !contains(github.event.pull_request.body, '[x] <!---ci_exclude_aarch64')}}
-    uses: ./.github/workflows/regression.yml
-    secrets: inherit
-    with:
-      runner_type: altinity-on-demand, altinity-regression-tester-aarch64
-      commit: eadf0647501a547d57c49b71ca256e10c6c304f6
-      arch: aarch64
-      build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
-      workflow_config: ${{ needs.config_workflow.outputs.data }}
-
-  SignRelease:
-    needs: [config_workflow, build_amd_release]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_sign.yml
-    secrets: inherit
-    with:
-      test_name: Sign release
-      runner_type: altinity-style-checker
-      data: ${{ needs.config_workflow.outputs.data }}
-  SignAarch64:
-    needs: [config_workflow, build_arm_release]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_sign.yml
-    secrets: inherit
-    with:
-      test_name: Sign aarch64
-      runner_type: altinity-style-checker-aarch64
-      data: ${{ needs.config_workflow.outputs.data }}
-
-  FinishCIReport:
-    if: ${{ !cancelled() }}
-    needs:
-      - config_workflow
-      - dockers_build_amd
-      - dockers_build_arm
-      - build_amd_binary
-      - clickhouse_keeper_jepsen
-      - SignRelease
-      - SignAarch64
-      - RegressionTestsRelease
-      - RegressionTestsAarch64
-      - GrypeScanServer
-      - GrypeScanKeeper
-    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
-    steps:
-      - name: Check out repository code
-        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
-        with:
-          clear-repository: true
-      - name: Finalize workflow report
-        if: ${{ !cancelled() }}
-        uses: ./.github/actions/create_workflow_report
-        with:
-          workflow_config: ${{ needs.config_workflow.outputs.data }}
-          final: true