diff --git a/.github/actions/cleanup_docker/action.yml b/.github/actions/cleanup_docker/action.yml new file mode 100644 index 0000000000000..0fdf104fb8b2d --- /dev/null +++ b/.github/actions/cleanup_docker/action.yml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Prune all docker images and containers on Github runners +description: Installs breeze and recreates current python image from artifact +inputs: + docker-volume-location: + description: File system location where to move docker space to - defaults to second disk which has 75GB + default: /mnt/var-lib-docker +runs: + using: composite + steps: + - name: Cleanup docker + run: ./.github/actions/cleanup_docker/cleanup_docker.sh + env: + TARGET_DOCKER_VOLUME_LOCATION: ${{ inputs.docker-volume-location }} + shell: bash diff --git a/scripts/ci/cleanup_docker.sh b/.github/actions/cleanup_docker/cleanup_docker.sh similarity index 94% rename from scripts/ci/cleanup_docker.sh rename to .github/actions/cleanup_docker/cleanup_docker.sh index 8f195fe755040..cd41efccdf9b6 100755 --- a/scripts/ci/cleanup_docker.sh +++ b/.github/actions/cleanup_docker/cleanup_docker.sh @@ -23,6 +23,7 @@ function cleanup_docker { if [ -n "${TARGET_DOCKER_VOLUME_LOCATION}" ]; then echo "Mounting ${TARGET_DOCKER_VOLUME_LOCATION} to /var/lib/docker" sudo mkdir -p "${TARGET_DOCKER_VOLUME_LOCATION}" /var/lib/docker + sudo chmod 710 "${TARGET_DOCKER_VOLUME_LOCATION}" /var/lib/docker sudo mount --bind "${TARGET_DOCKER_VOLUME_LOCATION}" /var/lib/docker fi sudo systemctl start docker diff --git a/.github/actions/prepare_all_ci_images/action.yml b/.github/actions/prepare_all_ci_images/action.yml index 25a695b4f974e..ddb3935384e6d 100644 --- a/.github/actions/prepare_all_ci_images/action.yml +++ b/.github/actions/prepare_all_ci_images/action.yml @@ -22,20 +22,14 @@ inputs: python-versions-list-as-string: description: 'Stringified array of all Python versions to test - separated by spaces.' required: true - docker-volume-location: - description: File system location where to move docker space to - default: /mnt/var-lib-docker platform: description: 'Platform for the build - linux/amd64 or linux/arm64' required: true runs: using: "composite" steps: - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: ${{ inputs.docker-volume-location }} - shell: bash + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker # TODO: Currently we cannot loop through the list of python versions and have dynamic list of # tasks. Instead we hardcode all possible python versions and they - but # this should be implemented in stash action as list of keys to download. diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index b29e1f7afa3a1..b8d8293956373 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -38,12 +38,8 @@ outputs: runs: using: "composite" steps: - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker - shell: bash + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 3dba80a024d1f..3694429cbfcc6 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -141,11 +141,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index 86c1c19738ac0..b387fa84e05b1 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -115,11 +115,8 @@ jobs: with: fetch-depth: 2 persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index f23b9c7246310..5698008ec7dbb 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -76,11 +76,8 @@ jobs: # Need to fetch all history for selective checks tests fetch-depth: 0 persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -100,11 +97,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: Setup pnpm uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # v4.0.0 with: @@ -170,11 +164,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Setup node" uses: actions/setup-node@v4 with: @@ -241,11 +232,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -299,11 +287,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -381,11 +366,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 39ef0a44f19b0..54a71f43ab998 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -129,11 +129,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index bf4baa0af7fd5..7e73b4c47dbff 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -323,11 +323,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Download docs prepared as artifacts" uses: actions/download-artifact@v4 with: @@ -407,11 +404,8 @@ jobs: with: fetch-depth: 2 persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0799ee8397fec..cd7aa8966514e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,11 +146,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: Fetch incoming commit ${{ github.sha }} with its parent uses: actions/checkout@v4 with: diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index e6af5339c5230..e076076262754 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -103,11 +103,8 @@ jobs: with: # Needed to perform push action persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} @@ -194,11 +191,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Free up disk space" shell: bash run: ./scripts/tools/free_up_disk_space.sh diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 0e6cd9f11aee0..47230c556c216 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -78,7 +78,6 @@ jobs: with: platform: "linux/amd64" python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }} - docker-volume-location: "" # TODO(jscheffl): Understand why it fails here and fix it - name: "Verify all CI images ${{ inputs.python-versions-list-as-string }}" run: breeze ci-image verify --run-in-parallel - name: "Source constraints" diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index e45d86ae576f5..381418724c5c4 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -96,11 +96,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 4bd13547b7f2c..ec625221df987 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -136,12 +136,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker - if: inputs.upload-package-artifact == 'true' + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Cleanup dist and context file" shell: bash run: rm -fv ./dist/* ./docker-context-files/* @@ -224,11 +220,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 5d0e3ec1d13c1..97f9bbbc7e02f 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -125,11 +125,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -204,11 +201,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 287c6fd9790ee..a4dc55f426291 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -59,11 +59,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: @@ -101,11 +98,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - - name: "Cleanup docker" - # Move docker space to second partition to have more space - run: ./scripts/ci/cleanup_docker.sh - env: - TARGET_DOCKER_VOLUME_LOCATION: /mnt/var-lib-docker + - name: Cleanup docker + uses: ./.github/actions/cleanup_docker - name: "Install Breeze" uses: ./.github/actions/breeze with: