From 28c5a1bf1dd5f357071c2e775bf7c48ed816e42f Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 2 Jul 2023 09:37:09 +0400 Subject: [PATCH] Refactor handling of artifacts (#1930) --- .github/actions/load-image/action.yml | 7 +++-- .../workflows/docker-build-test-upload.yml | 27 +++++++------------ .github/workflows/docker-merge-tags.yml | 6 ++--- .github/workflows/docker-tag-push.yml | 4 +-- .github/workflows/docker-wiki-update.yml | 10 +++---- Makefile | 2 +- 6 files changed, 24 insertions(+), 32 deletions(-) diff --git a/.github/actions/load-image/action.yml b/.github/actions/load-image/action.yml index 6ce7c65b2..55fa8c978 100644 --- a/.github/actions/load-image/action.yml +++ b/.github/actions/load-image/action.yml @@ -18,13 +18,12 @@ runs: uses: actions/download-artifact@v3 with: name: ${{ inputs.image }}-${{ inputs.platform }} - path: /tmp/ + path: /tmp/jupyter/images/ - name: Load downloaded image to docker 📥 run: | - docker load --input /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar + docker load --input /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar docker image ls --all shell: bash - name: Delete the file 🗑️ - run: rm -f /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar + run: rm /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar shell: bash - if: always() diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index 880c4fd78..d3c7f26a1 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -44,9 +44,7 @@ jobs: docker kill $(docker ps --quiet) || true docker rm $(docker ps --all --quiet) || true docker system prune --all --force - rm -rf /tmp/tags/ - rm -rf /tmp/hist_lines/ - rm -rf /tmp/manifests/ + rm -rf /tmp/jupyter/ shell: bash - name: Load parent built image to Docker 📥 @@ -70,44 +68,39 @@ jobs: - name: Write tags file 🏷 run: | - python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/tags/ --owner ${{ env.OWNER }} + python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --owner ${{ env.OWNER }} shell: bash - name: Upload tags file 💾 uses: actions/upload-artifact@v3 with: name: ${{ inputs.image }}-${{ inputs.platform }}-tags - path: /tmp/tags/${{ inputs.platform }}-${{ inputs.image }}.txt + path: /tmp/jupyter/tags/${{ inputs.platform }}-${{ inputs.image }}.txt retention-days: 3 - name: Write manifest and build history file 🏷 - run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner ${{ env.OWNER }} + run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --owner ${{ env.OWNER }} shell: bash - name: Upload manifest file 💾 uses: actions/upload-artifact@v3 with: name: ${{ inputs.image }}-${{ inputs.platform }}-manifest - path: /tmp/manifests/${{ inputs.platform }}-${{ inputs.image }}-*.md + path: /tmp/jupyter/manifests/${{ inputs.platform }}-${{ inputs.image }}-*.md retention-days: 3 - name: Upload build history line 💾 uses: actions/upload-artifact@v3 with: name: ${{ inputs.image }}-${{ inputs.platform }}-history_line - path: /tmp/hist_lines/${{ inputs.platform }}-${{ inputs.image }}-*.txt + path: /tmp/jupyter/hist_lines/${{ inputs.platform }}-${{ inputs.image }}-*.txt retention-days: 3 - name: Save image as a tar for later use 💾 - run: docker save ${{ env.OWNER }}/${{ inputs.image }} -o /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar + run: | + mkdir -p /tmp/jupyter/images/ + docker save ${{ env.OWNER }}/${{ inputs.image }} -o /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar shell: bash - name: Upload image as artifact 💾 uses: actions/upload-artifact@v3 with: name: ${{ inputs.image }}-${{ inputs.platform }} - path: /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar + path: /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar retention-days: 3 - - # Self-hosted runners share a state (whole VM) between runs - - name: Cleanup artifacts 🗑️ - run: | - rm -f /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar - shell: bash - if: always() diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/docker-merge-tags.yml index d977b8895..7287b0cf5 100644 --- a/.github/workflows/docker-merge-tags.yml +++ b/.github/workflows/docker-merge-tags.yml @@ -32,12 +32,12 @@ jobs: uses: actions/download-artifact@v3 with: name: ${{ inputs.image }}-x86_64-tags - path: /tmp/tags/ + path: /tmp/jupyter/tags/ - name: Download aarch64 tags file 📥 uses: actions/download-artifact@v3 with: name: ${{ inputs.image }}-aarch64-tags - path: /tmp/tags/ + path: /tmp/jupyter/tags/ # Docker might be stuck when pulling images # https://github.com/docker/for-mac/issues/2083 @@ -57,5 +57,5 @@ jobs: - name: Merge tags for the images 🔀 if: github.repository == 'jupyter/docker-stacks' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule') - run: python3 -m tagging.merge_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/tags/ + run: python3 -m tagging.merge_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ shell: bash diff --git a/.github/workflows/docker-tag-push.yml b/.github/workflows/docker-tag-push.yml index bd0baa0f2..75e4a7324 100644 --- a/.github/workflows/docker-tag-push.yml +++ b/.github/workflows/docker-tag-push.yml @@ -48,9 +48,9 @@ jobs: uses: actions/download-artifact@v3 with: name: ${{ inputs.image }}-${{ inputs.platform }}-tags - path: /tmp/tags/ + path: /tmp/jupyter/tags/ - name: Apply tags to the loaded image 🏷 - run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/tags/ --platform ${{ inputs.platform }} --owner ${{ env.OWNER }} + run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --owner ${{ env.OWNER }} - name: Push Images to Docker Hub 📤 if: github.repository == 'jupyter/docker-stacks' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule') diff --git a/.github/workflows/docker-wiki-update.yml b/.github/workflows/docker-wiki-update.yml index 9ae75dc88..d5747c64c 100644 --- a/.github/workflows/docker-wiki-update.yml +++ b/.github/workflows/docker-wiki-update.yml @@ -20,12 +20,12 @@ jobs: - name: Download all manifests and history lines 📥 uses: ./.github/actions/download-manifests with: - histLineDir: /tmp/hist_lines/ - manifestDir: /tmp/manifests/ + histLineDir: /tmp/jupyter/hist_lines/ + manifestDir: /tmp/jupyter/manifests/ - name: Display structure of downloaded files 🔍️ run: | - ls -R /tmp/hist_lines/ - ls -R /tmp/manifests/ + ls -R /tmp/jupyter/hist_lines/ + ls -R /tmp/jupyter/manifests/ shell: bash - name: Checkout Wiki Repo 📃 @@ -35,7 +35,7 @@ jobs: path: wiki/ - name: Update wiki page 🏷 - run: python3 -m tagging.update_wiki_page --wiki-dir wiki/ --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ + run: python3 -m tagging.update_wiki_page --wiki-dir wiki/ --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ shell: bash - name: Push Wiki to GitHub 📤 diff --git a/Makefile b/Makefile index 8a71110ab..c1a33edb0 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ linkcheck-docs: ## check broken links hook/%: ## run post-build hooks for an image python3 -m tagging.tag_image --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \ - python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner "$(OWNER)" + python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --owner "$(OWNER)" hook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images