From d5d197f10aa34aed1b1eef936f8ef30063c61af7 Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:26:30 -0300 Subject: [PATCH 01/11] fix push manifests --- .github/workflows/build-develop.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index a6485ac31..0d4d0b440 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -9,6 +9,7 @@ on: push: branches: - develop + - fix-pipeline env: # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) @@ -74,6 +75,7 @@ jobs: runs-on: ubuntu-latest outputs: version_number: ${{ env.VERSION }} + commit_hash: ${{ env.COMMIT }} steps: - uses: actions/checkout@v2 @@ -107,6 +109,7 @@ jobs: run: | echo "VERSION=`cat ${{github.workspace}}/build/VERSION`" >> $GITHUB_ENV echo "::set-output name=version::$(cat ${{github.workspace}}/build/VERSION)" + echo "COMMIT=`git rev-parse --short HEAD`" >> $GITHUB_ENV - name: Build working-directory: ${{github.workspace}}/build @@ -338,7 +341,7 @@ jobs: file: ./docker/Dockerfile.crashhandler platforms: linux/amd64 push: true - tags: ${{ env.IMAGE_NAME }}:${{ needs.unit-tests-linux.outputs.version_number }} , ${{ env.IMAGE_NAME }}:${{ env.REF_TAG }} , ${{ env.IMAGE_NAME2 }}:amd64 + tags: ${{ env.IMAGE_NAME }}:${{ needs.unit-tests-linux.outputs.version_number }} , ${{ env.IMAGE_NAME }}:${{ env.REF_TAG }} , ${{ env.IMAGE_NAME2 }}:amd64-${{needs.unit-tests-linux.outputs.commit_hash}} build-app-image-x64: needs: [ package-amd64 ] @@ -832,7 +835,7 @@ jobs: file: ./docker/Dockerfile.crashhandler platforms: linux/arm64 push: true - tags: ${{ env.IMAGE_NAME }}:arm64 + tags: ${{ env.IMAGE_NAME }}:arm64-${{needs.unit-tests-linux.outputs.commit_hash}} build-args: | ghtoken=${{ secrets.GITHUB_TOKEN }} @@ -922,8 +925,8 @@ jobs: - name: Create docker manifest run: | docker manifest create ghcr.io/ns1labs/pktvisor:latest-develop \ - ghcr.io/ns1labs/pktvisor:amd64 \ - ghcr.io/ns1labs/pktvisor:arm64 + ghcr.io/ns1labs/pktvisor:amd64-${{needs.unit-tests-linux.outputs.commit_hash}} \ + ghcr.io/ns1labs/pktvisor:arm64-${{needs.unit-tests-linux.outputs.commit_hash}} - name: Push manifest to ghcr.io run: | From 09905cf9b4d337f73069fd32f7eade8f8c28bd09 Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Wed, 11 Jan 2023 14:06:37 -0300 Subject: [PATCH 02/11] test --- .github/workflows/clean-runners.yml | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/clean-runners.yml b/.github/workflows/clean-runners.yml index d84749db1..a39b5efc6 100644 --- a/.github/workflows/clean-runners.yml +++ b/.github/workflows/clean-runners.yml @@ -15,3 +15,43 @@ jobs: GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }} run: | gh api -H "Accept: application/vnd.github.v3+json" /repos/ns1labs/pktvisor/actions/runners -q '.runners[] | {id,status,busy} | select((.busy == false) and (.status == "offline")) | {id} | .[]' --paginate | xargs -I {} gh api --method DELETE -H "Accept: application/vnd.github.v3+json" /repos/ns1labs/pktvisor/actions/runners/{} + + - name: Delete arm64 images from ghcr.io + uses: bots-house/ghcr-delete-image-action@v1.0.0 + with: + owner: ns1labs + name: pktvisor + token: ${{ secrets.GITHUB_TOKEN }} + tag: arm64-d5d197f + + - name: Delete amd64 images from ghcr.io + uses: bots-house/ghcr-delete-image-action@v1.0.0 + with: + owner: ns1labs + name: pktvisor + token: ${{ secrets.GITHUB_TOKEN }} + tag: amd64-d5d197f + + - name: Delete arm64 images from ghcr.io + uses: bots-house/ghcr-delete-image-action@v1.0.0 + with: + owner: ns1labs + name: pktvisor + token: ${{ secrets.GITHUB_TOKEN }} + tag: arm64 + + - name: Delete amd64 images from ghcr.io + uses: bots-house/ghcr-delete-image-action@v1.0.0 + with: + owner: ns1labs + name: pktvisor + token: ${{ secrets.GITHUB_TOKEN }} + tag: amd64 + + - name: Delete latest-develop images from ghcr.io + uses: bots-house/ghcr-delete-image-action@v1.0.0 + with: + owner: ns1labs + name: pktvisor + token: ${{ secrets.GITHUB_TOKEN }} + tag: latest-develop From fc712745584eb7d5678dd4dd3a2ad1d04dddb264 Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Wed, 11 Jan 2023 14:09:28 -0300 Subject: [PATCH 03/11] test --- .github/workflows/clean-runners.yml | 70 ++++++++++++++--------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/clean-runners.yml b/.github/workflows/clean-runners.yml index a39b5efc6..010557a58 100644 --- a/.github/workflows/clean-runners.yml +++ b/.github/workflows/clean-runners.yml @@ -16,42 +16,42 @@ jobs: run: | gh api -H "Accept: application/vnd.github.v3+json" /repos/ns1labs/pktvisor/actions/runners -q '.runners[] | {id,status,busy} | select((.busy == false) and (.status == "offline")) | {id} | .[]' --paginate | xargs -I {} gh api --method DELETE -H "Accept: application/vnd.github.v3+json" /repos/ns1labs/pktvisor/actions/runners/{} - - name: Delete arm64 images from ghcr.io - uses: bots-house/ghcr-delete-image-action@v1.0.0 - with: - owner: ns1labs - name: pktvisor - token: ${{ secrets.GITHUB_TOKEN }} - tag: arm64-d5d197f +# - name: Delete arm64 images from ghcr.io +# uses: bots-house/ghcr-delete-image-action@v1.0.0 +# with: +# owner: ns1labs +# name: pktvisor +# token: ${{ secrets.GITHUB_TOKEN }} +# tag: arm64-d5d197f - - name: Delete amd64 images from ghcr.io - uses: bots-house/ghcr-delete-image-action@v1.0.0 - with: - owner: ns1labs - name: pktvisor - token: ${{ secrets.GITHUB_TOKEN }} - tag: amd64-d5d197f +# - name: Delete amd64 images from ghcr.io +# uses: bots-house/ghcr-delete-image-action@v1.0.0 +# with: +# owner: ns1labs +# name: pktvisor +# token: ${{ secrets.GITHUB_TOKEN }} +# tag: amd64-d5d197f - - name: Delete arm64 images from ghcr.io - uses: bots-house/ghcr-delete-image-action@v1.0.0 - with: - owner: ns1labs - name: pktvisor - token: ${{ secrets.GITHUB_TOKEN }} - tag: arm64 +# - name: Delete arm64 images from ghcr.io +# uses: bots-house/ghcr-delete-image-action@v1.0.0 +# with: +# owner: ns1labs +# name: pktvisor +# token: ${{ secrets.GITHUB_TOKEN }} +# tag: arm64 - - name: Delete amd64 images from ghcr.io - uses: bots-house/ghcr-delete-image-action@v1.0.0 - with: - owner: ns1labs - name: pktvisor - token: ${{ secrets.GITHUB_TOKEN }} - tag: amd64 +# - name: Delete amd64 images from ghcr.io +# uses: bots-house/ghcr-delete-image-action@v1.0.0 +# with: +# owner: ns1labs +# name: pktvisor +# token: ${{ secrets.GITHUB_TOKEN }} +# tag: amd64 - - name: Delete latest-develop images from ghcr.io - uses: bots-house/ghcr-delete-image-action@v1.0.0 - with: - owner: ns1labs - name: pktvisor - token: ${{ secrets.GITHUB_TOKEN }} - tag: latest-develop +# - name: Delete latest-develop images from ghcr.io +# uses: bots-house/ghcr-delete-image-action@v1.0.0 +# with: +# owner: ns1labs +# name: pktvisor +# token: ${{ secrets.GITHUB_TOKEN }} +# tag: latest-develop From 7116b7941a70b7fff6282e29a214c6e280ca0972 Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Wed, 11 Jan 2023 14:48:10 -0300 Subject: [PATCH 04/11] clear cache --- .github/workflows/clean-runners.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/clean-runners.yml b/.github/workflows/clean-runners.yml index 010557a58..359d12ce2 100644 --- a/.github/workflows/clean-runners.yml +++ b/.github/workflows/clean-runners.yml @@ -16,6 +16,25 @@ jobs: run: | gh api -H "Accept: application/vnd.github.v3+json" /repos/ns1labs/pktvisor/actions/runners -q '.runners[] | {id,status,busy} | select((.busy == false) and (.status == "offline")) | {id} | .[]' --paginate | xargs -I {} gh api --method DELETE -H "Accept: application/vnd.github.v3+json" /repos/ns1labs/pktvisor/actions/runners/{} + - name: Clear cache + uses: actions/github-script@v6 + with: + script: | + console.log("About to clear") + const caches = await github.rest.actions.getActionsCacheList({ + owner: context.repo.owner, + repo: context.repo.repo, + }) + for (const cache of caches.data.actions_caches) { + console.log(cache) + github.rest.actions.deleteActionsCacheById({ + owner: context.repo.owner, + repo: context.repo.repo, + cache_id: cache.id, + }) + } + console.log("Clear completed") + # - name: Delete arm64 images from ghcr.io # uses: bots-house/ghcr-delete-image-action@v1.0.0 # with: From 9409a9c36c3005e7b0bf094fcc2cb23f42425160 Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Wed, 11 Jan 2023 15:40:27 -0300 Subject: [PATCH 05/11] fix --- .github/workflows/clean-runners.yml | 74 +++++++---------------------- 1 file changed, 17 insertions(+), 57 deletions(-) diff --git a/.github/workflows/clean-runners.yml b/.github/workflows/clean-runners.yml index 359d12ce2..f0663b156 100644 --- a/.github/workflows/clean-runners.yml +++ b/.github/workflows/clean-runners.yml @@ -16,61 +16,21 @@ jobs: run: | gh api -H "Accept: application/vnd.github.v3+json" /repos/ns1labs/pktvisor/actions/runners -q '.runners[] | {id,status,busy} | select((.busy == false) and (.status == "offline")) | {id} | .[]' --paginate | xargs -I {} gh api --method DELETE -H "Accept: application/vnd.github.v3+json" /repos/ns1labs/pktvisor/actions/runners/{} - - name: Clear cache - uses: actions/github-script@v6 - with: - script: | - console.log("About to clear") - const caches = await github.rest.actions.getActionsCacheList({ - owner: context.repo.owner, - repo: context.repo.repo, - }) - for (const cache of caches.data.actions_caches) { - console.log(cache) - github.rest.actions.deleteActionsCacheById({ - owner: context.repo.owner, - repo: context.repo.repo, - cache_id: cache.id, - }) - } - console.log("Clear completed") - -# - name: Delete arm64 images from ghcr.io -# uses: bots-house/ghcr-delete-image-action@v1.0.0 -# with: -# owner: ns1labs -# name: pktvisor -# token: ${{ secrets.GITHUB_TOKEN }} -# tag: arm64-d5d197f - -# - name: Delete amd64 images from ghcr.io -# uses: bots-house/ghcr-delete-image-action@v1.0.0 -# with: -# owner: ns1labs -# name: pktvisor -# token: ${{ secrets.GITHUB_TOKEN }} -# tag: amd64-d5d197f - -# - name: Delete arm64 images from ghcr.io -# uses: bots-house/ghcr-delete-image-action@v1.0.0 -# with: -# owner: ns1labs -# name: pktvisor -# token: ${{ secrets.GITHUB_TOKEN }} -# tag: arm64 - -# - name: Delete amd64 images from ghcr.io -# uses: bots-house/ghcr-delete-image-action@v1.0.0 -# with: -# owner: ns1labs -# name: pktvisor -# token: ${{ secrets.GITHUB_TOKEN }} -# tag: amd64 - -# - name: Delete latest-develop images from ghcr.io -# uses: bots-house/ghcr-delete-image-action@v1.0.0 +# - name: Clear cache +# uses: actions/github-script@v6 # with: -# owner: ns1labs -# name: pktvisor -# token: ${{ secrets.GITHUB_TOKEN }} -# tag: latest-develop +# script: | +# console.log("About to clear") +# const caches = await github.rest.actions.getActionsCacheList({ +# owner: context.repo.owner, +# repo: context.repo.repo, +# }) +# for (const cache of caches.data.actions_caches) { +# console.log(cache) +# github.rest.actions.deleteActionsCacheById({ +# owner: context.repo.owner, +# repo: context.repo.repo, +# cache_id: cache.id, +# }) +# } +# console.log("Clear completed") From a892e3393eba2752121e3e7d9088dc0467dd08c0 Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Wed, 11 Jan 2023 15:45:24 -0300 Subject: [PATCH 06/11] add amend and image clean at end --- .github/workflows/build-develop.yml | 30 +++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index 0d4d0b440..9b5a12405 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -925,8 +925,8 @@ jobs: - name: Create docker manifest run: | docker manifest create ghcr.io/ns1labs/pktvisor:latest-develop \ - ghcr.io/ns1labs/pktvisor:amd64-${{needs.unit-tests-linux.outputs.commit_hash}} \ - ghcr.io/ns1labs/pktvisor:arm64-${{needs.unit-tests-linux.outputs.commit_hash}} + --amend ghcr.io/ns1labs/pktvisor:amd64-${{needs.unit-tests-linux.outputs.commit_hash}} \ + --amend ghcr.io/ns1labs/pktvisor:arm64-${{needs.unit-tests-linux.outputs.commit_hash}} - name: Push manifest to ghcr.io run: | @@ -959,5 +959,27 @@ jobs: --all \ docker://ghcr.io/ns1labs/pktvisor:latest-develop \ docker://docker.io/ns1labs/pktvisor:${{ needs.unit-tests-linux.outputs.version_number }} - - + + - name: Delete arm64 images from ghcr.io + uses: bots-house/ghcr-delete-image-action@v1.0.0 + with: + owner: ns1labs + name: pktvisor + token: ${{ secrets.GITHUB_TOKEN }} + tag: arm64-${{needs.unit-tests-linux.outputs.commit_hash}} + + - name: Delete amd64 images from ghcr.io + uses: bots-house/ghcr-delete-image-action@v1.0.0 + with: + owner: ns1labs + name: pktvisor + token: ${{ secrets.GITHUB_TOKEN }} + tag: amd64-${{needs.unit-tests-linux.outputs.commit_hash}} + + - name: Delete latest-develop images from ghcr.io + uses: bots-house/ghcr-delete-image-action@v1.0.0 + with: + owner: ns1labs + name: pktvisor + token: ${{ secrets.GITHUB_TOKEN }} + tag: latest-develop From 3ba276ef34fb4ea7fde61c6060d7e909e45e5581 Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Wed, 11 Jan 2023 17:21:37 -0300 Subject: [PATCH 07/11] test with v2 --- .github/workflows/build-develop.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index 9b5a12405..ae62fcdbe 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -828,16 +828,14 @@ jobs: - name: Build + push - pktvisor (multi-arch) env: IMAGE_NAME: ghcr.io/ns1labs/pktvisor - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: builder: ${{ steps.buildx.outputs.name }} context: . file: ./docker/Dockerfile.crashhandler platforms: linux/arm64 push: true - tags: ${{ env.IMAGE_NAME }}:arm64-${{needs.unit-tests-linux.outputs.commit_hash}} - build-args: | - ghtoken=${{ secrets.GITHUB_TOKEN }} + tags: ${{ env.IMAGE_NAME }}:arm64-${{needs.unit-tests-linux.outputs.commit_hash}}, ${{ env.IMAGE_NAME }}:arm64 remove-github-runner-arm64: needs: [ spinup-arm64-runner, package-arm64 ] From 3d9e1efe995610fef5ddc0cea2d0e33f100ced8a Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Wed, 11 Jan 2023 18:29:57 -0300 Subject: [PATCH 08/11] update ami --- .github/hosted-runner/arm64/vars.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/hosted-runner/arm64/vars.tf b/.github/hosted-runner/arm64/vars.tf index 6430a67ba..98e99ba9f 100644 --- a/.github/hosted-runner/arm64/vars.tf +++ b/.github/hosted-runner/arm64/vars.tf @@ -16,7 +16,7 @@ variable "aws_region" { variable "amis" { type = map default = { - us-east-1 = "ami-070650c005cce4203" //ubuntu 22.04 arm64 on us-east-1 + us-east-1 = "ami-0f972b3903f44791c" //ubuntu 20.04 arm64 on us-east-1 } } From 879185f1eb9809270ab5e9a679119d4fc7c0cf52 Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Wed, 11 Jan 2023 18:56:41 -0300 Subject: [PATCH 09/11] test --- .github/workflows/build-develop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index ae62fcdbe..5dfd9b24d 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -910,7 +910,7 @@ jobs: pushing-manifests: needs: [ unit-tests-linux, remove-github-runner-arm64 ] - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 if: github.event_name != 'pull_request' steps: - name: Login to GitHub Container Registry From 52321213d91d25035ffddcde32251b4283191b68 Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Thu, 12 Jan 2023 08:36:43 -0300 Subject: [PATCH 10/11] test packing in tar --- .github/workflows/build-develop.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index 5dfd9b24d..40ed7b142 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -826,16 +826,22 @@ jobs: sed -i -e "s/CP_URL/$ESCAPED_REPLACE/g" docker/run.sh - name: Build + push - pktvisor (multi-arch) + id: docker_build env: IMAGE_NAME: ghcr.io/ns1labs/pktvisor - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} context: . file: ./docker/Dockerfile.crashhandler platforms: linux/arm64 - push: true - tags: ${{ env.IMAGE_NAME }}:arm64-${{needs.unit-tests-linux.outputs.commit_hash}}, ${{ env.IMAGE_NAME }}:arm64 + tags: ${{ env.IMAGE_NAME }}:arm64-${{needs.unit-tests-linux.outputs.commit_hash}} + outputs: type=docker,dest=/tmp/arm64.tar + + - name: Load image + run: | + docker load --input /tmp/arm64.tar + docker image ls -a remove-github-runner-arm64: needs: [ spinup-arm64-runner, package-arm64 ] @@ -923,8 +929,8 @@ jobs: - name: Create docker manifest run: | docker manifest create ghcr.io/ns1labs/pktvisor:latest-develop \ - --amend ghcr.io/ns1labs/pktvisor:amd64-${{needs.unit-tests-linux.outputs.commit_hash}} \ - --amend ghcr.io/ns1labs/pktvisor:arm64-${{needs.unit-tests-linux.outputs.commit_hash}} + ghcr.io/ns1labs/pktvisor:amd64-${{needs.unit-tests-linux.outputs.commit_hash}} \ + ghcr.io/ns1labs/pktvisor:arm64-${{needs.unit-tests-linux.outputs.commit_hash}} - name: Push manifest to ghcr.io run: | From b7842a7cb4cc54f393ad6cf2b9b61d7aa58f9bf7 Mon Sep 17 00:00:00 2001 From: "Everton H. Taques" <97463920+etaques@users.noreply.github.com> Date: Thu, 12 Jan 2023 09:40:25 -0300 Subject: [PATCH 11/11] add push after load --- .github/workflows/build-develop.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index 40ed7b142..8dfb7b23b 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -839,9 +839,12 @@ jobs: outputs: type=docker,dest=/tmp/arm64.tar - name: Load image + env: + IMAGE_NAME: ghcr.io/ns1labs/pktvisor run: | docker load --input /tmp/arm64.tar docker image ls -a + docker push ${{ env.IMAGE_NAME }}:arm64-${{needs.unit-tests-linux.outputs.commit_hash}} remove-github-runner-arm64: needs: [ spinup-arm64-runner, package-arm64 ]