From 4656becf39fc4b5794f8eebadb85037546dc965a Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Thu, 28 Apr 2022 19:09:05 +0000 Subject: [PATCH 1/3] build multiplat manifests with buildah Signed-off-by: Evan Baker --- .pipelines/containers/container-template.yaml | 27 +++-- .pipelines/containers/manifest-template.yaml | 11 +- .pipelines/pipeline.yaml | 100 ++++++++++-------- .pipelines/windows-image.yaml | 2 +- Makefile | 35 ++---- 5 files changed, 89 insertions(+), 86 deletions(-) diff --git a/.pipelines/containers/container-template.yaml b/.pipelines/containers/container-template.yaml index 4298ebdebf..8c8dff0ee7 100644 --- a/.pipelines/containers/container-template.yaml +++ b/.pipelines/containers/container-template.yaml @@ -13,21 +13,30 @@ steps: addPipelineData: false - script: | - docker run --privileged --rm tonistiigi/binfmt --install arm64 + . /etc/os-release + echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list + curl -L "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key" | sudo apt-key add - + sudo apt update -y + sudo apt install -y podman buildah skopeo qemu-user-static binfmt-support + sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes + name: container_env + displayName: Prepare Environment + +- script: | export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }}) make ${{ parameters.name }}-image OS=${{ parameters.os }} ARCH=${{ parameters.arch }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }} TAG=$PLATFORM_TAG name: image_build displayName: Image Build retryCountOnTaskFailure: 3 -- script: | - wget https://github.com/aquasecurity/trivy/releases/download/v0.18.1/trivy_0.18.1_Linux-64bit.tar.gz - tar -zxvf trivy*.tar.gz - mkdir -p ./trivy-cache - export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }}) - sudo ./trivy --exit-code 1 --cache-dir ./trivy-cache --severity HIGH,CRITICAL $(IMAGE_REGISTRY)/$(make ${{ parameters.name }}-image-name):$PLATFORM_TAG - name: trivy - displayName: Vulnerability Scan +# - script: | +# wget https://github.com/aquasecurity/trivy/releases/download/v0.18.1/trivy_0.18.1_Linux-64bit.tar.gz +# tar -zxvf trivy*.tar.gz +# mkdir -p ./trivy-cache +# export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }}) +# sudo ./trivy --exit-code 1 --cache-dir ./trivy-cache --severity HIGH,CRITICAL $(IMAGE_REGISTRY)/$(make ${{ parameters.name }}-image-name):$PLATFORM_TAG +# name: trivy +# displayName: Vulnerability Scan - script: | function auto-retry() diff --git a/.pipelines/containers/manifest-template.yaml b/.pipelines/containers/manifest-template.yaml index 85e1fde64d..0e780e58cf 100644 --- a/.pipelines/containers/manifest-template.yaml +++ b/.pipelines/containers/manifest-template.yaml @@ -12,6 +12,15 @@ steps: command: 'login' addPipelineData: false +- script: | + . /etc/os-release + echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list + curl -L "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key" | sudo apt-key add - + sudo apt update -y + sudo apt install -y podman buildah skopeo + name: manifest_env + displayName: Prepare Environment + - script: | make ${{ parameters.name }}-multiarch-manifest-create OSES="${{ parameters.oses }}" ARCHES="${{ parameters.arches }}" TAG=${{ parameters.tag }} make ${{ parameters.name }}-image-info TAG=${{ parameters.tag }} @@ -28,7 +37,7 @@ steps: printf "Attempt $i/$attempts - " && "$@" && break || sleep 3 && i=$[$i+1] && false done } - make multiarch-manifest-push-docker REGISTRY=$IMAGE_REGISTRY IMAGE=$(make ${{ parameters.name }}-image-name) TAG=${{ parameters.tag }} + make multiarch-manifest-push REGISTRY=$IMAGE_REGISTRY IMAGE=$(make ${{ parameters.name }}-image-name) TAG=${{ parameters.tag }} auto-retry make container-pull REGISTRY=$IMAGE_REGISTRY IMAGE=$(make ${{ parameters.name }}-image-name) TAG=${{ parameters.tag }} name: manifest_push displayName: Manifest Push diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 377503daaf..01fc9e2b63 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -23,6 +23,8 @@ stages: BUILD_NUMBER=$(Build.BuildNumber) echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})" echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)" + cat /etc/os-release + uname -a sudo chown -R $(whoami):$(whoami) . go version go env @@ -32,10 +34,58 @@ stages: displayName: "Set environmental variables" condition: always() - - stage: build_and_test - displayName: Build ACN and Test + - stage: test + displayName: Test dependsOn: - setup + jobs: + - job: test + displayName: Run Tests + variables: + STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + steps: + - script: | + make tools + # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout + { { { { + sudo -E env "PATH=$PATH" make test-all; + echo $? >&3; + } | tee >(build/tools/bin/go-junit-report > report.xml) >&4; + } 3>&1; + } | { read xs; exit $xs; } + } 4>&1 + retryCountOnTaskFailure: 3 + name: "Test" + displayName: "Run Tests" + + - bash: | + build/tools/bin/gocov convert coverage.out > coverage.json + build/tools/bin/gocov-xml < coverage.json > coverage.xml + name: "Coverage" + displayName: "Generate Coverage Reports" + condition: always() + + - task: PublishTestResults@2 + inputs: + testRunner: JUnit + testResultsFiles: report.xml + displayName: "Publish Test Results" + condition: always() + + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: coverage.xml + displayName: "Publish Code Coverage Results" + condition: always() + + - stage: binaries + displayName: Build Binaries + dependsOn: + - setup + - test jobs: - job: build displayName: Build Binaries @@ -86,55 +136,13 @@ stages: - publish: ./test/apimodels/ artifact: clusterdefinitions - - job: test - displayName: Run Tests - variables: - STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] - pool: - name: "$(BUILD_POOL_NAME_DEFAULT)" - steps: - - script: | - make tools - # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout - { { { { - sudo -E env "PATH=$PATH" make test-all; - echo $? >&3; - } | tee >(build/tools/bin/go-junit-report > report.xml) >&4; - } 3>&1; - } | { read xs; exit $xs; } - } 4>&1 - retryCountOnTaskFailure: 3 - name: "Test" - displayName: "Run Tests" - - - bash: | - build/tools/bin/gocov convert coverage.out > coverage.json - build/tools/bin/gocov-xml < coverage.json > coverage.xml - name: "Coverage" - displayName: "Generate Coverage Reports" - condition: always() - - - task: PublishTestResults@2 - inputs: - testRunner: JUnit - testResultsFiles: report.xml - displayName: "Publish Test Results" - condition: always() - - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: coverage.xml - displayName: "Publish Code Coverage Results" - condition: always() - - template: windows-image.yaml - stage: containerize displayName: Build Images dependsOn: - setup - - build_and_test + - test jobs: - job: containerize displayName: Build Images @@ -296,6 +304,4 @@ stages: BUILD_NUMBER=${BUILD_NUMBER//./-} echo Deleting storage container with name acn-$BUILD_NUMBER and account name $(STORAGE_ACCOUNT_NAME) az storage container delete -n acn-$BUILD_NUMBER --account-name $(STORAGE_ACCOUNT_NAME) - echo Pruning old docker images... - sudo docker system prune -f displayName: Cleanup remote Azure storage container diff --git a/.pipelines/windows-image.yaml b/.pipelines/windows-image.yaml index f7107747ed..5fee9f04e8 100644 --- a/.pipelines/windows-image.yaml +++ b/.pipelines/windows-image.yaml @@ -3,7 +3,7 @@ stages: displayName: Build Windows Images dependsOn: - setup - - build_and_test + - test jobs: - job: NPM displayName: Build NPM diff --git a/Makefile b/Makefile index e87929bdfc..3a35adcea5 100644 --- a/Makefile +++ b/Makefile @@ -349,25 +349,16 @@ azure-cnm-plugin-image: azure-cnm-plugin ## build the azure-cnm plugin container ## This section is for building multi-arch/os container image manifests. -multiarch-image-pull-docker: # util target to pull all variants of a multi-arch/os image - $(foreach OS,$(OSES),$(foreach ARCH,$(ARCHES),docker pull $(REGISTRY)/$(IMAGE):$(OS)-$(ARCH)-$(TAG);)) - -multiarch-manifest-create-docker: # util target to compose multiarch container manifests from os/arch images. - docker manifest create \ +multiarch-manifest-create: # util target to compose multiarch container manifests from os/arch images. + $(CONTAINER_BUILDER) manifest create \ $(REGISTRY)/$(IMAGE):$(TAG) \ $(foreach OS,$(OSES),$(foreach ARCH,$(ARCHES),$(REGISTRY)/$(IMAGE):$(OS)-$(ARCH)-$(TAG))) -multiarch-manifest-push-docker: # util target to push multiarch container manifest. - docker manifest push --purge $(REGISTRY)/$(IMAGE):$(TAG) +multiarch-manifest-push: # util target to push multiarch container manifest. + $(CONTAINER_BUILDER) manifest push $(REGISTRY)/$(IMAGE):$(TAG) docker://$(REGISTRY)/$(IMAGE):$(TAG) cni-manager-multiarch-manifest-create: ## build cni-manager multi-arch container manifest. - $(MAKE) multiarch-image-pull-docker \ - OSES="$(OSES)" \ - ARCHES="$(ARCHES)" \ - REGISTRY=$(IMAGE_REGISTRY) \ - IMAGE=$(CNI_IMAGE) \ - TAG=$(TAG) - $(MAKE) multiarch-manifest-create-docker \ + $(MAKE) multiarch-manifest-create \ OSES="$(OSES)" \ ARCHES="$(ARCHES)" \ REGISTRY=$(IMAGE_REGISTRY) \ @@ -375,13 +366,7 @@ cni-manager-multiarch-manifest-create: ## build cni-manager multi-arch container TAG=$(TAG) cns-multiarch-manifest-create: ## build azure-cns multi-arch container manifest. - $(MAKE) multiarch-image-pull-docker \ - OSES="$(OSES)" \ - ARCHES="$(ARCHES)" \ - REGISTRY=$(IMAGE_REGISTRY) \ - IMAGE=$(CNS_IMAGE) \ - TAG=$(TAG) - $(MAKE) multiarch-manifest-create-docker \ + $(MAKE) multiarch-manifest-create \ OSES="$(OSES)" \ ARCHES="$(ARCHES)" \ REGISTRY=$(IMAGE_REGISTRY) \ @@ -389,13 +374,7 @@ cns-multiarch-manifest-create: ## build azure-cns multi-arch container manifest. TAG=$(TAG) npm-multiarch-manifest-create: ## build azure-npm multi-arch container manifest. - $(MAKE) multiarch-image-pull-docker \ - OSES="$(OSES)" \ - ARCHES="$(ARCHES)" \ - REGISTRY=$(IMAGE_REGISTRY) \ - IMAGE=$(NPM_IMAGE) \ - TAG=$(TAG) - $(MAKE) multiarch-manifest-create-docker \ + $(MAKE) multiarch-manifest-create \ OSES="$(OSES)" \ ARCHES="$(ARCHES)" \ REGISTRY=$(IMAGE_REGISTRY) \ From 1f9efc98ec2b07654d4f417672458fa5428a2cdb Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Tue, 3 May 2022 22:14:06 +0000 Subject: [PATCH 2/3] copy container in to docker-daemon cache and re-enable trivvy Signed-off-by: Evan Baker --- .pipelines/containers/container-template.yaml | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/.pipelines/containers/container-template.yaml b/.pipelines/containers/container-template.yaml index 8c8dff0ee7..bb9c500222 100644 --- a/.pipelines/containers/container-template.yaml +++ b/.pipelines/containers/container-template.yaml @@ -13,11 +13,6 @@ steps: addPipelineData: false - script: | - . /etc/os-release - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list - curl -L "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key" | sudo apt-key add - - sudo apt update -y - sudo apt install -y podman buildah skopeo qemu-user-static binfmt-support sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes name: container_env displayName: Prepare Environment @@ -29,14 +24,16 @@ steps: displayName: Image Build retryCountOnTaskFailure: 3 -# - script: | -# wget https://github.com/aquasecurity/trivy/releases/download/v0.18.1/trivy_0.18.1_Linux-64bit.tar.gz -# tar -zxvf trivy*.tar.gz -# mkdir -p ./trivy-cache -# export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }}) -# sudo ./trivy --exit-code 1 --cache-dir ./trivy-cache --severity HIGH,CRITICAL $(IMAGE_REGISTRY)/$(make ${{ parameters.name }}-image-name):$PLATFORM_TAG -# name: trivy -# displayName: Vulnerability Scan +- script: | + export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }}) + export REF=$(IMAGE_REGISTRY)/$(make ${{ parameters.name }}-image-name):$PLATFORM_TAG + skopeo copy containers-storage:$REF docker-daemon:$REF + wget https://github.com/aquasecurity/trivy/releases/download/v0.18.1/trivy_0.18.1_Linux-64bit.tar.gz + tar -zxvf trivy*.tar.gz + mkdir -p ./trivy-cache + sudo ./trivy --exit-code 1 --cache-dir ./trivy-cache --severity HIGH,CRITICAL $REF + name: trivy + displayName: Vulnerability Scan - script: | function auto-retry() From fcf5dc8a81fbf8484192266bd7645172cb4278c7 Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Tue, 3 May 2022 23:37:03 +0000 Subject: [PATCH 3/3] set -e Signed-off-by: Evan Baker --- .pipelines/containers/container-template.yaml | 15 +++++-------- .pipelines/containers/manifest-template.yaml | 22 +++---------------- .pipelines/pipeline.yaml | 1 + 3 files changed, 9 insertions(+), 29 deletions(-) diff --git a/.pipelines/containers/container-template.yaml b/.pipelines/containers/container-template.yaml index bb9c500222..8eb3d34f20 100644 --- a/.pipelines/containers/container-template.yaml +++ b/.pipelines/containers/container-template.yaml @@ -13,11 +13,13 @@ steps: addPipelineData: false - script: | + set -e sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes name: container_env displayName: Prepare Environment - script: | + set -e export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }}) make ${{ parameters.name }}-image OS=${{ parameters.os }} ARCH=${{ parameters.arch }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }} TAG=$PLATFORM_TAG name: image_build @@ -25,6 +27,7 @@ steps: retryCountOnTaskFailure: 3 - script: | + set -e export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }}) export REF=$(IMAGE_REGISTRY)/$(make ${{ parameters.name }}-image-name):$PLATFORM_TAG skopeo copy containers-storage:$REF docker-daemon:$REF @@ -36,18 +39,10 @@ steps: displayName: Vulnerability Scan - script: | - function auto-retry() - { - export i="1" - export attempts="300" - false - while [[ $? -ne 0 ]] && [[ $i -lt $attempts ]]; do - printf "Attempt $i/$attempts - " && "$@" && break || sleep 3 && i=$[$i+1] && false - done - } + set -e export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }}) make ${{ parameters.name }}-image-push PLATFORM=${{ parameters.os }}/${{ parameters.arch }} TAG=$PLATFORM_TAG - auto-retry make ${{ parameters.name }}-image-pull PLATFORM=${{ parameters.os }}/${{ parameters.arch }} TAG=$PLATFORM_TAG + make ${{ parameters.name }}-image-pull PLATFORM=${{ parameters.os }}/${{ parameters.arch }} TAG=$PLATFORM_TAG retryCountOnTaskFailure: 3 name: image_push displayName: Push Images diff --git a/.pipelines/containers/manifest-template.yaml b/.pipelines/containers/manifest-template.yaml index 0e780e58cf..aa3c44e2c9 100644 --- a/.pipelines/containers/manifest-template.yaml +++ b/.pipelines/containers/manifest-template.yaml @@ -13,32 +13,16 @@ steps: addPipelineData: false - script: | - . /etc/os-release - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list - curl -L "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key" | sudo apt-key add - - sudo apt update -y - sudo apt install -y podman buildah skopeo - name: manifest_env - displayName: Prepare Environment - -- script: | + set -e make ${{ parameters.name }}-multiarch-manifest-create OSES="${{ parameters.oses }}" ARCHES="${{ parameters.arches }}" TAG=${{ parameters.tag }} make ${{ parameters.name }}-image-info TAG=${{ parameters.tag }} name: manifest_build displayName: Manifest Build - script: | - function auto-retry() - { - export i="1" - export attempts="300" - false - while [[ $? -ne 0 ]] && [[ $i -lt $attempts ]]; do - printf "Attempt $i/$attempts - " && "$@" && break || sleep 3 && i=$[$i+1] && false - done - } + set -e make multiarch-manifest-push REGISTRY=$IMAGE_REGISTRY IMAGE=$(make ${{ parameters.name }}-image-name) TAG=${{ parameters.tag }} - auto-retry make container-pull REGISTRY=$IMAGE_REGISTRY IMAGE=$(make ${{ parameters.name }}-image-name) TAG=${{ parameters.tag }} + make container-pull REGISTRY=$IMAGE_REGISTRY IMAGE=$(make ${{ parameters.name }}-image-name) TAG=${{ parameters.tag }} name: manifest_push displayName: Manifest Push diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 01fc9e2b63..50fc83b204 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -285,6 +285,7 @@ stages: - "ubuntu_18_04_linux_e2e" - "windows_19_03_e2e" - "windows_20_04_e2e" + - "windows_20_22_e2e" - "ubuntu_18_04_linux_dualstack_e2e" # - "windows_20_04_dualstack_e2e" jobs: