Skip to content

build multiplat manifests with buildah #1356

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 4, 2022
Merged

build multiplat manifests with buildah #1356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4656bec
build multiplat manifests with buildah
rbtr Apr 28, 2022
1f9efc9
copy container in to docker-daemon cache and re-enable trivvy
rbtr May 3, 2022
fcf5dc8
set -e
rbtr May 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 14 additions & 13 deletions .pipelines/containers/container-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,35 +13,36 @@ steps:
addPipelineData: false

- script: |
docker run --privileged --rm tonistiigi/binfmt --install arm64
set -e
sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes
name: container_env
displayName: Prepare Environment

- script: |
set -e
export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }})
make ${{ parameters.name }}-image OS=${{ parameters.os }} ARCH=${{ parameters.arch }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }} TAG=$PLATFORM_TAG
name: image_build
displayName: Image Build
retryCountOnTaskFailure: 3

- script: |
set -e
export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }})
export REF=$(IMAGE_REGISTRY)/$(make ${{ parameters.name }}-image-name):$PLATFORM_TAG
skopeo copy containers-storage:$REF docker-daemon:$REF
Copy link
Contributor

@jaer-tsun jaer-tsun May 4, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice!

wget https://github.com/aquasecurity/trivy/releases/download/v0.18.1/trivy_0.18.1_Linux-64bit.tar.gz
tar -zxvf trivy*.tar.gz
mkdir -p ./trivy-cache
export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }})
sudo ./trivy --exit-code 1 --cache-dir ./trivy-cache --severity HIGH,CRITICAL $(IMAGE_REGISTRY)/$(make ${{ parameters.name }}-image-name):$PLATFORM_TAG
sudo ./trivy --exit-code 1 --cache-dir ./trivy-cache --severity HIGH,CRITICAL $REF
name: trivy
displayName: Vulnerability Scan

- script: |
function auto-retry()
{
export i="1"
export attempts="300"
false
while [[ $? -ne 0 ]] && [[ $i -lt $attempts ]]; do
printf "Attempt $i/$attempts - " && "$@" && break || sleep 3 && i=$[$i+1] && false
done
}
set -e
export PLATFORM_TAG=$(make container-platform-tag TAG=${{ parameters.tag }} PLATFORM=${{ parameters.os }}/${{ parameters.arch }})
make ${{ parameters.name }}-image-push PLATFORM=${{ parameters.os }}/${{ parameters.arch }} TAG=$PLATFORM_TAG
auto-retry make ${{ parameters.name }}-image-pull PLATFORM=${{ parameters.os }}/${{ parameters.arch }} TAG=$PLATFORM_TAG
make ${{ parameters.name }}-image-pull PLATFORM=${{ parameters.os }}/${{ parameters.arch }} TAG=$PLATFORM_TAG
retryCountOnTaskFailure: 3
name: image_push
displayName: Push Images
Expand Down
15 changes: 4 additions & 11 deletions .pipelines/containers/manifest-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,16 @@ steps:
addPipelineData: false

- script: |
set -e
make ${{ parameters.name }}-multiarch-manifest-create OSES="${{ parameters.oses }}" ARCHES="${{ parameters.arches }}" TAG=${{ parameters.tag }}
make ${{ parameters.name }}-image-info TAG=${{ parameters.tag }}
name: manifest_build
displayName: Manifest Build

- script: |
function auto-retry()
{
export i="1"
export attempts="300"
false
while [[ $? -ne 0 ]] && [[ $i -lt $attempts ]]; do
printf "Attempt $i/$attempts - " && "$@" && break || sleep 3 && i=$[$i+1] && false
done
}
make multiarch-manifest-push-docker REGISTRY=$IMAGE_REGISTRY IMAGE=$(make ${{ parameters.name }}-image-name) TAG=${{ parameters.tag }}
auto-retry make container-pull REGISTRY=$IMAGE_REGISTRY IMAGE=$(make ${{ parameters.name }}-image-name) TAG=${{ parameters.tag }}
set -e
make multiarch-manifest-push REGISTRY=$IMAGE_REGISTRY IMAGE=$(make ${{ parameters.name }}-image-name) TAG=${{ parameters.tag }}
make container-pull REGISTRY=$IMAGE_REGISTRY IMAGE=$(make ${{ parameters.name }}-image-name) TAG=${{ parameters.tag }}
name: manifest_push
displayName: Manifest Push

Expand Down
101 changes: 54 additions & 47 deletions .pipelines/pipeline.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ stages:
BUILD_NUMBER=$(Build.BuildNumber)
echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})"
echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)"
cat /etc/os-release
uname -a
sudo chown -R $(whoami):$(whoami) .
go version
go env
Expand All @@ -32,10 +34,58 @@ stages:
displayName: "Set environmental variables"
condition: always()

- stage: build_and_test
displayName: Build ACN and Test
- stage: test
displayName: Test
dependsOn:
- setup
jobs:
- job: test
displayName: Run Tests
variables:
STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ]
pool:
name: "$(BUILD_POOL_NAME_DEFAULT)"
steps:
- script: |
make tools
# run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
{ { { {
sudo -E env "PATH=$PATH" make test-all;
echo $? >&3;
} | tee >(build/tools/bin/go-junit-report > report.xml) >&4;
} 3>&1;
} | { read xs; exit $xs; }
} 4>&1
retryCountOnTaskFailure: 3
name: "Test"
displayName: "Run Tests"

- bash: |
build/tools/bin/gocov convert coverage.out > coverage.json
build/tools/bin/gocov-xml < coverage.json > coverage.xml
name: "Coverage"
displayName: "Generate Coverage Reports"
condition: always()

- task: PublishTestResults@2
inputs:
testRunner: JUnit
testResultsFiles: report.xml
displayName: "Publish Test Results"
condition: always()

- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: coverage.xml
displayName: "Publish Code Coverage Results"
condition: always()

- stage: binaries
displayName: Build Binaries
dependsOn:
- setup
- test
jobs:
- job: build
displayName: Build Binaries
Expand Down Expand Up @@ -86,55 +136,13 @@ stages:
- publish: ./test/apimodels/
artifact: clusterdefinitions

- job: test
displayName: Run Tests
variables:
STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ]
pool:
name: "$(BUILD_POOL_NAME_DEFAULT)"
steps:
- script: |
make tools
# run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
{ { { {
sudo -E env "PATH=$PATH" make test-all;
echo $? >&3;
} | tee >(build/tools/bin/go-junit-report > report.xml) >&4;
} 3>&1;
} | { read xs; exit $xs; }
} 4>&1
retryCountOnTaskFailure: 3
name: "Test"
displayName: "Run Tests"

- bash: |
build/tools/bin/gocov convert coverage.out > coverage.json
build/tools/bin/gocov-xml < coverage.json > coverage.xml
name: "Coverage"
displayName: "Generate Coverage Reports"
condition: always()

- task: PublishTestResults@2
inputs:
testRunner: JUnit
testResultsFiles: report.xml
displayName: "Publish Test Results"
condition: always()

- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: coverage.xml
displayName: "Publish Code Coverage Results"
condition: always()

- template: windows-image.yaml

- stage: containerize
displayName: Build Images
dependsOn:
- setup
- build_and_test
- test
jobs:
- job: containerize
displayName: Build Images
Expand Down Expand Up @@ -277,6 +285,7 @@ stages:
- "ubuntu_18_04_linux_e2e"
- "windows_19_03_e2e"
- "windows_20_04_e2e"
- "windows_20_22_e2e"
- "ubuntu_18_04_linux_dualstack_e2e"
# - "windows_20_04_dualstack_e2e"
jobs:
Expand All @@ -296,6 +305,4 @@ stages:
BUILD_NUMBER=${BUILD_NUMBER//./-}
echo Deleting storage container with name acn-$BUILD_NUMBER and account name $(STORAGE_ACCOUNT_NAME)
az storage container delete -n acn-$BUILD_NUMBER --account-name $(STORAGE_ACCOUNT_NAME)
echo Pruning old docker images...
sudo docker system prune -f
displayName: Cleanup remote Azure storage container
2 changes: 1 addition & 1 deletion .pipelines/windows-image.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ stages:
displayName: Build Windows Images
dependsOn:
- setup
- build_and_test
- test
jobs:
- job: NPM
displayName: Build NPM
Expand Down
35 changes: 7 additions & 28 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,53 +349,32 @@ azure-cnm-plugin-image: azure-cnm-plugin ## build the azure-cnm plugin container

## This section is for building multi-arch/os container image manifests.

multiarch-image-pull-docker: # util target to pull all variants of a multi-arch/os image
$(foreach OS,$(OSES),$(foreach ARCH,$(ARCHES),docker pull $(REGISTRY)/$(IMAGE):$(OS)-$(ARCH)-$(TAG);))

multiarch-manifest-create-docker: # util target to compose multiarch container manifests from os/arch images.
docker manifest create \
multiarch-manifest-create: # util target to compose multiarch container manifests from os/arch images.
$(CONTAINER_BUILDER) manifest create \
$(REGISTRY)/$(IMAGE):$(TAG) \
$(foreach OS,$(OSES),$(foreach ARCH,$(ARCHES),$(REGISTRY)/$(IMAGE):$(OS)-$(ARCH)-$(TAG)))

multiarch-manifest-push-docker: # util target to push multiarch container manifest.
docker manifest push --purge $(REGISTRY)/$(IMAGE):$(TAG)
multiarch-manifest-push: # util target to push multiarch container manifest.
$(CONTAINER_BUILDER) manifest push $(REGISTRY)/$(IMAGE):$(TAG) docker://$(REGISTRY)/$(IMAGE):$(TAG)

cni-manager-multiarch-manifest-create: ## build cni-manager multi-arch container manifest.
$(MAKE) multiarch-image-pull-docker \
OSES="$(OSES)" \
ARCHES="$(ARCHES)" \
REGISTRY=$(IMAGE_REGISTRY) \
IMAGE=$(CNI_IMAGE) \
TAG=$(TAG)
$(MAKE) multiarch-manifest-create-docker \
$(MAKE) multiarch-manifest-create \
OSES="$(OSES)" \
ARCHES="$(ARCHES)" \
REGISTRY=$(IMAGE_REGISTRY) \
IMAGE=$(CNI_IMAGE) \
TAG=$(TAG)

cns-multiarch-manifest-create: ## build azure-cns multi-arch container manifest.
$(MAKE) multiarch-image-pull-docker \
OSES="$(OSES)" \
ARCHES="$(ARCHES)" \
REGISTRY=$(IMAGE_REGISTRY) \
IMAGE=$(CNS_IMAGE) \
TAG=$(TAG)
$(MAKE) multiarch-manifest-create-docker \
$(MAKE) multiarch-manifest-create \
OSES="$(OSES)" \
ARCHES="$(ARCHES)" \
REGISTRY=$(IMAGE_REGISTRY) \
IMAGE=$(CNS_IMAGE) \
TAG=$(TAG)

npm-multiarch-manifest-create: ## build azure-npm multi-arch container manifest.
$(MAKE) multiarch-image-pull-docker \
OSES="$(OSES)" \
ARCHES="$(ARCHES)" \
REGISTRY=$(IMAGE_REGISTRY) \
IMAGE=$(NPM_IMAGE) \
TAG=$(TAG)
$(MAKE) multiarch-manifest-create-docker \
$(MAKE) multiarch-manifest-create \
OSES="$(OSES)" \
ARCHES="$(ARCHES)" \
REGISTRY=$(IMAGE_REGISTRY) \
Expand Down