From 3b922a4d54e99bb7832f202aa3e5e1fc76237267 Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Fri, 28 Apr 2023 09:32:02 -0400 Subject: [PATCH 1/6] MULTIARCH-3440: refine multiarch support for test-unit and test-e2e using dockerfile and add ppc64le build to Makefile - Update the .goreleaser.yaml to include ppc64le - Update the Dockerfile to include wget and pigz dependency - Update the Dockerfile to consider arch it's built on - Update the util.sh to be arch agnostic - Update the cross build for ppc64le Signed-off-by: Paul Bastide --- .goreleaser.yaml | 1 + Dockerfile | 9 ++++--- Makefile | 6 ++++- test/e2e/e2e-simple.sh | 6 ++++- test/e2e/lib/util.sh | 53 ++++++++++++++++++++++++++++++++---------- 5 files changed, 58 insertions(+), 17 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 9bf9298bd..077e3274a 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -11,6 +11,7 @@ builds: - linux goarch: - amd64 + - ppc64le env: - CGO_ENABLED=1 flags: diff --git a/Dockerfile b/Dockerfile index 171bfcca8..c7683f71b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,15 +14,18 @@ ARG DNF_LIST="\ git \ gpgme-devel \ libassuan-devel \ + wget \ + pigz \ " ################################################################################# -# Build UBI8 Builder +# Build UBI8 Builder with multi-arch support RUN set -ex \ + && ARCH=$(arch | sed 's|x86_64|amd64|g') \ && dnf install -y --nodocs --setopt=install_weak_deps=false ${DNF_LIST} \ && dnf clean all -y \ - && GO_VERSION=go1.19.5 \ - && curl -sL https://golang.org/dl/${GO_VERSION}.linux-amd64.tar.gz \ + && GO_VERSION=go1.19.5 \ + && curl -sL https://golang.org/dl/${GO_VERSION}.linux-${ARCH}.tar.gz \ | tar xzvf - --directory /usr/local/ \ && /usr/local/go/bin/go version \ && ln -f /usr/local/go/bin/go /usr/bin/go diff --git a/Makefile b/Makefile index bb2307ef7..3f7cb7c84 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,11 @@ cross-build-linux-amd64: +@GOOS=linux GOARCH=amd64 $(MAKE) "$(GO_BUILD_FLAGS)" --no-print-directory build GO_BUILD_BINDIR=$(GO_BUILD_BINDIR)/linux-amd64 .PHONY: cross-build-linux-amd64 -cross-build: cross-build-linux-amd64 +cross-build-linux-ppc64le: + +@GOOS=linux GOARCH=ppc64le $(MAKE) "$(GO_BUILD_FLAGS)" --no-print-directory build GO_BUILD_BINDIR=$(GO_BUILD_BINDIR)/linux-amd64 +.PHONY: cross-build-linux-ppc64le + +cross-build: cross-build-linux-amd64 cross-build-linux-ppc64le .PHONY: cross-build hack-build: clean diff --git a/test/e2e/e2e-simple.sh b/test/e2e/e2e-simple.sh index 114e4cc7a..285f8f6d3 100755 --- a/test/e2e/e2e-simple.sh +++ b/test/e2e/e2e-simple.sh @@ -32,8 +32,12 @@ METADATA_OCI_CATALOG="oci://${DIR}/artifacts/rhop-ctlg-oci" TARGET_CATALOG_NAME="target-name" TARGET_CATALOG_TAG="target-tag" - GOBIN=$HOME/go/bin +# Check if this is running with a different location +if [ ! -d $GOBIN ] +then + GOBIN=/usr/local/go/bin/ +fi PATH=$PATH:$GOBIN mkdir -p $DATA_TMP diff --git a/test/e2e/lib/util.sh b/test/e2e/lib/util.sh index a93bbca03..7fc9c4748 100644 --- a/test/e2e/lib/util.sh +++ b/test/e2e/lib/util.sh @@ -43,18 +43,47 @@ function cleanup_conn() { # install_deps will install crane and registry2 in go bin dir function install_deps() { - pushd ${DATA_TMP} - GOFLAGS=-mod=mod go install github.com/google/go-containerregistry/cmd/crane@latest - popd - crane export registry:2 registry2.tar - tar xvf registry2.tar bin/registry - mv bin/registry $GOBIN - crane export quay.io/operator-framework/opm@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19 opm.tar - tar xvf opm.tar bin/opm - mv bin/opm $GOBIN - rm -f registry2.tar opm.tar - wget -O $GOBIN/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 - chmod +x $GOBIN/jq + if [ "$(arch)" == "x86_64" ] + then + pushd ${DATA_TMP} + GOFLAGS=-mod=mod go install github.com/google/go-containerregistry/cmd/crane@latest + popd + crane export registry:2 registry2.tar + tar xvf registry2.tar bin/registry + mv bin/registry $GOBIN + crane export quay.io/operator-framework/opm@sha256:d31c6ea5c50be93d6eb94d2b508f0208e84a308c011c6454ebf291d48b37df19 opm.tar + tar xvf opm.tar bin/opm + mv bin/opm $GOBIN + rm -f registry2.tar opm.tar + wget -O $GOBIN/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 + chmod +x $GOBIN/jq + else + pushd ${DATA_TMP} + # Creates a temp directory + mkdir -p test/e2e/operator-test.deps + cd test/e2e/operator-test.deps + + curl -o $GOBIN/opm -L https://github.com/operator-framework/operator-registry/releases/download/v1.26.5/linux-$(arch)-opm + chmod +x $GOBIN/opm + + # Serves a local registry + # When PR https://github.com/google/go-containerregistry/pull/1680 + # is in a release the following few lines won't be necessary. + git clone https://github.com/google/go-containerregistry.git + cd go-containerregistry + git checkout $(git describe --tags) # latest tag + go build ./cmd/crane + mv crane $GOBIN/ + + crane export $(arch)/registry:2 registry2.tar + tar xvf registry2.tar bin/registry + cp bin/registry ../ + mv bin/registry $GOBIN + + cd .. + rm -rf go-containerregistry/ + popd + fi } # setup_reg will configure and start registry2 processes From d43bacdad080403d83a4af84fe5a9d82de59a186 Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Wed, 3 May 2023 08:53:04 -0400 Subject: [PATCH 2/6] MULTIARCH-3440: Update per review Signed-off-by: Paul Bastide Co-authored-by: Jan Schintag <43986265+jschintag@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3f7cb7c84..886af506e 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ cross-build-linux-amd64: .PHONY: cross-build-linux-amd64 cross-build-linux-ppc64le: - +@GOOS=linux GOARCH=ppc64le $(MAKE) "$(GO_BUILD_FLAGS)" --no-print-directory build GO_BUILD_BINDIR=$(GO_BUILD_BINDIR)/linux-amd64 + +@GOOS=linux GOARCH=ppc64le $(MAKE) "$(GO_BUILD_FLAGS)" --no-print-directory build GO_BUILD_BINDIR=$(GO_BUILD_BINDIR)/linux-ppc64le .PHONY: cross-build-linux-ppc64le cross-build: cross-build-linux-amd64 cross-build-linux-ppc64le From 407325d32017890f7245654620e4ac9fcb77f431 Mon Sep 17 00:00:00 2001 From: Jan Schintag <43986265+jschintag@users.noreply.github.com> Date: Mon, 22 May 2023 20:52:16 +0200 Subject: [PATCH 3/6] MULTIARCH-3441: Enable builds for s390x (#1) - Update the .goreleaser.yaml to include s390x - Update the cross build for s390x - Link gcc to s390x-linux-gnu-gcc binary On s390x go compiler seems to expect the gcc binary at s390x-linux-gnu-gcc binary. However on rhel it is not installed there. Signed-off-by: Jan Schintag --- .goreleaser.yaml | 1 + Dockerfile | 4 +++- Makefile | 16 ++++++++++------ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 077e3274a..34df5c317 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -12,6 +12,7 @@ builds: goarch: - amd64 - ppc64le + - s390x env: - CGO_ENABLED=1 flags: diff --git a/Dockerfile b/Dockerfile index c7683f71b..04a90d551 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,7 @@ ARG DNF_LIST="\ ################################################################################# # Build UBI8 Builder with multi-arch support +# Link gcc to /usr/bin/s390x-linux-gnu-gcc as go requires it on s390x RUN set -ex \ && ARCH=$(arch | sed 's|x86_64|amd64|g') \ && dnf install -y --nodocs --setopt=install_weak_deps=false ${DNF_LIST} \ @@ -28,7 +29,8 @@ RUN set -ex \ && curl -sL https://golang.org/dl/${GO_VERSION}.linux-${ARCH}.tar.gz \ | tar xzvf - --directory /usr/local/ \ && /usr/local/go/bin/go version \ - && ln -f /usr/local/go/bin/go /usr/bin/go + && ln -f /usr/local/go/bin/go /usr/bin/go \ + && ln /usr/bin/gcc /usr/bin/s390x-linux-gnu-gcc WORKDIR /build ENTRYPOINT ["make"] diff --git a/Makefile b/Makefile index 886af506e..fcb22fdff 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,11 @@ cross-build-linux-ppc64le: +@GOOS=linux GOARCH=ppc64le $(MAKE) "$(GO_BUILD_FLAGS)" --no-print-directory build GO_BUILD_BINDIR=$(GO_BUILD_BINDIR)/linux-ppc64le .PHONY: cross-build-linux-ppc64le -cross-build: cross-build-linux-amd64 cross-build-linux-ppc64le +cross-build-linux-s390x: + +@GOOS=linux GOARCH=s390x $(MAKE) "$(GO_BUILD_FLAGS)" --no-print-directory build GO_BUILD_BINDIR=$(GO_BUILD_BINDIR)/linux-s390x +.PHONY: cross-build-linux-s390x + +cross-build: cross-build-linux-amd64 cross-build-linux-ppc64le cross-build-linux-s390x .PHONY: cross-build hack-build: clean @@ -75,12 +79,12 @@ publish-catalog: @cd test/operator && make .PHONY: publish-catalog -format: +format: $(GO) fmt ./pkg/... $(GO) fmt ./cmd/... .PHONY: format -vet: - $(GO) vet $(GO_BUILD_FLAGS) ./pkg/... - $(GO) vet $(GO_BUILD_FLAGS) ./cmd/... -.PHONY: vet \ No newline at end of file +vet: + $(GO) vet $(GO_BUILD_FLAGS) ./pkg/... + $(GO) vet $(GO_BUILD_FLAGS) ./cmd/... +.PHONY: vet From 0bec3590d8996526627072f926d9f9b2d686cf61 Mon Sep 17 00:00:00 2001 From: Jan Schintag <43986265+jschintag@users.noreply.github.com> Date: Wed, 24 May 2023 20:10:12 +0200 Subject: [PATCH 4/6] MULTIARCH-3441: Enable builds for s390x (#1) (#2) - Update the .goreleaser.yaml to include s390x - Update the cross build for s390x - Link gcc to s390x-linux-gnu-gcc binary On s390x go compiler seems to expect the gcc binary at s390x-linux-gnu-gcc binary. However on rhel it is not installed there. Signed-off-by: Jan Schintag --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index 04a90d551..0f9014e06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,13 @@ RUN set -ex \ && ln -f /usr/local/go/bin/go /usr/bin/go \ && ln /usr/bin/gcc /usr/bin/s390x-linux-gnu-gcc +################################################################################# +# Link gcc to /usr/bin/s390x-linux-gnu-gcc as go requires it on s390x +RUN ARCH=$(arch | sed 's|x86_64|amd64|g') \ + && [ "${ARCH}" == "s390x" ] \ + && ln /usr/bin/gcc /usr/bin/s390x-linux-gnu-gcc \ + || echo "Not running on s390x, skip linking gcc binary" + WORKDIR /build ENTRYPOINT ["make"] CMD [] From 738047a0cd1c3b244294d5fa7072436c4c52a303 Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Wed, 24 May 2023 14:57:58 -0400 Subject: [PATCH 5/6] MULTIARCH-3440: update the code to use go-containerregistry for all arches and add arm64 Signed-off-by: Paul Bastide --- .goreleaser.yaml | 1 + Dockerfile | 9 +++------ test/e2e/lib/util.sh | 35 ++++++++++++++++------------------- 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 34df5c317..52a88e552 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -13,6 +13,7 @@ builds: - amd64 - ppc64le - s390x + - arm64 env: - CGO_ENABLED=1 flags: diff --git a/Dockerfile b/Dockerfile index 0f9014e06..016b79620 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,22 +20,19 @@ ARG DNF_LIST="\ ################################################################################# # Build UBI8 Builder with multi-arch support -# Link gcc to /usr/bin/s390x-linux-gnu-gcc as go requires it on s390x RUN set -ex \ - && ARCH=$(arch | sed 's|x86_64|amd64|g') \ + && ARCH=$(arch | sed 's|x86_64|amd64|g' | sed 's|aarch64|arm64|g') \ && dnf install -y --nodocs --setopt=install_weak_deps=false ${DNF_LIST} \ && dnf clean all -y \ && GO_VERSION=go1.19.5 \ && curl -sL https://golang.org/dl/${GO_VERSION}.linux-${ARCH}.tar.gz \ | tar xzvf - --directory /usr/local/ \ && /usr/local/go/bin/go version \ - && ln -f /usr/local/go/bin/go /usr/bin/go \ - && ln /usr/bin/gcc /usr/bin/s390x-linux-gnu-gcc + && ln -f /usr/local/go/bin/go /usr/bin/go ################################################################################# # Link gcc to /usr/bin/s390x-linux-gnu-gcc as go requires it on s390x -RUN ARCH=$(arch | sed 's|x86_64|amd64|g') \ - && [ "${ARCH}" == "s390x" ] \ +RUN [ "$(arch)" == "s390x" ] \ && ln /usr/bin/gcc /usr/bin/s390x-linux-gnu-gcc \ || echo "Not running on s390x, skip linking gcc binary" diff --git a/test/e2e/lib/util.sh b/test/e2e/lib/util.sh index 7fc9c4748..ab52b6a83 100644 --- a/test/e2e/lib/util.sh +++ b/test/e2e/lib/util.sh @@ -55,33 +55,30 @@ function install_deps() { tar xvf opm.tar bin/opm mv bin/opm $GOBIN rm -f registry2.tar opm.tar - wget -O $GOBIN/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 - chmod +x $GOBIN/jq + wget -O $GOBIN/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 + chmod +x $GOBIN/jq else + # non-x86_64 flow pushd ${DATA_TMP} - # Creates a temp directory - mkdir -p test/e2e/operator-test.deps - cd test/e2e/operator-test.deps - curl -o $GOBIN/opm -L https://github.com/operator-framework/operator-registry/releases/download/v1.26.5/linux-$(arch)-opm + # For ppc64le, this is compiled with Power9 compatibility (does not run on Power8) + ARCH=$(arch | sed 's|aarch64|arm64|g') + curl -o $GOBIN/opm -L https://github.com/operator-framework/operator-registry/releases/download/v1.27.1/linux-${ARCH}-opm chmod +x $GOBIN/opm - # Serves a local registry - # When PR https://github.com/google/go-containerregistry/pull/1680 - # is in a release the following few lines won't be necessary. - git clone https://github.com/google/go-containerregistry.git - cd go-containerregistry - git checkout $(git describe --tags) # latest tag - go build ./cmd/crane - mv crane $GOBIN/ - - crane export $(arch)/registry:2 registry2.tar + GOFLAGS=-mod=mod go install github.com/google/go-containerregistry/cmd/crane@latest + mv ~/go/bin/crane $GOBIN/ + + if [ "${ARCH}" == "arm64" ] + then + crane export --platform linux/arm64/v8 registry:2 registry2.tar + else + crane export ${ARCH}/registry:2 registry2.tar + fi tar xvf registry2.tar bin/registry - cp bin/registry ../ mv bin/registry $GOBIN + rm registry2.tar - cd .. - rm -rf go-containerregistry/ popd fi } From 4ac106a3e3f27f1a30ace497d7177dcebc0044ba Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Thu, 1 Jun 2023 20:34:06 -0400 Subject: [PATCH 6/6] MULTIARCH-3440: Adding cross-build for arm64 Signed-off-by: Paul Bastide --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fcb22fdff..1986fa379 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,11 @@ cross-build-linux-s390x: +@GOOS=linux GOARCH=s390x $(MAKE) "$(GO_BUILD_FLAGS)" --no-print-directory build GO_BUILD_BINDIR=$(GO_BUILD_BINDIR)/linux-s390x .PHONY: cross-build-linux-s390x -cross-build: cross-build-linux-amd64 cross-build-linux-ppc64le cross-build-linux-s390x +cross-build-linux-arm64: + +@GOOS=linux GOARCH=arm64 $(MAKE) "$(GO_BUILD_FLAGS)" --no-print-directory build GO_BUILD_BINDIR=$(GO_BUILD_BINDIR)/linux-arm64 +.PHONY: cross-build-linux-arm64 + +cross-build: cross-build-linux-amd64 cross-build-linux-ppc64le cross-build-linux-s390x cross-build-linux-arm64 .PHONY: cross-build hack-build: clean