fix(devops): build multi platform images

.github/workflows/build-&-publish-docker-image.yml

@@ -31,43 +31,36 @@ jobs:
 
           echo "BRANCH=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV
           echo "SHA=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo $GITHUB_SHA || echo '${{ github.event.pull_request.head.sha }}')" >> $GITHUB_ENV
-
+          
       - name: Setup go 1.17
         uses: actions/setup-go@v2
         with:
           go-version: '1.17' # The Go version to download (if necessary) and use.
 
-      # - uses: actions/cache@v2
-      #   with:
-      #     path: ~/go/pkg/mod
-      #     key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-      #     restore-keys: |
-      #       ${{ runner.os }}-go-
-
-
       - name: Clone blobber
         uses: actions/checkout@v1
 
-      - name: Build blobber_base
-        run: ./docker.local/bin/build.base.sh
-
-      - name: Build blobber
-        run: ./docker.local/bin/build.blobber.sh
-
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      - name: Push blobber
+      - name: Build base
         run: |
-          docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG
-          docker push ${BLOBBER_REGISTRY}:$TAG || { sleep 10 && docker push ${BLOBBER_REGISTRY}:$TAG; }
+          export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base"
+          export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
+          ./docker.local/bin/build.base.sh
 
+      - name: Build blobber
+        run: |
           SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
-          docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA
-          docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA  || { sleep 10 && docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA; }
+
+          export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base"
+          export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:${TAG} -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}"
+          export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
+
+          ./docker.local/bin/build.blobber.sh
 
   validator:
     runs-on: [self-hosted, build]
@@ -89,36 +82,31 @@ jobs:
         with:
           go-version: '1.17' # The Go version to download (if necessary) and use.
 
-      # - uses: actions/cache@v2
-      #   with:
-      #     path: ~/go/pkg/mod
-      #     key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-      #     restore-keys: |
-      #       ${{ runner.os }}-go-
-
       - name: Clone blobber
         uses: actions/checkout@v1
 
-      - name: Build blobber_base
-        run: ./docker.local/bin/build.base.sh
-
-      - name: Build validator
-        run: ./docker.local/bin/build.validator.sh
-
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      - name: Push validator
+      - name: Build base
         run: |
-          docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG
-          docker push ${VALIDATOR_REGISTRY}:$TAG || { sleep 10 && docker push ${VALIDATOR_REGISTRY}:$TAG; }
+          export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base"
+          export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
+          ./docker.local/bin/build.base.sh
 
+      - name: Build validator
+        run: |
           SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8)
-          docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA
-          docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA || { sleep 10 && docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA; }
+
+          export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base"
+          export DOCKER_IMAGE_VALIDATOR="-t ${VALIDATOR_REGISTRY}:${TAG} -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}"
+          export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push"
+
+          ./docker.local/bin/build.validator.sh
+
 
   system-tests:
     if: github.event_name != 'workflow_dispatch'

.gitignore

@@ -14,3 +14,4 @@ __debug_bin
 dev.local/data
 out/
 **/tmp/
+*.tar.gz

docker.local/base.Dockerfile

@@ -2,6 +2,7 @@ FROM golang:1.17.1-alpine3.14 as blobber_base
 
 LABEL zchain="blobber"
 
+# https://mirrors.alpinelinux.org/
 # RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
 
 RUN echo "https://mirrors.aliyun.com/alpine/v3.14/main" >> /etc/apk/repositories
@@ -16,21 +17,25 @@ RUN echo "https://uk.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/reposit
 RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories
 RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories
 
-RUN echo "https://mirror.yandex.ru/alpine/v3.14/main" >> /etc/apk/repositories
-RUN echo "https://mirror.yandex.ru/alpine/v3.14/community" >> /etc/apk/repositories
 
 
-RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep
+RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep gmp gmp-dev openssl-dev 
 
 # Install Herumi's cryptography
-RUN apk add gmp gmp-dev openssl-dev && \
-    cd /tmp && \
-    wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \
-    wget -O - https://github.com/herumi/bls/archive/master.tar.gz | tar xz && \
-    mv mcl* mcl && \
-    mv bls* bls && \
-    make -C mcl -j $(nproc) lib/libmclbn256.so install && \
-    cp mcl/lib/libmclbn256.so /usr/local/lib && \
-    make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install && \
-    rm -R /tmp/mcl && \
-    rm -R /tmp/bls
+WORKDIR /tmp
+
+COPY ./docker.local/bin/mcl.tar.gz ./
+COPY ./docker.local/bin/bls.tar.gz ./
+
+RUN tar zxvf mcl.tar.gz && rm mcl.tar.gz && mv mcl* mcl
+
+RUN tar zxvf bls.tar.gz && rm bls.tar.gz && mv bls* bls  
+
+RUN make -C mcl -j $(nproc) lib/libmclbn256.so install 
+RUN cp mcl/lib/libmclbn256.so /usr/local/lib 
+
+RUN make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install 
+
+RUN rm -R /tmp/mcl && rm -R /tmp/bls
+
+#ENV GOPROXY=https://goproxy.cn

docker.local/bin/build.base.sh

@@ -3,6 +3,29 @@ set -e
 
 GIT_COMMIT=$(git rev-list -1 HEAD)
 echo $GIT_COMMIT
+echo "1> set DOCKER_IMAGE & DOCKER_BUILD"
+if [ -z "$DOCKER_BUILD" ]; then  
+    if [ "x86_64" != "$(uname -m)" ]; then
+        #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use
+        DOCKER_BUILD="buildx build --platform linux/arm64"
+    else
+        DOCKER_BUILD="build"
+    fi
+fi
 
+if [ -z "$DOCKER_IMAGE_BASE" ]; then  
+    DOCKER_IMAGE_BASE="blobber_base"
+fi
+echo "  DOCKER_BUILD=$DOCKER_BUILD"
+echo "  DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE"
 
-docker build --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t blobber_base
+echo ""
+echo "2> download herumi"
+
+[ ! -f ./docker.local/bin/mcl.tar.gz ] && wget -O ./docker.local/bin/mcl.tar.gz https://github.com/herumi/mcl/archive/refs/tags/v1.57.tar.gz 
+
+[ ! -f ./docker.local/bin/bls.tar.gz ] && wget -O ./docker.local/bin/bls.tar.gz https://github.com/herumi/bls/archive/refs/tags/v1.22.tar.gz
+
+echo ""
+echo "3> docker build"
+DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t $DOCKER_IMAGE_BASE

docker.local/bin/build.blobber.sh

@@ -4,22 +4,29 @@ set -e
 GIT_COMMIT=$(git rev-list -1 HEAD)
 echo $GIT_COMMIT
 
-cmd="build"
+echo "1> set DOCKER_IMAGE & DOCKER_BUILD"
+if [ -z "$DOCKER_BUILD" ]; then  
+    if [ "x86_64" != "$(uname -m)" ]; then
+        #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use
+        DOCKER_BUILD="buildx build --platform linux/arm64"
+    else
+        DOCKER_BUILD="build"
+    fi
+fi
 
-for arg in "$@"
-do
-    case $arg in
-        -m1|--m1|m1)
-        echo "The build will be performed for Apple M1 chip"
-        cmd="buildx build --platform linux/amd64"
-        shift
-        ;;
-    esac
-done
+if [ -z "$DOCKER_IMAGE_BASE" ]; then  
+    DOCKER_IMAGE_BASE="blobber_base"
+fi
 
-# [ -d ./gosdk ] && rm -rf gosdk
-# cp -r ../gosdk ./
+if [ -z "$DOCKER_IMAGE_BLOBBER" ]; then  
+    DOCKER_IMAGE_BLOBBER="-t blobber"
+fi
 
+echo "  DOCKER_BUILD=$DOCKER_BUILD"
+echo "  DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE"
+echo "  DOCKER_IMAGE_BLOBBER=$DOCKER_IMAGE_BLOBBER"
 
-docker $cmd --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/blobber.Dockerfile . -t blobber
+echo ""
 
+echo "2> docker build blobber"
+DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE -f docker.local/blobber.Dockerfile . $DOCKER_IMAGE_BLOBBER

docker.local/bin/build.validator.sh

@@ -4,21 +4,28 @@ set -e
 GIT_COMMIT=$(git rev-list -1 HEAD)
 echo $GIT_COMMIT
 
-cmd="build"
+echo "1> set DOCKER_IMAGE & DOCKER_BUILD"
+if [ -z "$DOCKER_BUILD" ]; then  
+    if [ "x86_64" != "$(uname -m)" ]; then
+        #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use
+        DOCKER_BUILD="buildx build --platform linux/arm64"
+    else
+        DOCKER_BUILD="build"
+    fi
+fi
 
-for arg in "$@"
-do
-    case $arg in
-        -m1|--m1|m1)
-        echo "The build will be performed for Apple M1 chip"
-        cmd="buildx build --platform linux/amd64"
-        shift
-        ;;
-    esac
-done
+if [ -z "$DOCKER_IMAGE_BASE" ]; then  
+    DOCKER_IMAGE_BASE="blobber_base"
+fi
 
-# [ -d ./gosdk ] && rm -rf gosdk
-# cp -r ../gosdk ./
+if [ -z "$DOCKER_IMAGE_VALIDATOR" ]; then  
+    DOCKER_IMAGE_VALIDATOR="-t validator"
+fi
 
-docker $cmd  --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/validator.Dockerfile . -t validator
+echo "  DOCKER_BUILD=$DOCKER_BUILD"
+echo "  DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE"
+echo "  DOCKER_IMAGE_VALIDATOR=$DOCKER_IMAGE_VALIDATOR"
 
+echo ""
+echo "2> docker build validator"
+DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE -f docker.local/validator.Dockerfile . $DOCKER_IMAGE_VALIDATOR

docker.local/blobber.Dockerfile

@@ -1,4 +1,6 @@
-FROM blobber_base as blobber_build
+# syntax=docker/dockerfile:1
+ARG DOCKER_IMAGE_BASE
+FROM $DOCKER_IMAGE_BASE  as blobber_build
 LABEL zchain="blobber"
 
 ENV SRC_DIR=/0chain

docker.local/validator.Dockerfile

@@ -1,9 +1,11 @@
-FROM blobber_base as validator_build
+# syntax=docker/dockerfile:1
+ARG DOCKER_IMAGE_BASE
+FROM $DOCKER_IMAGE_BASE as validator_build
 
 LABEL zchain="validator"
 
 
-ENV SRC_DIR=/blobber
+ENV SRC_DIR=/0chain
 ENV GO111MODULE=on
 #ENV GOPROXY=https://goproxy.cn,direct 
 
@@ -25,6 +27,6 @@ RUN apk add gmp gmp-dev openssl-dev
 COPY --from=validator_build  /usr/local/lib/libmcl*.so \
                         /usr/local/lib/libbls*.so \
                         /usr/local/lib/
-ENV APP_DIR=/blobber
+ENV APP_DIR=/validator
 WORKDIR $APP_DIR
-COPY --from=validator_build $APP_DIR/code/go/0chain.net/validator/validator $APP_DIR/bin/validator
+COPY --from=validator_build /0chain/code/go/0chain.net/validator/validator $APP_DIR/bin/validator