From 76b8b777e339571788c3627b77656ea63617fbc4 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 11:06:37 +0800 Subject: [PATCH 01/23] fix(devops): build multi platform images --- .../build-&-publish-docker-image.yml | 18 ++++++++---- docker.local/bin/build.base.sh | 9 +++++- docker.local/bin/build.blobber.sh | 28 ++++++------------- docker.local/bin/build.validator.sh | 27 ++++++------------ 4 files changed, 39 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 71bb8a4ce..d574f7ca1 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -31,7 +31,7 @@ jobs: echo "BRANCH=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo ${GITHUB_REF#refs/*/} || echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV echo "SHA=$([ -z '${{ github.event.pull_request.head.sha }}' ] && echo $GITHUB_SHA || echo '${{ github.event.pull_request.head.sha }}')" >> $GITHUB_ENV - + - name: Setup go 1.17 uses: actions/setup-go@v2 with: @@ -49,10 +49,14 @@ jobs: uses: actions/checkout@v1 - name: Build blobber_base - run: ./docker.local/bin/build.base.sh + run: | + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" + ./docker.local/bin/build.base.sh - name: Build blobber - run: ./docker.local/bin/build.blobber.sh + run: | + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" + ./docker.local/bin/build.blobber.sh - name: Login to Docker Hub uses: docker/login-action@v1 @@ -100,10 +104,14 @@ jobs: uses: actions/checkout@v1 - name: Build blobber_base - run: ./docker.local/bin/build.base.sh + run: | + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" + ./docker.local/bin/build.base.sh - name: Build validator - run: ./docker.local/bin/build.validator.sh + run: | + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" + ./docker.local/bin/build.validator.sh - name: Login to Docker Hub uses: docker/login-action@v1 diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh index 252909943..f4435bc46 100755 --- a/docker.local/bin/build.base.sh +++ b/docker.local/bin/build.base.sh @@ -4,5 +4,12 @@ set -e GIT_COMMIT=$(git rev-list -1 HEAD) echo $GIT_COMMIT +if [ -z "$DOCKER_BUILD" ]; then + if [ "x86_64" != "$(uname -m)" ]; then + DOCKER_BUILD="buildx build --platform linux/arm64" + else + DOCKER_BUILD="build" + fi +fi -docker build --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t blobber_base \ No newline at end of file +docker $DOCKER_BUILD --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t blobber_base \ No newline at end of file diff --git a/docker.local/bin/build.blobber.sh b/docker.local/bin/build.blobber.sh index 64daed5d1..40a6f2e96 100755 --- a/docker.local/bin/build.blobber.sh +++ b/docker.local/bin/build.blobber.sh @@ -4,22 +4,12 @@ set -e GIT_COMMIT=$(git rev-list -1 HEAD) echo $GIT_COMMIT -cmd="build" - -for arg in "$@" -do - case $arg in - -m1|--m1|m1) - echo "The build will be performed for Apple M1 chip" - cmd="buildx build --platform linux/amd64" - shift - ;; - esac -done - -# [ -d ./gosdk ] && rm -rf gosdk -# cp -r ../gosdk ./ - - -docker $cmd --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/blobber.Dockerfile . -t blobber - +if [ -z "$DOCKER_BUILD" ]; then + if [ "x86_64" != "$(uname -m)" ]; then + DOCKER_BUILD="buildx build --platform linux/arm64" + else + DOCKER_BUILD="build" + fi +fi + +docker $DOCKER_BUILD --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/blobber.Dockerfile . -t blobber \ No newline at end of file diff --git a/docker.local/bin/build.validator.sh b/docker.local/bin/build.validator.sh index 162828098..281e86576 100755 --- a/docker.local/bin/build.validator.sh +++ b/docker.local/bin/build.validator.sh @@ -4,21 +4,12 @@ set -e GIT_COMMIT=$(git rev-list -1 HEAD) echo $GIT_COMMIT -cmd="build" - -for arg in "$@" -do - case $arg in - -m1|--m1|m1) - echo "The build will be performed for Apple M1 chip" - cmd="buildx build --platform linux/amd64" - shift - ;; - esac -done - -# [ -d ./gosdk ] && rm -rf gosdk -# cp -r ../gosdk ./ - -docker $cmd --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/validator.Dockerfile . -t validator - +if [ -z "$DOCKER_BUILD" ]; then + if [ "x86_64" != "$(uname -m)" ]; then + DOCKER_BUILD="buildx build --platform linux/arm64" + else + DOCKER_BUILD="build" + fi +fi + +docker $DOCKER_BUILD --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/validator.Dockerfile . -t validator \ No newline at end of file From 9c304002cfc90dcecc9141ebd2bea525a4858a56 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 11:10:58 +0800 Subject: [PATCH 02/23] fix(devops): build multi platform images --- .github/workflows/build-&-publish-docker-image.yml | 4 ++-- docker.local/bin/build.base.sh | 1 + docker.local/bin/build.blobber.sh | 1 + docker.local/bin/build.validator.sh | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index d574f7ca1..0786b562e 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -51,11 +51,11 @@ jobs: - name: Build blobber_base run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" + docker buildx create --use ./docker.local/bin/build.base.sh - name: Build blobber run: | - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" ./docker.local/bin/build.blobber.sh - name: Login to Docker Hub @@ -106,11 +106,11 @@ jobs: - name: Build blobber_base run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" + docker buildx create --use ./docker.local/bin/build.base.sh - name: Build validator run: | - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" ./docker.local/bin/build.validator.sh - name: Login to Docker Hub diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh index f4435bc46..0906c525c 100755 --- a/docker.local/bin/build.base.sh +++ b/docker.local/bin/build.base.sh @@ -6,6 +6,7 @@ echo $GIT_COMMIT if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then + docker buildx create --use DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" diff --git a/docker.local/bin/build.blobber.sh b/docker.local/bin/build.blobber.sh index 40a6f2e96..bc373839e 100755 --- a/docker.local/bin/build.blobber.sh +++ b/docker.local/bin/build.blobber.sh @@ -6,6 +6,7 @@ echo $GIT_COMMIT if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then + docker buildx create --use DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" diff --git a/docker.local/bin/build.validator.sh b/docker.local/bin/build.validator.sh index 281e86576..a06822f93 100755 --- a/docker.local/bin/build.validator.sh +++ b/docker.local/bin/build.validator.sh @@ -6,6 +6,7 @@ echo $GIT_COMMIT if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then + docker buildx create --use DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" From 363353396a796e2e0c99e1f18f817d40c9d28517 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 11:17:48 +0800 Subject: [PATCH 03/23] fix(devops): build multi platform images --- docker.local/base.Dockerfile | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index 583b31c20..f5af60d55 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -23,14 +23,19 @@ RUN echo "https://mirror.yandex.ru/alpine/v3.14/community" >> /etc/apk/repositor RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep # Install Herumi's cryptography -RUN apk add gmp gmp-dev openssl-dev && \ - cd /tmp && \ +RUN apk add gmp gmp-dev openssl-dev + + +RUN cd /tmp && \ wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ wget -O - https://github.com/herumi/bls/archive/master.tar.gz | tar xz && \ - mv mcl* mcl && \ - mv bls* bls && \ + mv mcl* mcl && mv bls* bls + +RUN cd /tmp && \ make -C mcl -j $(nproc) lib/libmclbn256.so install && \ - cp mcl/lib/libmclbn256.so /usr/local/lib && \ - make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install && \ - rm -R /tmp/mcl && \ - rm -R /tmp/bls + cp mcl/lib/libmclbn256.so /usr/local/lib + +RUN cd /tmp && \ + make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install + +RUN cd /tmp && rm -R /tmp/mcl && rm -R /tmp/bls From 92ed1ebeb6f350f4f4fafa3327f5816edd4349d8 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 11:32:11 +0800 Subject: [PATCH 04/23] fix(devops): build multi platform images --- docker.local/base.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index f5af60d55..4e6c8e58c 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -38,4 +38,4 @@ RUN cd /tmp && \ RUN cd /tmp && \ make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install -RUN cd /tmp && rm -R /tmp/mcl && rm -R /tmp/bls +RUN rm -R /tmp/mcl && rm -R /tmp/bls From 92ac2d5426c1a81011a6ac35449991bd59f6f0ef Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 14:28:42 +0800 Subject: [PATCH 05/23] fix(devops): improved base.Dockerfile --- docker.local/base.Dockerfile | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index 4e6c8e58c..e8908d6ed 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -2,6 +2,7 @@ FROM golang:1.17.1-alpine3.14 as blobber_base LABEL zchain="blobber" +# https://mirrors.alpinelinux.org/ # RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories RUN echo "https://mirrors.aliyun.com/alpine/v3.14/main" >> /etc/apk/repositories @@ -10,14 +11,12 @@ RUN echo "https://mirrors.aliyun.com/alpine/v3.14/community" >> /etc/apk/reposit RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/main" >> /etc/apk/repositories RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/community" >> /etc/apk/repositories -RUN echo "https://uk.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories -RUN echo "https://uk.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories +# RUN echo "https://uk.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories +# RUN echo "https://uk.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories -RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories -RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories +# RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories +# RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories -RUN echo "https://mirror.yandex.ru/alpine/v3.14/main" >> /etc/apk/repositories -RUN echo "https://mirror.yandex.ru/alpine/v3.14/community" >> /etc/apk/repositories RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep @@ -25,17 +24,16 @@ RUN apk add --update --no-cache build-base linux-headers git cmake bash perl gre # Install Herumi's cryptography RUN apk add gmp gmp-dev openssl-dev +WORKDIR /tmp +RUN wget https://github.com/herumi/mcl/archive/master.tar.gz +RUN tar zxvf master.tar.gz && mv mcl* mcl -RUN cd /tmp && \ - wget -O - https://github.com/herumi/mcl/archive/master.tar.gz | tar xz && \ - wget -O - https://github.com/herumi/bls/archive/master.tar.gz | tar xz && \ - mv mcl* mcl && mv bls* bls +RUN wget https://github.com/herumi/bls/archive/master.tar.gz +RUN tar zxvf master.tar.gz && && mv bls* bls -RUN cd /tmp && \ - make -C mcl -j $(nproc) lib/libmclbn256.so install && \ - cp mcl/lib/libmclbn256.so /usr/local/lib +RUN make -C mcl -j $(nproc) lib/libmclbn256.so install +RUN cp mcl/lib/libmclbn256.so /usr/local/lib -RUN cd /tmp && \ - make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install +RUN make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install RUN rm -R /tmp/mcl && rm -R /tmp/bls From b39b3516abd14d2b4497fc4fa3482846af7bc8b9 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 15:34:03 +0800 Subject: [PATCH 06/23] fix(devops): remove master.tar.gz --- docker.local/base.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index e8908d6ed..88731cdbd 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -25,10 +25,10 @@ RUN apk add --update --no-cache build-base linux-headers git cmake bash perl gre RUN apk add gmp gmp-dev openssl-dev WORKDIR /tmp -RUN wget https://github.com/herumi/mcl/archive/master.tar.gz +RUN rm -rf master.tar.gz && wget https://github.com/herumi/mcl/archive/master.tar.gz RUN tar zxvf master.tar.gz && mv mcl* mcl -RUN wget https://github.com/herumi/bls/archive/master.tar.gz +RUN rm -rf master.tar.gz && wget https://github.com/herumi/bls/archive/master.tar.gz RUN tar zxvf master.tar.gz && && mv bls* bls RUN make -C mcl -j $(nproc) lib/libmclbn256.so install From a12d24415b67b6cab348005dd31a0db8fb034253 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 15:36:23 +0800 Subject: [PATCH 07/23] fix(devops): remove master.tar.gz --- docker.local/base.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index 88731cdbd..3fe097214 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -26,10 +26,10 @@ RUN apk add gmp gmp-dev openssl-dev WORKDIR /tmp RUN rm -rf master.tar.gz && wget https://github.com/herumi/mcl/archive/master.tar.gz -RUN tar zxvf master.tar.gz && mv mcl* mcl +RUN tar zxvf master.tar.gz && mv mcl* mcl RUN rm -rf master.tar.gz && wget https://github.com/herumi/bls/archive/master.tar.gz -RUN tar zxvf master.tar.gz && && mv bls* bls +RUN tar zxvf master.tar.gz && mv bls* bls RUN make -C mcl -j $(nproc) lib/libmclbn256.so install RUN cp mcl/lib/libmclbn256.so /usr/local/lib From fd14bf696838472cbf0d76153eaf0e778c7940cf Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 16:49:38 +0800 Subject: [PATCH 08/23] fix(devops): download herumi on shell instead of docker --- .gitignore | 1 + docker.local/base.Dockerfile | 20 +++++++++++--------- docker.local/bin/build.base.sh | 4 ++++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 3ba7a8d29..152b4072b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ __debug_bin dev.local/data out/ **/tmp/ +*.tar.gz diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index 3fe097214..b373c0d7f 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -3,13 +3,13 @@ FROM golang:1.17.1-alpine3.14 as blobber_base LABEL zchain="blobber" # https://mirrors.alpinelinux.org/ -# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories -RUN echo "https://mirrors.aliyun.com/alpine/v3.14/main" >> /etc/apk/repositories -RUN echo "https://mirrors.aliyun.com/alpine/v3.14/community" >> /etc/apk/repositories +# RUN echo "https://mirrors.aliyun.com/alpine/v3.14/main" >> /etc/apk/repositories +# RUN echo "https://mirrors.aliyun.com/alpine/v3.14/community" >> /etc/apk/repositories -RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/main" >> /etc/apk/repositories -RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/community" >> /etc/apk/repositories +# RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/main" >> /etc/apk/repositories +# RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/community" >> /etc/apk/repositories # RUN echo "https://uk.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories # RUN echo "https://uk.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories @@ -25,11 +25,13 @@ RUN apk add --update --no-cache build-base linux-headers git cmake bash perl gre RUN apk add gmp gmp-dev openssl-dev WORKDIR /tmp -RUN rm -rf master.tar.gz && wget https://github.com/herumi/mcl/archive/master.tar.gz -RUN tar zxvf master.tar.gz && mv mcl* mcl -RUN rm -rf master.tar.gz && wget https://github.com/herumi/bls/archive/master.tar.gz -RUN tar zxvf master.tar.gz && mv bls* bls +COPY ./docker.local/bin/mcl-master.tar.gz ./ +COPY ./docker.local/bin/bls-master.tar.gz ./ + +RUN tar zxvf mcl-master.tar.gz && mv mcl-master mcl + +RUN tar zxvf bls-master.tar.gz && mv bls-master bls RUN make -C mcl -j $(nproc) lib/libmclbn256.so install RUN cp mcl/lib/libmclbn256.so /usr/local/lib diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh index 0906c525c..7010e07bb 100755 --- a/docker.local/bin/build.base.sh +++ b/docker.local/bin/build.base.sh @@ -13,4 +13,8 @@ if [ -z "$DOCKER_BUILD" ]; then fi fi +[ ! -f ./docker.local/bin/mcl-master.tar.gz ] && wget -O ./docker.local/bin/mcl-master.tar.gz https://github.com/herumi/mcl/archive/master.tar.gz + +[ ! -f ./docker.local/bin/bls-master.tar.gz ] && wget -O ./docker.local/bin/bls-master.tar.gz https://github.com/herumi/bls/archive/master.tar.gz + docker $DOCKER_BUILD --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t blobber_base \ No newline at end of file From 2db5195cb088d5391113d3abc4c6a93f3f70f2f4 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 17:04:28 +0800 Subject: [PATCH 09/23] fix(devops): fixed docker buildx layer --- .github/workflows/build-&-publish-docker-image.yml | 3 ++- docker.local/base.Dockerfile | 10 +++++----- docker.local/bin/build.base.sh | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 0786b562e..61d260667 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -51,7 +51,8 @@ jobs: - name: Build blobber_base run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx create --use + docker buildx create --name blobber_buildx + docker buildx use blobber_buildx ./docker.local/bin/build.base.sh - name: Build blobber diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index b373c0d7f..a1bcad214 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -3,13 +3,13 @@ FROM golang:1.17.1-alpine3.14 as blobber_base LABEL zchain="blobber" # https://mirrors.alpinelinux.org/ -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories -# RUN echo "https://mirrors.aliyun.com/alpine/v3.14/main" >> /etc/apk/repositories -# RUN echo "https://mirrors.aliyun.com/alpine/v3.14/community" >> /etc/apk/repositories +RUN echo "https://mirrors.aliyun.com/alpine/v3.14/main" >> /etc/apk/repositories +RUN echo "https://mirrors.aliyun.com/alpine/v3.14/community" >> /etc/apk/repositories -# RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/main" >> /etc/apk/repositories -# RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/community" >> /etc/apk/repositories +RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/main" >> /etc/apk/repositories +RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/community" >> /etc/apk/repositories # RUN echo "https://uk.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories # RUN echo "https://uk.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh index 7010e07bb..531544eca 100755 --- a/docker.local/bin/build.base.sh +++ b/docker.local/bin/build.base.sh @@ -6,7 +6,8 @@ echo $GIT_COMMIT if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then - docker buildx create --use + docker buildx create --name blobber_buildx + docker buildx use blobber_buildx DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" From 7d54e69c90d5b11c1ea69c6a7ffb7adbeae711b3 Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 17:05:01 +0800 Subject: [PATCH 10/23] fix(devops): fixed docker buildx layer --- .github/workflows/build-&-publish-docker-image.yml | 2 +- docker.local/bin/build.base.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 61d260667..50ee27bdf 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -51,7 +51,7 @@ jobs: - name: Build blobber_base run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx create --name blobber_buildx + docker buildx create --name blobber_buildx --use docker buildx use blobber_buildx ./docker.local/bin/build.base.sh diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh index 531544eca..9569832b1 100755 --- a/docker.local/bin/build.base.sh +++ b/docker.local/bin/build.base.sh @@ -6,7 +6,7 @@ echo $GIT_COMMIT if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then - docker buildx create --name blobber_buildx + docker buildx create --name blobber_buildx --use docker buildx use blobber_buildx DOCKER_BUILD="buildx build --platform linux/arm64" else From d82d9c137e882d07ec5303811a577cc0dfe3cb4c Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 17:10:46 +0800 Subject: [PATCH 11/23] fix(devops): fixed docker buildx layer --- .github/workflows/build-&-publish-docker-image.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 50ee27bdf..85583fc33 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -52,11 +52,12 @@ jobs: run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" docker buildx create --name blobber_buildx --use - docker buildx use blobber_buildx ./docker.local/bin/build.base.sh - name: Build blobber run: | + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" + docker buildx create --name blobber_buildx --use ./docker.local/bin/build.blobber.sh - name: Login to Docker Hub @@ -107,11 +108,13 @@ jobs: - name: Build blobber_base run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx create --use + docker buildx create --name blobber_buildx --use ./docker.local/bin/build.base.sh - name: Build validator run: | + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" + docker buildx create --name blobber_buildx --use ./docker.local/bin/build.validator.sh - name: Login to Docker Hub From eff42895d9dfb24ae88a17b7b79ccd463d81838c Mon Sep 17 00:00:00 2001 From: Lz Date: Thu, 3 Mar 2022 17:16:01 +0800 Subject: [PATCH 12/23] fix(devops): fixed docker buildx layer --- .github/workflows/build-&-publish-docker-image.yml | 8 ++++---- docker.local/bin/build.base.sh | 3 +-- docker.local/bin/build.blobber.sh | 2 +- docker.local/bin/build.validator.sh | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 85583fc33..0c58fe006 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -51,13 +51,13 @@ jobs: - name: Build blobber_base run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx create --name blobber_buildx --use + docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use ./docker.local/bin/build.base.sh - name: Build blobber run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx create --name blobber_buildx --use + docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use ./docker.local/bin/build.blobber.sh - name: Login to Docker Hub @@ -108,13 +108,13 @@ jobs: - name: Build blobber_base run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx create --name blobber_buildx --use + docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use ./docker.local/bin/build.base.sh - name: Build validator run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx create --name blobber_buildx --use + docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use ./docker.local/bin/build.validator.sh - name: Login to Docker Hub diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh index 9569832b1..1274f4173 100755 --- a/docker.local/bin/build.base.sh +++ b/docker.local/bin/build.base.sh @@ -6,8 +6,7 @@ echo $GIT_COMMIT if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then - docker buildx create --name blobber_buildx --use - docker buildx use blobber_buildx + docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" diff --git a/docker.local/bin/build.blobber.sh b/docker.local/bin/build.blobber.sh index bc373839e..e435a96b7 100755 --- a/docker.local/bin/build.blobber.sh +++ b/docker.local/bin/build.blobber.sh @@ -6,7 +6,7 @@ echo $GIT_COMMIT if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then - docker buildx create --use + docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" diff --git a/docker.local/bin/build.validator.sh b/docker.local/bin/build.validator.sh index a06822f93..bb7e797ba 100755 --- a/docker.local/bin/build.validator.sh +++ b/docker.local/bin/build.validator.sh @@ -6,7 +6,7 @@ echo $GIT_COMMIT if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then - docker buildx create --use + docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" From 6a6cc9f69afcb711c3b9944ac42a97d3b5af39c4 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 4 Mar 2022 06:47:39 +0800 Subject: [PATCH 13/23] fix(devops): fixed docker buildx builder --- .github/workflows/build-&-publish-docker-image.yml | 5 +++-- docker.local/base.Dockerfile | 8 ++++---- docker.local/bin/build.base.sh | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 0c58fe006..3edc9eecf 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -108,13 +108,14 @@ jobs: - name: Build blobber_base run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use + docker buildx use blobber_buildx --default || docker buildx create --name blobber_buildx --use --default ./docker.local/bin/build.base.sh - name: Build validator run: | export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use + docker buildx use blobber_buildx --default || docker buildx create --name blobber_buildx --use --default + ./docker.local/bin/build.validator.sh - name: Login to Docker Hub diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index a1bcad214..7ce8e5849 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -11,11 +11,11 @@ RUN echo "https://mirrors.aliyun.com/alpine/v3.14/community" >> /etc/apk/reposit RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/main" >> /etc/apk/repositories RUN echo "https://sjc.edge.kernel.org/alpine/v3.14/community" >> /etc/apk/repositories -# RUN echo "https://uk.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories -# RUN echo "https://uk.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories +RUN echo "https://uk.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories +RUN echo "https://uk.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories -# RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories -# RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories +RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories +RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh index 1274f4173..61c799b89 100755 --- a/docker.local/bin/build.base.sh +++ b/docker.local/bin/build.base.sh @@ -6,7 +6,7 @@ echo $GIT_COMMIT if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then - docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use + docker buildx use blobber_buildx --default || docker buildx create --name blobber_buildx --use --default DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" From 1c90bddc2daf489c2f8b8a611c86da38dde4c99b Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 4 Mar 2022 15:51:41 +0800 Subject: [PATCH 14/23] fix(devops): test cache options for buildx --- .github/workflows/build-&-publish-docker-image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 3edc9eecf..308a514f6 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -107,14 +107,14 @@ jobs: - name: Build blobber_base run: | - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx use blobber_buildx --default || docker buildx create --name blobber_buildx --use --default + export DOCKER_BUILD="buildx build --platform linux/arm64 --cache-to=type=gha" + docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use ./docker.local/bin/build.base.sh - name: Build validator run: | - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx use blobber_buildx --default || docker buildx create --name blobber_buildx --use --default + export DOCKER_BUILD="buildx build --platform linux/arm64 --cache-from=type=gha" + docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use ./docker.local/bin/build.validator.sh From 6c12677b1f0888b60de1ed6e63fa28335294605f Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 4 Mar 2022 15:55:44 +0800 Subject: [PATCH 15/23] fix(devops): test --driver for buildx --- .github/workflows/build-&-publish-docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 308a514f6..0d892f35d 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -107,13 +107,13 @@ jobs: - name: Build blobber_base run: | - export DOCKER_BUILD="buildx build --platform linux/arm64 --cache-to=type=gha" + export DOCKER_BUILD="buildx build --platform linux/arm64 --cache-to=type=gha --driver docker-container" docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use ./docker.local/bin/build.base.sh - name: Build validator run: | - export DOCKER_BUILD="buildx build --platform linux/arm64 --cache-from=type=gha" + export DOCKER_BUILD="buildx build --platform linux/arm64 --cache-from=type=gha --driver docker-container" docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use ./docker.local/bin/build.validator.sh From 658f786de47e955618f74673bfc8ac9affc6046c Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 4 Mar 2022 15:57:24 +0800 Subject: [PATCH 16/23] fix(devops): test --driver for buildx --- docker.local/base.Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index 7ce8e5849..3306e6c73 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -19,11 +19,9 @@ RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repos -RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep +RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep add gmp gmp-dev openssl-dev # Install Herumi's cryptography -RUN apk add gmp gmp-dev openssl-dev - WORKDIR /tmp COPY ./docker.local/bin/mcl-master.tar.gz ./ From 72005054bf916022f4722a9bc22f8c7db511b149 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 4 Mar 2022 16:00:39 +0800 Subject: [PATCH 17/23] fix(devops): test --driver for buildx --- docker.local/base.Dockerfile | 2 +- docker.local/bin/build.base.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index 3306e6c73..cf86d9f46 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -19,7 +19,7 @@ RUN echo "https://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repos -RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep add gmp gmp-dev openssl-dev +RUN apk add --update --no-cache build-base linux-headers git cmake bash perl grep gmp gmp-dev openssl-dev # Install Herumi's cryptography WORKDIR /tmp diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh index 61c799b89..c45bfd955 100755 --- a/docker.local/bin/build.base.sh +++ b/docker.local/bin/build.base.sh @@ -6,15 +6,16 @@ echo $GIT_COMMIT if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then - docker buildx use blobber_buildx --default || docker buildx create --name blobber_buildx --use --default + docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" fi fi - +echo "1> download herumi" [ ! -f ./docker.local/bin/mcl-master.tar.gz ] && wget -O ./docker.local/bin/mcl-master.tar.gz https://github.com/herumi/mcl/archive/master.tar.gz [ ! -f ./docker.local/bin/bls-master.tar.gz ] && wget -O ./docker.local/bin/bls-master.tar.gz https://github.com/herumi/bls/archive/master.tar.gz +echo "2> docker build" docker $DOCKER_BUILD --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t blobber_base \ No newline at end of file From 469eb2af273e8d04b01aa5eb2e75c4da9edccffb Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 4 Mar 2022 20:29:41 +0800 Subject: [PATCH 18/23] fix(devops): fixed build stage name issue --- .../build-&-publish-docker-image.yml | 79 +++++++------------ docker.local/base.Dockerfile | 2 + docker.local/bin/build.base.sh | 20 +++-- docker.local/bin/build.blobber.sh | 20 ++++- docker.local/bin/build.validator.sh | 19 ++++- docker.local/blobber.Dockerfile | 4 +- docker.local/validator.Dockerfile | 10 ++- 7 files changed, 90 insertions(+), 64 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 0d892f35d..e8255cda8 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -37,43 +37,34 @@ jobs: with: go-version: '1.17' # The Go version to download (if necessary) and use. - # - uses: actions/cache@v2 - # with: - # path: ~/go/pkg/mod - # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - # restore-keys: | - # ${{ runner.os }}-go- - - - name: Clone blobber uses: actions/checkout@v1 - - name: Build blobber_base - run: | - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use - ./docker.local/bin/build.base.sh - - - name: Build blobber - run: | - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64" - docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use - ./docker.local/bin/build.blobber.sh - - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Push blobber + - name: Build base run: | - docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG - docker push ${BLOBBER_REGISTRY}:$TAG || { sleep 10 && docker push ${BLOBBER_REGISTRY}:$TAG; } + export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + ./docker.local/bin/build.base.sh + - name: Build blobber + run: | SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) - docker tag blobber:latest ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA - docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA || { sleep 10 && docker push ${BLOBBER_REGISTRY}:$TAG-$SHORT_SHA; } + + export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base" + export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:$TAG -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + + ./docker.local/bin/build.blobber.sh + + - name: Push blobber + run: | + validator: runs-on: [self-hosted, build] @@ -95,43 +86,31 @@ jobs: with: go-version: '1.17' # The Go version to download (if necessary) and use. - # - uses: actions/cache@v2 - # with: - # path: ~/go/pkg/mod - # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - # restore-keys: | - # ${{ runner.os }}-go- - - name: Clone blobber uses: actions/checkout@v1 - - name: Build blobber_base - run: | - export DOCKER_BUILD="buildx build --platform linux/arm64 --cache-to=type=gha --driver docker-container" - docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use - ./docker.local/bin/build.base.sh - - - name: Build validator - run: | - export DOCKER_BUILD="buildx build --platform linux/arm64 --cache-from=type=gha --driver docker-container" - docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use - - ./docker.local/bin/build.validator.sh - - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Push validator + - name: Build base run: | - docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG - docker push ${VALIDATOR_REGISTRY}:$TAG || { sleep 10 && docker push ${VALIDATOR_REGISTRY}:$TAG; } + export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + ./docker.local/bin/build.base.sh + - name: Build validator + run: | SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) - docker tag validator:latest ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA - docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA || { sleep 10 && docker push ${VALIDATOR_REGISTRY}:$TAG-$SHORT_SHA; } + + export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base" + export DOCKER_IMAGE_BLOBBER="-t ${VALIDATOR_REGISTRY}:$TAG -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + + ./docker.local/bin/build.validator.sh + system-tests: if: github.event_name != 'workflow_dispatch' diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index cf86d9f46..659a692e3 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -37,3 +37,5 @@ RUN cp mcl/lib/libmclbn256.so /usr/local/lib RUN make MCL_DIR=$(pwd)/mcl -C bls -j $(nproc) install RUN rm -R /tmp/mcl && rm -R /tmp/bls + +#ENV GOPROXY=https://goproxy.cn diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh index c45bfd955..918f0ddf0 100755 --- a/docker.local/bin/build.base.sh +++ b/docker.local/bin/build.base.sh @@ -3,19 +3,29 @@ set -e GIT_COMMIT=$(git rev-list -1 HEAD) echo $GIT_COMMIT - +echo "1> set DOCKER_IMAGE & DOCKER_BUILD" if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then - docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use + #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" fi fi -echo "1> download herumi" + +if [ -z "$DOCKER_IMAGE_BASE" ]; then + DOCKER_IMAGE_BASE="blobber_base" +fi +echo " DOCKER_BUILD=$DOCKER_BUILD" +echo " DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" + +echo "" +echo "2> download herumi" + [ ! -f ./docker.local/bin/mcl-master.tar.gz ] && wget -O ./docker.local/bin/mcl-master.tar.gz https://github.com/herumi/mcl/archive/master.tar.gz [ ! -f ./docker.local/bin/bls-master.tar.gz ] && wget -O ./docker.local/bin/bls-master.tar.gz https://github.com/herumi/bls/archive/master.tar.gz -echo "2> docker build" -docker $DOCKER_BUILD --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t blobber_base \ No newline at end of file +echo "" +echo "3> docker build" +DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/base.Dockerfile . -t $DOCKER_IMAGE_BASE \ No newline at end of file diff --git a/docker.local/bin/build.blobber.sh b/docker.local/bin/build.blobber.sh index e435a96b7..93f4b36dc 100755 --- a/docker.local/bin/build.blobber.sh +++ b/docker.local/bin/build.blobber.sh @@ -4,13 +4,29 @@ set -e GIT_COMMIT=$(git rev-list -1 HEAD) echo $GIT_COMMIT +echo "1> set DOCKER_IMAGE & DOCKER_BUILD" if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then - docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use + #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" fi fi -docker $DOCKER_BUILD --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/blobber.Dockerfile . -t blobber \ No newline at end of file +if [ -z "$DOCKER_IMAGE_BASE" ]; then + DOCKER_IMAGE_BASE="blobber_base" +fi + +if [ -z "$DOCKER_IMAGE_BLOBBER" ]; then + DOCKER_IMAGE_BLOBBER="-t blobber" +fi + +echo " DOCKER_BUILD=$DOCKER_BUILD" +echo " DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" +echo " DOCKER_IMAGE_BLOBBER=$DOCKER_IMAGE_BLOBBER" + +echo "" + +echo "2> docker build blobber" +DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE -f docker.local/blobber.Dockerfile . $DOCKER_IMAGE_BLOBBER \ No newline at end of file diff --git a/docker.local/bin/build.validator.sh b/docker.local/bin/build.validator.sh index bb7e797ba..e91b1b8c4 100755 --- a/docker.local/bin/build.validator.sh +++ b/docker.local/bin/build.validator.sh @@ -4,13 +4,28 @@ set -e GIT_COMMIT=$(git rev-list -1 HEAD) echo $GIT_COMMIT +echo "1> set DOCKER_IMAGE & DOCKER_BUILD" if [ -z "$DOCKER_BUILD" ]; then if [ "x86_64" != "$(uname -m)" ]; then - docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use + #docker buildx use blobber_buildx || docker buildx create --name blobber_buildx --use DOCKER_BUILD="buildx build --platform linux/arm64" else DOCKER_BUILD="build" fi fi -docker $DOCKER_BUILD --build-arg GIT_COMMIT=$GIT_COMMIT -f docker.local/validator.Dockerfile . -t validator \ No newline at end of file +if [ -z "$DOCKER_IMAGE_BASE" ]; then + DOCKER_IMAGE_BASE="blobber_base" +fi + +if [ -z "$DOCKER_IMAGE_VALIDATOR" ]; then + DOCKER_IMAGE_VALIDATOR="-t validator" +fi + +echo " DOCKER_BUILD=$DOCKER_BUILD" +echo " DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" +echo " DOCKER_IMAGE_VALIDATOR=$DOCKER_IMAGE_VALIDATOR" + +echo "" +echo "2> docker build validator" +DOCKER_BUILDKIT=1 docker $DOCKER_BUILD --progress=plain --build-arg GIT_COMMIT=$GIT_COMMIT --build-arg DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE -f docker.local/validator.Dockerfile . $DOCKER_IMAGE_VALIDATOR \ No newline at end of file diff --git a/docker.local/blobber.Dockerfile b/docker.local/blobber.Dockerfile index db01b09fa..86409499d 100644 --- a/docker.local/blobber.Dockerfile +++ b/docker.local/blobber.Dockerfile @@ -1,4 +1,6 @@ -FROM blobber_base as blobber_build +# syntax=docker/dockerfile:1 +ARG DOCKER_IMAGE_BASE +FROM $DOCKER_IMAGE_BASE as blobber_build LABEL zchain="blobber" ENV SRC_DIR=/0chain diff --git a/docker.local/validator.Dockerfile b/docker.local/validator.Dockerfile index cc3f284c4..a84ad4184 100644 --- a/docker.local/validator.Dockerfile +++ b/docker.local/validator.Dockerfile @@ -1,9 +1,11 @@ -FROM blobber_base as validator_build +# syntax=docker/dockerfile:1 +ARG DOCKER_IMAGE_BASE +FROM $DOCKER_IMAGE_BASE as validator_build LABEL zchain="validator" -ENV SRC_DIR=/blobber +ENV SRC_DIR=/0chain ENV GO111MODULE=on #ENV GOPROXY=https://goproxy.cn,direct @@ -25,6 +27,6 @@ RUN apk add gmp gmp-dev openssl-dev COPY --from=validator_build /usr/local/lib/libmcl*.so \ /usr/local/lib/libbls*.so \ /usr/local/lib/ -ENV APP_DIR=/blobber +ENV APP_DIR=/validator WORKDIR $APP_DIR -COPY --from=validator_build $APP_DIR/code/go/0chain.net/validator/validator $APP_DIR/bin/validator \ No newline at end of file +COPY --from=validator_build /0chain/code/go/0chain.net/validator/validator $APP_DIR/bin/validator \ No newline at end of file From 5ec65722f79857f1cbe8e7bc62ef9339de4a50f9 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 4 Mar 2022 20:33:54 +0800 Subject: [PATCH 19/23] fix(devops): build image for darwin/amd64 --- .github/workflows/build-&-publish-docker-image.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index e8255cda8..71ee668e6 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -49,7 +49,7 @@ jobs: - name: Build base run: | export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base" - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64,darwin/amd64 --push" ./docker.local/bin/build.base.sh - name: Build blobber @@ -58,14 +58,10 @@ jobs: export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base" export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:$TAG -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}" - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64,darwin/amd64 --push" ./docker.local/bin/build.blobber.sh - - name: Push blobber - run: | - - validator: runs-on: [self-hosted, build] steps: @@ -98,7 +94,7 @@ jobs: - name: Build base run: | export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base" - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64,darwin/amd64 --push" ./docker.local/bin/build.base.sh - name: Build validator @@ -107,7 +103,7 @@ jobs: export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base" export DOCKER_IMAGE_BLOBBER="-t ${VALIDATOR_REGISTRY}:$TAG -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}" - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64,darwin/amd64 --push" ./docker.local/bin/build.validator.sh From c93a7cac082e764170c8e6a636383238095a578c Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 4 Mar 2022 20:37:03 +0800 Subject: [PATCH 20/23] fix(devops): removed darwin/amd64 --- .github/workflows/build-&-publish-docker-image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 71ee668e6..8bf274bc8 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -49,7 +49,7 @@ jobs: - name: Build base run: | export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base" - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64,darwin/amd64 --push" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" ./docker.local/bin/build.base.sh - name: Build blobber @@ -58,7 +58,7 @@ jobs: export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base" export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:$TAG -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}" - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64,darwin/amd64 --push" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" ./docker.local/bin/build.blobber.sh @@ -94,7 +94,7 @@ jobs: - name: Build base run: | export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base" - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64,darwin/amd64 --push" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" ./docker.local/bin/build.base.sh - name: Build validator @@ -103,7 +103,7 @@ jobs: export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base" export DOCKER_IMAGE_BLOBBER="-t ${VALIDATOR_REGISTRY}:$TAG -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}" - export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64,darwin/amd64 --push" + export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" ./docker.local/bin/build.validator.sh From bf3363a421d2ddc387d5cfae24c2706cb3e8682b Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 4 Mar 2022 20:42:27 +0800 Subject: [PATCH 21/23] fix(devops): format github action --- .github/workflows/build-&-publish-docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 8bf274bc8..69a698670 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -57,7 +57,7 @@ jobs: SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) export DOCKER_IMAGE_BASE="${BLOBBER_REGISTRY}:base" - export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:$TAG -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}" + export DOCKER_IMAGE_BLOBBER="-t ${BLOBBER_REGISTRY}:${TAG} -t ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA}" export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" ./docker.local/bin/build.blobber.sh @@ -102,7 +102,7 @@ jobs: SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base" - export DOCKER_IMAGE_BLOBBER="-t ${VALIDATOR_REGISTRY}:$TAG -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}" + export DOCKER_IMAGE_BLOBBER="-t ${VALIDATOR_REGISTRY}:${TAG} -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}" export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" ./docker.local/bin/build.validator.sh From 5d64947b2dff28b47688c1c002835cddd16df352 Mon Sep 17 00:00:00 2001 From: Lz Date: Fri, 4 Mar 2022 20:46:12 +0800 Subject: [PATCH 22/23] fix(devops): fixed DOCKER_IMAGE_VALIDATOR --- .github/workflows/build-&-publish-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 69a698670..333cf9287 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -102,7 +102,7 @@ jobs: SHORT_SHA=$(echo ${{ env.SHA }} | head -c 8) export DOCKER_IMAGE_BASE="${VALIDATOR_REGISTRY}:base" - export DOCKER_IMAGE_BLOBBER="-t ${VALIDATOR_REGISTRY}:${TAG} -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}" + export DOCKER_IMAGE_VALIDATOR="-t ${VALIDATOR_REGISTRY}:${TAG} -t ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA}" export DOCKER_BUILD="buildx build --platform linux/amd64,linux/arm64 --push" ./docker.local/bin/build.validator.sh From ea5d144a47478eec148328b4b67423eb0e7b023e Mon Sep 17 00:00:00 2001 From: Lz Date: Sat, 5 Mar 2022 16:12:12 +0800 Subject: [PATCH 23/23] fix(devops): fixed bls/mcl tags --- docker.local/base.Dockerfile | 8 ++++---- docker.local/bin/build.base.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker.local/base.Dockerfile b/docker.local/base.Dockerfile index 659a692e3..b9c562b5b 100644 --- a/docker.local/base.Dockerfile +++ b/docker.local/base.Dockerfile @@ -24,12 +24,12 @@ RUN apk add --update --no-cache build-base linux-headers git cmake bash perl gre # Install Herumi's cryptography WORKDIR /tmp -COPY ./docker.local/bin/mcl-master.tar.gz ./ -COPY ./docker.local/bin/bls-master.tar.gz ./ +COPY ./docker.local/bin/mcl.tar.gz ./ +COPY ./docker.local/bin/bls.tar.gz ./ -RUN tar zxvf mcl-master.tar.gz && mv mcl-master mcl +RUN tar zxvf mcl.tar.gz && rm mcl.tar.gz && mv mcl* mcl -RUN tar zxvf bls-master.tar.gz && mv bls-master bls +RUN tar zxvf bls.tar.gz && rm bls.tar.gz && mv bls* bls RUN make -C mcl -j $(nproc) lib/libmclbn256.so install RUN cp mcl/lib/libmclbn256.so /usr/local/lib diff --git a/docker.local/bin/build.base.sh b/docker.local/bin/build.base.sh index 918f0ddf0..1e813fa5e 100755 --- a/docker.local/bin/build.base.sh +++ b/docker.local/bin/build.base.sh @@ -22,9 +22,9 @@ echo " DOCKER_IMAGE_BASE=$DOCKER_IMAGE_BASE" echo "" echo "2> download herumi" -[ ! -f ./docker.local/bin/mcl-master.tar.gz ] && wget -O ./docker.local/bin/mcl-master.tar.gz https://github.com/herumi/mcl/archive/master.tar.gz +[ ! -f ./docker.local/bin/mcl.tar.gz ] && wget -O ./docker.local/bin/mcl.tar.gz https://github.com/herumi/mcl/archive/refs/tags/v1.57.tar.gz -[ ! -f ./docker.local/bin/bls-master.tar.gz ] && wget -O ./docker.local/bin/bls-master.tar.gz https://github.com/herumi/bls/archive/master.tar.gz +[ ! -f ./docker.local/bin/bls.tar.gz ] && wget -O ./docker.local/bin/bls.tar.gz https://github.com/herumi/bls/archive/refs/tags/v1.22.tar.gz echo "" echo "3> docker build"