From 7a66592b93c55d84084335f217c2c2f6d9df2f46 Mon Sep 17 00:00:00 2001 From: Quentin McGaw Date: Tue, 2 Nov 2021 17:23:32 -0400 Subject: [PATCH] chore(ci): Dockerfile improvements and faster CI (#1934) - Accelerate the CI build - Make CI caching a possibility for the future to further accelerate build - Smaller image of 186MB instead of 3GB+ (uncompressed amd64) - Cache Polkadot JS NPM dependencies (package.json + package-lock.json) - Image versions as build arguments `DEBIAN_VERSION=bullseye-slim` and `GO_VERSION=1.15-buster` - Add `.dockerignore` file for better use of the Docker laying caching - Compact commands together for smaller image sizes and smaller build cache size - Use only `wget` and not `curl` in builder - Re-order instructions in final image to maximize Docker layer caching efficiency - Remove unused `GOBIN() string` function from `ci.go` since it's unused --- .dockerignore | 15 ++++ .github/workflows/docker-grandpa.yml | 1 + .github/workflows/docker-js.yml | 1 + .github/workflows/docker-rpc.yml | 1 + .github/workflows/docker-stable.yml | 1 + .github/workflows/docker-stress.yml | 1 + Dockerfile | 76 +++++++++---------- scripts/ci.go | 9 --- scripts/docker-entrypoint.sh | 2 +- .../polkadotjs_test/start_polkadotjs_test.go | 3 +- 10 files changed, 57 insertions(+), 53 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..4fb5eec073 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +/.git +/.githooks +/.github +/docs +/.deepsource.toml +/.dockerignore +/.gitattributes +/.gitignore +/.releaserc +/.codecov.yml +/docker-compose.yml +/Dockerfile +/Dockerfile.staging +/prometheus.yml +/README.md diff --git a/.github/workflows/docker-grandpa.yml b/.github/workflows/docker-grandpa.yml index c1a0a0ff07..e6a0c63f48 100644 --- a/.github/workflows/docker-grandpa.yml +++ b/.github/workflows/docker-grandpa.yml @@ -20,6 +20,7 @@ jobs: with: load: true file: ./Dockerfile + target: builder platforms: linux/amd64 push: false tags: chainsafe/gossamer:test diff --git a/.github/workflows/docker-js.yml b/.github/workflows/docker-js.yml index ff733b0a95..7a3ade8064 100644 --- a/.github/workflows/docker-js.yml +++ b/.github/workflows/docker-js.yml @@ -20,6 +20,7 @@ jobs: with: load: true file: ./Dockerfile + target: builder platforms: linux/amd64 push: false tags: chainsafe/gossamer:test diff --git a/.github/workflows/docker-rpc.yml b/.github/workflows/docker-rpc.yml index a09d6c7b9c..5ed5d61573 100644 --- a/.github/workflows/docker-rpc.yml +++ b/.github/workflows/docker-rpc.yml @@ -21,6 +21,7 @@ jobs: load: true file: ./Dockerfile platforms: linux/amd64 + target: builder push: false tags: chainsafe/gossamer:test - diff --git a/.github/workflows/docker-stable.yml b/.github/workflows/docker-stable.yml index 4e9af6c57a..47e6b03ea2 100644 --- a/.github/workflows/docker-stable.yml +++ b/.github/workflows/docker-stable.yml @@ -20,6 +20,7 @@ jobs: with: load: true file: ./Dockerfile + target: builder platforms: linux/amd64 push: false tags: chainsafe/gossamer:test diff --git a/.github/workflows/docker-stress.yml b/.github/workflows/docker-stress.yml index 64c6d13196..d2571263e5 100644 --- a/.github/workflows/docker-stress.yml +++ b/.github/workflows/docker-stress.yml @@ -20,6 +20,7 @@ jobs: with: load: true file: ./Dockerfile + target: builder platforms: linux/amd64 push: false tags: chainsafe/gossamer:test diff --git a/Dockerfile b/Dockerfile index ff8a6d108d..82fbb4d2ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,64 +1,56 @@ -FROM ubuntu:18.04 as builder +ARG DEBIAN_VERSION=bullseye-slim +ARG GO_VERSION=1.15-buster + +FROM golang:${GO_VERSION} AS builder -# Install GCC RUN apt-get update && \ apt-get install -y \ gcc \ cmake \ - wget \ - curl \ - npm + wget # Install node source for polkadotjs tests -RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - - -# Install nodejs for polkadotjs tests -RUN apt-get update && \ - apt-get install -y \ - nodejs - -# Install Go -RUN wget https://dl.google.com/go/go1.15.5.linux-amd64.tar.gz -RUN tar -C /usr/local -xzf go1.15.5.linux-amd64.tar.gz +RUN wget -qO- https://deb.nodesource.com/setup_14.x | bash - && \ + apt-get install -y nodejs # Install subkey -RUN wget -P /usr/local/bin/ https://chainbridge.ams3.digitaloceanspaces.com/subkey-v2.0.0 -RUN mv /usr/local/bin/subkey-v2.0.0 /usr/local/bin/subkey -RUN chmod +x /usr/local/bin/subkey - -# Configure go env vars -ENV GO111MODULE=on -ENV GOPATH=/gocode -ENV GOROOT=/usr/local/go -ENV PATH=$PATH:$GOPATH/bin:$GOROOT/bin +RUN wget -O /usr/local/bin/subkey https://chainbridge.ams3.digitaloceanspaces.com/subkey-v2.0.0 && \ + chmod +x /usr/local/bin/subkey -# Prepare structure and change dir -RUN mkdir -p $GOPATH/src/github.com/ChainSafe/gossamer -WORKDIR $GOPATH/src/github.com/ChainSafe/gossamer +# Polkadot JS dependencies +WORKDIR /go/src/github.com/ChainSafe/gossamer/tests/polkadotjs_test +COPY tests/polkadotjs_test/package.json tests/polkadotjs_test/package-lock.json ./ +RUN npm install -# Add go mod lock files and gossamer default config -COPY go.mod . -COPY go.sum . +WORKDIR /go/src/github.com/ChainSafe/gossamer -# Get go mods +# Go dependencies +COPY go.mod go.sum ./ RUN go mod download -# Copy gossamer sources -COPY . $GOPATH/src/github.com/ChainSafe/gossamer +# Prepare libwasmer.so for COPY +RUN cp /go/pkg/mod/github.com/wasmerio/go-ext-wasm@*/wasmer/libwasmer.so libwasmer.so -# Install js dependencies for polkadot.js tests -RUN cd $GOPATH/src/github.com/ChainSafe/gossamer/tests/polkadotjs_test && npm install +# Copy gossamer sources +COPY . . # Build RUN GOBIN=$GOPATH/src/github.com/ChainSafe/gossamer/bin go run scripts/ci.go install -# Create symlink -RUN ln -s $GOPATH/src/github.com/ChainSafe/gossamer/bin/gossamer /usr/local/gossamer +# Final stage based on Debian +FROM debian:${DEBIAN_VERSION} + +WORKDIR /gossamer -# Give permissions -RUN chmod +x $GOPATH/src/github.com/ChainSafe/gossamer/scripts/docker-entrypoint.sh +# Install libwasmer.so +ENV LD_LIBRARY_PATH=/lib:/usr/lib +COPY --from=builder /go/src/github.com/ChainSafe/gossamer/libwasmer.so /lib/libwasmer.so -# Expose gossamer command and port -ENTRYPOINT ["/gocode/src/github.com/ChainSafe/gossamer/scripts/docker-entrypoint.sh"] -CMD ["/usr/local/gossamer"] EXPOSE 7001 8546 8540 + +ENTRYPOINT ["/gossamer/docker-entrypoint.sh"] +CMD ["/gossamer/bin/gossamer"] + +COPY chain /gossamer/chain +COPY scripts/docker-entrypoint.sh /gossamer/docker-entrypoint.sh +COPY --from=builder /go/src/github.com/ChainSafe/gossamer/bin/gossamer /gossamer/bin/gossamer diff --git a/scripts/ci.go b/scripts/ci.go index 06c22ad644..7a805020de 100755 --- a/scripts/ci.go +++ b/scripts/ci.go @@ -78,13 +78,4 @@ func install(debug bool) { if err := cmd.Run(); err != nil { log.Fatal("Error: Could not build Gossamer. ", "error: ", err, ", cmd: ", cmd) } - -} - -// GOBIN returns the GOBIN environment variable -func GOBIN() string { - if os.Getenv("GOBIN") == "" { - log.Fatal("GOBIN is not set") - } - return os.Getenv("GOBIN") } diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 3abb78976b..e268d21569 100755 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -22,7 +22,7 @@ BASE_PATH=~/gossamer-dev set -euxo pipefail if [ ! -f $BASE_PATH/genesis_created ]; then - /usr/local/gossamer init --genesis=/gocode/src/github.com/ChainSafe/gossamer/chain/gssmr/genesis.json + /gossamer/bin/gossamer init --genesis=/gossamer/chain/gssmr/genesis.json touch $BASE_PATH/genesis_created; fi; diff --git a/tests/polkadotjs_test/start_polkadotjs_test.go b/tests/polkadotjs_test/start_polkadotjs_test.go index e9353c365b..7416fa2e58 100644 --- a/tests/polkadotjs_test/start_polkadotjs_test.go +++ b/tests/polkadotjs_test/start_polkadotjs_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/ChainSafe/gossamer/tests/utils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -44,7 +45,7 @@ func TestStartGossamerAndPolkadotAPI(t *testing.T) { command := "npx mocha ./test --timeout 30000" parts := strings.Fields(command) data, err := exec.Command(parts[0], parts[1:]...).Output() - require.NoError(t, err, string(data)) + assert.NoError(t, err, string(data)) //uncomment this to see log results from javascript tests //fmt.Printf("%s\n", data)