chore(ci): Dockerfile improvements and faster CI (#1934)

- Accelerate the CI build
- Make CI caching a possibility for the future to further accelerate build
- Smaller image of 186MB instead of 3GB+ (uncompressed amd64)
- Cache Polkadot JS NPM dependencies (package.json + package-lock.json)
- Image versions as build arguments `DEBIAN_VERSION=bullseye-slim` and `GO_VERSION=1.15-buster`
- Add `.dockerignore` file for better use of the Docker laying caching
- Compact commands together for smaller image sizes and smaller build cache size
- Use only `wget` and not `curl` in builder
- Re-order instructions in final image to maximize Docker layer caching efficiency
- Remove unused `GOBIN() string` function from `ci.go` since it's unused

.dockerignore

@@ -0,0 +1,15 @@
+/.git
+/.githooks
+/.github
+/docs
+/.deepsource.toml
+/.dockerignore
+/.gitattributes
+/.gitignore
+/.releaserc
+/.codecov.yml
+/docker-compose.yml
+/Dockerfile
+/Dockerfile.staging
+/prometheus.yml
+/README.md

.github/workflows/docker-grandpa.yml

@@ -20,6 +20,7 @@ jobs:
         with:
           load: true
           file: ./Dockerfile
+          target: builder
           platforms: linux/amd64
           push: false
           tags: chainsafe/gossamer:test

.github/workflows/docker-js.yml

@@ -20,6 +20,7 @@ jobs:
           with:
             load: true
             file: ./Dockerfile
+            target: builder
             platforms: linux/amd64
             push: false
             tags: chainsafe/gossamer:test

.github/workflows/docker-rpc.yml

@@ -21,6 +21,7 @@ jobs:
           load: true
           file: ./Dockerfile
           platforms: linux/amd64
+          target: builder
           push: false
           tags: chainsafe/gossamer:test
       -

.github/workflows/docker-stable.yml

@@ -20,6 +20,7 @@ jobs:
         with:
           load: true
           file: ./Dockerfile
+          target: builder
           platforms: linux/amd64
           push: false
           tags: chainsafe/gossamer:test

.github/workflows/docker-stress.yml

@@ -20,6 +20,7 @@ jobs:
         with:
           load: true
           file: ./Dockerfile
+          target: builder
           platforms: linux/amd64
           push: false
           tags: chainsafe/gossamer:test

Dockerfile

@@ -1,64 +1,56 @@
-FROM ubuntu:18.04 as builder
+ARG DEBIAN_VERSION=bullseye-slim
+ARG GO_VERSION=1.15-buster
+
+FROM golang:${GO_VERSION} AS builder
 
-# Install GCC
 RUN apt-get update && \
     apt-get install -y \
     gcc \
     cmake \
-    wget \
-    curl \
-    npm 
+    wget
 
 # Install node source for polkadotjs tests
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
-
-# Install nodejs for polkadotjs tests
-RUN apt-get update && \
-    apt-get install -y \
-    nodejs
-
-# Install Go
-RUN wget https://dl.google.com/go/go1.15.5.linux-amd64.tar.gz
-RUN tar -C /usr/local -xzf go1.15.5.linux-amd64.tar.gz
+RUN wget -qO- https://deb.nodesource.com/setup_14.x | bash - && \
+    apt-get install -y nodejs
 
 # Install subkey
-RUN wget -P /usr/local/bin/ https://chainbridge.ams3.digitaloceanspaces.com/subkey-v2.0.0
-RUN mv /usr/local/bin/subkey-v2.0.0 /usr/local/bin/subkey
-RUN chmod +x /usr/local/bin/subkey
-
-# Configure go env vars
-ENV GO111MODULE=on
-ENV GOPATH=/gocode
-ENV GOROOT=/usr/local/go
-ENV PATH=$PATH:$GOPATH/bin:$GOROOT/bin
+RUN wget -O /usr/local/bin/subkey https://chainbridge.ams3.digitaloceanspaces.com/subkey-v2.0.0 && \
+    chmod +x /usr/local/bin/subkey
 
-# Prepare structure and change dir
-RUN mkdir -p $GOPATH/src/github.com/ChainSafe/gossamer
-WORKDIR $GOPATH/src/github.com/ChainSafe/gossamer
+# Polkadot JS dependencies
+WORKDIR /go/src/github.com/ChainSafe/gossamer/tests/polkadotjs_test
+COPY tests/polkadotjs_test/package.json tests/polkadotjs_test/package-lock.json ./
+RUN npm install
 
-# Add go mod lock files and gossamer default config
-COPY go.mod .
-COPY go.sum .
+WORKDIR /go/src/github.com/ChainSafe/gossamer
 
-# Get go mods
+# Go dependencies
+COPY go.mod go.sum ./
 RUN go mod download
 
-# Copy gossamer sources
-COPY . $GOPATH/src/github.com/ChainSafe/gossamer
+# Prepare libwasmer.so for COPY
+RUN cp /go/pkg/mod/github.com/wasmerio/go-ext-wasm@*/wasmer/libwasmer.so libwasmer.so
 
-# Install js dependencies for polkadot.js tests
-RUN cd $GOPATH/src/github.com/ChainSafe/gossamer/tests/polkadotjs_test && npm install
+# Copy gossamer sources
+COPY . .
 
 # Build
 RUN GOBIN=$GOPATH/src/github.com/ChainSafe/gossamer/bin go run scripts/ci.go install
 
-# Create symlink
-RUN ln -s $GOPATH/src/github.com/ChainSafe/gossamer/bin/gossamer /usr/local/gossamer
+# Final stage based on Debian
+FROM debian:${DEBIAN_VERSION}
+
+WORKDIR /gossamer
 
-# Give permissions
-RUN chmod +x $GOPATH/src/github.com/ChainSafe/gossamer/scripts/docker-entrypoint.sh
+# Install libwasmer.so
+ENV LD_LIBRARY_PATH=/lib:/usr/lib
+COPY --from=builder /go/src/github.com/ChainSafe/gossamer/libwasmer.so /lib/libwasmer.so
 
-# Expose gossamer command and port
-ENTRYPOINT ["/gocode/src/github.com/ChainSafe/gossamer/scripts/docker-entrypoint.sh"]
-CMD ["/usr/local/gossamer"]
 EXPOSE 7001 8546 8540
+
+ENTRYPOINT ["/gossamer/docker-entrypoint.sh"]
+CMD ["/gossamer/bin/gossamer"]
+
+COPY chain /gossamer/chain
+COPY scripts/docker-entrypoint.sh /gossamer/docker-entrypoint.sh
+COPY --from=builder /go/src/github.com/ChainSafe/gossamer/bin/gossamer /gossamer/bin/gossamer

scripts/ci.go

@@ -78,13 +78,4 @@ func install(debug bool) {
 	if err := cmd.Run(); err != nil {
 		log.Fatal("Error: Could not build Gossamer. ", "error: ", err, ", cmd: ", cmd)
 	}
-
-}
-
-// GOBIN returns the GOBIN environment variable
-func GOBIN() string {
-	if os.Getenv("GOBIN") == "" {
-		log.Fatal("GOBIN is not set")
-	}
-	return os.Getenv("GOBIN")
 }

scripts/docker-entrypoint.sh

@@ -22,7 +22,7 @@ BASE_PATH=~/gossamer-dev
 set -euxo pipefail
 
 if [ ! -f $BASE_PATH/genesis_created ]; then
-	/usr/local/gossamer init --genesis=/gocode/src/github.com/ChainSafe/gossamer/chain/gssmr/genesis.json
+	/gossamer/bin/gossamer init --genesis=/gossamer/chain/gssmr/genesis.json
 	touch $BASE_PATH/genesis_created;
 fi;
 

tests/polkadotjs_test/start_polkadotjs_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/ChainSafe/gossamer/tests/utils"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -44,7 +45,7 @@ func TestStartGossamerAndPolkadotAPI(t *testing.T) {
 	command := "npx mocha ./test --timeout 30000"
 	parts := strings.Fields(command)
 	data, err := exec.Command(parts[0], parts[1:]...).Output()
-	require.NoError(t, err, string(data))
+	assert.NoError(t, err, string(data))
 
 	//uncomment this to see log results from javascript tests
 	//fmt.Printf("%s\n", data)