feat: improve CI workflow (#1367)

* testing cache * modified go file to test cache * removed deprecated cleanup job * modified git clone of genesis-tinkerer * add naming to step * modified to cache linux * test cache umeed v103 * add name to cache * update from tinkered_genesis to mainnet_tinkered_genesis * add cache for tinkered genesis * add missing variable * remove os from cache key * add needed step for current umeed * testing need dependencie * change dependencie * transfered umeed build to tests workflow * moved to use fixed value to grab builded bin * add debug * add debug * add debug * fix filepath umeed build * add debug * add executable permission to umeed bins * add make * set umeed as executable * add chmod after download bin * fix run * turn to old way of compile umeed * removed debug flag * fix output path * fix output path * add dependencie between workflow files * remove comments and turn back on unit-cover * remove go file diff to test workflows * add parallel multistage building to e2e dockerfile * modified go file to test * modified another go file to test * add cache docker to e2e * removed debug flag from e2e * added multiple building stages to e2e docker * add no cache again * turn back changes from umee e2e docker * add needed debug flag to have sh in the distroless * removed debug tag from make docker build since it is basically mandatory to have sh at the e2e * removed change diff in *.go file * Update contrib/scripts/download-mainnet-umeed.sh Co-authored-by: toteki <63419657+toteki@users.noreply.github.com> * Update contrib/scripts/tikerer-mainnet-genesis.sh Co-authored-by: toteki <63419657+toteki@users.noreply.github.com> * Update contrib/scripts/tikerer-mainnet-genesis.sh Co-authored-by: toteki <63419657+toteki@users.noreply.github.com> * move to tinker-mainnet-genesis.sh * modified *.go file to test CI * removed *.go file diff Co-authored-by: toteki <63419657+toteki@users.noreply.github.com>
umee-network · Sep 14, 2022 · 71b0d68 · 71b0d68
1 parent 36e618b
commit 71b0d68
Show file tree

Hide file tree

Showing 11 changed files with 138 additions and 102 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,5 +1,6 @@
 name: Build
 on:
+  workflow_call:
   pull_request:
   push:
     branches:
@@ -41,5 +42,6 @@ jobs:
       - uses: actions/upload-artifact@v3
         if: env.GIT_DIFF
         with:
-          name: umeed ${{ matrix.targetos }} ${{ matrix.arch }}
+          name: umeed-${{ matrix.targetos }}-${{ matrix.arch }}
           path: cmd/umeed/umeed
+          retention-days: 1
diff --git a/.github/workflows/simulations.yml b/.github/workflows/simulations.yml
@@ -7,14 +7,6 @@ on:
       - release/**
 
 jobs:
-  cleanup-runs:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: rokroskar/workflow-run-cleanup-action@master
-        env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-    if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main'"
-
   install-dependencies:
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -7,14 +7,6 @@ on:
       - release/**
 
 jobs:
-  cleanup-runs:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: rokroskar/workflow-run-cleanup-action@master
-        env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-    if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main'"
-
   install-tparse:
     runs-on: ubuntu-latest
     steps:
@@ -74,10 +66,21 @@ jobs:
         with:
           go-version: 1.19
           cache: true
+
+      # In this step, this action saves a list of existing images,
+      # the cache is created without them in the post run.
+      # It also restores the cache if it exists.
+      - name: cache docker layer
+        uses: satackey/action-docker-layer-caching@v0.0.11
+        if: env.GIT_DIFF
+        # Ignore the failure of a step and avoid terminating the job.
+        continue-on-error: true
+
       - name: Build Docker Image
         if: env.GIT_DIFF
         run: |
-          make docker-build-debug
+          DOCKER_BUILDKIT=1 make docker-build
+
       - name: Test E2E
         if: env.GIT_DIFF
         run: |
@@ -104,7 +107,11 @@ jobs:
         run: |
           cd price-feeder && make test-unit
 
+  build-umeed:
+    uses: ./.github/workflows/build.yml
+
   liveness-test:
+    needs: build-umeed
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
@@ -116,46 +123,67 @@ jobs:
             **/**.go
             go.mod
             go.sum
-      - uses: actions/setup-go@v3
+
+      - name: gets umeed bin
         if: env.GIT_DIFF
+        uses: actions/download-artifact@v3
         with:
-          go-version: 1.19
-          cache: true
-      - name: Build umeed
-        if: env.GIT_DIFF
-        run: |
-          make build
+          name: umeed-linux-amd64
+          path: ./build
+
       - name: Start single node chain
         if: env.GIT_DIFF
         run: |
+          chmod +x ./build/umeed
           ./contrib/scripts/single-node.sh
       - name: Test Local Network Liveness
         if: env.GIT_DIFF
         run: |
           sleep 1m
           ./contrib/scripts/test_localnet_liveness.sh 50 5 50 localhost:26657
 
-#  mainnet-upgrade-v1-v3:
-#    runs-on: ubuntu-latest
-#    timeout-minutes: 20
-#    steps:
-#      - uses: actions/checkout@v3
-#      - uses: technote-space/get-diff-action@v6.1.0
-#        with:
-#          PATTERNS: |
-#            **/**.go
-#            go.mod
-#            go.sum
-#      - uses: actions/setup-go@v3
-#        if: env.GIT_DIFF
-#        with:
-#          go-version: 1.19
-#          cache: true
-#      - name: Build umeed
-#        if: env.GIT_DIFF
-#        run: |
-#          make build
-#      - name: Fork mainnet and upgrade from v1.0.3 to v.3.x.x
-#        if: env.GIT_DIFF
-#        run: |
-#          ./contrib/scripts/umeemainnet_fork.sh
+  mainnet-upgrade-v1-v3:
+    needs: build-umeed
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v6.1.0
+        with:
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+
+      - name: gets umeed bin
+        if: env.GIT_DIFF
+        uses: actions/download-artifact@v3
+        with:
+          name: umeed-linux-amd64
+          path: ./build
+
+      - name: cache umeed mainnet
+        uses: actions/cache@v3
+        id: cache-umeed-v103
+        with:
+          path: ./contrib/scripts/umeed-releases/umeed-v1.0.3-linux-amd64/umeed
+          key: umeed-v103
+      - name: download mainnet binary
+        if: steps.cache-umeed-v103.outputs.cache-hit != 'true'
+        run: |
+          ./contrib/scripts/download-mainnet-umeed.sh
+      - name: cache mainnet tinkered genesis
+        uses: actions/cache@v3
+        id: cache-tinkered-genesis
+        with:
+          path: ./contrib/scripts/mainnet_tinkered_genesis.json
+          key: mainnet-tinkered-genesis
+      - name: tinkerer mainnet genesis
+        if: steps.cache-tinkered-genesis.outputs.cache-hit != 'true'
+        run: |
+          ./contrib/scripts/tinker-mainnet-genesis.sh
+      - name: Fork mainnet and upgrade from v1.0.3 to v.3.x.x
+        if: env.GIT_DIFF
+        run: |
+          chmod +x ./build/umeed
+          ./contrib/scripts/umeemainnet_fork.sh
diff --git a/.gitignore b/.gitignore
@@ -17,7 +17,7 @@ profile.out
 contrib/scripts/node-data
 contrib/scripts/umeemainnet.*
 contrib/scripts/cosmos-genesis-tinkerer
-contrib/scripts/tinkered_genesis.json
+contrib/scripts/mainnet_tinkered_genesis.json
 contrib/scripts/preprocessing.json
 contrib/scripts/umeed-releases
 # preprocessing can appear from the shell curdir that is cosmos-genesis-tinkerer is running

diff --git a/Makefile b/Makefile
@@ -108,16 +108,13 @@ clean:
 docker-build:
 	@docker build -t umeenet/umeed-e2e -f umee.e2e.Dockerfile .
 
-docker-build-debug:
-	@docker build -t umeenet/umeed-e2e --build-arg IMG_TAG=debug -f umee.e2e.Dockerfile .
-
 docker-push-hermes:
 	@cd tests/e2e/docker; docker build -t ghcr.io/umee-network/hermes-e2e:latest -f hermes.Dockerfile .; docker push ghcr.io/umee-network/hermes-e2e:latest
 
 docker-push-gaia:
 	@cd tests/e2e/docker; docker build -t ghcr.io/umee-network/gaia-e2e:latest -f gaia.Dockerfile .; docker push ghcr.io/umee-network/gaia-e2e:latest
 
-.PHONY: docker-build docker-build-debug docker-push-hermes docker-push-gaia
+.PHONY: docker-build docker-push-hermes docker-push-gaia
 
 ###############################################################################
 ##                              Tests & Linting                              ##

diff --git a/contrib/scripts/download-mainnet-umeed.sh b/contrib/scripts/download-mainnet-umeed.sh
@@ -0,0 +1,26 @@
+#!/bin/bash -eu
+
+# Download the umeed binary with the same version as mainnet and unpack it
+
+# USAGE: ./download-mainnet-umeed.sh
+
+CWD="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+UMEED_BIN_MAINNET_URL_TARBALL=${UMEED_BIN_MAINNET_URL_TARBALL:-"https://github.com/umee-network/umee/releases/download/v1.0.3/umeed-v1.0.3-linux-amd64.tar.gz"}
+UMEED_BIN_MAINNET=${UMEED_BIN_MAINNET:-"$CWD/umeed-releases/umeed-v1.0.3-linux-amd64/umeed"}
+
+# Checks for the umeed v1 file
+if [ ! -f "$UMEED_BIN_MAINNET" ]; then
+  echo "$UMEED_BIN_MAINNET doesn't exist"
+
+  if [ -z $UMEED_BIN_MAINNET_URL_TARBALL ]; then
+    echo You need to set the UMEED_BIN_MAINNET_URL_TARBALL variable
+    exit 1
+  fi
+
+  UMEED_RELEASES_PATH=$CWD/umeed-releases
+  mkdir -p $UMEED_RELEASES_PATH
+  wget -c $UMEED_BIN_MAINNET_URL_TARBALL -O - | tar -xz -C $UMEED_RELEASES_PATH
+
+  UMEED_BIN_MAINNET_BASENAME=$(basename $UMEED_BIN_MAINNET_URL_TARBALL .tar.gz)
+  UMEED_BIN_MAINNET=$UMEED_RELEASES_PATH/$UMEED_BIN_MAINNET_BASENAME/umeed
+fi
diff --git a/contrib/scripts/process_genesis.sh b/contrib/scripts/process_genesis.sh
@@ -11,7 +11,7 @@ PYTHON_CLI="${PYTHON_CLI:-python3}"
 if [ -d "$COSMOS_GENESIS_TINKERER_PATH" ]; then
   echo "$COSMOS_GENESIS_TINKERER_PATH exists."
 else
-  git clone https://github.com/umee-network/cosmos-genesis-tinkerer.git $CWD/cosmos-genesis-tinkerer
+  git clone --depth 1 --sparse https://github.com/umee-network/cosmos-genesis-tinkerer.git $CWD/cosmos-genesis-tinkerer
   pip install -r $COSMOS_GENESIS_TINKERER_PATH/requirements.txt
 fi
 

diff --git a/contrib/scripts/single-node.sh b/contrib/scripts/single-node.sh
@@ -41,9 +41,9 @@ hdir="$CHAIN_DIR/$CHAIN_ID"
 
 if ! command -v jq &> /dev/null
 then
-    echo "⚠️ jq command could not be found!"
-    echo "Install it by checking https://stedolan.github.io/jq/download/"
-    exit 1
+  echo "⚠️ jq command could not be found!"
+  echo "Install it by checking https://stedolan.github.io/jq/download/"
+  exit 1
 fi
 
 echo "--- Chain ID = $CHAIN_ID"

diff --git a/contrib/scripts/tinker-mainnet-genesis.sh b/contrib/scripts/tinker-mainnet-genesis.sh
@@ -0,0 +1,31 @@
+#!/bin/bash -eu
+
+# tinker the mainnet genesis by swapping polychain -> priv_validator_key
+
+# USAGE: ./tinker-mainnet-genesis.sh
+
+CWD="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+UMEEMAINNET_GENESIS_PATH="${UMEEMAINNET_GENESIS_PATH:-$CWD/mainnet_tinkered_genesis.json}"
+MAINNET_EXPORTED_GENESIS_URL="${MAINNET_EXPORTED_GENESIS_URL:-"https://storage.googleapis.com/umeedropzone/jul-28-umee-1-export.json.gz"}"
+EXPORTED_GENESIS_UNPROCESSED="${EXPORTED_GENESIS_UNPROCESSED:-$CWD/umeemainnet.genesis.json}"
+EXPORTED_GENESIS_UNZIPED="${EXPORTED_GENESIS_UNZIPED:-$CWD/umeemainnet.genesis.json.gz}"
+
+# Checks for the tinkered genesis file
+if [ ! -f "$UMEEMAINNET_GENESIS_PATH" ]; then
+  echo "$UMEEMAINNET_GENESIS_PATH doesn't exist"
+
+  if [ ! -f "$EXPORTED_GENESIS_UNPROCESSED" ]; then
+
+    if [ ! -f $EXPORTED_GENESIS_UNZIPED ]; then
+      echo "$EXPORTED_GENESIS_UNZIPED doesn't exist, we need to curl it"
+      curl $MAINNET_EXPORTED_GENESIS_URL > $EXPORTED_GENESIS_UNZIPED
+    fi
+
+    echo "$EXPORTED_GENESIS_UNPROCESSED doesn't exist, we need to unpack"
+    gunzip -k $EXPORTED_GENESIS_UNZIPED
+  fi
+
+  echo "$EXPORTED_GENESIS_UNPROCESSED exists and ready to be processed"
+  EXPORTED_GENESIS_UNPROCESSED_PATH=$EXPORTED_GENESIS_UNPROCESSED COSMOS_GENESIS_TINKERER_SCRIPT=umeemainnet-fork.py EXPORTED_GENESIS_PROCESSED_PATH=$UMEEMAINNET_GENESIS_PATH $CWD/process_genesis.sh
+fi
diff --git a/contrib/scripts/umeemainnet_fork.sh b/contrib/scripts/umeemainnet_fork.sh
@@ -8,39 +8,21 @@
 # USAGE: ./umeemainnet_fork.sh
 
 CWD="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
-UMEED_BIN_MAINNET_URL_TARBALL=${UMEED_BIN_MAINNET_URL_TARBALL:-"https://github.com/umee-network/umee/releases/download/v1.0.3/umeed-v1.0.3-linux-amd64.tar.gz"}
-UMEED_BIN_MAINNET=${UMEED_BIN_MAINNET:-"$CWD/umeed-releases/umeed-v1.0.3-linux-amd64/umeed"}
-
-# Checks for the umeed v1 file
-if [ ! -f "$UMEED_BIN_MAINNET" ]; then
-  echo "$UMEED_BIN_MAINNET doesn't exist"
-
-  if [ -z $UMEED_BIN_MAINNET_URL_TARBALL ]; then
-    echo You need to set the UMEED_BIN_MAINNET_URL_TARBALL variable
-    exit 1
-  fi
-
-  UMEED_RELEASES_PATH=$CWD/umeed-releases
-  mkdir -p $UMEED_RELEASES_PATH
-  wget -c $UMEED_BIN_MAINNET_URL_TARBALL -O - | tar -xz -C $UMEED_RELEASES_PATH
-
-  UMEED_BIN_MAINNET_BASENAME=$(basename $UMEED_BIN_MAINNET_URL_TARBALL .tar.gz)
-  UMEED_BIN_MAINNET=$UMEED_RELEASES_PATH/$UMEED_BIN_MAINNET_BASENAME/umeed
-fi
 
 CHAIN_ID="${CHAIN_ID:-umeemain-local-testnet}"
 FORK_DIR="${FORK_DIR:-$CWD}"
 CHAIN_DIR="${CHAIN_DIR:-$FORK_DIR/node-data}"
 LOG_LEVEL="${LOG_LEVEL:-debug}"
 BLOCK_TIME="${BLOCK_TIME:-6}"
 UPGRADE_TITLE="${UPGRADE_TITLE:-"v1.0-v3.0"}"
-MAINNET_EXPORTED_GENESIS_URL="${MAINNET_EXPORTED_GENESIS_URL:-"https://storage.googleapis.com/umeedropzone/jul-28-umee-1-export.json.gz"}"
 UMEED_BIN_CURRENT="${UMEED_BIN_CURRENT:-$FORK_DIR/../../build/umeed}"
-UMEEMAINNET_GENESIS_PATH="${UMEEMAINNET_GENESIS_PATH:-$CWD/tinkered_genesis.json}"
+UMEEMAINNET_GENESIS_PATH="${UMEEMAINNET_GENESIS_PATH:-$CWD/mainnet_tinkered_genesis.json}"
 NODE_PRIV_KEY="${NODE_PRIV_KEY:-$FORK_DIR/priv_validator_key.json}"
 SEC_AWAIT_NODE_START="${SEC_AWAIT_NODE_START:-80}"
 
 # Loads another sources
+. $CWD/download-mainnet-umeed.sh
+UMEEMAINNET_GENESIS_PATH=$UMEEMAINNET_GENESIS_PATH . $CWD/tinker-mainnet-genesis.sh
 . $CWD/blocks.sh
 
 nodeHome="$CHAIN_DIR/$CHAIN_ID"
@@ -85,26 +67,7 @@ else
 fi
 
 # Checks for the tikered genesis file
-if [ ! -f "$UMEEMAINNET_GENESIS_PATH" ]; then
-  echo "$UMEEMAINNET_GENESIS_PATH doesn't exist"
-  EXPORTED_GENESIS_UNPROCESSED=$CWD/umeemainnet.genesis.json
 
-  if [ ! -f "$EXPORTED_GENESIS_UNPROCESSED" ]; then
-
-    EXPORTED_GENESIS_UNZIPED=$CWD/umeemainnet.genesis.json.gz
-
-    if [ ! -f $EXPORTED_GENESIS_UNZIPED ]; then
-      echo "$EXPORTED_GENESIS_UNZIPED doesn't exist, we need to curl it"
-      curl $MAINNET_EXPORTED_GENESIS_URL > $EXPORTED_GENESIS_UNZIPED
-    fi
-
-    echo "$EXPORTED_GENESIS_UNPROCESSED doesn't exist, we need to unpack"
-    gunzip -k $EXPORTED_GENESIS_UNZIPED
-  fi
-
-  echo "$EXPORTED_GENESIS_UNPROCESSED exists and ready to be processed"
-  EXPORTED_GENESIS_UNPROCESSED_PATH=$EXPORTED_GENESIS_UNPROCESSED COSMOS_GENESIS_TINKERER_SCRIPT=umeemainnet-fork.py EXPORTED_GENESIS_PROCESSED_PATH=$UMEEMAINNET_GENESIS_PATH $CWD/process_genesis.sh
-fi
 
 echo Remove everything from the $CHAIN_DIR
 rm -rf $CHAIN_DIR

diff --git a/umee.e2e.Dockerfile b/umee.e2e.Dockerfile
@@ -1,5 +1,3 @@
-ARG IMG_TAG=latest
-
 # Fetch base packages
 FROM golang:1.19-alpine AS base-builder
 ENV PACKAGES make git libc-dev gcc linux-headers
@@ -27,8 +25,7 @@ RUN git clone https://github.com/umee-network/peggo.git
 RUN cd peggo && git checkout ${PEGGO_VERSION} && make build && cp ./build/peggo /usr/local/bin/
 
 # Add to a distroless container
-FROM gcr.io/distroless/cc:$IMG_TAG
-ARG IMG_TAG
+FROM gcr.io/distroless/cc:debug
 COPY --from=umeed-builder /go/bin/umeed /usr/local/bin/
 COPY --from=umeed-builder /go/bin/price-feeder /usr/local/bin/
 COPY --from=peggo-builder /usr/local/bin/peggo /usr/local/bin/