feat: e2e test suite (cosmos#1253)

.github/workflows/release-sims.yml

@@ -26,7 +26,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v2.1.5
         with:
-          go-version: 1.16
+          go-version: 1.17
       - name: Install runsim
         run: export GO111MODULE="on" && go get github.com/cosmos/tools/cmd/runsim@v1.0.0
       - uses: actions/cache@v2.1.7

.github/workflows/sims.yml

@@ -18,7 +18,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v2.1.5
         with:
-          go-version: 1.16
+          go-version: 1.17
       - name: Install runsim
         run: export GO111MODULE="on" && go get github.com/cosmos/tools/cmd/runsim@v1.0.0
       - uses: actions/cache@v2.1.7
@@ -33,7 +33,7 @@ jobs:
 #      - uses: actions/checkout@v2.4.0
 #      - uses: actions/setup-go@v2.1.5
 #        with:
-#          go-version: 1.16
+#          go-version: 1.17
 #      - uses: technote-space/get-diff-action@v5.0.2
 #        with:
 #          PATTERNS: |
@@ -63,7 +63,7 @@ jobs:
       - uses: actions/checkout@v2.4.0
       - uses: actions/setup-go@v2.1.5
         with:
-          go-version: 1.16
+          go-version: 1.17
       - uses: actions/cache@v2.1.7
         with:
           path: ~/go/bin
@@ -80,7 +80,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v2.1.5
         with:
-          go-version: 1.16
+          go-version: 1.17
       - uses: actions/checkout@v2.4.0
       - uses: technote-space/get-diff-action@v5.0.2
         with:
@@ -105,7 +105,7 @@ jobs:
       - uses: actions/checkout@v2.4.0
       - uses: actions/setup-go@v2.1.5
         with:
-          go-version: 1.16
+          go-version: 1.17
       - uses: technote-space/get-diff-action@v5.0.2
         with:
           PATTERNS: |
@@ -130,7 +130,7 @@ jobs:
       - uses: actions/checkout@v2.4.0
       - uses: actions/setup-go@v2.1.5
         with:
-          go-version: 1.16
+          go-version: 1.17
       - uses: technote-space/get-diff-action@v5.0.2
         with:
           PATTERNS: |
@@ -144,4 +144,4 @@ jobs:
       - name: test liveness
         run: |
           ./contrib/localnet-blocks-test.sh 100 5 50 localhost
-        if: "env.GIT_DIFF != ''"
+        if: "env.GIT_DIFF != ''"

.github/workflows/test.yml

@@ -18,7 +18,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v2.1.5
         with:
-          go-version: 1.16
+          go-version: 1.17
       - uses: actions/checkout@v2.4.0
       - uses: technote-space/get-diff-action@v5.0.2
         with:
@@ -31,7 +31,7 @@ jobs:
           make build
       - name: test & coverage report creation
         run: |
-          go test ./... -mod=readonly -timeout 12m -race -coverprofile=coverage.txt -covermode=atomic -tags='ledger test_ledger_mock'
+          make test-unit-cover
         if: "env.GIT_DIFF != ''"
       - name: filter out DONTCOVER
         run: |
@@ -48,4 +48,27 @@ jobs:
         with:
           file: ./coverage.txt # optional
           fail_ci_if_error: true
-        if: "env.GIT_DIFF != ''"
+        if: "env.GIT_DIFF != ''"
+
+  test-e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/setup-go@v2.2.0
+        with:
+          go-version: 1.17
+      - uses: actions/checkout@v2
+      - uses: technote-space/get-diff-action@v6.0.1
+        with:
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+      - name: Build Docker Image
+        run: |
+          make docker-build-debug
+        if: env.GIT_DIFF
+      - name: Test E2E
+        run: |
+          make test-e2e
+        if: env.GIT_DIFF

.golangci.yml

@@ -1,7 +1,7 @@
 run:
   tests: false
-#   # timeout for analysis, e.g. 30s, 5m, default is 1m
-#   timeout: 5m
+  skip-dirs:
+    - tests/e2e
 
 linters:
   disable-all: true

Makefile

@@ -170,22 +170,39 @@ sync-docs:
 
 include sims.mk
 
-test: test-unit test-build
-
-test-all: check test-race test-cover
-
-test-unit:
-	@VERSION=$(VERSION) go test -mod=readonly -tags='ledger test_ledger_mock' ./...
-
-test-race:
-	@VERSION=$(VERSION) go test -mod=readonly -race -tags='ledger test_ledger_mock' ./...
+PACKAGES_UNIT=$(shell go list ./... | grep -v -e '/tests/e2e')
+PACKAGES_E2E=$(shell go list ./... | grep '/e2e')
+TEST_PACKAGES=./...
+TEST_TARGETS := test-unit test-unit-cover test-race test-e2e
+
+test-unit: ARGS=-timeout=5m -tags='norace'
+test-unit: TEST_PACKAGES=$(PACKAGES_UNIT)
+test-unit-cover: ARGS=-timeout=5m -tags='norace' -coverprofile=coverage.txt -covermode=atomic
+test-unit-cover: TEST_PACKAGES=$(PACKAGES_UNIT)
+test-race: ARGS=-timeout=5m -race
+test-race: TEST_PACKAGES=$(PACKAGES_UNIT)
+test-e2e: ARGS=-timeout=25m -v
+test-e2e: TEST_PACKAGES=$(PACKAGES_E2E)
+$(TEST_TARGETS): run-tests
+
+run-tests:
+ifneq (,$(shell which tparse 2>/dev/null))
+	@echo "--> Running tests"
+	@go test -mod=readonly -json $(ARGS) $(TEST_PACKAGES) | tparse
+else
+	@echo "--> Running tests"
+	@go test -mod=readonly $(ARGS) $(TEST_PACKAGES)
+endif
 
-test-cover:
-	@go test -mod=readonly -timeout 30m -race -coverprofile=coverage.txt -covermode=atomic -tags='ledger test_ledger_mock' ./...
+.PHONY: run-tests $(TEST_TARGETS)
 
-benchmark:
-	@go test -mod=readonly -bench=. ./...
+docker-build-debug:
+	@docker build -t cosmos/gaiad-e2e --build-arg IMG_TAG=debug -f e2e.Dockerfile .
 
+# TODO: Push this to the Cosmos Dockerhub so we don't have to keep building it
+# in CI.
+docker-build-hermes:
+	@cd tests/e2e/docker; docker build -t cosmos/hermes-e2e:latest -f hermes.Dockerfile .
 
 ###############################################################################
 ###                                Linting                                  ###
@@ -229,7 +246,6 @@ test-docker-push: test-docker
 .PHONY: all build-linux install format lint \
 	go-mod-cache draw-deps clean build \
 	setup-transactions setup-contract-tests-data start-gaia run-lcd-contract-tests contract-tests \
-	test test-all test-build test-cover test-unit test-race \
 	benchmark \
 	build-docker-gaiadnode localnet-start localnet-stop \
-	docker-single-node
+	docker-single-node docker-build-debug docker-build-hermes

docs/validators/validator-setup.md

@@ -136,7 +136,7 @@ When attempting to perform routine maintenance or planning for an upcoming coord
 the block.
 
 ## Advanced configuration
-You can find more advanced information about running a node or a validator on the [Tendermint Core documentation](https://docs.tendermint.com/master/nodes/).
+You can find more advanced information about running a node or a validator on the [Tendermint Core documentation](https://docs.tendermint.com/v0.35/nodes/).
 
 ## Common Problems
 
@@ -181,4 +181,4 @@ LimitNOFILE=4096
 
 [Install]
 WantedBy=multi-user.target
-```
+```

e2e.Dockerfile

@@ -0,0 +1,19 @@
+ARG IMG_TAG=latest
+
+# Compile the gaiad binary
+FROM golang:1.17-alpine AS gaiad-builder
+WORKDIR /src/app/
+COPY go.mod go.sum* ./
+RUN go mod download
+COPY . .
+ENV PACKAGES curl make git libc-dev bash gcc linux-headers eudev-dev python3
+RUN apk add --no-cache $PACKAGES
+RUN CGO_ENABLED=0 make install
+
+# Add to a distroless container
+FROM gcr.io/distroless/cc:$IMG_TAG
+ARG IMG_TAG
+COPY --from=gaiad-builder /go/bin/gaiad /usr/local/bin/
+EXPOSE 26656 26657 1317 9090
+
+ENTRYPOINT ["gaiad", "start"]

go.mod

@@ -4,12 +4,15 @@ go 1.17
 
 require (
 	github.com/cosmos/cosmos-sdk v0.44.6
+	github.com/cosmos/go-bip39 v1.0.0
 	github.com/cosmos/ibc-go/v2 v2.0.3
 	github.com/gorilla/mux v1.8.0
+	github.com/ory/dockertest/v3 v3.8.1
 	github.com/gravity-devs/liquidity v1.4.6
 	github.com/rakyll/statik v0.1.7
 	github.com/spf13/cast v1.4.1
 	github.com/spf13/cobra v1.3.0
+	github.com/spf13/viper v1.10.0
 	github.com/strangelove-ventures/packet-forward-middleware v1.0.1
 	github.com/stretchr/testify v1.7.0
 	github.com/tendermint/tendermint v0.34.15
@@ -19,19 +22,23 @@ require (
 require (
 	filippo.io/edwards25519 v1.0.0-beta.2 // indirect
 	github.com/99designs/keyring v1.1.6 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect
 	github.com/DataDog/zstd v1.4.5 // indirect
+	github.com/Microsoft/go-winio v0.5.1 // indirect
+	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/Workiva/go-datastructures v1.0.53 // indirect
 	github.com/armon/go-metrics v0.3.10 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bgentry/speakeasy v0.1.0 // indirect
 	github.com/btcsuite/btcd v0.22.0-beta // indirect
+	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/coinbase/rosetta-sdk-go v0.7.0 // indirect
 	github.com/confio/ics23/go v0.6.6 // indirect
+	github.com/containerd/continuity v0.2.0 // indirect
 	github.com/cosmos/btcutil v1.0.4 // indirect
-	github.com/cosmos/go-bip39 v1.0.0 // indirect
 	github.com/cosmos/iavl v0.17.3 // indirect
 	github.com/cosmos/ledger-cosmos-go v0.11.1 // indirect
 	github.com/cosmos/ledger-go v0.9.2 // indirect
@@ -41,6 +48,10 @@ require (
 	github.com/dgraph-io/badger/v2 v2.2007.2 // indirect
 	github.com/dgraph-io/ristretto v0.0.3 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
+	github.com/docker/cli v20.10.11+incompatible // indirect
+	github.com/docker/docker v20.10.7+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
@@ -55,6 +66,7 @@ require (
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/google/btree v1.0.0 // indirect
 	github.com/google/orderedcode v0.0.1 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/handlers v1.5.1 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
@@ -67,6 +79,7 @@ require (
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hdevalence/ed25519consensus v0.0.0-20210204194344-59a8610d2b87 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/improbable-eng/grpc-web v0.14.1 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
@@ -81,7 +94,11 @@ require (
 	github.com/minio/highwayhash v1.0.2 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
 	github.com/mtibben/percent v0.2.1 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/runc v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
@@ -95,16 +112,19 @@ require (
 	github.com/rs/cors v1.8.0 // indirect
 	github.com/rs/zerolog v1.23.0 // indirect
 	github.com/sasha-s/go-deadlock v0.2.1-0.20190427202633-1595213edefa // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.10.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
 	github.com/tendermint/btcd v0.1.1 // indirect
 	github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15 // indirect
 	github.com/tendermint/go-amino v0.16.0 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/zondax/hid v0.9.0 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
 	golang.org/x/crypto v0.0.0-20210915214749-c084706c2272 // indirect
@@ -122,6 +142,8 @@ require (
 )
 
 replace (
+	github.com/cosmos/cosmos-sdk => github.com/cosmos/cosmos-sdk v0.44.6
+	github.com/cosmos/ibc-go/v2 => github.com/cosmos/ibc-go/v2 v2.0.3
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	github.com/tecbot/gorocksdb => github.com/cosmos/gorocksdb v1.2.0
 	google.golang.org/grpc => google.golang.org/grpc v1.33.2