Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: Adds parallel GitHub actions for e2e tests to get rid of flaky tests #4977

Merged
merged 15 commits into from Dec 21, 2021
12 changes: 9 additions & 3 deletions .bingo/Variables.mk
Expand Up @@ -23,11 +23,11 @@ $(ALERTMANAGER): $(BINGO_DIR)/alertmanager.mod
@echo "(re)installing $(GOBIN)/alertmanager-v0.20.0"
@cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=alertmanager.mod -o=$(GOBIN)/alertmanager-v0.20.0 "github.com/prometheus/alertmanager/cmd/alertmanager"

BINGO := $(GOBIN)/bingo-v0.4.3
BINGO := $(GOBIN)/bingo-v0.5.2
$(BINGO): $(BINGO_DIR)/bingo.mod
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
@echo "(re)installing $(GOBIN)/bingo-v0.4.3"
@cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=bingo.mod -o=$(GOBIN)/bingo-v0.4.3 "github.com/bwplotka/bingo"
@echo "(re)installing $(GOBIN)/bingo-v0.5.2"
@cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=bingo.mod -o=$(GOBIN)/bingo-v0.5.2 "github.com/bwplotka/bingo"

FAILLINT := $(GOBIN)/faillint-v1.8.0
$(FAILLINT): $(BINGO_DIR)/faillint.mod
Expand Down Expand Up @@ -59,6 +59,12 @@ $(GOLANGCI_LINT): $(BINGO_DIR)/golangci-lint.mod
@echo "(re)installing $(GOBIN)/golangci-lint-v1.39.1-0.20210330125642-6844f6abf817"
@cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=golangci-lint.mod -o=$(GOBIN)/golangci-lint-v1.39.1-0.20210330125642-6844f6abf817 "github.com/golangci/golangci-lint/cmd/golangci-lint"

GOTESPLIT := $(GOBIN)/gotesplit-v0.1.2
$(GOTESPLIT): $(BINGO_DIR)/gotesplit.mod
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
@echo "(re)installing $(GOBIN)/gotesplit-v0.1.2"
@cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=gotesplit.mod -o=$(GOBIN)/gotesplit-v0.1.2 "github.com/Songmu/gotesplit/cmd/gotesplit"

HUGO := $(GOBIN)/hugo-v0.80.0
$(HUGO): $(BINGO_DIR)/hugo.mod
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
Expand Down
2 changes: 1 addition & 1 deletion .bingo/bingo.mod
Expand Up @@ -2,4 +2,4 @@ module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT

go 1.14

require github.com/bwplotka/bingo v0.4.3
require github.com/bwplotka/bingo v0.5.2
5 changes: 5 additions & 0 deletions .bingo/gotesplit.mod
@@ -0,0 +1,5 @@
module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT

go 1.17

require github.com/Songmu/gotesplit v0.1.2 // cmd/gotesplit
4 changes: 3 additions & 1 deletion .bingo/variables.env
Expand Up @@ -10,7 +10,7 @@ fi

ALERTMANAGER="${GOBIN}/alertmanager-v0.20.0"

BINGO="${GOBIN}/bingo-v0.4.3"
BINGO="${GOBIN}/bingo-v0.5.2"

FAILLINT="${GOBIN}/faillint-v1.8.0"

Expand All @@ -22,6 +22,8 @@ GOJSONTOYAML="${GOBIN}/gojsontoyaml-v0.0.0-20191212081931-bf2969bbd742"

GOLANGCI_LINT="${GOBIN}/golangci-lint-v1.39.1-0.20210330125642-6844f6abf817"

GOTESPLIT="${GOBIN}/gotesplit-v0.1.2"

HUGO="${GOBIN}/hugo-v0.80.0"

JB="${GOBIN}/jb-v0.4.0"
Expand Down
7 changes: 6 additions & 1 deletion .github/workflows/go.yaml
Expand Up @@ -55,6 +55,11 @@ jobs:
- name: Linting & vetting.
run: make go-lint
e2e:
strategy:
fail-fast: true
matrix:
parallelism: [4]
index: [0,1,2,3]
runs-on: ubuntu-latest
name: Thanos end-to-end tests
env:
Expand All @@ -74,4 +79,4 @@ jobs:
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

- name: Run e2e docker-based tests.
run: make test-e2e
run: make test-e2e GH_PARALLEL=${{ matrix.parallelism }} GH_INDEX=${{ matrix.index }}
2 changes: 1 addition & 1 deletion CHANGELOG.md
Expand Up @@ -12,7 +12,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
- [#4970](https://github.com/thanos-io/thanos/pull/4970) Added a new flag `exclude-delete` to `tools bucket ls`, which excludes blocks marked for deletion.

### Added

- [#4977](https://github.com/thanos-io/thanos/pull/4977) Build: Upgrade to `bingo v0.5.2` and implements `gotesplit` to allow for parallelism in our GitHub e2e tests.
- [#4228](https://github.com/thanos-io/thanos/pull/4228) Tools `thanos bucket inspect`: Add flag `--output` to provide output method (table,csv,tsv).
- [#4680](https://github.com/thanos-io/thanos/pull/4680) Query: add `exemplar.partial-response` flag to control partial response.
- [#4679](https://github.com/thanos-io/thanos/pull/4679) Added `enable-feature` flag to enable negative offsets and @ modifier, similar to Prometheus.
Expand Down
9 changes: 6 additions & 3 deletions Makefile
Expand Up @@ -6,6 +6,9 @@ DOCKER_IMAGE_REPO ?= quay.io/thanos/thanos
DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))-$(shell date +%Y-%m-%d)-$(shell git rev-parse --short HEAD)
DOCKER_CI_TAG ?= test

GH_PARALLEL ?= 1
GH_INDEX ?= 0

BASE_DOCKER_SHA=''
arch = $(shell uname -m)
# Run `DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect quay.io/prometheus/busybox:latest` to get SHA or
Expand Down Expand Up @@ -276,15 +279,15 @@ test-local:

.PHONY: test-e2e
test-e2e: ## Runs all Thanos e2e docker-based e2e tests from test/e2e. Required access to docker daemon.
test-e2e: docker
test-e2e: docker $(GOTESPLIT)
@echo ">> cleaning docker environment."
@docker system prune -f --volumes
@echo ">> cleaning e2e test garbage."
@rm -rf ./test/e2e/e2e_*
@echo ">> running /test/e2e tests."
# NOTE(bwplotka):
# * If you see errors on CI (timeouts), but not locally, try to add -parallel 1 to limit to single CPU to reproduce small 1CPU machine.
@go test $(GOTEST_OPTS) ./test/e2e/...
# * If you see errors on CI (timeouts), but not locally, try to add -parallel 1 (Wiard note: to the GOTEST_OPTS arg) to limit to single CPU to reproduce small 1CPU machine.
@$(GOTESPLIT) -total ${GH_PARALLEL} -index ${GH_INDEX} ./test/e2e/... -- ${GOTEST_OPTS}

.PHONY: test-e2e-local
test-e2e-local: ## Runs all thanos e2e tests locally.
Expand Down