From e63ab14b1ef996402e897f425eaecf84decbe667 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 8 Jun 2023 14:58:21 -0400 Subject: [PATCH] ci: merge Go coverage reports before upload (#10666) Attempting to fix the state of codecov action checks right now, which are behaving very erratically. Using the new functionality in Go 1.20 to merge multiple reports, so now the unit & E2E coverage data reports are stored as artifacts and then downloaded, merged, and finally uploaded to codecov as a new job. Additionally, add a `codecov.yml` config and try to turn down the aggressiveness of it for CI checks. Signed-off-by: Milas Bowman --- .github/workflows/ci.yml | 80 ++++++++++++++++++++++++++++++---------- Dockerfile | 16 ++++---- Makefile | 28 +++++++------- codecov.yml | 21 +++++++++++ docker-bake.hcl | 19 ++++++---- pkg/e2e/framework.go | 22 ++++++----- 6 files changed, 127 insertions(+), 59 deletions(-) create mode 100644 codecov.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bc7de5189..1b4d645f2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,6 @@ on: default: "false" env: - DESTDIR: "./bin" DOCKER_CLI_VERSION: "20.10.17" permissions: @@ -103,7 +102,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: compose - path: ${{ env.DESTDIR }}/* + path: ./bin/release/* if-no-files-found: error test: @@ -124,13 +123,15 @@ jobs: *.cache-from=type=gha,scope=test *.cache-to=type=gha,scope=test - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + name: Gather coverage data + uses: actions/upload-artifact@v3 + with: + name: coverage-data-unit + path: bin/coverage/unit/ + if-no-files-found: error e2e: runs-on: ubuntu-latest - env: - DESTDIR: "./bin/build" strategy: fail-fast: false matrix: @@ -179,11 +180,17 @@ jobs: name: Test plugin mode if: ${{ matrix.mode == 'plugin' }} run: | - rm -rf ./covdatafiles - mkdir ./covdatafiles - make e2e-compose GOCOVERDIR=covdatafiles - go tool covdata textfmt -i=covdatafiles -o=coverage.out - + rm -rf ./bin/coverage/e2e + mkdir -p ./bin/coverage/e2e + make e2e-compose GOCOVERDIR=bin/coverage/e2e TEST_FLAGS="-v" + - + name: Gather coverage data + if: ${{ matrix.mode == 'plugin' }} + uses: actions/upload-artifact@v3 + with: + name: coverage-data-e2e + path: bin/coverage/e2e/ + if-no-files-found: error - name: Test standalone mode if: ${{ matrix.mode == 'standalone' }} @@ -196,9 +203,44 @@ jobs: if: ${{ matrix.mode == 'cucumber'}} run: | make test-cucumber - - - name: Upload coverage to Codecov + + coverage: + runs-on: ubuntu-22.04 + needs: + - test + - e2e + steps: + # codecov won't process the report without the source code available + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: 'go.mod' + check-latest: true + - name: Download unit test coverage + uses: actions/download-artifact@v3 + with: + name: coverage-data-unit + path: coverage/unit + - name: Download E2E test coverage + uses: actions/download-artifact@v3 + with: + name: coverage-data-e2e + path: coverage/e2e + - name: Merge coverage reports + run: | + go tool covdata textfmt -i=./coverage/unit,./coverage/e2e -o ./coverage.txt + - name: Store coverage report in GitHub Actions + uses: actions/upload-artifact@v3 + with: + name: go-covdata-txt + path: ./coverage.txt + if-no-files-found: error + - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 + with: + files: ./coverage.txt release: permissions: @@ -216,10 +258,10 @@ jobs: uses: actions/download-artifact@v3 with: name: compose - path: ${{ env.DESTDIR }} + path: bin/release - name: Create checksums - working-directory: ${{ env.DESTDIR }} + working-directory: bin/release run: | find . -type f -print0 | sort -z | xargs -r0 shasum -a 256 -b | sed 's# \*\./# *#' > $RUNNER_TEMP/checksums.txt shasum -a 256 -U -c $RUNNER_TEMP/checksums.txt @@ -227,21 +269,21 @@ jobs: cat checksums.txt | while read sum file; do echo "$sum $file" > ${file#\*}.sha256; done - name: License - run: cp packaging/* ${{ env.DESTDIR }}/ + run: cp packaging/* bin/release/ - name: List artifacts run: | - tree -nh ${{ env.DESTDIR }} + tree -nh bin/release - name: Check artifacts run: | - find ${{ env.DESTDIR }} -type f -exec file -e ascii -- {} + + find bin/release -type f -exec file -e ascii -- {} + - name: GitHub Release if: startsWith(github.ref, 'refs/tags/v') uses: ncipollo/release-action@58ae73b360456532aafd58ee170c045abbeaee37 # v1.10.0 with: - artifacts: ${{ env.DESTDIR }}/* + artifacts: bin/release/* generateReleaseNotes: true draft: true token: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 86797f46fa..3ded2db4a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -84,8 +84,8 @@ RUN --mount=type=bind,target=. \ --mount=type=bind,from=osxcross,src=/osxsdk,target=/xx-sdk \ xx-go --wrap && \ if [ "$(xx-info os)" == "darwin" ]; then export CGO_ENABLED=1; fi && \ - make build GO_BUILDTAGS="$BUILD_TAGS" DESTDIR=/usr/bin && \ - xx-verify --static /usr/bin/docker-compose + make build GO_BUILDTAGS="$BUILD_TAGS" DESTDIR=/out && \ + xx-verify --static /out/docker-compose FROM build-base AS lint ARG BUILD_TAGS @@ -100,11 +100,13 @@ ARG BUILD_TAGS RUN --mount=type=bind,target=. \ --mount=type=cache,target=/root/.cache \ --mount=type=cache,target=/go/pkg/mod \ - go test -tags "$BUILD_TAGS" -v -coverprofile=/tmp/coverage.txt -covermode=atomic $(go list $(TAGS) ./... | grep -vE 'e2e') && \ - go tool cover -func=/tmp/coverage.txt + rm -rf /tmp/coverage && \ + mkdir -p /tmp/coverage && \ + go test -tags "$BUILD_TAGS" -v -cover -covermode=atomic $(go list $(TAGS) ./... | grep -vE 'e2e') -args -test.gocoverdir="/tmp/coverage" && \ + go tool covdata percent -i=/tmp/coverage FROM scratch AS test-coverage -COPY --from=test /tmp/coverage.txt /coverage.txt +COPY --from=test --link /tmp/coverage / FROM base AS license-set ARG LICENSE_FILES @@ -162,11 +164,11 @@ RUN --mount=target=/context \ EOT FROM scratch AS binary-unix -COPY --link --from=build /usr/bin/docker-compose / +COPY --link --from=build /out/docker-compose / FROM binary-unix AS binary-darwin FROM binary-unix AS binary-linux FROM scratch AS binary-windows -COPY --link --from=build /usr/bin/docker-compose /docker-compose.exe +COPY --link --from=build /out/docker-compose /docker-compose.exe FROM binary-$TARGETOS AS binary # enable scanning for this stage ARG BUILDKIT_SBOM_SCAN_STAGE=true diff --git a/Makefile b/Makefile index 432b34cce9..9c9f45dba6 100644 --- a/Makefile +++ b/Makefile @@ -25,22 +25,10 @@ else DETECTED_OS = $(shell uname -s) endif -ifeq ($(DETECTED_OS),Linux) - MOBY_DOCKER=/usr/bin/docker -endif -ifeq ($(DETECTED_OS),Darwin) - MOBY_DOCKER=/Applications/Docker.app/Contents/Resources/bin/docker -endif ifeq ($(DETECTED_OS),Windows) BINARY_EXT=.exe endif -TEST_COVERAGE_FLAGS = -coverprofile=coverage.out -covermode=atomic -ifneq ($(DETECTED_OS),Windows) - # go race detector requires gcc on Windows so not used by default - # https://github.com/golang/go/issues/27089 - TEST_COVERAGE_FLAGS += -race -endif BUILD_FLAGS?= TEST_FLAGS?= E2E_TEST?= @@ -50,13 +38,23 @@ else endif BUILDX_CMD ?= docker buildx -DESTDIR ?= ./bin/build + +# DESTDIR overrides the output path for binaries and other artifacts +# this is used by docker/docker-ce-packaging for the apt/rpm builds, +# so it's important that the resulting binary ends up EXACTLY at the +# path $DESTDIR/docker-compose when specified. +# +# See https://github.com/docker/docker-ce-packaging/blob/e43fbd37e48fde49d907b9195f23b13537521b94/rpm/SPECS/docker-compose-plugin.spec#L47 +# +# By default, all artifacts go to subdirectories under ./bin/ in the +# repo root, e.g. ./bin/build, ./bin/coverage, ./bin/release. +DESTDIR ?= all: build .PHONY: build ## Build the compose cli-plugin build: - GO111MODULE=on go build $(BUILD_FLAGS) -trimpath -tags "$(GO_BUILDTAGS)" -ldflags "$(GO_LDFLAGS)" -o "$(DESTDIR)/docker-compose$(BINARY_EXT)" ./cmd + GO111MODULE=on go build $(BUILD_FLAGS) -trimpath -tags "$(GO_BUILDTAGS)" -ldflags "$(GO_LDFLAGS)" -o "$(or $(DESTDIR),./bin/build)/docker-compose$(BINARY_EXT)" ./cmd .PHONY: binary binary: @@ -69,7 +67,7 @@ binary-with-coverage: .PHONY: install install: binary mkdir -p ~/.docker/cli-plugins - install bin/build/docker-compose ~/.docker/cli-plugins/docker-compose + install $(or $(DESTDIR),./bin/build)/docker-compose ~/.docker/cli-plugins/docker-compose .PHONY: e2e-compose e2e-compose: ## Run end to end local tests in plugin mode. Set E2E_TEST=TestName to run a single test diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000000..a66912f32e --- /dev/null +++ b/codecov.yml @@ -0,0 +1,21 @@ +coverage: + status: + project: + default: + informational: true + target: auto + threshold: 2% + patch: + default: + informational: true + +comment: + require_changes: true + +ignore: + - "packaging" + - "docs" + - "bin" + - "e2e" + - "pkg/e2e" + - "**/*_test.go" diff --git a/docker-bake.hcl b/docker-bake.hcl index 872216d1e8..5c6522d3a8 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -25,13 +25,16 @@ variable "DOCS_FORMATS" { default = "md,yaml" } -# Defines the output folder +# Defines the output folder to override the default behavior. +# See Makefile for details, this is generally only useful for +# the packaging scripts and care should be taken to not break +# them. variable "DESTDIR" { default = "" } -function "bindir" { +function "outdir" { params = [defaultdir] - result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}" + result = DESTDIR != "" ? DESTDIR : "${defaultdir}" } # Special target: https://github.com/docker/metadata-action#bake-definition @@ -84,23 +87,23 @@ target "vendor-update" { target "test" { inherits = ["_common"] target = "test-coverage" - output = [bindir("coverage")] + output = [outdir("./bin/coverage/unit")] } target "binary-with-coverage" { inherits = ["_common"] target = "binary" args = { - BUILD_FLAGS = "-cover" + BUILD_FLAGS = "-cover -covermode=atomic" } - output = [bindir("build")] + output = [outdir("./bin/build")] platforms = ["local"] } target "binary" { inherits = ["_common"] target = "binary" - output = [bindir("build")] + output = [outdir("./bin/build")] platforms = ["local"] } @@ -124,7 +127,7 @@ target "binary-cross" { target "release" { inherits = ["binary-cross"] target = "release" - output = [bindir("release")] + output = [outdir("./bin/release")] } target "docs-validate" { diff --git a/pkg/e2e/framework.go b/pkg/e2e/framework.go index 7ef45f48d1..cb3255d0b5 100644 --- a/pkg/e2e/framework.go +++ b/pkg/e2e/framework.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "net/http" "os" "path/filepath" @@ -134,7 +135,7 @@ func initializePlugins(t testing.TB, configDir string) { require.NoError(t, os.MkdirAll(filepath.Join(configDir, "cli-plugins"), 0o755), "Failed to create cli-plugins directory") composePlugin, err := findExecutable(DockerComposeExecutableName) - if os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { t.Logf("WARNING: docker-compose cli-plugin not found") } @@ -161,20 +162,21 @@ func dirContents(dir string) []string { } func findExecutable(executableName string) (string, error) { - _, filename, _, _ := runtime.Caller(0) - root := filepath.Join(filepath.Dir(filename), "..", "..") - buildPath := filepath.Join(root, "bin", "build") - - bin, err := filepath.Abs(filepath.Join(buildPath, executableName)) - if err != nil { - return "", err + bin := os.Getenv("COMPOSE_E2E_BIN_PATH") + if bin == "" { + _, filename, _, _ := runtime.Caller(0) + buildPath := filepath.Join(filepath.Dir(filename), "..", "..", "bin", "build") + var err error + bin, err = filepath.Abs(filepath.Join(buildPath, executableName)) + if err != nil { + return "", err + } } if _, err := os.Stat(bin); err == nil { return bin, nil } - - return "", errors.Wrap(os.ErrNotExist, "executable not found") + return "", fmt.Errorf("looking for %q: %w", bin, fs.ErrNotExist) } func findPluginExecutable(pluginExecutableName string) (string, error) {