ci: merge Go coverage reports before upload (#10666)

Attempting to fix the state of codecov action checks right now,
which are behaving very erratically.

Using the new functionality in Go 1.20 to merge multiple reports,
so now the unit & E2E coverage data reports are stored as artifacts
and then downloaded, merged, and finally uploaded to codecov as a
new job.

Additionally, add a `codecov.yml` config and try to turn down the
aggressiveness of it for CI checks.

Signed-off-by: Milas Bowman <milas.bowman@docker.com>

.github/workflows/ci.yml

@@ -19,7 +19,6 @@ on:
         default: "false"
 
 env:
-  DESTDIR: "./bin"
   DOCKER_CLI_VERSION: "20.10.17"
 
 permissions:
@@ -103,7 +102,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: compose
-          path: ${{ env.DESTDIR }}/*
+          path: ./bin/release/*
           if-no-files-found: error
 
   test:
@@ -124,13 +123,15 @@ jobs:
             *.cache-from=type=gha,scope=test
             *.cache-to=type=gha,scope=test
       -
-        name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        name: Gather coverage data
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-data-unit
+          path: bin/coverage/unit/
+          if-no-files-found: error
 
   e2e:
     runs-on: ubuntu-latest
-    env:
-      DESTDIR: "./bin/build"
     strategy:
       fail-fast: false
       matrix:
@@ -179,11 +180,17 @@ jobs:
         name: Test plugin mode
         if: ${{ matrix.mode == 'plugin' }}
         run: |
-          rm -rf ./covdatafiles
-          mkdir ./covdatafiles
-          make e2e-compose GOCOVERDIR=covdatafiles
-          go tool covdata textfmt -i=covdatafiles -o=coverage.out
-
+          rm -rf ./bin/coverage/e2e
+          mkdir -p ./bin/coverage/e2e
+          make e2e-compose GOCOVERDIR=bin/coverage/e2e TEST_FLAGS="-v"
+      -
+        name: Gather coverage data
+        if: ${{ matrix.mode == 'plugin' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-data-e2e
+          path: bin/coverage/e2e/
+          if-no-files-found: error
       -
         name: Test standalone mode
         if: ${{ matrix.mode == 'standalone' }}
@@ -196,9 +203,44 @@ jobs:
         if: ${{ matrix.mode == 'cucumber'}}
         run: |
           make test-cucumber
-      -
-        name: Upload coverage to Codecov
+
+  coverage:
+    runs-on: ubuntu-22.04
+    needs:
+      - test
+      - e2e
+    steps:
+      # codecov won't process the report without the source code available
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: Download unit test coverage
+        uses: actions/download-artifact@v3
+        with:
+          name: coverage-data-unit
+          path: coverage/unit
+      - name: Download E2E test coverage
+        uses: actions/download-artifact@v3
+        with:
+          name: coverage-data-e2e
+          path: coverage/e2e
+      - name: Merge coverage reports
+        run: |
+          go tool covdata textfmt -i=./coverage/unit,./coverage/e2e -o ./coverage.txt
+      - name: Store coverage report in GitHub Actions
+        uses: actions/upload-artifact@v3
+        with:
+          name: go-covdata-txt
+          path: ./coverage.txt
+          if-no-files-found: error
+      - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
+        with:
+          files: ./coverage.txt
 
   release:
     permissions:
@@ -216,32 +258,32 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: compose
-          path: ${{ env.DESTDIR }}
+          path: bin/release
       -
         name: Create checksums
-        working-directory: ${{ env.DESTDIR }}
+        working-directory: bin/release
         run: |
           find . -type f -print0 | sort -z | xargs -r0 shasum -a 256 -b | sed 's# \*\./# *#' > $RUNNER_TEMP/checksums.txt
           shasum -a 256 -U -c $RUNNER_TEMP/checksums.txt
           mv $RUNNER_TEMP/checksums.txt .
           cat checksums.txt | while read sum file; do echo "$sum $file" > ${file#\*}.sha256; done
       -
         name: License
-        run: cp packaging/* ${{ env.DESTDIR }}/
+        run: cp packaging/* bin/release/
       -
         name: List artifacts
         run: |
-          tree -nh ${{ env.DESTDIR }}
+          tree -nh bin/release
       -
         name: Check artifacts
         run: |
-          find ${{ env.DESTDIR }} -type f -exec file -e ascii -- {} +
+          find bin/release -type f -exec file -e ascii -- {} +
       -
         name: GitHub Release
         if: startsWith(github.ref, 'refs/tags/v')
         uses: ncipollo/release-action@58ae73b360456532aafd58ee170c045abbeaee37 # v1.10.0
         with:
-          artifacts: ${{ env.DESTDIR }}/*
+          artifacts: bin/release/*
           generateReleaseNotes: true
           draft: true
           token: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -84,8 +84,8 @@ RUN --mount=type=bind,target=. \
     --mount=type=bind,from=osxcross,src=/osxsdk,target=/xx-sdk \
     xx-go --wrap && \
     if [ "$(xx-info os)" == "darwin" ]; then export CGO_ENABLED=1; fi && \
-    make build GO_BUILDTAGS="$BUILD_TAGS" DESTDIR=/usr/bin && \
-    xx-verify --static /usr/bin/docker-compose
+    make build GO_BUILDTAGS="$BUILD_TAGS" DESTDIR=/out && \
+    xx-verify --static /out/docker-compose
 
 FROM build-base AS lint
 ARG BUILD_TAGS
@@ -100,11 +100,13 @@ ARG BUILD_TAGS
 RUN --mount=type=bind,target=. \
     --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/go/pkg/mod \
-    go test -tags "$BUILD_TAGS" -v -coverprofile=/tmp/coverage.txt -covermode=atomic $(go list  $(TAGS) ./... | grep -vE 'e2e') && \
-    go tool cover -func=/tmp/coverage.txt
+    rm -rf /tmp/coverage && \
+    mkdir -p /tmp/coverage && \
+    go test -tags "$BUILD_TAGS" -v -cover -covermode=atomic $(go list  $(TAGS) ./... | grep -vE 'e2e') -args -test.gocoverdir="/tmp/coverage" && \
+    go tool covdata percent -i=/tmp/coverage
 
 FROM scratch AS test-coverage
-COPY --from=test /tmp/coverage.txt /coverage.txt
+COPY --from=test --link /tmp/coverage /
 
 FROM base AS license-set
 ARG LICENSE_FILES
@@ -162,11 +164,11 @@ RUN --mount=target=/context \
 EOT
 
 FROM scratch AS binary-unix
-COPY --link --from=build /usr/bin/docker-compose /
+COPY --link --from=build /out/docker-compose /
 FROM binary-unix AS binary-darwin
 FROM binary-unix AS binary-linux
 FROM scratch AS binary-windows
-COPY --link --from=build /usr/bin/docker-compose /docker-compose.exe
+COPY --link --from=build /out/docker-compose /docker-compose.exe
 FROM binary-$TARGETOS AS binary
 # enable scanning for this stage
 ARG BUILDKIT_SBOM_SCAN_STAGE=true

Makefile

@@ -25,22 +25,10 @@ else
     DETECTED_OS = $(shell uname -s)
 endif
 
-ifeq ($(DETECTED_OS),Linux)
-	MOBY_DOCKER=/usr/bin/docker
-endif
-ifeq ($(DETECTED_OS),Darwin)
-	MOBY_DOCKER=/Applications/Docker.app/Contents/Resources/bin/docker
-endif
 ifeq ($(DETECTED_OS),Windows)
 	BINARY_EXT=.exe
 endif
 
-TEST_COVERAGE_FLAGS = -coverprofile=coverage.out -covermode=atomic
-ifneq ($(DETECTED_OS),Windows)
-	# go race detector requires gcc on Windows so not used by default
-	# https://github.com/golang/go/issues/27089
-	TEST_COVERAGE_FLAGS += -race
-endif
 BUILD_FLAGS?=
 TEST_FLAGS?=
 E2E_TEST?=
@@ -50,13 +38,23 @@ else
 endif
 
 BUILDX_CMD ?= docker buildx
-DESTDIR ?= ./bin/build
+
+# DESTDIR overrides the output path for binaries and other artifacts
+# this is used by docker/docker-ce-packaging for the apt/rpm builds,
+# so it's important that the resulting binary ends up EXACTLY at the
+# path $DESTDIR/docker-compose when specified.
+#
+# See https://github.com/docker/docker-ce-packaging/blob/e43fbd37e48fde49d907b9195f23b13537521b94/rpm/SPECS/docker-compose-plugin.spec#L47
+#
+# By default, all artifacts go to subdirectories under ./bin/ in the
+# repo root, e.g. ./bin/build, ./bin/coverage, ./bin/release.
+DESTDIR ?=
 
 all: build
 
 .PHONY: build ## Build the compose cli-plugin
 build:
-	GO111MODULE=on go build $(BUILD_FLAGS) -trimpath -tags "$(GO_BUILDTAGS)" -ldflags "$(GO_LDFLAGS)" -o "$(DESTDIR)/docker-compose$(BINARY_EXT)" ./cmd
+	GO111MODULE=on go build $(BUILD_FLAGS) -trimpath -tags "$(GO_BUILDTAGS)" -ldflags "$(GO_LDFLAGS)" -o "$(or $(DESTDIR),./bin/build)/docker-compose$(BINARY_EXT)" ./cmd
 
 .PHONY: binary
 binary:
@@ -69,7 +67,7 @@ binary-with-coverage:
 .PHONY: install
 install: binary
 	mkdir -p ~/.docker/cli-plugins
-	install bin/build/docker-compose ~/.docker/cli-plugins/docker-compose
+	install $(or $(DESTDIR),./bin/build)/docker-compose ~/.docker/cli-plugins/docker-compose
 
 .PHONY: e2e-compose
 e2e-compose: ## Run end to end local tests in plugin mode. Set E2E_TEST=TestName to run a single test

codecov.yml

@@ -0,0 +1,21 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+        target: auto
+        threshold: 2%
+    patch:
+      default:
+        informational: true
+
+comment:
+  require_changes: true
+
+ignore:
+  - "packaging"
+  - "docs"
+  - "bin"
+  - "e2e"
+  - "pkg/e2e"
+  - "**/*_test.go"

docker-bake.hcl

@@ -25,13 +25,16 @@ variable "DOCS_FORMATS" {
   default = "md,yaml"
 }
 
-# Defines the output folder
+# Defines the output folder to override the default behavior.
+# See Makefile for details, this is generally only useful for
+# the packaging scripts and care should be taken to not break
+# them.
 variable "DESTDIR" {
   default = ""
 }
-function "bindir" {
+function "outdir" {
   params = [defaultdir]
-  result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}"
+  result = DESTDIR != "" ? DESTDIR : "${defaultdir}"
 }
 
 # Special target: https://github.com/docker/metadata-action#bake-definition
@@ -84,23 +87,23 @@ target "vendor-update" {
 target "test" {
   inherits = ["_common"]
   target = "test-coverage"
-  output = [bindir("coverage")]
+  output = [outdir("./bin/coverage/unit")]
 }
 
 target "binary-with-coverage" {
   inherits = ["_common"]
   target = "binary"
   args = {
-    BUILD_FLAGS = "-cover"
+    BUILD_FLAGS = "-cover -covermode=atomic"
   }
-  output = [bindir("build")]
+  output = [outdir("./bin/build")]
   platforms = ["local"]
 }
 
 target "binary" {
   inherits = ["_common"]
   target = "binary"
-  output = [bindir("build")]
+  output = [outdir("./bin/build")]
   platforms = ["local"]
 }
 
@@ -124,7 +127,7 @@ target "binary-cross" {
 target "release" {
   inherits = ["binary-cross"]
   target = "release"
-  output = [bindir("release")]
+  output = [outdir("./bin/release")]
 }
 
 target "docs-validate" {

pkg/e2e/framework.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -134,7 +135,7 @@ func initializePlugins(t testing.TB, configDir string) {
 	require.NoError(t, os.MkdirAll(filepath.Join(configDir, "cli-plugins"), 0o755),
 		"Failed to create cli-plugins directory")
 	composePlugin, err := findExecutable(DockerComposeExecutableName)
-	if os.IsNotExist(err) {
+	if errors.Is(err, fs.ErrNotExist) {
 		t.Logf("WARNING: docker-compose cli-plugin not found")
 	}
 
@@ -161,20 +162,21 @@ func dirContents(dir string) []string {
 }
 
 func findExecutable(executableName string) (string, error) {
-	_, filename, _, _ := runtime.Caller(0)
-	root := filepath.Join(filepath.Dir(filename), "..", "..")
-	buildPath := filepath.Join(root, "bin", "build")
-
-	bin, err := filepath.Abs(filepath.Join(buildPath, executableName))
-	if err != nil {
-		return "", err
+	bin := os.Getenv("COMPOSE_E2E_BIN_PATH")
+	if bin == "" {
+		_, filename, _, _ := runtime.Caller(0)
+		buildPath := filepath.Join(filepath.Dir(filename), "..", "..", "bin", "build")
+		var err error
+		bin, err = filepath.Abs(filepath.Join(buildPath, executableName))
+		if err != nil {
+			return "", err
+		}
 	}
 
 	if _, err := os.Stat(bin); err == nil {
 		return bin, nil
 	}
-
-	return "", errors.Wrap(os.ErrNotExist, "executable not found")
+	return "", fmt.Errorf("looking for %q: %w", bin, fs.ErrNotExist)
 }
 
 func findPluginExecutable(pluginExecutableName string) (string, error) {