Include end-to-end test code coverage (#2293)

Merge code coverage from both unit and end-to-end integration tests to obtain a complete picture of correct code coverage. Also, merge code coverage from multiple platforms now that the new tool `go tool covdata` supports this. Since the end-to-end tests executes the standalone CLI binary, run `go build` with `-cover` for the go compiler to insert coverage profile statements in the compiled output. `GOCOVERDIR` is set when invoking the CLI to output coverage data to the specified directory (see [here](https://go.dev/testing/coverage/)). Thus, the CLI is compiled once with `-cover` added for running tests, then recompiled without `-cover` for release. Fixes #1996
Azure · May 25, 2023 · e6ece84 · e6ece84
1 parent 9c03285
commit e6ece84
Show file tree

Hide file tree

Showing 8 changed files with 171 additions and 36 deletions.
diff --git a/.gitignore b/.gitignore
@@ -17,8 +17,12 @@
 # GoLand temp files
 .idea 
 
-# Output of the go coverage tool, specifically when used with LiteIDE
+# go code coverage outputs
+# text format
 *.out
+# binary formats
+covcounters.*
+covmeta.*
 
 # Dependency directories (remove the comment below to include it)
 # vendor/

diff --git a/cli/azd/.vscode/cspell-azd-dictionary.txt b/cli/azd/.vscode/cspell-azd-dictionary.txt
@@ -66,6 +66,7 @@ funcapp
 functestapp
 functionapp
 GOARCH
+GOCOVERDIR
 godotenv
 golangci
 hotspot

diff --git a/cli/azd/ci-build.ps1 b/cli/azd/ci-build.ps1
@@ -1,8 +1,17 @@
 param(
     [string] $Version = (Get-Content "$PSScriptRoot/../version.txt"),
-    [string] $SourceVersion = (git rev-parse HEAD)
+    [string] $SourceVersion = (git rev-parse HEAD),
+    [switch] $CodeCoverageEnabled
 )
 
+# Remove any previously built binaries
+go clean
+
+if ($LASTEXITCODE) {
+    Write-Host "Error running go clean"
+    exit $LASTEXITCODE
+}
+
 # On Windows, use the goversioninfo tool to embed the version information into the executable.
 if ($IsWindows) {
     Write-Host "Windows build, set version info and run 'go generate'"
@@ -47,6 +56,10 @@ $buildFlags = @(
     "-asmflags=-trimpath"
 )
 
+if ($CodeCoverageEnabled) {
+    $buildFlags += "-cover"
+}
+
 # Build constraint tags
 # cfi: Enable Control Flow Integrity (CFI),
 # cfg: Enable Control Flow Guard (CFG),

diff --git a/cli/azd/ci-test.ps1 b/cli/azd/ci-test.ps1
@@ -1,14 +1,60 @@
 param(
-    [string] $Timeout = '90m',
-    [string] $CoverageFileOut = 'cover.out',
-    [string] $Package = './...',
-    [switch] $ShortMode
+    [switch] $ShortMode,
+    [string] $UnitTestCoverageDir = 'cover-unit',
+    [string] $IntegrationTestTimeout = '90m',
+    [string] $IntegrationTestCoverageDir = 'cover-int'
 )
 
-$goTest = "$(go env GOPATH)/bin/gotestsum -- -timeout $Timeout -v -coverprofile='$CoverageFileOut' $Package"
+$ErrorActionPreference = 'Stop'
+
+$gopath = go env GOPATH
+if ($LASTEXITCODE) {
+    throw "go env GOPATH failed with exit code: $LASTEXITCODE, stdout: $gopath"
+}
+
+$gotestsumBinary = "gotestsum"
+if ($IsWindows) {
+    $gotestsumBinary += ".exe"
+}
+
+$gotestsum = Join-Path $gopath "bin" $gotestsumBinary
+if (-not (Test-Path $gotestsum)) {
+    throw "gotestsum is not installed at $gotestsum"
+}
+
+function New-EmptyDirectory {
+    param([string]$Path) 
+    if (Test-Path $Path) {
+        Remove-Item -Force -Recurse $Path | Out-Null
+    }
+
+    New-Item -ItemType Directory -Force -Path $Path
+}
+$unitCoverDir = New-EmptyDirectory -Path $UnitTestCoverageDir
+Write-Host "Running unit tests..."
+
+# --test.gocoverdir is currently a "under-the-cover" way to pass the coverage directory to a test binary
+# See https://github.com/golang/go/issues/51430#issuecomment-1344711300
+#
+# This may be improved in go1.21 with an official 'go test' flag.
+& $gotestsum -- ./... -short -v -cover -args --test.gocoverdir="$($unitCoverDir.FullName)"
+if ($LASTEXITCODE) {
+    exit $LASTEXITCODE
+}
 
 if ($ShortMode) {
-    $goTest = $goTest + " -short"
+    Write-Host "Short mode, skipping integration tests"
+    exit 0
 }
 
-Invoke-Expression $goTest
+Write-Host "Running integration tests..."
+$intCoverDir = New-EmptyDirectory -Path $IntegrationTestCoverageDir
+
+# GOCOVERDIR enables any binaries (in this case, azd.exe) built with '-cover',
+# to write out coverage output to the specific directory.
+$env:GOCOVERDIR = $intCoverDir.FullName
+
+& $gotestsum -- ./test/... -v -timeout $IntegrationTestTimeout
+if ($LASTEXITCODE) {
+    exit $LASTEXITCODE
+}
diff --git a/cli/azd/cmd/cobra_builder_test.go b/cli/azd/cmd/cobra_builder_test.go
@@ -39,6 +39,8 @@ func Test_BuildAndRunSimpleCommand(t *testing.T) {
 	require.NotNil(t, cmd)
 	require.NoError(t, err)
 
+	// Disable args processing from os:args
+	cmd.SetArgs([]string{})
 	err = cmd.ExecuteContext(context.Background())
 
 	require.NoError(t, err)

diff --git a/cli/azd/test/azdcli/cli.go b/cli/azd/test/azdcli/cli.go
@@ -96,6 +96,12 @@ func NewCLI(t *testing.T) *CLI {
 	buildOnce.Do(func() {
 		cmd := exec.Command("go", "build")
 		cmd.Dir = filepath.Dir(cliPath)
+
+		// Build with coverage if GOCOVERDIR is specified.
+		if os.Getenv("GOCOVERDIR") != "" {
+			cmd.Args = append(cmd.Args, "-cover")
+		}
+
 		output, err := cmd.CombinedOutput()
 		if err != nil {
 			panic(fmt.Errorf(

diff --git a/eng/pipelines/release-cli.yml b/eng/pipelines/release-cli.yml
@@ -50,7 +50,6 @@ stages:
               BuildOutputName: azd
               SetExecutableBit: true
               SetShieldInfo: true
-              GenerateCoverage: true
               BuildLinuxPackages: true
               AZURE_DEV_CI_OS: lin
               Codeql.Enabled: true
@@ -93,8 +92,9 @@ stages:
               arguments: >-
                 -Version $(CLI_VERSION)
                 -SourceVersion $(Build.SourceVersion)
+                -CodeCoverageEnabled
               workingDirectory: cli/azd
-            displayName: Build Go Binary
+            displayName: Build Go Binary (Coverage enabled)
 
           - template: /eng/pipelines/templates/steps/build-msi.yml
             parameters:
@@ -133,6 +133,9 @@ stages:
               pwsh: true
               targetType: filePath
               filePath: cli/azd/ci-test.ps1
+              arguments: >-
+                -UnitTestCoverageDir './cover-$(AZURE_DEV_CI_OS)/unit'
+                -IntegrationTestCoverageDir './cover-$(AZURE_DEV_CI_OS)/int'
               workingDirectory: cli/azd
             displayName: Test Go Binary
             env:
@@ -161,32 +164,16 @@ stages:
             displayName: Publish test results
             condition: succeededOrFailed()
 
-          - pwsh: |
-              go install github.com/axw/gocov/gocov@latest
-              if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-              go install github.com/AlekSi/gocov-xml@latest
-              if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-              ~/go/bin/gocov convert cover.out | ~/go/bin/gocov-xml > coverage.xml
-            workingDirectory: cli/azd
-            condition: >-
-              and(
-                succeeded(),
-                eq(variables['GenerateCoverage'], 'true'),
-                ne(variables['Skip.LiveTest'], 'true')
-              )
-            displayName: Generate coverage
-
-          - task: PublishCodeCoverageResults@1
-            condition: >-
-              and(
-                succeeded(),
-                eq(variables['GenerateCoverage'], 'true'),
-                ne(variables['Skip.LiveTest'], 'true')
-              )
+          - task: PowerShell@2
             inputs:
-              codeCoverageTool: Cobertura
-              summaryFileLocation: '$(Build.SourcesDirectory)/**/coverage.xml'
-            displayName: Publish Code Coverage to DevOps
+              pwsh: true
+              targetType: filePath
+              filePath: cli/azd/ci-build.ps1
+              arguments: >-
+                -Version $(CLI_VERSION)
+                -SourceVersion $(Build.SourceVersion)
+              workingDirectory: cli/azd
+            displayName: Build Go Binary
 
           - pwsh: Move-Item $(BuildOutputName) $(BuildTarget)
             workingDirectory: cli/azd
@@ -212,6 +199,10 @@ stages:
                 -Version "$(CLI_VERSION)"
             displayName: Set shield info
 
+          - publish: cli/azd/cover-$(AZURE_DEV_CI_OS)
+            artifact: cover-$(AZURE_DEV_CI_OS)
+            displayName: Upload code coverage
+
           - publish: eng/shields/standalone.json
             condition: and(succeeded(), eq(variables['SetShieldInfo'], 'true'))
             artifact: shield-standalone
@@ -376,6 +367,68 @@ stages:
             artifact: docs
             displayName: Upload generated documentation
 
+  - stage: CodeCoverage_Upload
+    dependsOn: BuildAndTest
+    jobs:
+    - job: Upload
+      pool:
+        name: azsdk-pool-mms-ubuntu-2004-general
+        vmImage: MMSUbuntu20.04
+      steps:
+        - template: /eng/pipelines/templates/steps/setup-go.yml
+        - template: /eng/pipelines/templates/steps/download-artifacts.yml
+          parameters:
+            Artifacts:
+            - cover-win
+            - cover-lin
+            - cover-mac
+        - pwsh: |
+            New-Item -ItemType Directory -Force -Path cover
+            New-Item -ItemType Directory -Force -Path cover-int
+            New-Item -ItemType Directory -Force -Path cover-unit
+
+            $unitCoverage = (Get-ChildItem cover-*/unit).FullName -join ","
+            $integrationCoverage = (Get-ChildItem cover-*/int).FullName -join ","
+
+            # Merge unit test coverage across platforms
+            go tool covdata merge -i="$unitCoverage" -o cover-unit
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
+            # Merge integration test coverage across platforms
+            go tool covdata merge -i="$integrationCoverage" -o cover-int
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
+            # Merge unit and integration code coverage
+            go tool covdata merge -i="cover-unit,cover-int" -o cover
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
+            # Convert to text format
+            go tool covdata textfmt -i=cover -o cover.out
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
+            go install github.com/axw/gocov/gocov@latest
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+            go install github.com/AlekSi/gocov-xml@latest
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+            
+            ~/go/bin/gocov convert cover.out | ~/go/bin/gocov-xml > coverage.xml
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          displayName: Merge code coverage files
+
+        - publish: cover-unit
+          artifact: cover-unit
+          displayName: Upload unit test code coverage
+
+        - publish: cover-int
+          artifact: cover-int
+          displayName: Upload integration test code coverage
+
+        - task: PublishCodeCoverageResults@1
+          inputs:
+            codeCoverageTool: Cobertura
+            summaryFileLocation: '$(Build.SourcesDirectory)/**/coverage.xml'
+          displayName: Publish Code Coverage to DevOps
+
   - stage: Sign
     dependsOn: BuildAndTest
     jobs:

diff --git a/eng/pipelines/templates/steps/download-artifacts.yml b/eng/pipelines/templates/steps/download-artifacts.yml
@@ -0,0 +1,10 @@
+parameters:
+- name: Artifacts
+  type: object
+
+steps:
+  - ${{ each artifact in parameters.Artifacts }}:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: ${{ artifact }}
+        path: ${{ artifact }}