Add best-effort coverage reports per test type (#1915)

Ensure that each executed test type marks some content as covered, so we
can have an idea of the files covered by tests, and the test types executed.
Before we reported if a test type has been executed by marking a line in a
manifest as covered, but this fails if the manifest doesn't have enough lines.

At the moment the following files are fully or partially marked as covered if an
specific type of test is executed:

Pipeline tests:
- No changes in this PR, we have coverage based on ES stats.
Asset tests:
- All files under the kibana directory.
Policy tests:
- All manifests.
- All template files in the executed data stream.
Static tests:
- Sample events.
System tests
- All manifests.
- Fields files.

The rest of non-development files are marked as not covered.

Author: jsoriano

cmd/testrunner.go

@@ -175,6 +175,8 @@ func testRunnerAssetCommandAction(cmd *cobra.Command, args []string) error {
 		PackageRootPath:  packageRootPath,
 		KibanaClient:     kibanaClient,
 		GlobalTestConfig: globalTestConfig.Asset,
+		WithCoverage:     testCoverage,
+		CoverageType:     testCoverageFormat,
 	})
 
 	results, err := testrunner.RunSuite(ctx, runner)
@@ -264,6 +266,8 @@ func testRunnerStaticCommandAction(cmd *cobra.Command, args []string) error {
 		DataStreams:        dataStreams,
 		FailOnMissingTests: failOnMissing,
 		GlobalTestConfig:   globalTestConfig.Static,
+		WithCoverage:       testCoverage,
+		CoverageType:       testCoverageFormat,
 	})
 
 	results, err := testrunner.RunSuite(ctx, runner)
@@ -572,6 +576,8 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 		DeferCleanup:               deferCleanup,
 		RunIndependentElasticAgent: false,
 		GlobalTestConfig:           globalTestConfig.System,
+		WithCoverage:               testCoverage,
+		CoverageType:               testCoverageFormat,
 	})
 
 	logger.Debugf("Running suite...")
@@ -683,6 +689,8 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error {
 		FailOnMissingTests: failOnMissing,
 		GenerateTestResult: generateTestResult,
 		GlobalTestConfig:   globalTestConfig.Policy,
+		WithCoverage:       testCoverage,
+		CoverageType:       testCoverageFormat,
 	})
 
 	results, err := testrunner.RunSuite(ctx, runner)

internal/packages/assets.go

@@ -34,6 +34,7 @@ type Asset struct {
 	ID         string    `json:"id"`
 	Type       AssetType `json:"type"`
 	DataStream string
+	SourcePath string
 }
 
 // String method returns a string representation of the asset
@@ -181,8 +182,9 @@ func loadFileBasedAssets(kibanaAssetsFolderPath string, assetType AssetType) ([]
 		}
 
 		asset := Asset{
-			ID:   assetID,
-			Type: assetType,
+			ID:         assetID,
+			Type:       assetType,
+			SourcePath: assetPath,
 		}
 		assets = append(assets, asset)
 	}

internal/testrunner/coberturacoverage.go

@@ -8,8 +8,6 @@ import (
 	"bytes"
 	"encoding/xml"
 	"fmt"
-	"path/filepath"
-	"sort"
 )
 
 func init() {
@@ -200,64 +198,3 @@ func (c *CoberturaCoverage) Merge(other CoverageReport) error {
 	}
 	return nil
 }
-
-func transformToCoberturaReport(details *testCoverageDetails, baseFolder string, timestamp int64) *CoberturaCoverage {
-	var classes []*CoberturaClass
-	lineNumberTestType := lineNumberPerTestType(string(details.testType))
-
-	// sort data streams to ensure same ordering in coverage arrays
-	sortedDataStreams := make([]string, 0, len(details.dataStreams))
-	for dataStream := range details.dataStreams {
-		sortedDataStreams = append(sortedDataStreams, dataStream)
-	}
-	sort.Strings(sortedDataStreams)
-
-	for _, dataStream := range sortedDataStreams {
-		testCases := details.dataStreams[dataStream]
-
-		if dataStream == "" && details.packageType == "integration" {
-			continue // ignore tests running in the package context (not data stream), mostly referring to installed assets
-		}
-
-		var methods []*CoberturaMethod
-		var lines []*CoberturaLine
-
-		if len(testCases) == 0 {
-			methods = append(methods, &CoberturaMethod{
-				Name:  "Missing",
-				Lines: []*CoberturaLine{{Number: lineNumberTestType, Hits: 0}},
-			})
-			lines = append(lines, []*CoberturaLine{{Number: lineNumberTestType, Hits: 0}}...)
-		} else {
-			methods = append(methods, &CoberturaMethod{
-				Name:  "OK",
-				Lines: []*CoberturaLine{{Number: lineNumberTestType, Hits: 1}},
-			})
-			lines = append(lines, []*CoberturaLine{{Number: lineNumberTestType, Hits: 1}}...)
-		}
-
-		fileName := filepath.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml")
-		if dataStream == "" {
-			// input package
-			fileName = filepath.Join(baseFolder, details.packageName, "manifest.yml")
-		}
-
-		aClass := &CoberturaClass{
-			Name:     string(details.testType),
-			Filename: fileName,
-			Methods:  methods,
-			Lines:    lines,
-		}
-		classes = append(classes, aClass)
-	}
-
-	return &CoberturaCoverage{
-		Timestamp: timestamp,
-		Packages: []*CoberturaPackage{
-			{
-				Name:    details.packageName,
-				Classes: classes,
-			},
-		},
-	}
-}

internal/testrunner/coverage.go

@@ -0,0 +1,229 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package testrunner
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/elastic/elastic-package/internal/files"
+)
+
+// GenerateBasePackageCoverageReport generates a coverage report where all files under the root path are
+// marked as not covered. It ignores files under _dev directories.
+func GenerateBasePackageCoverageReport(pkgName, rootPath, format string) (CoverageReport, error) {
+	repoPath, err := files.FindRepositoryRootDirectory()
+	if err != nil {
+		return nil, fmt.Errorf("failed to find repository root directory: %w", err)
+	}
+
+	var coverage CoverageReport
+	err = filepath.WalkDir(rootPath, func(match string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			if d.Name() == "_dev" {
+				return fs.SkipDir
+			}
+			return nil
+		}
+
+		fileCoverage, err := generateBaseFileCoverageReport(repoPath, pkgName, match, format, false)
+		if err != nil {
+			return fmt.Errorf("failed to generate base coverage for \"%s\": %w", match, err)
+		}
+		if coverage == nil {
+			coverage = fileCoverage
+			return nil
+		}
+
+		err = coverage.Merge(fileCoverage)
+		if err != nil {
+			return fmt.Errorf("cannot merge coverages: %w", err)
+		}
+
+		return nil
+	})
+	// If the directory is not found, give it as valid, will return an empty coverage. This is also useful for mocked tests.
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return nil, fmt.Errorf("failed to walk package directory %s: %w", rootPath, err)
+	}
+	return coverage, nil
+}
+
+// GenerateBaseFileCoverageReport generates a coverage report for a given file, where all the file is marked as covered or uncovered.
+func GenerateBaseFileCoverageReport(pkgName, path, format string, covered bool) (CoverageReport, error) {
+	repoPath, err := files.FindRepositoryRootDirectory()
+	if err != nil {
+		return nil, fmt.Errorf("failed to find repository root directory: %w", err)
+	}
+
+	return generateBaseFileCoverageReport(repoPath, pkgName, path, format, covered)
+}
+
+// GenerateBaseFileCoverageReport generates a coverage report for all the files matching any of the given patterns. The complete
+// files are marked as fully covered or uncovered depending on the given value.
+func GenerateBaseFileCoverageReportGlob(pkgName string, patterns []string, format string, covered bool) (CoverageReport, error) {
+	repoPath, err := files.FindRepositoryRootDirectory()
+	if err != nil {
+		return nil, fmt.Errorf("failed to find repository root directory: %w", err)
+	}
+
+	var coverage CoverageReport
+	for _, pattern := range patterns {
+		matches, err := filepath.Glob(pattern)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, match := range matches {
+			fileCoverage, err := generateBaseFileCoverageReport(repoPath, pkgName, match, format, covered)
+			if err != nil {
+				return nil, fmt.Errorf("failed to generate base coverage for \"%s\": %w", match, err)
+			}
+			if coverage == nil {
+				coverage = fileCoverage
+				continue
+			}
+
+			err = coverage.Merge(fileCoverage)
+			if err != nil {
+				return nil, fmt.Errorf("cannot merge coverages: %w", err)
+			}
+		}
+	}
+	return coverage, nil
+}
+
+func generateBaseFileCoverageReport(repoPath, pkgName, path, format string, covered bool) (CoverageReport, error) {
+	switch format {
+	case "cobertura":
+		return generateBaseCoberturaFileCoverageReport(repoPath, pkgName, path, covered)
+	case "generic":
+		return generateBaseGenericFileCoverageReport(repoPath, pkgName, path, covered)
+	default:
+		return nil, fmt.Errorf("unknwon coverage format %s", format)
+	}
+}
+
+func generateBaseCoberturaFileCoverageReport(repoPath, pkgName, path string, covered bool) (*CoberturaCoverage, error) {
+	coveragePath, err := filepath.Rel(repoPath, path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to obtain path inside repository for %s", path)
+	}
+	ext := filepath.Ext(path)
+	class := CoberturaClass{
+		Name:     pkgName + "." + strings.TrimSuffix(filepath.Base(path), ext),
+		Filename: coveragePath,
+	}
+	pkg := CoberturaPackage{
+		Name: pkgName,
+		Classes: []*CoberturaClass{
+			&class,
+		},
+	}
+	coverage := CoberturaCoverage{
+		Sources: []*CoberturaSource{
+			{
+				Path: path,
+			},
+		},
+		Packages: []*CoberturaPackage{
+			&pkg,
+		},
+		Timestamp: time.Now().UnixNano(),
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %v", err)
+	}
+	defer f.Close()
+
+	hits := int64(0)
+	if covered {
+		hits = 1
+	}
+	lines, err := countReaderLines(f)
+	if err != nil {
+		return nil, fmt.Errorf("failed to count lines in file: %w", err)
+	}
+	for i := range lines {
+		line := CoberturaLine{
+			Number: i + 1,
+			Hits:   hits,
+		}
+		class.Lines = append(class.Lines, &line)
+	}
+	coverage.LinesValid = int64(lines)
+	coverage.LinesCovered = int64(lines) * hits
+
+	return &coverage, nil
+}
+
+func generateBaseGenericFileCoverageReport(repoPath, _, path string, covered bool) (*GenericCoverage, error) {
+	coveragePath, err := filepath.Rel(repoPath, path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to obtain path inside repository for %s", path)
+	}
+	file := GenericFile{
+		Path: coveragePath,
+	}
+	coverage := GenericCoverage{
+		Version:   1,
+		Timestamp: time.Now().UnixNano(),
+		TestType:  fmt.Sprintf("Coverage for %s", coveragePath),
+		Files: []*GenericFile{
+			&file,
+		},
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %v", err)
+	}
+	defer f.Close()
+
+	lines, err := countReaderLines(f)
+	if err != nil {
+		return nil, fmt.Errorf("failed to count lines in file: %w", err)
+	}
+	for i := range lines {
+		line := GenericLine{
+			LineNumber: int64(i) + 1,
+			Covered:    covered,
+		}
+		file.Lines = append(file.Lines, &line)
+	}
+
+	return &coverage, nil
+}
+
+func countReaderLines(r io.Reader) (int, error) {
+	count := 0
+	buffered := bufio.NewReader(r)
+	for {
+		c, _, err := buffered.ReadRune()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return 0, fmt.Errorf("failed to read rune: %w", err)
+		}
+		if c != '\n' {
+			continue
+		}
+		count += 1
+	}
+	return count, nil
+}