From c7bf4c94c02a8db2faa0f3559970bbcec3e2e890 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Dec 2023 12:11:56 +0100 Subject: [PATCH 01/35] Update paths coverage --- internal/testrunner/coverageoutput.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index 72658ade2f..2da23226e7 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -346,7 +346,7 @@ func transformToCoberturaReport(details *testCoverageDetails) *CoberturaCoverage aClass := &CoberturaClass{ Name: string(details.testType), - Filename: path.Join(details.packageName, dataStream), + Filename: path.Join(details.packageName, "data_stream", dataStream), Methods: methods, } classes = append(classes, aClass) From bd5a5004970896865ff55148327b38d421526cfa Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Dec 2023 13:41:45 +0100 Subject: [PATCH 02/35] Add manifest.yml into the path --- internal/testrunner/coverageoutput.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index 2da23226e7..c6f7cd3d9c 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -346,7 +346,7 @@ func transformToCoberturaReport(details *testCoverageDetails) *CoberturaCoverage aClass := &CoberturaClass{ Name: string(details.testType), - Filename: path.Join(details.packageName, "data_stream", dataStream), + Filename: path.Join(details.packageName, "data_stream", dataStream, "manifest.yml"), Methods: methods, } classes = append(classes, aClass) From 25cd56f04a37bcc68f8392a82373e9bcbd0a9065 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Dec 2023 14:09:26 +0100 Subject: [PATCH 03/35] Add base folder --- internal/testrunner/coverageoutput.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index c6f7cd3d9c..cf1d880d3d 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -12,9 +12,11 @@ import ( "os" "path" "path/filepath" + "strings" "time" "github.com/elastic/elastic-package/internal/builder" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/multierror" ) @@ -240,10 +242,21 @@ func WriteCoverage(packageRootPath, packageName string, testType TestType, resul return fmt.Errorf("can't collect test coverage details: %w", err) } + dir, err := files.FindRepositoryRootDirectory() + if err != nil { + return err + } + + relativePath := strings.TrimPrefix(packageRootPath, dir) + if strings.HasPrefix(relativePath, "/") { + relativePath = relativePath[1:] + } + baseFolder := strings.TrimSuffix(relativePath, packageName) + // Use provided cobertura report, or generate a custom report if not available. report := details.cobertura if report == nil { - report = transformToCoberturaReport(details) + report = transformToCoberturaReport(details, baseFolder) } err = writeCoverageReportFile(report, packageName) @@ -323,7 +336,7 @@ func verifyTestExpected(packageRootPath string, dataStreamName string, testType return true, nil } -func transformToCoberturaReport(details *testCoverageDetails) *CoberturaCoverage { +func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) *CoberturaCoverage { var classes []*CoberturaClass for dataStream, testCases := range details.dataStreams { if dataStream == "" { @@ -346,7 +359,7 @@ func transformToCoberturaReport(details *testCoverageDetails) *CoberturaCoverage aClass := &CoberturaClass{ Name: string(details.testType), - Filename: path.Join(details.packageName, "data_stream", dataStream, "manifest.yml"), + Filename: path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml"), Methods: methods, } classes = append(classes, aClass) From d7c89be0f655c5f8f7e4efe703c6d1e271e434cb Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Dec 2023 16:30:46 +0100 Subject: [PATCH 04/35] Get filepath with respect to working copy root --- internal/testrunner/runners/pipeline/coverage.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index 8735f96b3a..4551138650 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -12,6 +12,7 @@ import ( "time" "github.com/elastic/elastic-package/internal/elasticsearch/ingest" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/testrunner" ) @@ -44,6 +45,13 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe // and a default.yml pipeline). basePath := filepath.Dir(options.PackageRootPath) + dir, err := files.FindRepositoryRootDirectory() + if err != nil { + return nil, err + } + + basePath = basePath[:len(dir)] + coverage := &testrunner.CoberturaCoverage{ Sources: []*testrunner.CoberturaSource{ { From 64dfd9e48786edb0225433b45601fd8d7c824a1c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 2 Jan 2024 13:24:20 +0100 Subject: [PATCH 05/35] Fix linting --- internal/testrunner/coverageoutput.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index cf1d880d3d..b54565f628 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -248,9 +248,7 @@ func WriteCoverage(packageRootPath, packageName string, testType TestType, resul } relativePath := strings.TrimPrefix(packageRootPath, dir) - if strings.HasPrefix(relativePath, "/") { - relativePath = relativePath[1:] - } + relativePath = strings.TrimPrefix(relativePath, "/") baseFolder := strings.TrimSuffix(relativePath, packageName) // Use provided cobertura report, or generate a custom report if not available. From bba2f9be31447202cc9e045fe3edb38d34c3f8df Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 12 Jan 2024 11:26:21 +0100 Subject: [PATCH 06/35] Remove Hits from Methods attribute not present in DTD --- internal/testrunner/coverageoutput.go | 2 -- internal/testrunner/runners/pipeline/coverage.go | 1 - 2 files changed, 3 deletions(-) diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index b54565f628..8957c50fb8 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -106,7 +106,6 @@ type CoberturaMethod struct { LineRate float32 `xml:"line-rate,attr"` BranchRate float32 `xml:"branch-rate,attr"` Complexity float32 `xml:"complexity,attr"` - Hits int64 `xml:"hits,attr"` Lines []*CoberturaLine `xml:"lines>line"` } @@ -150,7 +149,6 @@ func (c *CoberturaClass) merge(b *CoberturaClass) error { } // Update methods for idx := range b.Methods { - c.Methods[idx].Hits += b.Methods[idx].Hits for l := range b.Methods[idx].Lines { c.Methods[idx].Lines[l].Hits += b.Methods[idx].Lines[l].Hits } diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index 4551138650..772543d1a2 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -128,7 +128,6 @@ func coverageForSinglePipeline(pipeline ingest.Pipeline, stats ingest.PipelineSt } method := testrunner.CoberturaMethod{ Name: srcProc.Type, - Hits: pstats.Processors[idx].Stats.Count, } for num := srcProc.FirstLine; num <= srcProc.LastLine; num++ { line := &testrunner.CoberturaLine{ From 73c0dfef7f4a6fe86825972f5ce8341924fca371 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 12 Jan 2024 13:08:52 +0100 Subject: [PATCH 07/35] Add base folder in package name --- internal/testrunner/coverageoutput.go | 4 ++-- internal/testrunner/runners/pipeline/coverage.go | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index 8957c50fb8..722a90a596 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -247,7 +247,7 @@ func WriteCoverage(packageRootPath, packageName string, testType TestType, resul relativePath := strings.TrimPrefix(packageRootPath, dir) relativePath = strings.TrimPrefix(relativePath, "/") - baseFolder := strings.TrimSuffix(relativePath, packageName) + baseFolder := filepath.Dir(relativePath) // Use provided cobertura report, or generate a custom report if not available. report := details.cobertura @@ -365,7 +365,7 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) Timestamp: time.Now().UnixNano(), Packages: []*CoberturaPackage{ { - Name: details.packageName, + Name: strings.TrimSuffix(baseFolder, "/") + "." + details.packageName, Classes: classes, }, }, diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index 772543d1a2..34e4ccff0d 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -34,11 +34,6 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe return nil, fmt.Errorf("error fetching pipeline stats for code coverage calculations: %w", err) } - // Construct the Cobertura report. - pkg := &testrunner.CoberturaPackage{ - Name: options.TestFolder.Package + "." + options.TestFolder.DataStream, - } - // Use the package's parent directory as base path, so that the relative paths // for each class (pipeline) include the package name. This prevents paths for // different packages colliding (i.e. a lot of packages have a "log" datastream @@ -52,6 +47,15 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe basePath = basePath[:len(dir)] + relativePath := strings.TrimPrefix(options.PackageRootPath, dir) + relativePath = strings.TrimPrefix(relativePath, "/") + baseFolder := filepath.Dir(relativePath) + + // Construct the Cobertura report. + pkg := &testrunner.CoberturaPackage{ + Name: baseFolder + "." + options.TestFolder.Package + "." + options.TestFolder.DataStream, + } + coverage := &testrunner.CoberturaCoverage{ Sources: []*testrunner.CoberturaSource{ { From af490105e947d8d291a2f7d1f9c4787f87433ec5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 12 Jan 2024 13:10:17 +0100 Subject: [PATCH 08/35] Add different lines in manifest.yml for each type of test --- internal/testrunner/coverageoutput.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index 722a90a596..ac4979ba71 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -334,6 +334,16 @@ func verifyTestExpected(packageRootPath string, dataStreamName string, testType func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) *CoberturaCoverage { var classes []*CoberturaClass + var lineNumberPerTestType map[string]int = map[string]int{ + "asset": 1, + "pipeline": 2, + "system": 3, + "static": 4, + } + lineNumber, ok := lineNumberPerTestType[string(details.testType)] + if !ok { + lineNumber = 5 + } for dataStream, testCases := range details.dataStreams { if dataStream == "" { continue // ignore tests running in the package context (not data stream), mostly referring to installed assets @@ -344,12 +354,12 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) if len(testCases) == 0 { methods = append(methods, &CoberturaMethod{ Name: "Missing", - Lines: []*CoberturaLine{{Number: 1, Hits: 0}}, + Lines: []*CoberturaLine{{Number: lineNumber, Hits: 0}}, }) } else { methods = append(methods, &CoberturaMethod{ Name: "OK", - Lines: []*CoberturaLine{{Number: 1, Hits: 1}}, + Lines: []*CoberturaLine{{Number: lineNumber, Hits: 1}}, }) } From 9cb94d8e6b22dbce897f2835b30a2a8741301078 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 12 Jan 2024 13:10:34 +0100 Subject: [PATCH 09/35] Add lines into CoberturaClass --- internal/testrunner/coverageoutput.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index ac4979ba71..38ebbe8db0 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -350,23 +350,27 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) } var methods []*CoberturaMethod + var lines []*CoberturaLine if len(testCases) == 0 { methods = append(methods, &CoberturaMethod{ Name: "Missing", Lines: []*CoberturaLine{{Number: lineNumber, Hits: 0}}, }) + lines = append(lines, []*CoberturaLine{{Number: lineNumber, Hits: 0}}...) } else { methods = append(methods, &CoberturaMethod{ Name: "OK", Lines: []*CoberturaLine{{Number: lineNumber, Hits: 1}}, }) + lines = append(lines, []*CoberturaLine{{Number: lineNumber, Hits: 1}}...) } aClass := &CoberturaClass{ Name: string(details.testType), Filename: path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml"), Methods: methods, + Lines: lines, } classes = append(classes, aClass) } From 26ffdac5b7879d1bfb1977c498ebf574c849181e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 12 Jan 2024 13:13:59 +0100 Subject: [PATCH 10/35] Check coverage xml files --- .buildkite/pipeline.trigger.integration.tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 5a27da9986..95105a35bf 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -46,6 +46,7 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do echo " - build/test-results/*.xml" echo " - build/elastic-stack-dump/check-*/logs/*.log" echo " - build/elastic-stack-dump/check-*/logs/fleet-server-internal/**/*" + echo " - build/test-coverage/coverage-*.xml" if [[ $test =~ with-kind$ ]]; then echo " - build/kubectl-dump.txt" fi @@ -63,6 +64,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " provider: \"gcp\"" echo " artifact_paths:" echo " - build/test-results/*.xml" + echo " - build/test-coverage/coverage-*.xml" done popd > /dev/null @@ -79,6 +81,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " provider: \"gcp\"" echo " artifact_paths:" echo " - build/test-results/*.xml" + echo " - build/test-coverage/coverage-*.xml" done popd > /dev/null From d58434049c10a7fe00fb3cc1044dc76e0eb560f1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 12 Jan 2024 13:16:23 +0100 Subject: [PATCH 11/35] Remove hits from CoberturaMethod in tests --- internal/testrunner/coverageoutput_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/testrunner/coverageoutput_test.go b/internal/testrunner/coverageoutput_test.go index 5cdd234c30..2f561686ce 100644 --- a/internal/testrunner/coverageoutput_test.go +++ b/internal/testrunner/coverageoutput_test.go @@ -111,7 +111,6 @@ func TestCoberturaCoverage_Merge(t *testing.T) { Methods: []*CoberturaMethod{ { Name: "foo", - Hits: 2, Lines: []*CoberturaLine{ { Number: 13, @@ -125,7 +124,6 @@ func TestCoberturaCoverage_Merge(t *testing.T) { }, { Name: "bar", - Hits: 1, Lines: []*CoberturaLine{ { Number: 24, @@ -163,7 +161,6 @@ func TestCoberturaCoverage_Merge(t *testing.T) { Methods: []*CoberturaMethod{ { Name: "foo", - Hits: 1, Lines: []*CoberturaLine{ { Number: 13, @@ -177,7 +174,6 @@ func TestCoberturaCoverage_Merge(t *testing.T) { }, { Name: "bar", - Hits: 1, Lines: []*CoberturaLine{ { Number: 24, @@ -217,7 +213,6 @@ func TestCoberturaCoverage_Merge(t *testing.T) { Methods: []*CoberturaMethod{ { Name: "foo", - Hits: 3, Lines: []*CoberturaLine{ { Number: 13, @@ -231,7 +226,6 @@ func TestCoberturaCoverage_Merge(t *testing.T) { }, { Name: "bar", - Hits: 2, Lines: []*CoberturaLine{ { Number: 24, From d6e001ef25c20a9588a42123ad09dd41360806c7 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 12 Jan 2024 15:42:24 +0100 Subject: [PATCH 12/35] Replace / chars in paths for package names --- internal/testrunner/coverageoutput.go | 2 +- internal/testrunner/runners/pipeline/coverage.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index 38ebbe8db0..ce2a40d61b 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -379,7 +379,7 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) Timestamp: time.Now().UnixNano(), Packages: []*CoberturaPackage{ { - Name: strings.TrimSuffix(baseFolder, "/") + "." + details.packageName, + Name: strings.Replace(strings.TrimSuffix(baseFolder, "/"), "/", ".", -1) + "." + details.packageName, Classes: classes, }, }, diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index 34e4ccff0d..aaac508177 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -53,7 +53,7 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe // Construct the Cobertura report. pkg := &testrunner.CoberturaPackage{ - Name: baseFolder + "." + options.TestFolder.Package + "." + options.TestFolder.DataStream, + Name: strings.Replace(baseFolder, "/", ".", -1) + "." + options.TestFolder.Package + "." + options.TestFolder.DataStream, } coverage := &testrunner.CoberturaCoverage{ From e9e9826427e77b33d2db2fd4df54062d7a035ab6 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 12 Jan 2024 16:32:14 +0100 Subject: [PATCH 13/35] Avoid setting sources for test coverage pipeline --- internal/testrunner/runners/pipeline/coverage.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index aaac508177..532ad2c2e4 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -57,11 +57,11 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe } coverage := &testrunner.CoberturaCoverage{ - Sources: []*testrunner.CoberturaSource{ - { - Path: basePath, - }, - }, + // Sources: []*testrunner.CoberturaSource{ + // { + // Path: basePath, + // }, + // }, Packages: []*testrunner.CoberturaPackage{pkg}, Timestamp: time.Now().UnixNano(), } From 83ac6bf10a789d0bbd63e4d211411b98cbc87c0d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jan 2024 11:18:47 +0100 Subject: [PATCH 14/35] Add generic output - WIP --- cmd/testrunner.go | 9 +- internal/cobraext/flags.go | 5 +- internal/testrunner/coverageoutput.go | 50 +++++-- internal/testrunner/coverageoutput_test.go | 2 +- internal/testrunner/genericoutput.go | 132 ++++++++++++++++++ internal/testrunner/runners/asset/runner.go | 2 +- .../testrunner/runners/pipeline/coverage.go | 102 ++++++++++---- .../testrunner/runners/pipeline/runner.go | 4 + internal/testrunner/testrunner.go | 3 +- 9 files changed, 266 insertions(+), 43 deletions(-) create mode 100644 internal/testrunner/genericoutput.go diff --git a/cmd/testrunner.go b/cmd/testrunner.go index 242844257d..d088b0dcb1 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -71,6 +71,7 @@ func setupTestCommand() *cobraext.Command { cmd.PersistentFlags().StringP(cobraext.ReportFormatFlagName, "", string(formats.ReportFormatHuman), cobraext.ReportFormatFlagDescription) cmd.PersistentFlags().StringP(cobraext.ReportOutputFlagName, "", string(outputs.ReportOutputSTDOUT), cobraext.ReportOutputFlagDescription) cmd.PersistentFlags().BoolP(cobraext.TestCoverageFlagName, "", false, cobraext.TestCoverageFlagDescription) + cmd.PersistentFlags().StringP(cobraext.TestCoverageFormatFlagName, "", "cobertura", cobraext.TestCoverageFormatFlagDescription) cmd.PersistentFlags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription) cmd.PersistentFlags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription) cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar)) @@ -127,6 +128,11 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command return cobraext.FlagParsingError(err, cobraext.TestCoverageFlagName) } + testCoverageType, err := cmd.Flags().GetString(cobraext.TestCoverageFormatFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.TestCoverageFormatFlagName) + } + packageRootPath, found, err := packages.FindPackageRoot() if !found { return errors.New("package root not found") @@ -246,6 +252,7 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command DeferCleanup: deferCleanup, ServiceVariant: variantFlag, WithCoverage: testCoverage, + CoverageType: testCoverageType, }) results = append(results, r...) @@ -266,7 +273,7 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command } if testCoverage { - err := testrunner.WriteCoverage(packageRootPath, manifest.Name, runner.Type(), results) + err := testrunner.WriteCoverage(packageRootPath, manifest.Name, runner.Type(), results, testCoverageType) if err != nil { return fmt.Errorf("error writing test coverage: %w", err) } diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 4cd1580e08..e1dd88b027 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -190,7 +190,10 @@ const ( StatusExtraInfoFlagDescription = "show additional information (comma-separated values: \"%s\")" TestCoverageFlagName = "test-coverage" - TestCoverageFlagDescription = "generate Cobertura test coverage reports" + TestCoverageFlagDescription = "enable test coverage reports" + + TestCoverageFormatFlagName = "coverage-format" + TestCoverageFormatFlagDescription = "set format for coverage reports: \"cobertura,generic\" (default cobertura)" VariantFlagName = "variant" VariantFlagDescription = "service variant" diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index ce2a40d61b..83cf8d1c01 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -20,13 +20,19 @@ import ( "github.com/elastic/elastic-package/internal/multierror" ) +type CoverageReport interface { + TimeStamp() int64 + Merge(CoverageReport) error + Bytes() ([]byte, error) +} + const coverageDtd = `` type testCoverageDetails struct { packageName string testType TestType dataStreams map[string][]string // : - cobertura *CoberturaCoverage // For tests to provide custom Cobertura results. + coverage CoverageReport // For tests to provide custom Cobertura results. errors multierror.Error } @@ -47,12 +53,12 @@ func (tcd *testCoverageDetails) withTestResults(results []TestResult) *testCover tcd.dataStreams[result.DataStream] = []string{} } tcd.dataStreams[result.DataStream] = append(tcd.dataStreams[result.DataStream], result.Name) - if tcd.cobertura != nil && result.Coverage != nil { - if err := tcd.cobertura.merge(result.Coverage); err != nil { + if tcd.coverage != nil && result.Coverage != nil { + if err := tcd.coverage.Merge(result.Coverage); err != nil { tcd.errors = append(tcd.errors, fmt.Errorf("can't merge Cobertura coverage for test `%s`: %w", result.Name, err)) } - } else if tcd.cobertura == nil { - tcd.cobertura = result.Coverage + } else if tcd.coverage == nil { + tcd.coverage = result.Coverage } } return tcd @@ -115,10 +121,14 @@ type CoberturaLine struct { Hits int64 `xml:"hits,attr"` } -func (c *CoberturaCoverage) bytes() ([]byte, error) { +func (c *CoberturaCoverage) TimeStamp() int64 { + return c.Timestamp +} + +func (c *CoberturaCoverage) Bytes() ([]byte, error) { out, err := xml.MarshalIndent(&c, "", " ") if err != nil { - return nil, fmt.Errorf("unable to format test results as xUnit: %w", err) + return nil, fmt.Errorf("unable to format test results as Coverage: %w", err) } var buffer bytes.Buffer @@ -184,7 +194,12 @@ func (p *CoberturaPackage) merge(b *CoberturaPackage) error { } // merge merges two coverage reports. -func (c *CoberturaCoverage) merge(b *CoberturaCoverage) error { +func (c *CoberturaCoverage) Merge(other CoverageReport) error { + b, ok := other.(*CoberturaCoverage) + if !ok { + return fmt.Errorf("not able to assert report to be merged as CoberturaCoverage") + + } // Merge source paths for _, path := range b.Sources { found := false @@ -234,7 +249,7 @@ func (c *CoberturaCoverage) merge(b *CoberturaCoverage) error { // WriteCoverage function calculates test coverage for the given package. // It requires to execute tests for all data streams (same test type), so the coverage can be calculated properly. -func WriteCoverage(packageRootPath, packageName string, testType TestType, results []TestResult) error { +func WriteCoverage(packageRootPath, packageName string, testType TestType, results []TestResult, testCoverageType string) error { details, err := collectTestCoverageDetails(packageRootPath, packageName, testType, results) if err != nil { return fmt.Errorf("can't collect test coverage details: %w", err) @@ -250,9 +265,15 @@ func WriteCoverage(packageRootPath, packageName string, testType TestType, resul baseFolder := filepath.Dir(relativePath) // Use provided cobertura report, or generate a custom report if not available. - report := details.cobertura + report := details.coverage if report == nil { - report = transformToCoberturaReport(details, baseFolder) + switch testCoverageType { + case "cobertura": + report = transformToCoberturaReport(details, baseFolder) + case "generic": + report = transformToGenericCoverageReport(details, baseFolder) + } + } err = writeCoverageReportFile(report, packageName) @@ -274,6 +295,7 @@ func collectTestCoverageDetails(packageRootPath, packageName string, testType Te if len(details.errors) > 0 { return nil, details.errors } + fmt.Printf("Details for tests:\n%+v", details) return details, nil } @@ -386,7 +408,7 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) } } -func writeCoverageReportFile(report *CoberturaCoverage, packageName string) error { +func writeCoverageReportFile(report CoverageReport, packageName string) error { dest, err := testCoverageReportsDir() if err != nil { return fmt.Errorf("could not determine test coverage reports folder: %w", err) @@ -400,10 +422,10 @@ func writeCoverageReportFile(report *CoberturaCoverage, packageName string) erro } } - fileName := fmt.Sprintf("coverage-%s-%d-report.xml", packageName, report.Timestamp) + fileName := fmt.Sprintf("coverage-%s-%d-report.xml", packageName, report.TimeStamp()) filePath := filepath.Join(dest, fileName) - b, err := report.bytes() + b, err := report.Bytes() if err != nil { return fmt.Errorf("can't marshal test coverage report: %w", err) } diff --git a/internal/testrunner/coverageoutput_test.go b/internal/testrunner/coverageoutput_test.go index 2f561686ce..c92c70933f 100644 --- a/internal/testrunner/coverageoutput_test.go +++ b/internal/testrunner/coverageoutput_test.go @@ -257,7 +257,7 @@ func TestCoberturaCoverage_Merge(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := tt.rhs.merge(&tt.lhs) + err := tt.rhs.Merge(&tt.lhs) if !tt.wantErr { if !assert.NoError(t, err) { t.Fatal(err) diff --git a/internal/testrunner/genericoutput.go b/internal/testrunner/genericoutput.go new file mode 100644 index 0000000000..1c54b49764 --- /dev/null +++ b/internal/testrunner/genericoutput.go @@ -0,0 +1,132 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package testrunner + +import ( + "bytes" + "encoding/xml" + "fmt" + "path" + "time" +) + +// GenericCoverage is the root element for a Cobertura XML report. +type GenericCoverage struct { + XMLName xml.Name `xml:"coverage"` + Version int64 `xml:"version,attr"` + Files []*GenericFile `xml:"file"` + Timestamp int64 `xml:"-"` + TestType string `xml:",comment"` +} + +type GenericFile struct { + Path string `xml:"path,attr"` + Lines []*GenericLine `xml:"lineToCover"` +} + +type GenericLine struct { + LineNumber int64 `xml:"lineNumber,attr"` + Covered bool `xml:"covered,attr"` +} + +func (c *GenericCoverage) TimeStamp() int64 { + return c.Timestamp +} + +func (c *GenericCoverage) Bytes() ([]byte, error) { + out, err := xml.MarshalIndent(&c, "", " ") + if err != nil { + return nil, fmt.Errorf("unable to format test results as Coverage: %w", err) + } + + var buffer bytes.Buffer + buffer.WriteString(xml.Header) + buffer.WriteString("\n") + buffer.Write(out) + return buffer.Bytes(), nil +} + +func (c *GenericFile) merge(b *GenericFile) error { + // Merge files + for _, coverageLine := range b.Lines { + found := false + for _, existingLine := range c.Lines { + if existingLine.LineNumber == coverageLine.LineNumber { + found = true + break + } + } + if !found { + c.Lines = append(c.Lines, coverageLine) + } + } + return nil +} + +// merge merges two coverage reports. +func (c *GenericCoverage) Merge(other CoverageReport) error { + b, ok := other.(*GenericCoverage) + if !ok { + return fmt.Errorf("not able to assert report to be merged as GenericCoverage") + } + // Merge files + for _, coverageFile := range b.Files { + var target *GenericFile + for _, existingFile := range c.Files { + if existingFile.Path == coverageFile.Path { + target = existingFile + break + } + } + if target != nil { + if err := target.merge(coverageFile); err != nil { + return err + } + } else { + c.Files = append(c.Files, coverageFile) + } + } + return nil +} + +func transformToGenericCoverageReport(details *testCoverageDetails, baseFolder string) *GenericCoverage { + var lineNumberPerTestType map[string]int64 = map[string]int64{ + "asset": 1, + "pipeline": 2, + "system": 3, + "static": 4, + } + lineNumber, ok := lineNumberPerTestType[string(details.testType)] + if !ok { + lineNumber = 5 + } + var files []*GenericFile + for dataStream, testCases := range details.dataStreams { + if dataStream == "" { + continue // ignore tests running in the package context (not data stream), mostly referring to installed assets + } + + dataStreamPath := path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml") + + if len(testCases) == 0 { + files = append(files, &GenericFile{ + Path: dataStreamPath, + Lines: []*GenericLine{{LineNumber: lineNumber, Covered: false}}, + }) + } else { + files = append(files, &GenericFile{ + Path: dataStreamPath, + Lines: []*GenericLine{{LineNumber: lineNumber, Covered: true}}, + }) + } + } + + return &GenericCoverage{ + Timestamp: time.Now().UnixNano(), + Version: 1, + Files: files, + TestType: fmt.Sprintf("Coverage for %s test", details.testType), + } +} diff --git a/internal/testrunner/runners/asset/runner.go b/internal/testrunner/runners/asset/runner.go index d70ea19ec8..c7ec2b305f 100644 --- a/internal/testrunner/runners/asset/runner.go +++ b/internal/testrunner/runners/asset/runner.go @@ -118,7 +118,7 @@ func (r *runner) run() ([]testrunner.TestResult, error) { return result.WithError(fmt.Errorf("could not load expected package assets: %w", err)) } - results := make([]testrunner.TestResult, 0, len(expectedAssets)) + results := make([]testrunner.TestResult, 0, len(expectedAssets)+1) for _, e := range expectedAssets { rc := testrunner.NewResultComposer(testrunner.TestResult{ Name: fmt.Sprintf("%s %s is loaded", e.Type, e.ID), diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index 532ad2c2e4..ffa9abdbc7 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -18,7 +18,7 @@ import ( ) // GetPipelineCoverage returns a coverage report for the provided set of ingest pipelines. -func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipeline) (*testrunner.CoberturaCoverage, error) { +func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipeline) (testrunner.CoverageReport, error) { dataStreamPath, found, err := packages.FindDataStreamRootForPath(options.TestFolder.Path) if err != nil { return nil, fmt.Errorf("locating data_stream root failed: %w", err) @@ -56,51 +56,73 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe Name: strings.Replace(baseFolder, "/", ".", -1) + "." + options.TestFolder.Package + "." + options.TestFolder.DataStream, } - coverage := &testrunner.CoberturaCoverage{ - // Sources: []*testrunner.CoberturaSource{ - // { - // Path: basePath, - // }, - // }, - Packages: []*testrunner.CoberturaPackage{pkg}, - Timestamp: time.Now().UnixNano(), + if options.CoverageType == "cobertura" { + cobertura := &testrunner.CoberturaCoverage{ + // Sources: []*testrunner.CoberturaSource{ + // { + // Path: basePath, + // }, + // }, + Packages: []*testrunner.CoberturaPackage{pkg}, + Timestamp: time.Now().UnixNano(), + } + + // Calculate coverage for each pipeline + for _, pipeline := range pipelines { + covered, class, err := coberturaForSinglePipeline(pipeline, stats, basePath, dataStreamPath) + if err != nil { + return nil, fmt.Errorf("error calculating coverage for pipeline '%s': %w", pipeline.Filename(), err) + } + pkg.Classes = append(pkg.Classes, class) + cobertura.LinesValid += int64(len(class.Methods)) + cobertura.LinesCovered += covered + } + return cobertura, nil } - // Calculate coverage for each pipeline - for _, pipeline := range pipelines { - covered, class, err := coverageForSinglePipeline(pipeline, stats, basePath, dataStreamPath) - if err != nil { - return nil, fmt.Errorf("error calculating coverage for pipeline '%s': %w", pipeline.Filename(), err) + if options.CoverageType == "generic" { + coverage := &testrunner.GenericCoverage{ + Timestamp: time.Now().UnixNano(), + TestType: "Cobertura for pipeline test", } - pkg.Classes = append(pkg.Classes, class) - coverage.LinesValid += int64(len(class.Methods)) - coverage.LinesCovered += covered + + // Calculate coverage for each pipeline + for _, pipeline := range pipelines { + _, file, err := genericCoverageForSinglePipeline(pipeline, stats, basePath, dataStreamPath) + if err != nil { + return nil, fmt.Errorf("error calculating coverage for pipeline '%s': %w", pipeline.Filename(), err) + } + coverage.Files = append(coverage.Files, file) + } + return coverage, nil + } - return coverage, nil + + return nil, fmt.Errorf("unrecognised coverage type") } -func coverageForSinglePipeline(pipeline ingest.Pipeline, stats ingest.PipelineStatsMap, basePath, dataStreamPath string) (linesCovered int64, class *testrunner.CoberturaClass, err error) { +func pipelineDataForCoverage(pipeline ingest.Pipeline, stats ingest.PipelineStatsMap, basePath, dataStreamPath string) (string, string, []ingest.Processor, ingest.PipelineStats, error) { // Load the list of main processors from the pipeline source code, annotated with line numbers. src, err := pipeline.Processors() if err != nil { - return 0, nil, err + return "", "", nil, ingest.PipelineStats{}, err } pstats, found := stats[pipeline.Name] if !found { - return 0, nil, fmt.Errorf("pipeline '%s' not installed in Elasticsearch", pipeline.Name) + return "", "", nil, ingest.PipelineStats{}, fmt.Errorf("pipeline '%s' not installed in Elasticsearch", pipeline.Name) } // Ensure there is no inconsistency in the list of processors in stats vs obtained from source. if len(src) != len(pstats.Processors) { - return 0, nil, fmt.Errorf("processor count mismatch for %s (src:%d stats:%d)", pipeline.Filename(), len(src), len(pstats.Processors)) + return "", "", nil, ingest.PipelineStats{}, fmt.Errorf("processor count mismatch for %s (src:%d stats:%d)", pipeline.Filename(), len(src), len(pstats.Processors)) } for idx, st := range pstats.Processors { // Check that we have the expected type of processor, except for `compound` processors. // Elasticsearch will return a `compound` processor in the case of `foreach` and // any processor that defines `on_failure` processors. if st.Type != "compound" && st.Type != src[idx].Type { - return 0, nil, fmt.Errorf("processor type mismatch for %s processor %d (src:%s stats:%s)", pipeline.Filename(), idx, src[idx].Type, st.Type) + return "", "", nil, ingest.PipelineStats{}, fmt.Errorf("processor type mismatch for %s processor %d (src:%s stats:%s)", pipeline.Filename(), idx, src[idx].Type, st.Type) } } @@ -116,9 +138,41 @@ func coverageForSinglePipeline(pipeline ingest.Pipeline, stats ingest.PipelineSt pipelinePath := filepath.Join(dataStreamPath, "elasticsearch", "ingest_pipeline", pipeline.Filename()) pipelineRelPath, err := filepath.Rel(basePath, pipelinePath) if err != nil { - return 0, nil, fmt.Errorf("cannot create relative path to pipeline file. Package root: '%s', pipeline path: '%s': %w", basePath, pipelinePath, err) + return "", "", nil, ingest.PipelineStats{}, fmt.Errorf("cannot create relative path to pipeline file. Package root: '%s', pipeline path: '%s': %w", basePath, pipelinePath, err) + } + + return pipelineName, pipelineRelPath, src, pstats, nil +} + +func genericCoverageForSinglePipeline(pipeline ingest.Pipeline, stats ingest.PipelineStatsMap, basePath, dataStreamPath string) (linesCovered int64, class *testrunner.GenericFile, err error) { + _, pipelineRelPath, src, pstats, err := pipelineDataForCoverage(pipeline, stats, basePath, dataStreamPath) + if err != nil { + return 0, nil, err + } + // Report every pipeline as a "class". + file := &testrunner.GenericFile{ + Path: pipelineRelPath, } + for idx, srcProc := range src { + if pstats.Processors[idx].Stats.Count > 0 { + linesCovered++ + } + for num := srcProc.FirstLine; num <= srcProc.LastLine; num++ { + line := &testrunner.GenericLine{ + LineNumber: int64(num), + Covered: pstats.Processors[idx].Stats.Count > 0, + } + file.Lines = append(file.Lines, line) + } + } + return linesCovered, file, nil +} +func coberturaForSinglePipeline(pipeline ingest.Pipeline, stats ingest.PipelineStatsMap, basePath, dataStreamPath string) (linesCovered int64, class *testrunner.CoberturaClass, err error) { + pipelineName, pipelineRelPath, src, pstats, err := pipelineDataForCoverage(pipeline, stats, basePath, dataStreamPath) + if err != nil { + return 0, nil, err + } // Report every pipeline as a "class". class = &testrunner.CoberturaClass{ Name: pipelineName, diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index fce07f9fa9..12c6ea6d2b 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -116,6 +116,7 @@ func (r *runner) run() ([]testrunner.TestResult, error) { return nil, errors.New("missing Elasticsearch client") } + fmt.Printf("TestFolder.Path: %s\n", r.options.TestFolder.Path) dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.options.TestFolder.Path) if err != nil { return nil, fmt.Errorf("locating data_stream root failed: %w", err) @@ -135,6 +136,7 @@ func (r *runner) run() ([]testrunner.TestResult, error) { return nil, fmt.Errorf("failed to read manifest: %w", err) } + fmt.Printf("TestFolder.DataStream: %s\n", r.options.TestFolder.DataStream) dsManifest, err := packages.ReadDataStreamManifestFromPackageRoot(r.options.PackageRootPath, r.options.TestFolder.DataStream) if err != nil { return nil, fmt.Errorf("failed to read data stream manifest: %w", err) @@ -244,6 +246,8 @@ func (r *runner) run() ([]testrunner.TestResult, error) { results = append(results, tr) } + // Add empty coverage for data Streams wihtout tests + return results, nil } diff --git a/internal/testrunner/testrunner.go b/internal/testrunner/testrunner.go index 2c03e01341..d9df974b08 100644 --- a/internal/testrunner/testrunner.go +++ b/internal/testrunner/testrunner.go @@ -33,6 +33,7 @@ type TestOptions struct { DeferCleanup time.Duration ServiceVariant string WithCoverage bool + CoverageType string } // TestRunner is the interface all test runners must implement. @@ -92,7 +93,7 @@ type TestResult struct { Skipped *SkipConfig // Coverage details in Cobertura format (optional). - Coverage *CoberturaCoverage + Coverage CoverageReport } // ResultComposer wraps a TestResult and provides convenience methods for From 82dc88f30bec4bdfed4fe5353613de34239aaf6e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jan 2024 12:15:32 +0100 Subject: [PATCH 15/35] Some refactors --- internal/testrunner/coverageoutput.go | 53 ++++++++++++------- .../testrunner/runners/pipeline/coverage.go | 10 ++-- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index 83cf8d1c01..bae6679ec9 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -250,37 +250,42 @@ func (c *CoberturaCoverage) Merge(other CoverageReport) error { // WriteCoverage function calculates test coverage for the given package. // It requires to execute tests for all data streams (same test type), so the coverage can be calculated properly. func WriteCoverage(packageRootPath, packageName string, testType TestType, results []TestResult, testCoverageType string) error { + report, err := CreateCoverageReport(packageRootPath, packageName, testType, results, testCoverageType) + if err != nil { + return fmt.Errorf("can't create coverage report: %w", err) + } + + err = writeCoverageReportFile(report, packageName) + if err != nil { + return fmt.Errorf("can't write test coverage report file: %w", err) + } + return nil +} + +func CreateCoverageReport(packageRootPath, packageName string, testType TestType, results []TestResult, CoverageFormat string) (CoverageReport, error) { details, err := collectTestCoverageDetails(packageRootPath, packageName, testType, results) if err != nil { - return fmt.Errorf("can't collect test coverage details: %w", err) + return nil, fmt.Errorf("can't collect test coverage details: %w", err) } + if details.coverage != nil { + // Use provided cobertura report + return details.coverage, nil + } + + // generate a custom report if not available dir, err := files.FindRepositoryRootDirectory() if err != nil { - return err + return nil, err } relativePath := strings.TrimPrefix(packageRootPath, dir) relativePath = strings.TrimPrefix(relativePath, "/") baseFolder := filepath.Dir(relativePath) - // Use provided cobertura report, or generate a custom report if not available. - report := details.coverage - if report == nil { - switch testCoverageType { - case "cobertura": - report = transformToCoberturaReport(details, baseFolder) - case "generic": - report = transformToGenericCoverageReport(details, baseFolder) - } + report := transformToCoverageReport(details, baseFolder, CoverageFormat) - } - - err = writeCoverageReportFile(report, packageName) - if err != nil { - return fmt.Errorf("can't write test coverage report file: %w", err) - } - return nil + return report, nil } func collectTestCoverageDetails(packageRootPath, packageName string, testType TestType, results []TestResult) (*testCoverageDetails, error) { @@ -354,6 +359,18 @@ func verifyTestExpected(packageRootPath string, dataStreamName string, testType return true, nil } +func transformToCoverageReport(details *testCoverageDetails, baseFolder, coverageFormat string) CoverageReport { + if coverageFormat == "cobertura" { + return transformToCoberturaReport(details, baseFolder) + } + + if coverageFormat == "generic" { + return transformToGenericCoverageReport(details, baseFolder) + } + + return nil +} + func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) *CoberturaCoverage { var classes []*CoberturaClass var lineNumberPerTestType map[string]int = map[string]int{ diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index ffa9abdbc7..df263a67c7 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -58,11 +58,11 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe if options.CoverageType == "cobertura" { cobertura := &testrunner.CoberturaCoverage{ - // Sources: []*testrunner.CoberturaSource{ - // { - // Path: basePath, - // }, - // }, + Sources: []*testrunner.CoberturaSource{ + { + Path: basePath, + }, + }, Packages: []*testrunner.CoberturaPackage{pkg}, Timestamp: time.Now().UnixNano(), } From 811e2cd205624f13f9ffe7ae942624da888a2886 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jan 2024 13:36:15 +0100 Subject: [PATCH 16/35] Show tests for input packages Pending to load assets for input packages --- cmd/testrunner.go | 2 +- internal/packages/assets.go | 2 ++ internal/testrunner/coverageoutput.go | 31 ++++++++++++++++----------- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/cmd/testrunner.go b/cmd/testrunner.go index d088b0dcb1..58fbebf251 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -273,7 +273,7 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command } if testCoverage { - err := testrunner.WriteCoverage(packageRootPath, manifest.Name, runner.Type(), results, testCoverageType) + err := testrunner.WriteCoverage(packageRootPath, manifest.Name, manifest.Type, runner.Type(), results, testCoverageType) if err != nil { return fmt.Errorf("error writing test coverage: %w", err) } diff --git a/internal/packages/assets.go b/internal/packages/assets.go index 2ed2ec6ad6..4f3a98bc21 100644 --- a/internal/packages/assets.go +++ b/internal/packages/assets.go @@ -153,6 +153,8 @@ func loadElasticsearchAssets(pkgRootPath string) ([]Asset, error) { } } + // TODO add assets for input packages + return assets, nil } diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go index bae6679ec9..21df007e17 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coverageoutput.go @@ -30,14 +30,15 @@ const coverageDtd = ` : coverage CoverageReport // For tests to provide custom Cobertura results. errors multierror.Error } -func newTestCoverageDetails(packageName string, testType TestType) *testCoverageDetails { - return &testCoverageDetails{packageName: packageName, testType: testType, dataStreams: map[string][]string{}} +func newTestCoverageDetails(packageName, packageType string, testType TestType) *testCoverageDetails { + return &testCoverageDetails{packageName: packageName, packageType: packageType, testType: testType, dataStreams: map[string][]string{}} } func (tcd *testCoverageDetails) withUncoveredDataStreams(dataStreams []string) *testCoverageDetails { @@ -249,8 +250,8 @@ func (c *CoberturaCoverage) Merge(other CoverageReport) error { // WriteCoverage function calculates test coverage for the given package. // It requires to execute tests for all data streams (same test type), so the coverage can be calculated properly. -func WriteCoverage(packageRootPath, packageName string, testType TestType, results []TestResult, testCoverageType string) error { - report, err := CreateCoverageReport(packageRootPath, packageName, testType, results, testCoverageType) +func WriteCoverage(packageRootPath, packageName, packageType string, testType TestType, results []TestResult, testCoverageType string) error { + report, err := createCoverageReport(packageRootPath, packageName, packageType, testType, results, testCoverageType) if err != nil { return fmt.Errorf("can't create coverage report: %w", err) } @@ -262,14 +263,14 @@ func WriteCoverage(packageRootPath, packageName string, testType TestType, resul return nil } -func CreateCoverageReport(packageRootPath, packageName string, testType TestType, results []TestResult, CoverageFormat string) (CoverageReport, error) { - details, err := collectTestCoverageDetails(packageRootPath, packageName, testType, results) +func createCoverageReport(packageRootPath, packageName, packageType string, testType TestType, results []TestResult, CoverageFormat string) (CoverageReport, error) { + details, err := collectTestCoverageDetails(packageRootPath, packageName, packageType, testType, results) if err != nil { return nil, fmt.Errorf("can't collect test coverage details: %w", err) } if details.coverage != nil { - // Use provided cobertura report + // Use provided coverage report return details.coverage, nil } @@ -288,19 +289,18 @@ func CreateCoverageReport(packageRootPath, packageName string, testType TestType return report, nil } -func collectTestCoverageDetails(packageRootPath, packageName string, testType TestType, results []TestResult) (*testCoverageDetails, error) { +func collectTestCoverageDetails(packageRootPath, packageName, packageType string, testType TestType, results []TestResult) (*testCoverageDetails, error) { withoutTests, err := findDataStreamsWithoutTests(packageRootPath, testType) if err != nil { return nil, fmt.Errorf("can't find data streams without tests: %w", err) } - details := newTestCoverageDetails(packageName, testType). + details := newTestCoverageDetails(packageName, packageType, testType). withUncoveredDataStreams(withoutTests). withTestResults(results) if len(details.errors) > 0 { return nil, details.errors } - fmt.Printf("Details for tests:\n%+v", details) return details, nil } @@ -383,8 +383,9 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) if !ok { lineNumber = 5 } + for dataStream, testCases := range details.dataStreams { - if dataStream == "" { + if dataStream == "" && details.packageType == "integration" { continue // ignore tests running in the package context (not data stream), mostly referring to installed assets } @@ -405,9 +406,15 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) lines = append(lines, []*CoberturaLine{{Number: lineNumber, Hits: 1}}...) } + fileName := path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml") + if dataStream == "" { + // input package + fileName = path.Join(baseFolder, details.packageName, "manifest.yml") + } + aClass := &CoberturaClass{ Name: string(details.testType), - Filename: path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml"), + Filename: fileName, Methods: methods, Lines: lines, } From 46ef78bb570624f72af6179c6befde2119d75c9d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jan 2024 17:57:24 +0100 Subject: [PATCH 17/35] Refactor and tests --- cmd/testrunner.go | 7 + internal/testrunner/coberturacoverage.go | 196 +++++++++++++ ...tput_test.go => coberturacoverage_test.go} | 0 .../{coverageoutput.go => coveragereport.go} | 235 ++-------------- internal/testrunner/coveragereport_test.go | 257 ++++++++++++++++++ .../{genericoutput.go => genericcobertura.go} | 20 +- internal/testrunner/genericcobertura_test.go | 92 +++++++ .../testrunner/runners/pipeline/runner.go | 27 +- 8 files changed, 609 insertions(+), 225 deletions(-) create mode 100644 internal/testrunner/coberturacoverage.go rename internal/testrunner/{coverageoutput_test.go => coberturacoverage_test.go} (100%) rename internal/testrunner/{coverageoutput.go => coveragereport.go} (57%) create mode 100644 internal/testrunner/coveragereport_test.go rename internal/testrunner/{genericoutput.go => genericcobertura.go} (86%) create mode 100644 internal/testrunner/genericcobertura_test.go diff --git a/cmd/testrunner.go b/cmd/testrunner.go index 58fbebf251..b7834a2324 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -154,6 +154,7 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command signal.Enable() var testFolders []testrunner.TestFolder + // var dataStreamWithoutTests []testrunner.TestFolder if hasDataStreams && runner.CanRunPerDataStream() { var dataStreams []string // We check for the existence of the data streams flag before trying to @@ -240,6 +241,8 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command } } + // dataStreamsTested := map[string]bool{} + var results []testrunner.TestResult for _, folder := range testFolders { r, err := testrunner.Run(testType, testrunner.TestOptions{ @@ -255,6 +258,8 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command CoverageType: testCoverageType, }) + // dataStreamsTested[folder.DataStream] = true + results = append(results, r...) if err != nil { @@ -262,6 +267,8 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command } } + // for dataStream := datastreams + format := testrunner.TestReportFormat(reportFormat) report, err := testrunner.FormatReport(format, results) if err != nil { diff --git a/internal/testrunner/coberturacoverage.go b/internal/testrunner/coberturacoverage.go new file mode 100644 index 0000000000..622951a314 --- /dev/null +++ b/internal/testrunner/coberturacoverage.go @@ -0,0 +1,196 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package testrunner + +import ( + "bytes" + "encoding/xml" + "fmt" +) + +const coverageDtd = `` + +// CoberturaCoverage is the root element for a Cobertura XML report. +type CoberturaCoverage struct { + XMLName xml.Name `xml:"coverage"` + LineRate float32 `xml:"line-rate,attr"` + BranchRate float32 `xml:"branch-rate,attr"` + Version string `xml:"version,attr"` + Timestamp int64 `xml:"timestamp,attr"` + LinesCovered int64 `xml:"lines-covered,attr"` + LinesValid int64 `xml:"lines-valid,attr"` + BranchesCovered int64 `xml:"branches-covered,attr"` + BranchesValid int64 `xml:"branches-valid,attr"` + Complexity float32 `xml:"complexity,attr"` + Sources []*CoberturaSource `xml:"sources>source"` + Packages []*CoberturaPackage `xml:"packages>package"` +} + +// CoberturaSource represents a base path to the covered source code. +type CoberturaSource struct { + Path string `xml:",chardata"` +} + +// CoberturaPackage represents a package in a Cobertura XML report. +type CoberturaPackage struct { + Name string `xml:"name,attr"` + LineRate float32 `xml:"line-rate,attr"` + BranchRate float32 `xml:"branch-rate,attr"` + Complexity float32 `xml:"complexity,attr"` + Classes []*CoberturaClass `xml:"classes>class"` +} + +// CoberturaClass represents a class in a Cobertura XML report. +type CoberturaClass struct { + Name string `xml:"name,attr"` + Filename string `xml:"filename,attr"` + LineRate float32 `xml:"line-rate,attr"` + BranchRate float32 `xml:"branch-rate,attr"` + Complexity float32 `xml:"complexity,attr"` + Methods []*CoberturaMethod `xml:"methods>method"` + Lines []*CoberturaLine `xml:"lines>line"` +} + +// CoberturaMethod represents a method in a Cobertura XML report. +type CoberturaMethod struct { + Name string `xml:"name,attr"` + Signature string `xml:"signature,attr"` + LineRate float32 `xml:"line-rate,attr"` + BranchRate float32 `xml:"branch-rate,attr"` + Complexity float32 `xml:"complexity,attr"` + Lines []*CoberturaLine `xml:"lines>line"` +} + +// CoberturaLine represents a source line in a Cobertura XML report. +type CoberturaLine struct { + Number int `xml:"number,attr"` + Hits int64 `xml:"hits,attr"` +} + +func (c *CoberturaCoverage) TimeStamp() int64 { + return c.Timestamp +} + +func (c *CoberturaCoverage) Bytes() ([]byte, error) { + out, err := xml.MarshalIndent(&c, "", " ") + if err != nil { + return nil, fmt.Errorf("unable to format test results as Coverage: %w", err) + } + + var buffer bytes.Buffer + buffer.WriteString(xml.Header) + buffer.WriteString("\n") + buffer.WriteString(coverageDtd) + buffer.WriteString("\n") + buffer.Write(out) + return buffer.Bytes(), nil +} + +// merge merges two coverage reports for a given class. +func (c *CoberturaClass) merge(b *CoberturaClass) error { + // Check preconditions: classes should be the same. + equal := c.Name == b.Name && + c.Filename == b.Filename && + len(c.Lines) == len(b.Lines) && + len(c.Methods) == len(b.Methods) + for idx := range c.Lines { + equal = equal && c.Lines[idx].Number == b.Lines[idx].Number + } + for idx := range c.Methods { + equal = equal && c.Methods[idx].Name == b.Methods[idx].Name && + len(c.Methods[idx].Lines) == len(b.Methods[idx].Lines) + } + if !equal { + return fmt.Errorf("merging incompatible classes: %+v != %+v", *c, *b) + } + // Update methods + for idx := range b.Methods { + for l := range b.Methods[idx].Lines { + c.Methods[idx].Lines[l].Hits += b.Methods[idx].Lines[l].Hits + } + } + // Rebuild lines + c.Lines = nil + for _, m := range c.Methods { + c.Lines = append(c.Lines, m.Lines...) + } + return nil +} + +// merge merges two coverage reports for a given package. +func (p *CoberturaPackage) merge(b *CoberturaPackage) error { + // Merge classes + for _, class := range b.Classes { + var target *CoberturaClass + for _, existing := range p.Classes { + if existing.Name == class.Name { + target = existing + break + } + } + if target != nil { + if err := target.merge(class); err != nil { + return err + } + } else { + p.Classes = append(p.Classes, class) + } + } + return nil +} + +// merge merges two coverage reports. +func (c *CoberturaCoverage) Merge(other CoverageReport) error { + b, ok := other.(*CoberturaCoverage) + if !ok { + return fmt.Errorf("not able to assert report to be merged as CoberturaCoverage") + + } + // Merge source paths + for _, path := range b.Sources { + found := false + for _, existing := range c.Sources { + if found = existing.Path == path.Path; found { + break + } + } + if !found { + c.Sources = append(c.Sources, path) + } + } + + // Merge packages + for _, pkg := range b.Packages { + var target *CoberturaPackage + for _, existing := range c.Packages { + if existing.Name == pkg.Name { + target = existing + break + } + } + if target != nil { + if err := target.merge(pkg); err != nil { + return err + } + } else { + c.Packages = append(c.Packages, pkg) + } + } + + // Recalculate global line coverage count + c.LinesValid = 0 + c.LinesCovered = 0 + for _, pkg := range c.Packages { + for _, cls := range pkg.Classes { + for _, line := range cls.Lines { + c.LinesValid++ + if line.Hits > 0 { + c.LinesCovered++ + } + } + } + } + return nil +} diff --git a/internal/testrunner/coverageoutput_test.go b/internal/testrunner/coberturacoverage_test.go similarity index 100% rename from internal/testrunner/coverageoutput_test.go rename to internal/testrunner/coberturacoverage_test.go diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coveragereport.go similarity index 57% rename from internal/testrunner/coverageoutput.go rename to internal/testrunner/coveragereport.go index 21df007e17..08fb7514f9 100644 --- a/internal/testrunner/coverageoutput.go +++ b/internal/testrunner/coveragereport.go @@ -5,8 +5,6 @@ package testrunner import ( - "bytes" - "encoding/xml" "errors" "fmt" "os" @@ -26,7 +24,19 @@ type CoverageReport interface { Bytes() ([]byte, error) } -const coverageDtd = `` +func lineNumberPerTestType(testType string) int { + var lineNumberPerTestType map[string]int = map[string]int{ + "asset": 1, + "pipeline": 2, + "system": 3, + "static": 4, + } + lineNumber, ok := lineNumberPerTestType[testType] + if !ok { + lineNumber = 5 + } + return lineNumber +} type testCoverageDetails struct { packageName string @@ -65,193 +75,11 @@ func (tcd *testCoverageDetails) withTestResults(results []TestResult) *testCover return tcd } -// CoberturaCoverage is the root element for a Cobertura XML report. -type CoberturaCoverage struct { - XMLName xml.Name `xml:"coverage"` - LineRate float32 `xml:"line-rate,attr"` - BranchRate float32 `xml:"branch-rate,attr"` - Version string `xml:"version,attr"` - Timestamp int64 `xml:"timestamp,attr"` - LinesCovered int64 `xml:"lines-covered,attr"` - LinesValid int64 `xml:"lines-valid,attr"` - BranchesCovered int64 `xml:"branches-covered,attr"` - BranchesValid int64 `xml:"branches-valid,attr"` - Complexity float32 `xml:"complexity,attr"` - Sources []*CoberturaSource `xml:"sources>source"` - Packages []*CoberturaPackage `xml:"packages>package"` -} - -// CoberturaSource represents a base path to the covered source code. -type CoberturaSource struct { - Path string `xml:",chardata"` -} - -// CoberturaPackage represents a package in a Cobertura XML report. -type CoberturaPackage struct { - Name string `xml:"name,attr"` - LineRate float32 `xml:"line-rate,attr"` - BranchRate float32 `xml:"branch-rate,attr"` - Complexity float32 `xml:"complexity,attr"` - Classes []*CoberturaClass `xml:"classes>class"` -} - -// CoberturaClass represents a class in a Cobertura XML report. -type CoberturaClass struct { - Name string `xml:"name,attr"` - Filename string `xml:"filename,attr"` - LineRate float32 `xml:"line-rate,attr"` - BranchRate float32 `xml:"branch-rate,attr"` - Complexity float32 `xml:"complexity,attr"` - Methods []*CoberturaMethod `xml:"methods>method"` - Lines []*CoberturaLine `xml:"lines>line"` -} - -// CoberturaMethod represents a method in a Cobertura XML report. -type CoberturaMethod struct { - Name string `xml:"name,attr"` - Signature string `xml:"signature,attr"` - LineRate float32 `xml:"line-rate,attr"` - BranchRate float32 `xml:"branch-rate,attr"` - Complexity float32 `xml:"complexity,attr"` - Lines []*CoberturaLine `xml:"lines>line"` -} - -// CoberturaLine represents a source line in a Cobertura XML report. -type CoberturaLine struct { - Number int `xml:"number,attr"` - Hits int64 `xml:"hits,attr"` -} - -func (c *CoberturaCoverage) TimeStamp() int64 { - return c.Timestamp -} - -func (c *CoberturaCoverage) Bytes() ([]byte, error) { - out, err := xml.MarshalIndent(&c, "", " ") - if err != nil { - return nil, fmt.Errorf("unable to format test results as Coverage: %w", err) - } - - var buffer bytes.Buffer - buffer.WriteString(xml.Header) - buffer.WriteString("\n") - buffer.WriteString(coverageDtd) - buffer.WriteString("\n") - buffer.Write(out) - return buffer.Bytes(), nil -} - -// merge merges two coverage reports for a given class. -func (c *CoberturaClass) merge(b *CoberturaClass) error { - // Check preconditions: classes should be the same. - equal := c.Name == b.Name && - c.Filename == b.Filename && - len(c.Lines) == len(b.Lines) && - len(c.Methods) == len(b.Methods) - for idx := range c.Lines { - equal = equal && c.Lines[idx].Number == b.Lines[idx].Number - } - for idx := range c.Methods { - equal = equal && c.Methods[idx].Name == b.Methods[idx].Name && - len(c.Methods[idx].Lines) == len(b.Methods[idx].Lines) - } - if !equal { - return fmt.Errorf("merging incompatible classes: %+v != %+v", *c, *b) - } - // Update methods - for idx := range b.Methods { - for l := range b.Methods[idx].Lines { - c.Methods[idx].Lines[l].Hits += b.Methods[idx].Lines[l].Hits - } - } - // Rebuild lines - c.Lines = nil - for _, m := range c.Methods { - c.Lines = append(c.Lines, m.Lines...) - } - return nil -} - -// merge merges two coverage reports for a given package. -func (p *CoberturaPackage) merge(b *CoberturaPackage) error { - // Merge classes - for _, class := range b.Classes { - var target *CoberturaClass - for _, existing := range p.Classes { - if existing.Name == class.Name { - target = existing - break - } - } - if target != nil { - if err := target.merge(class); err != nil { - return err - } - } else { - p.Classes = append(p.Classes, class) - } - } - return nil -} - -// merge merges two coverage reports. -func (c *CoberturaCoverage) Merge(other CoverageReport) error { - b, ok := other.(*CoberturaCoverage) - if !ok { - return fmt.Errorf("not able to assert report to be merged as CoberturaCoverage") - - } - // Merge source paths - for _, path := range b.Sources { - found := false - for _, existing := range c.Sources { - if found = existing.Path == path.Path; found { - break - } - } - if !found { - c.Sources = append(c.Sources, path) - } - } - - // Merge packages - for _, pkg := range b.Packages { - var target *CoberturaPackage - for _, existing := range c.Packages { - if existing.Name == pkg.Name { - target = existing - break - } - } - if target != nil { - if err := target.merge(pkg); err != nil { - return err - } - } else { - c.Packages = append(c.Packages, pkg) - } - } - - // Recalculate global line coverage count - c.LinesValid = 0 - c.LinesCovered = 0 - for _, pkg := range c.Packages { - for _, cls := range pkg.Classes { - for _, line := range cls.Lines { - c.LinesValid++ - if line.Hits > 0 { - c.LinesCovered++ - } - } - } - } - return nil -} - // WriteCoverage function calculates test coverage for the given package. // It requires to execute tests for all data streams (same test type), so the coverage can be calculated properly. func WriteCoverage(packageRootPath, packageName, packageType string, testType TestType, results []TestResult, testCoverageType string) error { - report, err := createCoverageReport(packageRootPath, packageName, packageType, testType, results, testCoverageType) + timestamp := time.Now().UnixNano() + report, err := createCoverageReport(packageRootPath, packageName, packageType, testType, results, testCoverageType, timestamp) if err != nil { return fmt.Errorf("can't create coverage report: %w", err) } @@ -263,7 +91,7 @@ func WriteCoverage(packageRootPath, packageName, packageType string, testType Te return nil } -func createCoverageReport(packageRootPath, packageName, packageType string, testType TestType, results []TestResult, CoverageFormat string) (CoverageReport, error) { +func createCoverageReport(packageRootPath, packageName, packageType string, testType TestType, results []TestResult, CoverageFormat string, timestamp int64) (CoverageReport, error) { details, err := collectTestCoverageDetails(packageRootPath, packageName, packageType, testType, results) if err != nil { return nil, fmt.Errorf("can't collect test coverage details: %w", err) @@ -284,7 +112,7 @@ func createCoverageReport(packageRootPath, packageName, packageType string, test relativePath = strings.TrimPrefix(relativePath, "/") baseFolder := filepath.Dir(relativePath) - report := transformToCoverageReport(details, baseFolder, CoverageFormat) + report := transformToCoverageReport(details, baseFolder, CoverageFormat, timestamp) return report, nil } @@ -359,30 +187,21 @@ func verifyTestExpected(packageRootPath string, dataStreamName string, testType return true, nil } -func transformToCoverageReport(details *testCoverageDetails, baseFolder, coverageFormat string) CoverageReport { +func transformToCoverageReport(details *testCoverageDetails, baseFolder, coverageFormat string, timestamp int64) CoverageReport { if coverageFormat == "cobertura" { - return transformToCoberturaReport(details, baseFolder) + return transformToCoberturaReport(details, baseFolder, timestamp) } if coverageFormat == "generic" { - return transformToGenericCoverageReport(details, baseFolder) + return transformToGenericCoverageReport(details, baseFolder, timestamp) } return nil } -func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) *CoberturaCoverage { +func transformToCoberturaReport(details *testCoverageDetails, baseFolder string, timestamp int64) *CoberturaCoverage { var classes []*CoberturaClass - var lineNumberPerTestType map[string]int = map[string]int{ - "asset": 1, - "pipeline": 2, - "system": 3, - "static": 4, - } - lineNumber, ok := lineNumberPerTestType[string(details.testType)] - if !ok { - lineNumber = 5 - } + lineNumberTestType := lineNumberPerTestType(string(details.testType)) for dataStream, testCases := range details.dataStreams { if dataStream == "" && details.packageType == "integration" { @@ -395,15 +214,15 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) if len(testCases) == 0 { methods = append(methods, &CoberturaMethod{ Name: "Missing", - Lines: []*CoberturaLine{{Number: lineNumber, Hits: 0}}, + Lines: []*CoberturaLine{{Number: lineNumberTestType, Hits: 0}}, }) - lines = append(lines, []*CoberturaLine{{Number: lineNumber, Hits: 0}}...) + lines = append(lines, []*CoberturaLine{{Number: lineNumberTestType, Hits: 0}}...) } else { methods = append(methods, &CoberturaMethod{ Name: "OK", - Lines: []*CoberturaLine{{Number: lineNumber, Hits: 1}}, + Lines: []*CoberturaLine{{Number: lineNumberTestType, Hits: 1}}, }) - lines = append(lines, []*CoberturaLine{{Number: lineNumber, Hits: 1}}...) + lines = append(lines, []*CoberturaLine{{Number: lineNumberTestType, Hits: 1}}...) } fileName := path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml") @@ -422,7 +241,7 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string) } return &CoberturaCoverage{ - Timestamp: time.Now().UnixNano(), + Timestamp: timestamp, Packages: []*CoberturaPackage{ { Name: strings.Replace(strings.TrimSuffix(baseFolder, "/"), "/", ".", -1) + "." + details.packageName, diff --git a/internal/testrunner/coveragereport_test.go b/internal/testrunner/coveragereport_test.go new file mode 100644 index 0000000000..74bd8cad1e --- /dev/null +++ b/internal/testrunner/coveragereport_test.go @@ -0,0 +1,257 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package testrunner + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateCoverageReport(t *testing.T) { + tests := []struct { + name string + rootPath string + packageName string + packageType string + coverageFormat string + timestamp int64 + testType TestType + results []TestResult + expected CoverageReport + }{ + { + name: "generate custom cobertura coverage", + testType: "system", + rootPath: "/my/path/package", + packageName: "package", + packageType: "integration", + coverageFormat: "cobertura", + timestamp: 10, + results: []TestResult{ + { + Name: "test1", + Package: "package.myclass", + DataStream: "metrics", + TimeElapsed: 1 * time.Second, + Coverage: nil, + }, + { + Name: "test2", + Package: "package.myclass", + DataStream: "logs", + TimeElapsed: 2 * time.Second, + Coverage: nil, + }, + }, + expected: &CoberturaCoverage{ + Version: "", + Timestamp: 10, + Packages: []*CoberturaPackage{ + { + Name: "my.path.package", + Classes: []*CoberturaClass{ + { + Name: "system", + Filename: "my/path/package/data_stream/metrics/manifest.yml", + Methods: []*CoberturaMethod{ + { + Name: "OK", + Signature: "", + Lines: []*CoberturaLine{ + { + Number: 3, + Hits: 1, + }, + }, + }, + }, + Lines: []*CoberturaLine{ + { + Number: 3, + Hits: 1, + }, + }, + }, + { + Name: "system", + Filename: "my/path/package/data_stream/logs/manifest.yml", + Methods: []*CoberturaMethod{ + { + Name: "OK", + Signature: "", + Lines: []*CoberturaLine{ + { + Number: 3, + Hits: 1, + }, + }, + }, + }, + Lines: []*CoberturaLine{ + { + Number: 3, + Hits: 1, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "generate custom generic coverage", + testType: "system", + rootPath: "/my/path/package", + packageName: "package", + packageType: "integration", + coverageFormat: "generic", + timestamp: 10, + results: []TestResult{ + { + Name: "test1", + Package: "package.myclass", + DataStream: "metrics", + TimeElapsed: 1 * time.Second, + Coverage: nil, + }, + { + Name: "test2", + Package: "package.myclass", + DataStream: "logs", + TimeElapsed: 2 * time.Second, + Coverage: nil, + }, + }, + expected: &GenericCoverage{ + Version: 1, + Files: []*GenericFile{ + { + Path: "my/path/package/data_stream/metrics/manifest.yml", + Lines: []*GenericLine{ + { + LineNumber: 3, + Covered: true, + }, + }, + }, + { + Path: "my/path/package/data_stream/logs/manifest.yml", + Lines: []*GenericLine{ + { + LineNumber: 3, + Covered: true, + }, + }, + }, + }, + TestType: "Coverage for system test", + Timestamp: 10, + }, + }, + { + name: "generate custom generic coverage", + testType: "system", + rootPath: "/my/path/package", + packageName: "package", + packageType: "integration", + coverageFormat: "generic", + timestamp: 10, + results: []TestResult{ + { + Name: "test1", + Package: "package.myclass", + DataStream: "metrics", + TimeElapsed: 1 * time.Second, + Coverage: nil, + }, + }, + expected: &GenericCoverage{ + Version: 1, + Files: []*GenericFile{ + { + Path: "my/path/package/data_stream/metrics/manifest.yml", + Lines: []*GenericLine{ + { + LineNumber: 3, + Covered: true, + }, + }, + }, + }, + TestType: "Coverage for system test", + Timestamp: 10, + }, + }, + { + name: "use provided generic coverage", + testType: "system", + rootPath: "/my/path/package", + packageName: "package", + packageType: "integration", + coverageFormat: "generic", + timestamp: 10, + results: []TestResult{ + { + Name: "test1", + Package: "package.myclass", + DataStream: "metrics", + TimeElapsed: 1 * time.Second, + Coverage: &GenericCoverage{ + Version: 1, + Files: []*GenericFile{ + { + Path: "my/path/package/data_stream/metrics/foo.yml", + Lines: []*GenericLine{ + { + LineNumber: 1, + Covered: true, + }, + { + LineNumber: 2, + Covered: true, + }, + }, + }, + }, + TestType: "Coverage for system test", + Timestamp: 20, + }, + }, + }, + expected: &GenericCoverage{ + Version: 1, + Files: []*GenericFile{ + { + Path: "my/path/package/data_stream/metrics/foo.yml", + Lines: []*GenericLine{ + { + LineNumber: 1, + Covered: true, + }, + { + LineNumber: 2, + Covered: true, + }, + }, + }, + }, + TestType: "Coverage for system test", + Timestamp: 20, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + report, err := createCoverageReport(tt.rootPath, tt.packageName, tt.packageType, tt.testType, tt.results, tt.coverageFormat, tt.timestamp) + require.NoError(t, err) + assert.Equal(t, tt.expected, report) + }) + } +} diff --git a/internal/testrunner/genericoutput.go b/internal/testrunner/genericcobertura.go similarity index 86% rename from internal/testrunner/genericoutput.go rename to internal/testrunner/genericcobertura.go index 1c54b49764..2a6a3b53bd 100644 --- a/internal/testrunner/genericoutput.go +++ b/internal/testrunner/genericcobertura.go @@ -9,7 +9,6 @@ import ( "encoding/xml" "fmt" "path" - "time" ) // GenericCoverage is the root element for a Cobertura XML report. @@ -91,17 +90,8 @@ func (c *GenericCoverage) Merge(other CoverageReport) error { return nil } -func transformToGenericCoverageReport(details *testCoverageDetails, baseFolder string) *GenericCoverage { - var lineNumberPerTestType map[string]int64 = map[string]int64{ - "asset": 1, - "pipeline": 2, - "system": 3, - "static": 4, - } - lineNumber, ok := lineNumberPerTestType[string(details.testType)] - if !ok { - lineNumber = 5 - } +func transformToGenericCoverageReport(details *testCoverageDetails, baseFolder string, timestamp int64) *GenericCoverage { + lineNumberTestType := lineNumberPerTestType(string(details.testType)) var files []*GenericFile for dataStream, testCases := range details.dataStreams { if dataStream == "" { @@ -113,18 +103,18 @@ func transformToGenericCoverageReport(details *testCoverageDetails, baseFolder s if len(testCases) == 0 { files = append(files, &GenericFile{ Path: dataStreamPath, - Lines: []*GenericLine{{LineNumber: lineNumber, Covered: false}}, + Lines: []*GenericLine{{LineNumber: int64(lineNumberTestType), Covered: false}}, }) } else { files = append(files, &GenericFile{ Path: dataStreamPath, - Lines: []*GenericLine{{LineNumber: lineNumber, Covered: true}}, + Lines: []*GenericLine{{LineNumber: int64(lineNumberTestType), Covered: true}}, }) } } return &GenericCoverage{ - Timestamp: time.Now().UnixNano(), + Timestamp: timestamp, Version: 1, Files: files, TestType: fmt.Sprintf("Coverage for %s test", details.testType), diff --git a/internal/testrunner/genericcobertura_test.go b/internal/testrunner/genericcobertura_test.go new file mode 100644 index 0000000000..173d6ab381 --- /dev/null +++ b/internal/testrunner/genericcobertura_test.go @@ -0,0 +1,92 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package testrunner + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGenericCoverage_Merge(t *testing.T) { + tests := []struct { + name string + rhs, lhs, expected GenericCoverage + wantErr bool + }{ + { + name: "merge files", + rhs: GenericCoverage{ + Files: []*GenericFile{ + { + Path: "/a", + Lines: []*GenericLine{ + {LineNumber: 1, Covered: true}, + }, + }, + { + Path: "/c", + Lines: []*GenericLine{}, + }, + }, + }, + lhs: GenericCoverage{ + Files: []*GenericFile{ + { + Path: "/b", + Lines: []*GenericLine{ + {LineNumber: 1, Covered: true}, + }, + }, + { + Path: "/c", + Lines: []*GenericLine{ + {LineNumber: 1, Covered: false}, + {LineNumber: 2, Covered: false}, + }, + }, + }, + }, + expected: GenericCoverage{ + Files: []*GenericFile{ + { + Path: "/a", + Lines: []*GenericLine{ + {LineNumber: 1, Covered: true}, + }, + }, + { + Path: "/c", + Lines: []*GenericLine{ + {LineNumber: 1, Covered: false}, + {LineNumber: 2, Covered: false}, + }, + }, + { + Path: "/b", + Lines: []*GenericLine{ + {LineNumber: 1, Covered: true}, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.rhs.Merge(&tt.lhs) + if !tt.wantErr { + if !assert.NoError(t, err) { + t.Fatal(err) + } + } else { + if !assert.Error(t, err) { + t.Fatal("error expected") + } + } + assert.Equal(t, tt.expected, tt.rhs) + }) + } +} diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index 12c6ea6d2b..abbe4356df 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -246,9 +246,32 @@ func (r *runner) run() ([]testrunner.TestResult, error) { results = append(results, tr) } - // Add empty coverage for data Streams wihtout tests - return results, nil + // fmt.Printf(">>> testCaseFiles count (datastream %q): %d\n", r.options.TestFolder.DataStream, len(testCaseFiles)) + // if len(testCaseFiles) > 0 { + // return results, nil + // } + // expected, err := testrunner.VerifyTestExpected(r.options.PackageRootPath, r.options.TestFolder.DataStream, r.Type()) + // if err != nil { + // return nil, fmt.Errorf("can't verify if test is expected: %w", err) + // } + // if !expected { + // return results, nil + // } + // tr := testrunner.TestResult{ + // TestType: TestType, + // Package: r.options.TestFolder.Package, + // DataStream: r.options.TestFolder.DataStream, + // Name: r.options.TestFolder.DataStream, + // } + // startTime := time.Now() + // tr.TimeElapsed = time.Since(startTime) + // tr.Coverage, err = GetPipelineCoverage(r.options, r.pipelines) + // if err != nil { + // return nil, fmt.Errorf("error calculating pipeline coverage: %w", err) + // } + // results = append(results, tr) + // return results, nil } func (r *runner) listTestCaseFiles() ([]string, error) { From 00612aa360a180e4fb2bf5f9fc13e7d78ba575b5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jan 2024 19:21:20 +0100 Subject: [PATCH 18/35] Refactor --- .../testrunner/runners/pipeline/coverage.go | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index df263a67c7..f2445128df 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -69,7 +69,11 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe // Calculate coverage for each pipeline for _, pipeline := range pipelines { - covered, class, err := coberturaForSinglePipeline(pipeline, stats, basePath, dataStreamPath) + pipelineName, pipelineRelPath, src, pstats, err := pipelineDataForCoverage(pipeline, stats, basePath, dataStreamPath) + if err != nil { + return nil, err + } + covered, class, err := coberturaForSinglePipeline(pipelineName, pipelineRelPath, src, pstats) if err != nil { return nil, fmt.Errorf("error calculating coverage for pipeline '%s': %w", pipeline.Filename(), err) } @@ -88,7 +92,11 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe // Calculate coverage for each pipeline for _, pipeline := range pipelines { - _, file, err := genericCoverageForSinglePipeline(pipeline, stats, basePath, dataStreamPath) + _, pipelineRelPath, src, pstats, err := pipelineDataForCoverage(pipeline, stats, basePath, dataStreamPath) + if err != nil { + return nil, err + } + _, file, err := genericCoverageForSinglePipeline(pipelineRelPath, src, pstats) if err != nil { return nil, fmt.Errorf("error calculating coverage for pipeline '%s': %w", pipeline.Filename(), err) } @@ -144,12 +152,8 @@ func pipelineDataForCoverage(pipeline ingest.Pipeline, stats ingest.PipelineStat return pipelineName, pipelineRelPath, src, pstats, nil } -func genericCoverageForSinglePipeline(pipeline ingest.Pipeline, stats ingest.PipelineStatsMap, basePath, dataStreamPath string) (linesCovered int64, class *testrunner.GenericFile, err error) { - _, pipelineRelPath, src, pstats, err := pipelineDataForCoverage(pipeline, stats, basePath, dataStreamPath) - if err != nil { - return 0, nil, err - } - // Report every pipeline as a "class". +func genericCoverageForSinglePipeline(pipelineRelPath string, src []ingest.Processor, pstats ingest.PipelineStats) (linesCovered int64, class *testrunner.GenericFile, err error) { + // Report every pipeline as a "file". file := &testrunner.GenericFile{ Path: pipelineRelPath, } @@ -168,11 +172,7 @@ func genericCoverageForSinglePipeline(pipeline ingest.Pipeline, stats ingest.Pip return linesCovered, file, nil } -func coberturaForSinglePipeline(pipeline ingest.Pipeline, stats ingest.PipelineStatsMap, basePath, dataStreamPath string) (linesCovered int64, class *testrunner.CoberturaClass, err error) { - pipelineName, pipelineRelPath, src, pstats, err := pipelineDataForCoverage(pipeline, stats, basePath, dataStreamPath) - if err != nil { - return 0, nil, err - } +func coberturaForSinglePipeline(pipelineName, pipelineRelPath string, src []ingest.Processor, pstats ingest.PipelineStats) (linesCovered int64, class *testrunner.CoberturaClass, err error) { // Report every pipeline as a "class". class = &testrunner.CoberturaClass{ Name: pipelineName, From 738fc69e54f92abf7c568f109906358177734d66 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jan 2024 19:46:25 +0100 Subject: [PATCH 19/35] Ensure data streams are sorted - more tests added --- internal/testrunner/coveragereport.go | 12 ++++++- internal/testrunner/coveragereport_test.go | 42 +++++++++++++++++++--- internal/testrunner/genericcobertura.go | 23 +++++++++--- 3 files changed, 67 insertions(+), 10 deletions(-) diff --git a/internal/testrunner/coveragereport.go b/internal/testrunner/coveragereport.go index 08fb7514f9..d6e32f365a 100644 --- a/internal/testrunner/coveragereport.go +++ b/internal/testrunner/coveragereport.go @@ -10,6 +10,7 @@ import ( "os" "path" "path/filepath" + "sort" "strings" "time" @@ -203,7 +204,16 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string, var classes []*CoberturaClass lineNumberTestType := lineNumberPerTestType(string(details.testType)) - for dataStream, testCases := range details.dataStreams { + // sort data streams to ensure same ordering in coverage arrays + sortedDataStreams := make([]string, 0, len(details.dataStreams)) + for dataStream := range details.dataStreams { + sortedDataStreams = append(sortedDataStreams, dataStream) + } + sort.Strings(sortedDataStreams) + + for _, dataStream := range sortedDataStreams { + testCases := details.dataStreams[dataStream] + if dataStream == "" && details.packageType == "integration" { continue // ignore tests running in the package context (not data stream), mostly referring to installed assets } diff --git a/internal/testrunner/coveragereport_test.go b/internal/testrunner/coveragereport_test.go index 74bd8cad1e..18d8d5f4ed 100644 --- a/internal/testrunner/coveragereport_test.go +++ b/internal/testrunner/coveragereport_test.go @@ -57,7 +57,7 @@ func TestCreateCoverageReport(t *testing.T) { Classes: []*CoberturaClass{ { Name: "system", - Filename: "my/path/package/data_stream/metrics/manifest.yml", + Filename: "my/path/package/data_stream/logs/manifest.yml", Methods: []*CoberturaMethod{ { Name: "OK", @@ -79,7 +79,7 @@ func TestCreateCoverageReport(t *testing.T) { }, { Name: "system", - Filename: "my/path/package/data_stream/logs/manifest.yml", + Filename: "my/path/package/data_stream/metrics/manifest.yml", Methods: []*CoberturaMethod{ { Name: "OK", @@ -132,7 +132,7 @@ func TestCreateCoverageReport(t *testing.T) { Version: 1, Files: []*GenericFile{ { - Path: "my/path/package/data_stream/metrics/manifest.yml", + Path: "my/path/package/data_stream/logs/manifest.yml", Lines: []*GenericLine{ { LineNumber: 3, @@ -141,7 +141,7 @@ func TestCreateCoverageReport(t *testing.T) { }, }, { - Path: "my/path/package/data_stream/logs/manifest.yml", + Path: "my/path/package/data_stream/metrics/manifest.yml", Lines: []*GenericLine{ { LineNumber: 3, @@ -245,6 +245,40 @@ func TestCreateCoverageReport(t *testing.T) { Timestamp: 20, }, }, + { + name: "generic coverage for an input package", + testType: "asset", + rootPath: "/my/path/package", + packageName: "package", + packageType: "input", + coverageFormat: "generic", + timestamp: 10, + results: []TestResult{ + { + Name: "test1", + Package: "package.myclass", + DataStream: "", + TimeElapsed: 1 * time.Second, + Coverage: nil, + }, + }, + expected: &GenericCoverage{ + Version: 1, + Files: []*GenericFile{ + { + Path: "my/path/package/manifest.yml", + Lines: []*GenericLine{ + { + LineNumber: 1, + Covered: true, + }, + }, + }, + }, + TestType: "Coverage for asset test", + Timestamp: 10, + }, + }, } for _, tt := range tests { diff --git a/internal/testrunner/genericcobertura.go b/internal/testrunner/genericcobertura.go index 2a6a3b53bd..3c2ed60f8f 100644 --- a/internal/testrunner/genericcobertura.go +++ b/internal/testrunner/genericcobertura.go @@ -9,6 +9,7 @@ import ( "encoding/xml" "fmt" "path" + "sort" ) // GenericCoverage is the root element for a Cobertura XML report. @@ -93,21 +94,33 @@ func (c *GenericCoverage) Merge(other CoverageReport) error { func transformToGenericCoverageReport(details *testCoverageDetails, baseFolder string, timestamp int64) *GenericCoverage { lineNumberTestType := lineNumberPerTestType(string(details.testType)) var files []*GenericFile - for dataStream, testCases := range details.dataStreams { - if dataStream == "" { + // sort data streams to ensure same ordering in coverage arrays + sortedDataStreams := make([]string, 0, len(details.dataStreams)) + for dataStream := range details.dataStreams { + sortedDataStreams = append(sortedDataStreams, dataStream) + } + sort.Strings(sortedDataStreams) + + for _, dataStream := range sortedDataStreams { + if dataStream == "" && details.packageType == "integration" { continue // ignore tests running in the package context (not data stream), mostly referring to installed assets } + testCases := details.dataStreams[dataStream] - dataStreamPath := path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml") + fileName := path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml") + if dataStream == "" { + // input package + fileName = path.Join(baseFolder, details.packageName, "manifest.yml") + } if len(testCases) == 0 { files = append(files, &GenericFile{ - Path: dataStreamPath, + Path: fileName, Lines: []*GenericLine{{LineNumber: int64(lineNumberTestType), Covered: false}}, }) } else { files = append(files, &GenericFile{ - Path: dataStreamPath, + Path: fileName, Lines: []*GenericLine{{LineNumber: int64(lineNumberTestType), Covered: true}}, }) } From 522ffd6f048099f95e55279d88b5c9911f67780b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jan 2024 20:12:11 +0100 Subject: [PATCH 20/35] Add comments --- .buildkite/pipeline.trigger.integration.tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 95105a35bf..6cc396c901 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -46,7 +46,7 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do echo " - build/test-results/*.xml" echo " - build/elastic-stack-dump/check-*/logs/*.log" echo " - build/elastic-stack-dump/check-*/logs/fleet-server-internal/**/*" - echo " - build/test-coverage/coverage-*.xml" + echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package if [[ $test =~ with-kind$ ]]; then echo " - build/kubectl-dump.txt" fi @@ -64,7 +64,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " provider: \"gcp\"" echo " artifact_paths:" echo " - build/test-results/*.xml" - echo " - build/test-coverage/coverage-*.xml" + echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package done popd > /dev/null @@ -81,7 +81,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " provider: \"gcp\"" echo " artifact_paths:" echo " - build/test-results/*.xml" - echo " - build/test-coverage/coverage-*.xml" + echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package done popd > /dev/null From 9d153d30909d77ae21d9ceebbc5cb1b4f88de9e2 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jan 2024 20:48:02 +0100 Subject: [PATCH 21/35] Manage coverage paths for windows --- internal/testrunner/coberturacoverage.go | 69 +++++++++++++++ internal/testrunner/coveragereport.go | 84 ++++--------------- .../testrunner/runners/pipeline/coverage.go | 11 ++- 3 files changed, 92 insertions(+), 72 deletions(-) diff --git a/internal/testrunner/coberturacoverage.go b/internal/testrunner/coberturacoverage.go index 622951a314..3c1da8bf28 100644 --- a/internal/testrunner/coberturacoverage.go +++ b/internal/testrunner/coberturacoverage.go @@ -7,7 +7,15 @@ package testrunner import ( "bytes" "encoding/xml" + "errors" "fmt" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/elastic/elastic-package/internal/builder" ) const coverageDtd = `` @@ -194,3 +202,64 @@ func (c *CoberturaCoverage) Merge(other CoverageReport) error { } return nil } + +func transformToCoberturaReport(details *testCoverageDetails, baseFolder string, timestamp int64) *CoberturaCoverage { + var classes []*CoberturaClass + lineNumberTestType := lineNumberPerTestType(string(details.testType)) + + // sort data streams to ensure same ordering in coverage arrays + sortedDataStreams := make([]string, 0, len(details.dataStreams)) + for dataStream := range details.dataStreams { + sortedDataStreams = append(sortedDataStreams, dataStream) + } + sort.Strings(sortedDataStreams) + + for _, dataStream := range sortedDataStreams { + testCases := details.dataStreams[dataStream] + + if dataStream == "" && details.packageType == "integration" { + continue // ignore tests running in the package context (not data stream), mostly referring to installed assets + } + + var methods []*CoberturaMethod + var lines []*CoberturaLine + + if len(testCases) == 0 { + methods = append(methods, &CoberturaMethod{ + Name: "Missing", + Lines: []*CoberturaLine{{Number: lineNumberTestType, Hits: 0}}, + }) + lines = append(lines, []*CoberturaLine{{Number: lineNumberTestType, Hits: 0}}...) + } else { + methods = append(methods, &CoberturaMethod{ + Name: "OK", + Lines: []*CoberturaLine{{Number: lineNumberTestType, Hits: 1}}, + }) + lines = append(lines, []*CoberturaLine{{Number: lineNumberTestType, Hits: 1}}...) + } + + fileName := path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml") + if dataStream == "" { + // input package + fileName = path.Join(baseFolder, details.packageName, "manifest.yml") + } + + aClass := &CoberturaClass{ + Name: string(details.testType), + Filename: fileName, + Methods: methods, + Lines: lines, + } + classes = append(classes, aClass) + } + + return &CoberturaCoverage{ + Timestamp: timestamp, + Packages: []*CoberturaPackage{ + { + Name: strings.Replace(strings.TrimSuffix(baseFolder, "/"), "/", ".", -1) + "." + details.packageName, + Classes: classes, + }, + }, + } +} diff --git a/internal/testrunner/coveragereport.go b/internal/testrunner/coveragereport.go index d6e32f365a..58c1ad9484 100644 --- a/internal/testrunner/coveragereport.go +++ b/internal/testrunner/coveragereport.go @@ -8,9 +8,7 @@ import ( "errors" "fmt" "os" - "path" "path/filepath" - "sort" "strings" "time" @@ -104,20 +102,31 @@ func createCoverageReport(packageRootPath, packageName, packageType string, test } // generate a custom report if not available - dir, err := files.FindRepositoryRootDirectory() + baseFolder, err := GetBaseFolderPackageForCoverage(packageRootPath) if err != nil { return nil, err } - relativePath := strings.TrimPrefix(packageRootPath, dir) - relativePath = strings.TrimPrefix(relativePath, "/") - baseFolder := filepath.Dir(relativePath) - report := transformToCoverageReport(details, baseFolder, CoverageFormat, timestamp) return report, nil } +func GetBaseFolderPackageForCoverage(packageRootPath string) (string, error) { + dir, err := files.FindRepositoryRootDirectory() + if err != nil { + return "", err + } + + relativePath := strings.TrimPrefix(packageRootPath, dir) + relativePath = strings.TrimPrefix(relativePath, "/") // linux + relativePath = strings.TrimPrefix(relativePath, "\\") // windows + baseFolder := filepath.Dir(relativePath) + + // Force to show always "/" as file separator + return strings.ReplaceAll(baseFolder, "\\", "/"), nil +} + func collectTestCoverageDetails(packageRootPath, packageName, packageType string, testType TestType, results []TestResult) (*testCoverageDetails, error) { withoutTests, err := findDataStreamsWithoutTests(packageRootPath, testType) if err != nil { @@ -200,67 +209,6 @@ func transformToCoverageReport(details *testCoverageDetails, baseFolder, coverag return nil } -func transformToCoberturaReport(details *testCoverageDetails, baseFolder string, timestamp int64) *CoberturaCoverage { - var classes []*CoberturaClass - lineNumberTestType := lineNumberPerTestType(string(details.testType)) - - // sort data streams to ensure same ordering in coverage arrays - sortedDataStreams := make([]string, 0, len(details.dataStreams)) - for dataStream := range details.dataStreams { - sortedDataStreams = append(sortedDataStreams, dataStream) - } - sort.Strings(sortedDataStreams) - - for _, dataStream := range sortedDataStreams { - testCases := details.dataStreams[dataStream] - - if dataStream == "" && details.packageType == "integration" { - continue // ignore tests running in the package context (not data stream), mostly referring to installed assets - } - - var methods []*CoberturaMethod - var lines []*CoberturaLine - - if len(testCases) == 0 { - methods = append(methods, &CoberturaMethod{ - Name: "Missing", - Lines: []*CoberturaLine{{Number: lineNumberTestType, Hits: 0}}, - }) - lines = append(lines, []*CoberturaLine{{Number: lineNumberTestType, Hits: 0}}...) - } else { - methods = append(methods, &CoberturaMethod{ - Name: "OK", - Lines: []*CoberturaLine{{Number: lineNumberTestType, Hits: 1}}, - }) - lines = append(lines, []*CoberturaLine{{Number: lineNumberTestType, Hits: 1}}...) - } - - fileName := path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml") - if dataStream == "" { - // input package - fileName = path.Join(baseFolder, details.packageName, "manifest.yml") - } - - aClass := &CoberturaClass{ - Name: string(details.testType), - Filename: fileName, - Methods: methods, - Lines: lines, - } - classes = append(classes, aClass) - } - - return &CoberturaCoverage{ - Timestamp: timestamp, - Packages: []*CoberturaPackage{ - { - Name: strings.Replace(strings.TrimSuffix(baseFolder, "/"), "/", ".", -1) + "." + details.packageName, - Classes: classes, - }, - }, - } -} - func writeCoverageReportFile(report CoverageReport, packageName string) error { dest, err := testCoverageReportsDir() if err != nil { diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index f2445128df..684b84f28f 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -47,13 +47,16 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe basePath = basePath[:len(dir)] - relativePath := strings.TrimPrefix(options.PackageRootPath, dir) - relativePath = strings.TrimPrefix(relativePath, "/") - baseFolder := filepath.Dir(relativePath) + baseFolder, err := testrunner.GetBaseFolderPackageForCoverage(options.PackageRootPath) + if err != nil { + return nil, err + } + baseFolderAsPackage := strings.ReplaceAll(baseFolder, "/", ".") + baseFolderAsPackage = strings.ReplaceAll(baseFolderAsPackage, "\\", ".") // Construct the Cobertura report. pkg := &testrunner.CoberturaPackage{ - Name: strings.Replace(baseFolder, "/", ".", -1) + "." + options.TestFolder.Package + "." + options.TestFolder.DataStream, + Name: baseFolderAsPackage + "." + options.TestFolder.Package + "." + options.TestFolder.DataStream, } if options.CoverageType == "cobertura" { From 9193fd92bea70c01742c0d87675772e23f912d7a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jan 2024 21:18:05 +0100 Subject: [PATCH 22/35] Remove base folder from CoberturaPackage --- internal/testrunner/coberturacoverage.go | 8 +------- internal/testrunner/runners/pipeline/coverage.go | 9 +-------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/internal/testrunner/coberturacoverage.go b/internal/testrunner/coberturacoverage.go index 3c1da8bf28..98ad504a1c 100644 --- a/internal/testrunner/coberturacoverage.go +++ b/internal/testrunner/coberturacoverage.go @@ -7,15 +7,9 @@ package testrunner import ( "bytes" "encoding/xml" - "errors" "fmt" - "os" "path" - "path/filepath" "sort" - "strings" - - "github.com/elastic/elastic-package/internal/builder" ) const coverageDtd = `` @@ -257,7 +251,7 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string, Timestamp: timestamp, Packages: []*CoberturaPackage{ { - Name: strings.Replace(strings.TrimSuffix(baseFolder, "/"), "/", ".", -1) + "." + details.packageName, + Name: details.packageName, Classes: classes, }, }, diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index 684b84f28f..b43128e107 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -47,16 +47,9 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe basePath = basePath[:len(dir)] - baseFolder, err := testrunner.GetBaseFolderPackageForCoverage(options.PackageRootPath) - if err != nil { - return nil, err - } - - baseFolderAsPackage := strings.ReplaceAll(baseFolder, "/", ".") - baseFolderAsPackage = strings.ReplaceAll(baseFolderAsPackage, "\\", ".") // Construct the Cobertura report. pkg := &testrunner.CoberturaPackage{ - Name: baseFolderAsPackage + "." + options.TestFolder.Package + "." + options.TestFolder.DataStream, + Name: options.TestFolder.Package + "." + options.TestFolder.DataStream, } if options.CoverageType == "cobertura" { From 0a6eaeea1f9973128a04f50a1399e9d55b88394a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jan 2024 21:38:36 +0100 Subject: [PATCH 23/35] Use filepath for tests --- internal/testrunner/coveragereport.go | 6 +-- internal/testrunner/coveragereport_test.go | 57 ++++++++++++---------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/internal/testrunner/coveragereport.go b/internal/testrunner/coveragereport.go index 58c1ad9484..a1381220c3 100644 --- a/internal/testrunner/coveragereport.go +++ b/internal/testrunner/coveragereport.go @@ -119,11 +119,11 @@ func GetBaseFolderPackageForCoverage(packageRootPath string) (string, error) { } relativePath := strings.TrimPrefix(packageRootPath, dir) - relativePath = strings.TrimPrefix(relativePath, "/") // linux - relativePath = strings.TrimPrefix(relativePath, "\\") // windows + relativePath = strings.TrimPrefix(relativePath, "/") // Trim prefix for linux + relativePath = strings.TrimPrefix(relativePath, "\\") // Trim prefix for windows baseFolder := filepath.Dir(relativePath) - // Force to show always "/" as file separator + // Force to show always "/" as file separator in paths return strings.ReplaceAll(baseFolder, "\\", "/"), nil } diff --git a/internal/testrunner/coveragereport_test.go b/internal/testrunner/coveragereport_test.go index 18d8d5f4ed..04a06129c6 100644 --- a/internal/testrunner/coveragereport_test.go +++ b/internal/testrunner/coveragereport_test.go @@ -5,6 +5,8 @@ package testrunner import ( + "os" + "path/filepath" "testing" "time" @@ -13,6 +15,9 @@ import ( ) func TestCreateCoverageReport(t *testing.T) { + workDir, err := os.Getwd() + require.NoError(t, err) + packageRootPath := filepath.Join(workDir, "my", "path", "package") tests := []struct { name string rootPath string @@ -27,22 +32,22 @@ func TestCreateCoverageReport(t *testing.T) { { name: "generate custom cobertura coverage", testType: "system", - rootPath: "/my/path/package", - packageName: "package", + rootPath: packageRootPath, + packageName: "mypackage", packageType: "integration", coverageFormat: "cobertura", timestamp: 10, results: []TestResult{ { Name: "test1", - Package: "package.myclass", + Package: "mypackage", DataStream: "metrics", TimeElapsed: 1 * time.Second, Coverage: nil, }, { Name: "test2", - Package: "package.myclass", + Package: "mypackage", DataStream: "logs", TimeElapsed: 2 * time.Second, Coverage: nil, @@ -53,11 +58,11 @@ func TestCreateCoverageReport(t *testing.T) { Timestamp: 10, Packages: []*CoberturaPackage{ { - Name: "my.path.package", + Name: "mypackage", Classes: []*CoberturaClass{ { Name: "system", - Filename: "my/path/package/data_stream/logs/manifest.yml", + Filename: "internal/testrunner/my/path/mypackage/data_stream/logs/manifest.yml", Methods: []*CoberturaMethod{ { Name: "OK", @@ -79,7 +84,7 @@ func TestCreateCoverageReport(t *testing.T) { }, { Name: "system", - Filename: "my/path/package/data_stream/metrics/manifest.yml", + Filename: "internal/testrunner/my/path/mypackage/data_stream/metrics/manifest.yml", Methods: []*CoberturaMethod{ { Name: "OK", @@ -107,22 +112,22 @@ func TestCreateCoverageReport(t *testing.T) { { name: "generate custom generic coverage", testType: "system", - rootPath: "/my/path/package", - packageName: "package", + rootPath: packageRootPath, + packageName: "mypackage", packageType: "integration", coverageFormat: "generic", timestamp: 10, results: []TestResult{ { Name: "test1", - Package: "package.myclass", + Package: "mypackage", DataStream: "metrics", TimeElapsed: 1 * time.Second, Coverage: nil, }, { Name: "test2", - Package: "package.myclass", + Package: "mypackage", DataStream: "logs", TimeElapsed: 2 * time.Second, Coverage: nil, @@ -132,7 +137,7 @@ func TestCreateCoverageReport(t *testing.T) { Version: 1, Files: []*GenericFile{ { - Path: "my/path/package/data_stream/logs/manifest.yml", + Path: "internal/testrunner/my/path/mypackage/data_stream/logs/manifest.yml", Lines: []*GenericLine{ { LineNumber: 3, @@ -141,7 +146,7 @@ func TestCreateCoverageReport(t *testing.T) { }, }, { - Path: "my/path/package/data_stream/metrics/manifest.yml", + Path: "internal/testrunner/my/path/mypackage/data_stream/metrics/manifest.yml", Lines: []*GenericLine{ { LineNumber: 3, @@ -157,15 +162,15 @@ func TestCreateCoverageReport(t *testing.T) { { name: "generate custom generic coverage", testType: "system", - rootPath: "/my/path/package", - packageName: "package", + rootPath: packageRootPath, + packageName: "mypackage", packageType: "integration", coverageFormat: "generic", timestamp: 10, results: []TestResult{ { Name: "test1", - Package: "package.myclass", + Package: "mypackage", DataStream: "metrics", TimeElapsed: 1 * time.Second, Coverage: nil, @@ -175,7 +180,7 @@ func TestCreateCoverageReport(t *testing.T) { Version: 1, Files: []*GenericFile{ { - Path: "my/path/package/data_stream/metrics/manifest.yml", + Path: "internal/testrunner/my/path/mypackage/data_stream/metrics/manifest.yml", Lines: []*GenericLine{ { LineNumber: 3, @@ -191,22 +196,22 @@ func TestCreateCoverageReport(t *testing.T) { { name: "use provided generic coverage", testType: "system", - rootPath: "/my/path/package", - packageName: "package", + rootPath: packageRootPath, + packageName: "mypackage", packageType: "integration", coverageFormat: "generic", timestamp: 10, results: []TestResult{ { Name: "test1", - Package: "package.myclass", + Package: "mypackage", DataStream: "metrics", TimeElapsed: 1 * time.Second, Coverage: &GenericCoverage{ Version: 1, Files: []*GenericFile{ { - Path: "my/path/package/data_stream/metrics/foo.yml", + Path: "internal/testrunner/my/path/mypackage/data_stream/metrics/foo.yml", Lines: []*GenericLine{ { LineNumber: 1, @@ -228,7 +233,7 @@ func TestCreateCoverageReport(t *testing.T) { Version: 1, Files: []*GenericFile{ { - Path: "my/path/package/data_stream/metrics/foo.yml", + Path: "internal/testrunner/my/path/mypackage/data_stream/metrics/foo.yml", Lines: []*GenericLine{ { LineNumber: 1, @@ -248,15 +253,15 @@ func TestCreateCoverageReport(t *testing.T) { { name: "generic coverage for an input package", testType: "asset", - rootPath: "/my/path/package", - packageName: "package", + rootPath: packageRootPath, + packageName: "mypackage", packageType: "input", coverageFormat: "generic", timestamp: 10, results: []TestResult{ { Name: "test1", - Package: "package.myclass", + Package: "mypackage", DataStream: "", TimeElapsed: 1 * time.Second, Coverage: nil, @@ -266,7 +271,7 @@ func TestCreateCoverageReport(t *testing.T) { Version: 1, Files: []*GenericFile{ { - Path: "my/path/package/manifest.yml", + Path: "internal/testrunner/my/path/mypackage/manifest.yml", Lines: []*GenericLine{ { LineNumber: 1, From f938f1297e3ce9561a4f0ef4b99c9351e37872d3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Jan 2024 22:06:15 +0100 Subject: [PATCH 24/35] Update method to get base folder using filepath --- internal/testrunner/coveragereport.go | 8 +++-- internal/testrunner/coveragereport_test.go | 34 ---------------------- 2 files changed, 5 insertions(+), 37 deletions(-) diff --git a/internal/testrunner/coveragereport.go b/internal/testrunner/coveragereport.go index a1381220c3..83a183abbb 100644 --- a/internal/testrunner/coveragereport.go +++ b/internal/testrunner/coveragereport.go @@ -118,9 +118,11 @@ func GetBaseFolderPackageForCoverage(packageRootPath string) (string, error) { return "", err } - relativePath := strings.TrimPrefix(packageRootPath, dir) - relativePath = strings.TrimPrefix(relativePath, "/") // Trim prefix for linux - relativePath = strings.TrimPrefix(relativePath, "\\") // Trim prefix for windows + relativePath, err := filepath.Rel(dir, packageRootPath) + if err != nil { + return "", fmt.Errorf("cannot create relative path to package root path. Root directory: '%s', Package root path: '%s': %w", dir, packageRootPath, err) + } + // Remove latest folder (package) since coverage methods already add the package name in the paths baseFolder := filepath.Dir(relativePath) // Force to show always "/" as file separator in paths diff --git a/internal/testrunner/coveragereport_test.go b/internal/testrunner/coveragereport_test.go index 04a06129c6..e750c4df26 100644 --- a/internal/testrunner/coveragereport_test.go +++ b/internal/testrunner/coveragereport_test.go @@ -159,40 +159,6 @@ func TestCreateCoverageReport(t *testing.T) { Timestamp: 10, }, }, - { - name: "generate custom generic coverage", - testType: "system", - rootPath: packageRootPath, - packageName: "mypackage", - packageType: "integration", - coverageFormat: "generic", - timestamp: 10, - results: []TestResult{ - { - Name: "test1", - Package: "mypackage", - DataStream: "metrics", - TimeElapsed: 1 * time.Second, - Coverage: nil, - }, - }, - expected: &GenericCoverage{ - Version: 1, - Files: []*GenericFile{ - { - Path: "internal/testrunner/my/path/mypackage/data_stream/metrics/manifest.yml", - Lines: []*GenericLine{ - { - LineNumber: 3, - Covered: true, - }, - }, - }, - }, - TestType: "Coverage for system test", - Timestamp: 10, - }, - }, { name: "use provided generic coverage", testType: "system", From 41531acdcbd1e96f229477d435297fa7204d9988 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jan 2024 10:35:23 +0100 Subject: [PATCH 25/35] Use filepath for files in cverage reports --- internal/testrunner/coberturacoverage.go | 6 +++--- internal/testrunner/coveragereport.go | 4 +--- internal/testrunner/coveragereport_test.go | 14 +++++++------- internal/testrunner/genericcobertura.go | 6 +++--- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/internal/testrunner/coberturacoverage.go b/internal/testrunner/coberturacoverage.go index 98ad504a1c..87883eafc4 100644 --- a/internal/testrunner/coberturacoverage.go +++ b/internal/testrunner/coberturacoverage.go @@ -8,7 +8,7 @@ import ( "bytes" "encoding/xml" "fmt" - "path" + "path/filepath" "sort" ) @@ -232,10 +232,10 @@ func transformToCoberturaReport(details *testCoverageDetails, baseFolder string, lines = append(lines, []*CoberturaLine{{Number: lineNumberTestType, Hits: 1}}...) } - fileName := path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml") + fileName := filepath.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml") if dataStream == "" { // input package - fileName = path.Join(baseFolder, details.packageName, "manifest.yml") + fileName = filepath.Join(baseFolder, details.packageName, "manifest.yml") } aClass := &CoberturaClass{ diff --git a/internal/testrunner/coveragereport.go b/internal/testrunner/coveragereport.go index 83a183abbb..155c765eed 100644 --- a/internal/testrunner/coveragereport.go +++ b/internal/testrunner/coveragereport.go @@ -9,7 +9,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "time" "github.com/elastic/elastic-package/internal/builder" @@ -125,8 +124,7 @@ func GetBaseFolderPackageForCoverage(packageRootPath string) (string, error) { // Remove latest folder (package) since coverage methods already add the package name in the paths baseFolder := filepath.Dir(relativePath) - // Force to show always "/" as file separator in paths - return strings.ReplaceAll(baseFolder, "\\", "/"), nil + return baseFolder, nil } func collectTestCoverageDetails(packageRootPath, packageName, packageType string, testType TestType, results []TestResult) (*testCoverageDetails, error) { diff --git a/internal/testrunner/coveragereport_test.go b/internal/testrunner/coveragereport_test.go index e750c4df26..32f51f4116 100644 --- a/internal/testrunner/coveragereport_test.go +++ b/internal/testrunner/coveragereport_test.go @@ -62,7 +62,7 @@ func TestCreateCoverageReport(t *testing.T) { Classes: []*CoberturaClass{ { Name: "system", - Filename: "internal/testrunner/my/path/mypackage/data_stream/logs/manifest.yml", + Filename: filepath.Join("internal", "testrunner", "my", "path", "mypackage", "data_stream", "logs", "manifest.yml"), Methods: []*CoberturaMethod{ { Name: "OK", @@ -84,7 +84,7 @@ func TestCreateCoverageReport(t *testing.T) { }, { Name: "system", - Filename: "internal/testrunner/my/path/mypackage/data_stream/metrics/manifest.yml", + Filename: filepath.Join("internal", "testrunner", "my", "path", "mypackage", "data_stream", "metrics", "manifest.yml"), Methods: []*CoberturaMethod{ { Name: "OK", @@ -137,7 +137,7 @@ func TestCreateCoverageReport(t *testing.T) { Version: 1, Files: []*GenericFile{ { - Path: "internal/testrunner/my/path/mypackage/data_stream/logs/manifest.yml", + Path: filepath.Join("internal", "testrunner", "my", "path", "mypackage", "data_stream", "logs", "manifest.yml"), Lines: []*GenericLine{ { LineNumber: 3, @@ -146,7 +146,7 @@ func TestCreateCoverageReport(t *testing.T) { }, }, { - Path: "internal/testrunner/my/path/mypackage/data_stream/metrics/manifest.yml", + Path: filepath.Join("internal", "testrunner", "my", "path", "mypackage", "data_stream", "metrics", "manifest.yml"), Lines: []*GenericLine{ { LineNumber: 3, @@ -177,7 +177,7 @@ func TestCreateCoverageReport(t *testing.T) { Version: 1, Files: []*GenericFile{ { - Path: "internal/testrunner/my/path/mypackage/data_stream/metrics/foo.yml", + Path: filepath.Join("internal", "testrunner", "my", "path", "mypackage", "data_stream", "metrics", "foo.yml"), Lines: []*GenericLine{ { LineNumber: 1, @@ -199,7 +199,7 @@ func TestCreateCoverageReport(t *testing.T) { Version: 1, Files: []*GenericFile{ { - Path: "internal/testrunner/my/path/mypackage/data_stream/metrics/foo.yml", + Path: filepath.Join("internal", "testrunner", "my", "path", "mypackage", "data_stream", "metrics", "foo.yml"), Lines: []*GenericLine{ { LineNumber: 1, @@ -237,7 +237,7 @@ func TestCreateCoverageReport(t *testing.T) { Version: 1, Files: []*GenericFile{ { - Path: "internal/testrunner/my/path/mypackage/manifest.yml", + Path: filepath.Join("internal", "testrunner", "my", "path", "mypackage", "manifest.yml"), Lines: []*GenericLine{ { LineNumber: 1, diff --git a/internal/testrunner/genericcobertura.go b/internal/testrunner/genericcobertura.go index 3c2ed60f8f..00fee16bc9 100644 --- a/internal/testrunner/genericcobertura.go +++ b/internal/testrunner/genericcobertura.go @@ -8,7 +8,7 @@ import ( "bytes" "encoding/xml" "fmt" - "path" + "path/filepath" "sort" ) @@ -107,10 +107,10 @@ func transformToGenericCoverageReport(details *testCoverageDetails, baseFolder s } testCases := details.dataStreams[dataStream] - fileName := path.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml") + fileName := filepath.Join(baseFolder, details.packageName, "data_stream", dataStream, "manifest.yml") if dataStream == "" { // input package - fileName = path.Join(baseFolder, details.packageName, "manifest.yml") + fileName = filepath.Join(baseFolder, details.packageName, "manifest.yml") } if len(testCases) == 0 { From c961c39e753c0b7377dd57c0dfab015e9cf80afe Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jan 2024 11:25:19 +0100 Subject: [PATCH 26/35] Add package type as part of the coverage report file name --- internal/testrunner/coveragereport.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/coveragereport.go b/internal/testrunner/coveragereport.go index 155c765eed..18ac10fed2 100644 --- a/internal/testrunner/coveragereport.go +++ b/internal/testrunner/coveragereport.go @@ -82,7 +82,7 @@ func WriteCoverage(packageRootPath, packageName, packageType string, testType Te return fmt.Errorf("can't create coverage report: %w", err) } - err = writeCoverageReportFile(report, packageName) + err = writeCoverageReportFile(report, packageName, packageType) if err != nil { return fmt.Errorf("can't write test coverage report file: %w", err) } @@ -209,7 +209,7 @@ func transformToCoverageReport(details *testCoverageDetails, baseFolder, coverag return nil } -func writeCoverageReportFile(report CoverageReport, packageName string) error { +func writeCoverageReportFile(report CoverageReport, packageName, packageType string) error { dest, err := testCoverageReportsDir() if err != nil { return fmt.Errorf("could not determine test coverage reports folder: %w", err) @@ -223,7 +223,7 @@ func writeCoverageReportFile(report CoverageReport, packageName string) error { } } - fileName := fmt.Sprintf("coverage-%s-%d-report.xml", packageName, report.TimeStamp()) + fileName := fmt.Sprintf("coverage-%s-%s-%d-report.xml", packageName, packageType, report.TimeStamp()) filePath := filepath.Join(dest, fileName) b, err := report.Bytes() From 23701cd495e4ac65d92834224f9d809af3d3f015 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jan 2024 11:55:38 +0100 Subject: [PATCH 27/35] Add test type as part of the coverage report file name --- internal/testrunner/coveragereport.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/coveragereport.go b/internal/testrunner/coveragereport.go index 18ac10fed2..6c4c2dc2f0 100644 --- a/internal/testrunner/coveragereport.go +++ b/internal/testrunner/coveragereport.go @@ -82,7 +82,7 @@ func WriteCoverage(packageRootPath, packageName, packageType string, testType Te return fmt.Errorf("can't create coverage report: %w", err) } - err = writeCoverageReportFile(report, packageName, packageType) + err = writeCoverageReportFile(report, packageName, string(testType)) if err != nil { return fmt.Errorf("can't write test coverage report file: %w", err) } @@ -209,7 +209,7 @@ func transformToCoverageReport(details *testCoverageDetails, baseFolder, coverag return nil } -func writeCoverageReportFile(report CoverageReport, packageName, packageType string) error { +func writeCoverageReportFile(report CoverageReport, packageName, testType string) error { dest, err := testCoverageReportsDir() if err != nil { return fmt.Errorf("could not determine test coverage reports folder: %w", err) @@ -223,7 +223,7 @@ func writeCoverageReportFile(report CoverageReport, packageName, packageType str } } - fileName := fmt.Sprintf("coverage-%s-%s-%d-report.xml", packageName, packageType, report.TimeStamp()) + fileName := fmt.Sprintf("coverage-%s-%s-%d-report.xml", packageName, testType, report.TimeStamp()) filePath := filepath.Join(dest, fileName) b, err := report.Bytes() From a51cb2f5b66ba7908a7d051e846ca77fdc41a809 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jan 2024 12:59:37 +0100 Subject: [PATCH 28/35] Remove leftover from testing --- internal/testrunner/runners/asset/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/asset/runner.go b/internal/testrunner/runners/asset/runner.go index c7ec2b305f..d70ea19ec8 100644 --- a/internal/testrunner/runners/asset/runner.go +++ b/internal/testrunner/runners/asset/runner.go @@ -118,7 +118,7 @@ func (r *runner) run() ([]testrunner.TestResult, error) { return result.WithError(fmt.Errorf("could not load expected package assets: %w", err)) } - results := make([]testrunner.TestResult, 0, len(expectedAssets)+1) + results := make([]testrunner.TestResult, 0, len(expectedAssets)) for _, e := range expectedAssets { rc := testrunner.NewResultComposer(testrunner.TestResult{ Name: fmt.Sprintf("%s %s is loaded", e.Type, e.ID), From f667f8c7039df12b10fbb515c9b79c795cc178ca Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jan 2024 13:18:53 +0100 Subject: [PATCH 29/35] Remove debug leftovers --- cmd/testrunner.go | 6 ----- .../testrunner/runners/pipeline/runner.go | 27 ------------------- 2 files changed, 33 deletions(-) diff --git a/cmd/testrunner.go b/cmd/testrunner.go index b7834a2324..54fe9b792b 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -241,8 +241,6 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command } } - // dataStreamsTested := map[string]bool{} - var results []testrunner.TestResult for _, folder := range testFolders { r, err := testrunner.Run(testType, testrunner.TestOptions{ @@ -258,8 +256,6 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command CoverageType: testCoverageType, }) - // dataStreamsTested[folder.DataStream] = true - results = append(results, r...) if err != nil { @@ -267,8 +263,6 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command } } - // for dataStream := datastreams - format := testrunner.TestReportFormat(reportFormat) report, err := testrunner.FormatReport(format, results) if err != nil { diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index abbe4356df..fce07f9fa9 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -116,7 +116,6 @@ func (r *runner) run() ([]testrunner.TestResult, error) { return nil, errors.New("missing Elasticsearch client") } - fmt.Printf("TestFolder.Path: %s\n", r.options.TestFolder.Path) dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.options.TestFolder.Path) if err != nil { return nil, fmt.Errorf("locating data_stream root failed: %w", err) @@ -136,7 +135,6 @@ func (r *runner) run() ([]testrunner.TestResult, error) { return nil, fmt.Errorf("failed to read manifest: %w", err) } - fmt.Printf("TestFolder.DataStream: %s\n", r.options.TestFolder.DataStream) dsManifest, err := packages.ReadDataStreamManifestFromPackageRoot(r.options.PackageRootPath, r.options.TestFolder.DataStream) if err != nil { return nil, fmt.Errorf("failed to read data stream manifest: %w", err) @@ -247,31 +245,6 @@ func (r *runner) run() ([]testrunner.TestResult, error) { } return results, nil - // fmt.Printf(">>> testCaseFiles count (datastream %q): %d\n", r.options.TestFolder.DataStream, len(testCaseFiles)) - // if len(testCaseFiles) > 0 { - // return results, nil - // } - // expected, err := testrunner.VerifyTestExpected(r.options.PackageRootPath, r.options.TestFolder.DataStream, r.Type()) - // if err != nil { - // return nil, fmt.Errorf("can't verify if test is expected: %w", err) - // } - // if !expected { - // return results, nil - // } - // tr := testrunner.TestResult{ - // TestType: TestType, - // Package: r.options.TestFolder.Package, - // DataStream: r.options.TestFolder.DataStream, - // Name: r.options.TestFolder.DataStream, - // } - // startTime := time.Now() - // tr.TimeElapsed = time.Since(startTime) - // tr.Coverage, err = GetPipelineCoverage(r.options, r.pipelines) - // if err != nil { - // return nil, fmt.Errorf("error calculating pipeline coverage: %w", err) - // } - // results = append(results, tr) - // return results, nil } func (r *runner) listTestCaseFiles() ([]string, error) { From d98f7b4c4c0d52ba5209265a15c6b62ad3011744 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jan 2024 13:20:28 +0100 Subject: [PATCH 30/35] Ensure right paths for sources and filenames (pipeline coverage) --- .../testrunner/runners/pipeline/coverage.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index b43128e107..67a6245c55 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -40,19 +40,16 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe // and a default.yml pipeline). basePath := filepath.Dir(options.PackageRootPath) - dir, err := files.FindRepositoryRootDirectory() + repositoryRootDir, err := files.FindRepositoryRootDirectory() if err != nil { return nil, err } - basePath = basePath[:len(dir)] - - // Construct the Cobertura report. - pkg := &testrunner.CoberturaPackage{ - Name: options.TestFolder.Package + "." + options.TestFolder.DataStream, - } - if options.CoverageType == "cobertura" { + pkg := &testrunner.CoberturaPackage{ + Name: options.TestFolder.Package + "." + options.TestFolder.DataStream, + } + cobertura := &testrunner.CoberturaCoverage{ Sources: []*testrunner.CoberturaSource{ { @@ -65,7 +62,7 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe // Calculate coverage for each pipeline for _, pipeline := range pipelines { - pipelineName, pipelineRelPath, src, pstats, err := pipelineDataForCoverage(pipeline, stats, basePath, dataStreamPath) + pipelineName, pipelineRelPath, src, pstats, err := pipelineDataForCoverage(pipeline, stats, repositoryRootDir, dataStreamPath) if err != nil { return nil, err } @@ -88,7 +85,7 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe // Calculate coverage for each pipeline for _, pipeline := range pipelines { - _, pipelineRelPath, src, pstats, err := pipelineDataForCoverage(pipeline, stats, basePath, dataStreamPath) + _, pipelineRelPath, src, pstats, err := pipelineDataForCoverage(pipeline, stats, repositoryRootDir, dataStreamPath) if err != nil { return nil, err } From 9b6565421744b4008f3cd1fec55c9d3689938e7b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 17 Jan 2024 13:23:38 +0100 Subject: [PATCH 31/35] Rephrase comments --- internal/testrunner/coveragereport.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/coveragereport.go b/internal/testrunner/coveragereport.go index 6c4c2dc2f0..3aa79e59e5 100644 --- a/internal/testrunner/coveragereport.go +++ b/internal/testrunner/coveragereport.go @@ -41,7 +41,7 @@ type testCoverageDetails struct { packageType string testType TestType dataStreams map[string][]string // : - coverage CoverageReport // For tests to provide custom Cobertura results. + coverage CoverageReport // For tests to provide custom Coverage results. errors multierror.Error } @@ -64,7 +64,7 @@ func (tcd *testCoverageDetails) withTestResults(results []TestResult) *testCover tcd.dataStreams[result.DataStream] = append(tcd.dataStreams[result.DataStream], result.Name) if tcd.coverage != nil && result.Coverage != nil { if err := tcd.coverage.Merge(result.Coverage); err != nil { - tcd.errors = append(tcd.errors, fmt.Errorf("can't merge Cobertura coverage for test `%s`: %w", result.Name, err)) + tcd.errors = append(tcd.errors, fmt.Errorf("can't merge coverage for test `%s`: %w", result.Name, err)) } } else if tcd.coverage == nil { tcd.coverage = result.Coverage From aa9e30caf64c2f6d80813db754336598d37347e7 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jan 2024 10:46:06 +0100 Subject: [PATCH 32/35] Rename createCoverageReport parameter --- cmd/testrunner.go | 1 - internal/testrunner/coveragereport.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/testrunner.go b/cmd/testrunner.go index 54fe9b792b..58fbebf251 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -154,7 +154,6 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command signal.Enable() var testFolders []testrunner.TestFolder - // var dataStreamWithoutTests []testrunner.TestFolder if hasDataStreams && runner.CanRunPerDataStream() { var dataStreams []string // We check for the existence of the data streams flag before trying to diff --git a/internal/testrunner/coveragereport.go b/internal/testrunner/coveragereport.go index 3aa79e59e5..ca3c7dcd87 100644 --- a/internal/testrunner/coveragereport.go +++ b/internal/testrunner/coveragereport.go @@ -89,7 +89,7 @@ func WriteCoverage(packageRootPath, packageName, packageType string, testType Te return nil } -func createCoverageReport(packageRootPath, packageName, packageType string, testType TestType, results []TestResult, CoverageFormat string, timestamp int64) (CoverageReport, error) { +func createCoverageReport(packageRootPath, packageName, packageType string, testType TestType, results []TestResult, coverageFormat string, timestamp int64) (CoverageReport, error) { details, err := collectTestCoverageDetails(packageRootPath, packageName, packageType, testType, results) if err != nil { return nil, fmt.Errorf("can't collect test coverage details: %w", err) @@ -106,7 +106,7 @@ func createCoverageReport(packageRootPath, packageName, packageType string, test return nil, err } - report := transformToCoverageReport(details, baseFolder, CoverageFormat, timestamp) + report := transformToCoverageReport(details, baseFolder, coverageFormat, timestamp) return report, nil } From 549ffda7a2883a64de7c124ac93281b83b5cac64 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jan 2024 11:17:51 +0100 Subject: [PATCH 33/35] Check coverage format value set in the parameter --- cmd/testrunner.go | 13 +++++++++---- internal/cobraext/flags.go | 2 +- internal/testrunner/coberturacoverage.go | 4 ++++ internal/testrunner/coveragereport.go | 11 +++++++++++ internal/testrunner/genericcobertura.go | 4 ++++ 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/cmd/testrunner.go b/cmd/testrunner.go index 58fbebf251..f4308d421b 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "github.com/spf13/cobra" @@ -71,7 +72,7 @@ func setupTestCommand() *cobraext.Command { cmd.PersistentFlags().StringP(cobraext.ReportFormatFlagName, "", string(formats.ReportFormatHuman), cobraext.ReportFormatFlagDescription) cmd.PersistentFlags().StringP(cobraext.ReportOutputFlagName, "", string(outputs.ReportOutputSTDOUT), cobraext.ReportOutputFlagDescription) cmd.PersistentFlags().BoolP(cobraext.TestCoverageFlagName, "", false, cobraext.TestCoverageFlagDescription) - cmd.PersistentFlags().StringP(cobraext.TestCoverageFormatFlagName, "", "cobertura", cobraext.TestCoverageFormatFlagDescription) + cmd.PersistentFlags().StringP(cobraext.TestCoverageFormatFlagName, "", "cobertura", fmt.Sprintf(cobraext.TestCoverageFormatFlagDescription, strings.Join(testrunner.CoverageFormatsList(), ","))) cmd.PersistentFlags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription) cmd.PersistentFlags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription) cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar)) @@ -128,11 +129,15 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command return cobraext.FlagParsingError(err, cobraext.TestCoverageFlagName) } - testCoverageType, err := cmd.Flags().GetString(cobraext.TestCoverageFormatFlagName) + testCoverageFormat, err := cmd.Flags().GetString(cobraext.TestCoverageFormatFlagName) if err != nil { return cobraext.FlagParsingError(err, cobraext.TestCoverageFormatFlagName) } + if !slices.Contains(testrunner.CoverageFormatsList(), testCoverageFormat) { + return cobraext.FlagParsingError(fmt.Errorf("coverage format not available: %s", testCoverageFormat), cobraext.TestCoverageFormatFlagName) + } + packageRootPath, found, err := packages.FindPackageRoot() if !found { return errors.New("package root not found") @@ -252,7 +257,7 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command DeferCleanup: deferCleanup, ServiceVariant: variantFlag, WithCoverage: testCoverage, - CoverageType: testCoverageType, + CoverageType: testCoverageFormat, }) results = append(results, r...) @@ -273,7 +278,7 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command } if testCoverage { - err := testrunner.WriteCoverage(packageRootPath, manifest.Name, manifest.Type, runner.Type(), results, testCoverageType) + err := testrunner.WriteCoverage(packageRootPath, manifest.Name, manifest.Type, runner.Type(), results, testCoverageFormat) if err != nil { return fmt.Errorf("error writing test coverage: %w", err) } diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index e1dd88b027..2faaa4863d 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -193,7 +193,7 @@ const ( TestCoverageFlagDescription = "enable test coverage reports" TestCoverageFormatFlagName = "coverage-format" - TestCoverageFormatFlagDescription = "set format for coverage reports: \"cobertura,generic\" (default cobertura)" + TestCoverageFormatFlagDescription = "set format for coverage reports: %s" VariantFlagName = "variant" VariantFlagDescription = "service variant" diff --git a/internal/testrunner/coberturacoverage.go b/internal/testrunner/coberturacoverage.go index 87883eafc4..38e3a8e651 100644 --- a/internal/testrunner/coberturacoverage.go +++ b/internal/testrunner/coberturacoverage.go @@ -12,6 +12,10 @@ import ( "sort" ) +func init() { + registerCoverageReporterFormat("cobertura") +} + const coverageDtd = `` // CoberturaCoverage is the root element for a Cobertura XML report. diff --git a/internal/testrunner/coveragereport.go b/internal/testrunner/coveragereport.go index ca3c7dcd87..9a81c7cc31 100644 --- a/internal/testrunner/coveragereport.go +++ b/internal/testrunner/coveragereport.go @@ -22,6 +22,17 @@ type CoverageReport interface { Bytes() ([]byte, error) } +var coverageReportFormatters = []string{} + +// registerCoverageReporterFormat registers a test coverage report formatter. +func registerCoverageReporterFormat(name string) { + coverageReportFormatters = append(coverageReportFormatters, name) +} + +func CoverageFormatsList() []string { + return coverageReportFormatters +} + func lineNumberPerTestType(testType string) int { var lineNumberPerTestType map[string]int = map[string]int{ "asset": 1, diff --git a/internal/testrunner/genericcobertura.go b/internal/testrunner/genericcobertura.go index 00fee16bc9..8b17c4b4a9 100644 --- a/internal/testrunner/genericcobertura.go +++ b/internal/testrunner/genericcobertura.go @@ -12,6 +12,10 @@ import ( "sort" ) +func init() { + registerCoverageReporterFormat("generic") +} + // GenericCoverage is the root element for a Cobertura XML report. type GenericCoverage struct { XMLName xml.Name `xml:"coverage"` From 5579da4d4f80e097b16e35be3f46aef6112d5f00 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jan 2024 11:40:15 +0100 Subject: [PATCH 34/35] Update test scripts to use coverage format generic --- scripts/test-check-false-positives.sh | 2 +- scripts/test-check-packages.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test-check-false-positives.sh b/scripts/test-check-false-positives.sh index f09016441a..eb78480e7f 100755 --- a/scripts/test-check-false-positives.sh +++ b/scripts/test-check-false-positives.sh @@ -38,7 +38,7 @@ function check_expected_errors() { rm -f ${result_tests} ( cd "$package_root" - elastic-package test -v --report-format xUnit --report-output file --test-coverage --defer-cleanup 1s || true + elastic-package test -v --report-format xUnit --report-output file --test-coverage --coverage-format=generic --defer-cleanup 1s || true ) cat ${result_tests} | tr -d '\n' > ${results_no_spaces} diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index a9b116e3fd..f0a9c88a9c 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -107,7 +107,7 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s else # defer-cleanup is set to a short period to verify that the option is available - elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage + elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic fi ) cd - From 499953c0751ac1f23181453eef4ae2fb191bf9a6 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Jan 2024 13:01:37 +0100 Subject: [PATCH 35/35] Add missing version field --- internal/testrunner/runners/pipeline/coverage.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/testrunner/runners/pipeline/coverage.go b/internal/testrunner/runners/pipeline/coverage.go index 67a6245c55..42f1619602 100644 --- a/internal/testrunner/runners/pipeline/coverage.go +++ b/internal/testrunner/runners/pipeline/coverage.go @@ -79,6 +79,7 @@ func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipe if options.CoverageType == "generic" { coverage := &testrunner.GenericCoverage{ + Version: 1, Timestamp: time.Now().UnixNano(), TestType: "Cobertura for pipeline test", }