add Elastic Stack and Elastic Package version to test results report

cmd/testrunner.go

@@ -31,6 +31,7 @@ import (
 	"github.com/elastic/elastic-package/internal/testrunner/runners/policy"
 	"github.com/elastic/elastic-package/internal/testrunner/runners/static"
 	"github.com/elastic/elastic-package/internal/testrunner/runners/system"
+	"github.com/elastic/elastic-package/internal/version"
 )
 
 const testLongDescription = `Use this command to run tests on a package. Currently, the following types of tests are available:
@@ -174,6 +175,7 @@ func testRunnerAssetCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to read global config: %w", err)
 	}
 
+	cmd.Println(version.Version())
 	runner := asset.NewAssetTestRunner(asset.AssetTestRunnerOptions{
 		PackageRootPath:  packageRootPath,
 		KibanaClient:     kibanaClient,
@@ -262,6 +264,7 @@ func testRunnerStaticCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to read global config: %w", err)
 	}
 
+	cmd.Println(version.Version())
 	runner := static.NewStaticTestRunner(static.StaticTestRunnerOptions{
 		PackageRootPath:    packageRootPath,
 		DataStreams:        dataStreams,
@@ -380,6 +383,13 @@ func testRunnerPipelineCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to read global config: %w", err)
 	}
 
+	esClientInfo, err := esClient.Info(ctx)
+	if err != nil {
+		return fmt.Errorf("fetching stack version failed: %w", err)
+	}
+
+	cmd.Println(version.Version())
+	cmd.Printf("elastic-stack: %s\n", esClientInfo.Version.Number)
 	runner := pipeline.NewPipelineTestRunner(pipeline.PipelineTestRunnerOptions{
 		Profile:            profile,
 		PackageRootPath:    packageRootPath,
@@ -566,6 +576,13 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to read global config: %w", err)
 	}
 
+	stackVersion, err := kibanaClient.Version()
+	if err != nil {
+		return fmt.Errorf("fetching stack version failed: %w", err)
+	}
+
+	cmd.Println(version.Version())
+	cmd.Printf("elastic-stack: %s\n", stackVersion.Version())
 	runner := system.NewSystemTestRunner(system.SystemTestRunnerOptions{
 		Profile:            profile,
 		PackageRootPath:    packageRootPath,
@@ -691,6 +708,13 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to read global config: %w", err)
 	}
 
+	stackVersion, err := kibanaClient.Version()
+	if err != nil {
+		return fmt.Errorf("fetching stack version failed: %w", err)
+	}
+
+	cmd.Println(version.Version())
+	cmd.Printf("elastic-stack: %s\n", stackVersion.Version())
 	runner := policy.NewPolicyTestRunner(policy.PolicyTestRunnerOptions{
 		PackageRootPath:    packageRootPath,
 		KibanaClient:       kibanaClient,

cmd/version.go

@@ -6,7 +6,6 @@ package cmd
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/spf13/cobra"
 
@@ -29,13 +28,6 @@ func setupVersionCommand() *cobraext.Command {
 }
 
 func versionCommandAction(cmd *cobra.Command, args []string) error {
-	var sb strings.Builder
-	sb.WriteString("elastic-package ")
-	if version.Tag != "" {
-		sb.WriteString(version.Tag)
-		sb.WriteString(" ")
-	}
-	sb.WriteString(fmt.Sprintf("version-hash %s (build time: %s)", version.CommitHash, version.BuildTimeFormatted()))
-	fmt.Println(sb.String())
+	fmt.Println(version.Version())
 	return nil
 }

internal/cobraext/flags.go

@@ -154,7 +154,7 @@ const (
 	ProfileFormatFlagDescription = "format of the profiles list (table | json)"
 
 	ReportFormatFlagName        = "report-format"
-	ReportFormatFlagDescription = "format of test report, eg: human, xUnit"
+	ReportFormatFlagDescription = "format of test report, eg: human, xUnit, json"
 
 	ReportFullFlagName        = "full"
 	ReportFullFlagDescription = "whether to show the full report or a summary"

internal/testrunner/reporters/formats/json.go

@@ -0,0 +1,72 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package formats
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/elastic/elastic-package/internal/testrunner"
+)
+
+func init() {
+	testrunner.RegisterReporterFormat(ReportFormatJSON, reportJSONFormat)
+}
+
+const (
+	// ReportFormatJSON reports test results in a JSON format
+	ReportFormatJSON testrunner.TestReportFormat = "json"
+)
+
+type jsonResult struct {
+	Package        string `json:"package"`
+	DataStream     string `json:"data_stream,omitempty"`
+	TestType       string `json:"test_type"`
+	Name           string `json:"name"`
+	Result         string `json:"result"`
+	TimeElapsed    string `json:"time_elapsed"`
+	FailureDetails string `json:"failure_details,omitempty"`
+}
+
+func reportJSONFormat(results []testrunner.TestResult) (string, error) {
+	if len(results) == 0 {
+		return "No test results", nil
+	}
+
+	jsonReport := make([]jsonResult, 0, len(results))
+	for _, r := range results {
+		jsonResult := jsonResult{
+			Package:     r.Package,
+			DataStream:  r.DataStream,
+			TestType:    string(r.TestType),
+			Name:        r.Name,
+			TimeElapsed: r.TimeElapsed.String(),
+		}
+
+		if r.FailureMsg != "" {
+			jsonResult.FailureDetails = fmt.Sprintf("%s/%s %s:\n%s\n", r.Package, r.DataStream, r.Name, r.FailureDetails)
+		}
+
+		var result string
+		if r.ErrorMsg != "" {
+			result = fmt.Sprintf("ERROR: %s", r.ErrorMsg)
+		} else if r.FailureMsg != "" {
+			result = fmt.Sprintf("FAIL: %s", r.FailureMsg)
+		} else if r.Skipped != nil {
+			result = fmt.Sprintf("SKIPPED: %s", r.Skipped.String())
+		} else {
+			result = "PASS"
+		}
+		jsonResult.Result = result
+
+		jsonReport = append(jsonReport, jsonResult)
+	}
+
+	b, err := json.Marshal(jsonReport)
+	if err != nil {
+		return "", fmt.Errorf("marshaling test results to JSON: %w", err)
+	}
+	return string(b), nil
+}

internal/testrunner/reporters/outputs/file.go

@@ -39,8 +39,11 @@ func reportToFile(pkg, report string, testType testrunner.TestType, format testr
 	}
 
 	ext := "txt"
-	if format == formats.ReportFormatXUnit {
+	switch format {
+	case formats.ReportFormatXUnit:
 		ext = "xml"
+	case formats.ReportFormatJSON:
+		ext = "json"
 	}
 
 	fileName := fmt.Sprintf("%s-%s-%d.%s", pkg, testType, time.Now().UnixNano(), ext)

internal/version/version.go

@@ -5,8 +5,10 @@
 package version
 
 import (
+	"fmt"
 	"runtime/debug"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -32,8 +34,8 @@ func init() {
 	}
 }
 
-// BuildTimeFormatted method returns the build time preserving the RFC3339 format.
-func BuildTimeFormatted() string {
+// buildTimeFormatted method returns the build time preserving the RFC3339 format.
+func buildTimeFormatted() string {
 	if BuildTime == "unknown" {
 		return BuildTime
 	}
@@ -44,3 +46,14 @@ func BuildTimeFormatted() string {
 	}
 	return time.Unix(seconds, 0).Format(time.RFC3339)
 }
+
+func Version() string {
+	var sb strings.Builder
+	sb.WriteString("elastic-package ")
+	if Tag != "" {
+		sb.WriteString(Tag)
+		sb.WriteString(" ")
+	}
+	sb.WriteString(fmt.Sprintf("version-hash %s (build time: %s)", CommitHash, buildTimeFormatted()))
+	return sb.String()
+}