feat(cli): show required gates results (#2947)

kubeshop · Jul 14, 2023 · 837a6b5 · 837a6b5
1 parent 20b9567
commit 837a6b5
Show file tree

Hide file tree

Showing 13 changed files with 94 additions and 11 deletions.
diff --git a/cli/formatters/test_run.go b/cli/formatters/test_run.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/kubeshop/tracetest/cli/openapi"
 	"github.com/pterm/pterm"
+	"golang.org/x/exp/slices"
 )
 
 const (
@@ -94,7 +95,7 @@ func (f testRun) pretty(output TestRunOutput) string {
 		return f.formatSuccessfulTest(output.Test, output.Run)
 	}
 
-	if !output.Run.Result.GetAllPassed() {
+	if !output.Run.Result.GetAllPassed() || !output.Run.RequiredGatesResult.GetPassed() {
 		return f.formatFailedTest(output.Test, output.Run)
 	}
 
@@ -210,6 +211,24 @@ func (f testRun) formatFailedTest(test openapi.Test, run openapi.TestRun) string
 		}
 	}
 
+	if len(run.GetRequiredGatesResult().Required) > 0 {
+		buffer.WriteString("\n")
+		gatesPassed := run.RequiredGatesResult.GetPassed()
+		icon := f.getStateIcon(gatesPassed)
+		message = f.getColoredText(
+			gatesPassed,
+			f.formatMessage("\t%s %s\n", icon, "Required gates"),
+		)
+		buffer.WriteString(message)
+		for _, gateResult := range run.GetRequiredGatesResult().Required {
+			passed := slices.Index(run.RequiredGatesResult.Failed, gateResult) == -1
+			icon := f.getStateIcon(passed)
+			message := f.formatMessage("\t\t%s %s\n", icon, gateResult)
+			message = f.getColoredText(passed, message)
+			buffer.WriteString(message)
+		}
+	}
+
 	return buffer.String()
 }
 

diff --git a/cli/runner/test_runner.go b/cli/runner/test_runner.go
@@ -110,7 +110,7 @@ func (r testRunner) UpdateResult(ctx context.Context, result RunResult) (RunResu
 		return RunResult{}, err
 	}
 
-	passed := !isStateFailed(updated.GetState()) && updated.Result.GetAllPassed()
+	passed := !isStateFailed(updated.GetState()) && updated.Result.GetAllPassed() && updated.RequiredGatesResult.GetPassed()
 
 	return RunResult{
 		Resource: test,

diff --git a/cli/runner/transaction_runner.go b/cli/runner/transaction_runner.go
@@ -118,7 +118,7 @@ func (r transactionRunner) UpdateResult(ctx context.Context, result RunResult) (
 		}
 	}
 
-	passed := !isStateFailed(updated.GetState()) && allPassed
+	passed := !isStateFailed(updated.GetState()) && allPassed && updated.GetAllStepsRequiredGatesPassed()
 
 	return RunResult{
 		Resource: transaction,

diff --git a/examples/collector/tracetest-provision.yaml b/examples/collector/tracetest-provision.yaml
@@ -14,3 +14,11 @@ spec:
   name: OpenTelemetry Collector
   type: otlp
   default: true
+---
+type: TestRunner
+spec:
+  id: current
+  name: default
+  requiredGates:
+    - analyzer-score
+    - test-specs
diff --git a/examples/tracetest-jaeger/tracetest-provision.yaml b/examples/tracetest-jaeger/tracetest-provision.yaml
@@ -17,3 +17,11 @@ spec:
     endpoint: jaeger:16685
     tls:
       insecure: true
+---
+type: TestRunner
+spec:
+  id: current
+  name: default
+  requiredGates:
+    - analyzer-score
+    - test-specs
diff --git a/examples/tracetest-opensearch/tracetest-provision.yaml b/examples/tracetest-opensearch/tracetest-provision.yaml
@@ -17,3 +17,11 @@ spec:
     addresses:
       - http://opensearch:9200
     index: traces
+---
+type: TestRunner
+spec:
+  id: current
+  name: default
+  requiredGates:
+    - analyzer-score
+    - test-specs
diff --git a/examples/tracetest-provisioning-env/docker-compose.yml b/examples/tracetest-provisioning-env/docker-compose.yml
@@ -25,7 +25,7 @@ services:
         environment:
             TRACETEST_DEV: ${TRACETEST_DEV}
             # contents of `tracetest-provision.yaml` encoded as base64
-            TRACETEST_PROVISIONING: "LS0tCnR5cGU6IFBvbGxpbmdQcm9maWxlCnNwZWM6CiAgbmFtZTogRGVmYXVsdAogIHN0cmF0ZWd5OiBwZXJpb2RpYwogIGRlZmF1bHQ6IHRydWUKICBwZXJpb2RpYzoKICAgIHJldHJ5RGVsYXk6IDVzCiAgICB0aW1lb3V0OiAxMG0KCi0tLQp0eXBlOiBEYXRhU3RvcmUKc3BlYzoKICBuYW1lOiBKYWVnZXIKICB0eXBlOiBqYWVnZXIKICBkZWZhdWx0OiB0cnVlCiAgamFlZ2VyOgogICAgZW5kcG9pbnQ6IGphZWdlcjoxNjY4NQogICAgdGxzOgogICAgICBpbnNlY3VyZTogdHJ1ZQo="
+            TRACETEST_PROVISIONING: "LS0tCnR5cGU6IFBvbGxpbmdQcm9maWxlCnNwZWM6CiAgbmFtZTogRGVmYXVsdAogIHN0cmF0ZWd5OiBwZXJpb2RpYwogIGRlZmF1bHQ6IHRydWUKICBwZXJpb2RpYzoKICAgIHJldHJ5RGVsYXk6IDVzCiAgICB0aW1lb3V0OiAxMG0KCi0tLQp0eXBlOiBEYXRhU3RvcmUKc3BlYzoKICBuYW1lOiBKYWVnZXIKICB0eXBlOiBqYWVnZXIKICBkZWZhdWx0OiB0cnVlCiAgamFlZ2VyOgogICAgZW5kcG9pbnQ6IGphZWdlcjoxNjY4NQogICAgdGxzOgogICAgICBpbnNlY3VyZTogdHJ1ZQotLS0KdHlwZTogVGVzdFJ1bm5lcgpzcGVjOgogIGlkOiBjdXJyZW50CiAgbmFtZTogZGVmYXVsdAogIHJlcXVpcmVkR2F0ZXM6CiAgICAtIGFuYWx5emVyLXNjb3JlCiAgICAtIHRlc3Qtc3BlY3MK"
 
     postgres:
         image: postgres:14

diff --git a/examples/tracetest-provisioning-env/tracetest-provision.yaml b/examples/tracetest-provisioning-env/tracetest-provision.yaml
@@ -18,3 +18,11 @@ spec:
     endpoint: jaeger:16685
     tls:
       insecure: true
+---
+type: TestRunner
+spec:
+  id: current
+  name: default
+  requiredGates:
+    - analyzer-score
+    - test-specs
diff --git a/examples/tracetest-signoz/tracetest/tracetest-provision.yaml b/examples/tracetest-signoz/tracetest/tracetest-provision.yaml
@@ -13,3 +13,11 @@ type: DataStore
 spec:
   name: Signoz
   type: signoz
+---
+type: TestRunner
+spec:
+  id: current
+  name: default
+  requiredGates:
+    - analyzer-score
+    - test-specs
diff --git a/examples/tracetest-tempo/tracetest-provision.yaml b/examples/tracetest-tempo/tracetest-provision.yaml
@@ -19,3 +19,11 @@ spec:
       endpoint: tempo:9095
       tls:
         insecure: true
+---
+type: TestRunner
+spec:
+  id: current
+  name: default
+  requiredGates:
+    - analyzer-score
+    - test-specs
diff --git a/local-config/tracetest.provision.yaml b/local-config/tracetest.provision.yaml
@@ -21,3 +21,11 @@ spec:
   periodic:
     timeout: 30s
     retryDelay: 1s
+---
+type: TestRunner
+spec:
+  id: current
+  name: default
+  requiredGates:
+    - analyzer-score
+    - test-specs
diff --git a/server/transaction/transaction_run_repository.go b/server/transaction/transaction_run_repository.go
@@ -155,14 +155,14 @@ UPDATE transaction_runs SET
 	"last_error" = $4,
 	"pass" = $5,
 	"fail" = $6,
-	"all_steps_required_gates_passed" = $11,
+	"all_steps_required_gates_passed" = $7,
 
-	"metadata" = $7,
+	"metadata" = $8,
 
 	-- environment
-	"environment" = $8
+	"environment" = $9
 
-WHERE id = $9 AND transaction_id = $10
+WHERE id = $10 AND transaction_id = $11
 `
 
 func (td *RunRepository) UpdateRun(ctx context.Context, tr TransactionRun) error {
@@ -193,7 +193,7 @@ func (td *RunRepository) UpdateRun(ctx context.Context, tr TransactionRun) error
 	}
 
 	pass, fail := tr.ResultsCount()
-	AllStepsRequiredGatesPassed := tr.StepsGatesValidation()
+	allStepsRequiredGatesPassed := tr.StepsGatesValidation()
 
 	_, err = stmt.ExecContext(
 		ctx,
@@ -203,11 +203,11 @@ func (td *RunRepository) UpdateRun(ctx context.Context, tr TransactionRun) error
 		lastError,
 		pass,
 		fail,
+		allStepsRequiredGatesPassed,
 		jsonMetadata,
 		jsonEnvironment,
 		tr.ID,
 		tr.TransactionID,
-		AllStepsRequiredGatesPassed,
 	)
 
 	if err != nil {
@@ -424,7 +424,7 @@ func (td *RunRepository) readRunRow(row scanner) (TransactionRun, error) {
 
 	// checks if the flag exists, if it doesn't we use the fail field to determine if all steps passed
 	if allStepsRequiredGatesPassed == nil {
-		failed := fail.Valid && fail.Int32 == 0
+		failed := r.Fail == 0
 		allStepsRequiredGatesPassed = &failed
 	}
 

diff --git a/testing/cli-e2etest/environment/jaeger/server-setup/tracetest-provision.yaml b/testing/cli-e2etest/environment/jaeger/server-setup/tracetest-provision.yaml
@@ -7,3 +7,11 @@ spec:
   periodic:
     retryDelay: 500ms
     timeout: 1m
+---
+type: TestRunner
+spec:
+  id: current
+  name: default
+  requiredGates:
+    - analyzer-score
+    - test-specs