feature: Required Gates (#2922)

* feature(backend): Required Gates (#2888)

* feature(backend): Required Gates

* adding tests and fixing bugs

* adding tests and fixing bugs

* addressing PR comments

* addressing PR comments

* updating go.modsum

* feature(frontend): Required Gates (#2932)

* feature(frontend): Test Runner Configuration

* feature(frontend): cleanup

* feature(frontend): removing unnecessary prop

* feature(frontend): addressing PR review comments

api/testRunner.yaml

@@ -0,0 +1,51 @@
+openapi: 3.0.0
+components:
+  schemas:
+    TestRunnerResource:
+      type: object
+      properties:
+        type:
+          type: string
+          description: "Represents the type of this resource. It should always be set as 'TestRunner'."
+          enum:
+            - TestRunner
+        spec:
+          $ref: "#/components/schemas/TestRunner"
+    TestRunner:
+      type: object
+      properties:
+        id:
+          type: string
+          description: "ID of the test runner resource. It should always be set as 'current'."
+          enum:
+            - current
+        name:
+          type: string
+          description: "Name given for this test runner set"
+        requiredGates:
+          type: array
+          items:
+            $ref: "#/components/schemas/SupportedGates"
+    RequiredGatesResult:
+      type: object
+      properties:
+        required:
+          type: array
+          items:
+            $ref: "#/components/schemas/SupportedGates"
+        failed:
+          type: array
+          items:
+            $ref: "#/components/schemas/SupportedGates"
+        passed:
+          type: boolean
+      required:
+        - passed
+        - failed
+        - required
+    SupportedGates:
+      type: string
+      enum:
+        - analyzer-score
+        - analyzer-rules
+        - test-specs

api/tests.yaml

@@ -218,6 +218,8 @@ components:
                 type: string
               error:
                 type: string
+        requiredGatesResult:
+          $ref: "./testRunner.yaml#/components/schemas/RequiredGatesResult"
 
         metadata:
           type: object
@@ -242,6 +244,11 @@ components:
           type: array
           items:
             $ref: "./environments.yaml#/components/schemas/EnvironmentValue"
+        requiredGates:
+          type: array
+          nullable: true
+          items:
+            $ref: "./testRunner.yaml#/components/schemas/SupportedGates"
 
     AssertionResults:
       type: object

api/transactions.yaml

@@ -1,7 +1,6 @@
 openapi: 3.0.0
 components:
   schemas:
-
     TransactionResourceList:
       type: object
       properties:
@@ -86,3 +85,5 @@ components:
           type: integer
         fail:
           type: integer
+        allStepsRequiredGatesPassed:
+          type: boolean

go.work.sum

@@ -407,11 +407,3 @@ google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsA
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=
-k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I=
-k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM=
-k8s.io/cri-api v0.25.0/go.mod h1:J1rAyQkSJ2Q6I+aBMOVgg2/cbbebso6FNa0UagiR0kc=
-k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk=
-k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=

server/app/app.go

@@ -19,6 +19,7 @@ import (
 	"github.com/kubeshop/tracetest/server/environment"
 	"github.com/kubeshop/tracetest/server/executor"
 	"github.com/kubeshop/tracetest/server/executor/pollingprofile"
+	"github.com/kubeshop/tracetest/server/executor/testrunner"
 	"github.com/kubeshop/tracetest/server/executor/trigger"
 	httpServer "github.com/kubeshop/tracetest/server/http"
 	"github.com/kubeshop/tracetest/server/http/mappings"
@@ -198,6 +199,7 @@ func (app *App) Start(opts ...appOption) error {
 	linterRepo := analyzer.NewRepository(db)
 	testRepo := test.NewRepository(db)
 	runRepo := test.NewRunRepository(db)
+	testRunnerRepo := testrunner.NewRepository(db)
 
 	transactionsRepository := transaction.NewRepository(db, testRepo)
 	transactionRunRepository := transaction.NewRunRepository(db, runRepo)
@@ -209,6 +211,7 @@ func (app *App) Start(opts ...appOption) error {
 		pollingProfileRepo,
 		dataStoreRepo,
 		linterRepo,
+		testRunnerRepo,
 		testDB,
 		testRepo,
 		runRepo,
@@ -279,6 +282,7 @@ func (app *App) Start(opts ...appOption) error {
 	registerDemosResource(demoRepo, apiRouter, provisioner, tracer)
 	registerDataStoreResource(dataStoreRepo, apiRouter, provisioner, tracer)
 	registerAnalyzer(linterRepo, apiRouter, provisioner, tracer)
+	registerTestRunner(testRunnerRepo, apiRouter, provisioner, tracer)
 	registerTestResource(testRepo, apiRouter, provisioner, tracer)
 
 	isTracetestDev := os.Getenv("TRACETEST_DEV") != ""
@@ -379,6 +383,18 @@ func registerAnalyzer(linterRepo *analyzer.Repository, router *mux.Router, provi
 	provisioner.AddResourceProvisioner(manager)
 }
 
+func registerTestRunner(testRunnerRepo *testrunner.Repository, router *mux.Router, provisioner *provisioning.Provisioner, tracer trace.Tracer) {
+	manager := resourcemanager.New[testrunner.TestRunner](
+		testrunner.ResourceName,
+		testrunner.ResourceNamePlural,
+		testRunnerRepo,
+		resourcemanager.DisableDelete(),
+		resourcemanager.WithTracer(tracer),
+	)
+	manager.RegisterRoutes(router)
+	provisioner.AddResourceProvisioner(manager)
+}
+
 func registerTransactionResource(repo *transaction.Repository, router *mux.Router, provisioner *provisioning.Provisioner, tracer trace.Tracer) {
 	manager := resourcemanager.New[transaction.Transaction](
 		transaction.TransactionResourceName,

server/app/facade.go

@@ -7,6 +7,7 @@ import (
 	"github.com/kubeshop/tracetest/server/environment"
 	"github.com/kubeshop/tracetest/server/executor"
 	"github.com/kubeshop/tracetest/server/executor/pollingprofile"
+	"github.com/kubeshop/tracetest/server/executor/testrunner"
 	"github.com/kubeshop/tracetest/server/executor/trigger"
 	"github.com/kubeshop/tracetest/server/linter/analyzer"
 	"github.com/kubeshop/tracetest/server/model"
@@ -39,12 +40,12 @@ func (rf runnerFacade) StopTest(testID id.ID, runID int) {
 	})
 }
 
-func (rf runnerFacade) RunTest(ctx context.Context, test test.Test, rm test.RunMetadata, env environment.Environment) test.Run {
-	return rf.runner.Run(ctx, test, rm, env)
+func (rf runnerFacade) RunTest(ctx context.Context, test test.Test, rm test.RunMetadata, env environment.Environment, gates *[]testrunner.RequiredGate) test.Run {
+	return rf.runner.Run(ctx, test, rm, env, gates)
 }
 
-func (rf runnerFacade) RunTransaction(ctx context.Context, tr transaction.Transaction, rm test.RunMetadata, env environment.Environment) transaction.TransactionRun {
-	return rf.transactionRunner.Run(ctx, tr, rm, env)
+func (rf runnerFacade) RunTransaction(ctx context.Context, tr transaction.Transaction, rm test.RunMetadata, env environment.Environment, gates *[]testrunner.RequiredGate) transaction.TransactionRun {
+	return rf.transactionRunner.Run(ctx, tr, rm, env, gates)
 }
 
 func (rf runnerFacade) RunAssertions(ctx context.Context, request executor.AssertionRequest) {
@@ -55,6 +56,7 @@ func newRunnerFacades(
 	ppRepo *pollingprofile.Repository,
 	dsRepo *datastore.Repository,
 	lintRepo *analyzer.Repository,
+	trRepo *testrunner.Repository,
 	db model.Repository,
 	testRepo test.Repository,
 	runRepo test.RunRepository,
@@ -117,6 +119,7 @@ func newRunnerFacades(
 		dsRepo,
 		eventEmitter,
 		ppRepo,
+		trRepo,
 	)
 
 	transactionRunner := executor.NewTransactionRunner(

server/executor/runner.go

@@ -11,6 +11,7 @@ import (
 	"github.com/kubeshop/tracetest/server/analytics"
 	"github.com/kubeshop/tracetest/server/datastore"
 	"github.com/kubeshop/tracetest/server/environment"
+	"github.com/kubeshop/tracetest/server/executor/testrunner"
 	triggerer "github.com/kubeshop/tracetest/server/executor/trigger"
 	"github.com/kubeshop/tracetest/server/expression"
 	"github.com/kubeshop/tracetest/server/model"
@@ -31,14 +32,18 @@ type RunResult struct {
 }
 
 type Runner interface {
-	Run(context.Context, test.Test, test.RunMetadata, environment.Environment) test.Run
+	Run(context.Context, test.Test, test.RunMetadata, environment.Environment, *[]testrunner.RequiredGate) test.Run
 }
 
 type PersistentRunner interface {
 	Runner
 	WorkerPool
 }
 
+type TestRunnerGetter interface {
+	GetDefault(ctx context.Context) testrunner.TestRunner
+}
+
 func NewPersistentRunner(
 	triggers *triggerer.Registry,
 	runs test.RunRepository,
@@ -50,6 +55,7 @@ func NewPersistentRunner(
 	dsRepo resourcemanager.Current[datastore.DataStore],
 	eventEmitter EventEmitter,
 	ppGetter PollingProfileGetter,
+	trGetter TestRunnerGetter,
 ) PersistentRunner {
 	return persistentRunner{
 		triggers:            triggers,
@@ -62,6 +68,7 @@ func NewPersistentRunner(
 		subscriptionManager: subscriptionManager,
 		eventEmitter:        eventEmitter,
 		ppGetter:            ppGetter,
+		trGetter:            trGetter,
 		executeQueue:        make(chan execReq, 5),
 		exit:                make(chan bool, 1),
 	}
@@ -78,6 +85,7 @@ type persistentRunner struct {
 	dsRepo              resourcemanager.Current[datastore.DataStore]
 	eventEmitter        EventEmitter
 	ppGetter            PollingProfileGetter
+	trGetter            TestRunnerGetter
 
 	executeQueue chan execReq
 	exit         chan bool
@@ -139,7 +147,7 @@ func getNewCtx(ctx context.Context) context.Context {
 	return otel.GetTextMapPropagator().Extract(context.Background(), carrier)
 }
 
-func (r persistentRunner) Run(ctx context.Context, testObj test.Test, metadata test.RunMetadata, environment environment.Environment) test.Run {
+func (r persistentRunner) Run(ctx context.Context, testObj test.Test, metadata test.RunMetadata, environment environment.Environment, requiredGates *[]testrunner.RequiredGate) test.Run {
 	ctx, cancelCtx := context.WithCancel(
 		getNewCtx(ctx),
 	)
@@ -152,6 +160,14 @@ func (r persistentRunner) Run(ctx context.Context, testObj test.Test, metadata t
 
 	r.listenForStopRequests(ctx, cancelCtx, run)
 
+	// configuring required gates
+	if requiredGates == nil {
+		rg := r.trGetter.GetDefault(ctx).RequiredGates
+		requiredGates = &rg
+	}
+
+	run = run.ConfigureRequiredGates(*requiredGates)
+
 	ds := []expression.DataStore{expression.EnvironmentDataStore{
 		Values: environment.Values,
 	}}

server/executor/runner_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/kubeshop/tracetest/server/environment"
 	"github.com/kubeshop/tracetest/server/executor"
 	"github.com/kubeshop/tracetest/server/executor/pollingprofile"
+	"github.com/kubeshop/tracetest/server/executor/testrunner"
 	triggerer "github.com/kubeshop/tracetest/server/executor/trigger"
 	"github.com/kubeshop/tracetest/server/pkg/id"
 	"github.com/kubeshop/tracetest/server/subscription"
@@ -23,6 +24,14 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+type defaultTestRunnerGetter struct{}
+
+func (dpc defaultTestRunnerGetter) GetDefault(context.Context) testrunner.TestRunner {
+	testRunner := testrunner.DefaultTestRunner
+
+	return testRunner
+}
+
 func TestPersistentRunner(t *testing.T) {
 	t.Run("TestIsTriggerd", func(t *testing.T) {
 		t.Parallel()
@@ -99,7 +108,7 @@ func (f runnerFixture) run(tests []test.Test, ttl time.Duration) {
 	f.runner.Start(2)
 	time.Sleep(10 * time.Millisecond)
 	for _, testObj := range tests {
-		f.runner.Run(context.TODO(), testObj, test.RunMetadata{}, environment.Environment{})
+		f.runner.Run(context.TODO(), testObj, test.RunMetadata{}, environment.Environment{}, nil)
 	}
 	time.Sleep(ttl)
 	f.runner.Stop()
@@ -161,6 +170,7 @@ func runnerSetup(t *testing.T) runnerFixture {
 		getDataStoreRepositoryMock(t),
 		eventEmitter,
 		defaultProfileGetter{5 * time.Second, 30 * time.Second},
+		defaultTestRunnerGetter{},
 	)
 
 	mtp.Test(t)

server/executor/testrunner/main_test.go

@@ -0,0 +1,18 @@
+package testrunner_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/kubeshop/tracetest/server/testmock"
+)
+
+func TestMain(m *testing.M) {
+	testmock.StartTestEnvironment()
+
+	exitVal := m.Run()
+
+	testmock.StopTestEnvironment()
+
+	os.Exit(exitVal)
+}