feat: Adding var template support (#3053)

* feat: Adding template support * removing log * feat: Updating testing to use * moving environment data store to variable data store
kubeshop · Aug 15, 2023 · abd131a · abd131a
1 parent 7d508ad
commit abd131a
Show file tree

Hide file tree

Showing 53 changed files with 201 additions and 152 deletions.
diff --git a/examples/table-driven-test/test.yaml b/examples/table-driven-test/test.yaml
@@ -12,8 +12,8 @@ spec:
       - key: Content-Type
         value: application/json
       - key: X-Some-Header
-        value: ${env:someValue}
-      body: '{"id":${env:pokeid}}'
+        value: ${var:someValue}
+      body: '{"id":${var:pokeid}}'
 
   specs:
   - name: 'All Database Spans: Processing time is less than 100ms'

diff --git a/examples/tracetest-aws-step-functions/tests/exam.yaml b/examples/tracetest-aws-step-functions/tests/exam.yaml
@@ -12,9 +12,9 @@ spec:
           value: application/json
       body: |-
         {
-            "TaskToken": "${env:TASK_TOKEN}",
-            "ExamId": "${env:EXAM_ID}",
-            "IncidentId": "${env:INCIDENT_ID}",
+            "TaskToken": "${var:TASK_TOKEN}",
+            "ExamId": "${var:EXAM_ID}",
+            "IncidentId": "${var:INCIDENT_ID}",
             "Score": "80"
         }
   specs:

diff --git a/examples/tracetest-grafana-tempo-pokeshop/tests/post-mortem.yaml b/examples/tracetest-grafana-tempo-pokeshop/tests/post-mortem.yaml
@@ -5,9 +5,9 @@ spec:
   trigger:
     type: traceid
     traceid:
-      id: ${env:traceId}
+      id: ${var:traceId}
   specs:
-  - selector: span[tracetest.span.type="general" name="import pokemon"]
-    name: Validate there are no exceptions when importing pokemon.
-    assertions:
-    - attr:span.events   not-contains   "exception"
+    - selector: span[tracetest.span.type="general" name="import pokemon"]
+      name: Validate there are no exceptions when importing pokemon.
+      assertions:
+        - attr:span.events   not-contains   "exception"
diff --git a/examples/tracetest-k6/tests/test.yaml b/examples/tracetest-k6/tests/test.yaml
@@ -6,7 +6,7 @@ spec:
   trigger:
     type: traceid
     traceid:
-      id: ${env:TRACE_ID}
+      id: ${var:TRACE_ID}
   specs:
     - selector: span[tracetest.span.type="general" name="import pokemon"]
       assertions:

diff --git a/examples/tracetesting-kubernetes/setup-k3s.md b/examples/tracetesting-kubernetes/setup-k3s.md
@@ -169,7 +169,7 @@ spec:
   trigger:
     type: traceid # MUST BE traceide
     traceid:
-      id: ${env:asd} # MUST BE in line 8
+      id: ${var:asd} # MUST BE in line 8
   # the rest of the file can be modified as needed
   specs:
   - name: List span exists

diff --git a/examples/tracetesting-kubernetes/test.yaml b/examples/tracetesting-kubernetes/test.yaml
@@ -5,7 +5,7 @@ spec:
   trigger:
     type: traceid
     traceid:
-      id: ${env:asd}
+      id: ${var:asd}
   specs:
   - name: List span exists
     selector: span[tracetest.span.type="general" name="List"]

diff --git a/server/executor/assertion_runner.go b/server/executor/assertion_runner.go
@@ -111,7 +111,7 @@ func (e *defaultAssertionRunner) executeAssertions(ctx context.Context, req Job)
 		return test.Run{}, fmt.Errorf("trace not available")
 	}
 
-	ds := []expression.DataStore{expression.EnvironmentDataStore{
+	ds := []expression.DataStore{expression.VariableDataStore{
 		Values: req.Run.VariableSet.Values,
 	}}
 
@@ -123,7 +123,7 @@ func (e *defaultAssertionRunner) executeAssertions(ctx context.Context, req Job)
 
 	newVariableSet := createVariableSet(req.Run.VariableSet, outputs)
 
-	ds = []expression.DataStore{expression.EnvironmentDataStore{Values: newVariableSet.Values}}
+	ds = []expression.DataStore{expression.VariableDataStore{Values: newVariableSet.Values}}
 
 	assertionResult, allPassed := e.assertionExecutor.Assert(ctx, req.Test.Specs, *run.Trace, ds)
 

diff --git a/server/executor/runner.go b/server/executor/runner.go
@@ -120,7 +120,7 @@ func (r persistentRunner) ProcessItem(ctx context.Context, job Job) {
 	run.TraceID = traceID
 	r.handleDBError(run, r.updater.Update(ctx, run))
 
-	ds := []expression.DataStore{expression.EnvironmentDataStore{
+	ds := []expression.DataStore{expression.VariableDataStore{
 		Values: run.VariableSet.Values,
 	}}
 

diff --git a/server/expression/data_store.go b/server/expression/data_store.go
@@ -69,35 +69,20 @@ func (ds MetaAttributesDataStore) count() string {
 	return strconv.Itoa(len(ds.SelectedSpans))
 }
 
-type VariableDataStore map[string]string
-
-func (ds VariableDataStore) Source() string {
-	return "var"
-}
-
-func (ds VariableDataStore) Get(name string) (string, error) {
-	value, found := ds[name]
-	if !found {
-		return "", fmt.Errorf(`variable "%s" is not set`, name)
-	}
-
-	return value, nil
-}
-
-type EnvironmentDataStore struct {
+type VariableDataStore struct {
 	Values []variableset.VariableSetValue
 }
 
-func (ds EnvironmentDataStore) Source() string {
+func (ds VariableDataStore) Source() string {
 	return "env"
 }
 
-func (ds EnvironmentDataStore) Get(name string) (string, error) {
+func (ds VariableDataStore) Get(name string) (string, error) {
 	for _, v := range ds.Values {
 		if v.Key == name {
 			return v.Value, nil
 		}
 	}
 
-	return "", fmt.Errorf(`environment variable "%s" not found`, name)
+	return "", fmt.Errorf(`variable "%s" not found`, name)
 }
diff --git a/server/expression/executor.go b/server/expression/executor.go
@@ -166,6 +166,10 @@ func (e Executor) resolveTerm(term *Term) (value.Value, error) {
 		return e.resolveEnvironment(term.Environment)
 	}
 
+	if term.Variable != nil {
+		return e.resolveEnvironment((*Environment)(term.Variable))
+	}
+
 	if term.FunctionCall != nil {
 		return e.resolveFunctionCall(term.FunctionCall)
 	}

diff --git a/server/expression/executor_test.go b/server/expression/executor_test.go
@@ -21,7 +21,7 @@ type executorTestCase struct {
 
 	AttributeDataStore      expression.DataStore
 	MetaAttributesDataStore expression.DataStore
-	EnvironmentDataStore    expression.DataStore
+	VariableDataStore       expression.DataStore
 }
 
 func TestBasicExpressionExecution(t *testing.T) {
@@ -435,9 +435,19 @@ func TestFailureCases(t *testing.T) {
 			Name:                 "should_report_missing_environment_variable",
 			Query:                `env:test = "abc"`,
 			ShouldPass:           false,
-			ExpectedErrorMessage: `resolution error: environment variable "test" not found`,
+			ExpectedErrorMessage: `resolution error: variable "test" not found`,
 
-			EnvironmentDataStore: expression.EnvironmentDataStore{
+			VariableDataStore: expression.VariableDataStore{
+				Values: []variableset.VariableSetValue{},
+			},
+		},
+		{
+			Name:                 "should_report_missing_environment_variable",
+			Query:                `var:host = "abc"`,
+			ShouldPass:           false,
+			ExpectedErrorMessage: `resolution error: variable "host" not found`,
+
+			VariableDataStore: expression.VariableDataStore{
 				Values: []variableset.VariableSetValue{},
 			},
 		},
@@ -466,9 +476,19 @@ func TestFailureCases(t *testing.T) {
 			Name:                 "should_report_problem_resolving_array_item",
 			Query:                `["value", env:test, "anotherValue"] | get_index 0`,
 			ShouldPass:           false,
-			ExpectedErrorMessage: `resolution error: at index 1 of array: environment variable "test" not found`,
+			ExpectedErrorMessage: `resolution error: at index 1 of array: variable "test" not found`,
+
+			VariableDataStore: expression.VariableDataStore{
+				Values: []variableset.VariableSetValue{},
+			},
+		},
+		{
+			Name:                 "should_report_problem_resolving_array_item",
+			Query:                `["value", var:host, "anotherValue"] | get_index 0`,
+			ShouldPass:           false,
+			ExpectedErrorMessage: `resolution error: at index 1 of array: variable "host" not found`,
 
-			EnvironmentDataStore: expression.EnvironmentDataStore{
+			VariableDataStore: expression.VariableDataStore{
 				Values: []variableset.VariableSetValue{},
 			},
 		},
@@ -483,7 +503,7 @@ func executeResolveStatementTestCases(t *testing.T, testCases []executorTestCase
 			executor := expression.NewExecutor(
 				testCase.AttributeDataStore,
 				testCase.MetaAttributesDataStore,
-				testCase.EnvironmentDataStore,
+				testCase.VariableDataStore,
 			)
 			left, err := executor.ResolveStatement(testCase.Query)
 			debugMessage := fmt.Sprintf("left value: %s", left)

diff --git a/server/expression/linting/missing_variables.go b/server/expression/linting/missing_variables.go
@@ -18,6 +18,10 @@ func DetectMissingVariables(target interface{}, availableVariables []string) []s
 			if token.Type == expression.EnvironmentType {
 				variables = append(variables, token.Identifier)
 			}
+
+			if token.Type == expression.VariableType {
+				variables = append(variables, token.Identifier)
+			}
 		}
 
 		missingVariables = append(missingVariables, getSetDifference(variables, availableVariables)...)

diff --git a/server/expression/linting/missing_variables_test.go b/server/expression/linting/missing_variables_test.go
@@ -50,7 +50,7 @@ func TestMissingVariableDetection(t *testing.T) {
 			name:               "no_missing_variables_if_variable_exists",
 			availableVariables: []string{"SERVER_URL", "PORT", "TOKEN"},
 			object: HTTPRequest{
-				URL:    `${env:SERVER_URL}:${PORT}`,
+				URL:    `${env:SERVER_URL}:${var:PORT}`,
 				Method: "GET",
 				Auth: HTTPAuth{
 					Token: "abc",
@@ -62,7 +62,7 @@ func TestMissingVariableDetection(t *testing.T) {
 			name:               "missing_variables_if_variable_doesnt_exists",
 			availableVariables: []string{"SERVER_URL"},
 			object: HTTPRequest{
-				URL:    `${env:SERVER_URL}:${env:PORT}`,
+				URL:    `${env:SERVER_URL}:${var:PORT}`,
 				Method: "GET",
 				Auth: HTTPAuth{
 					Token: "abc",
@@ -86,7 +86,7 @@ func TestMissingVariableDetection(t *testing.T) {
 			name:               "missing_variables_in_inner_struct",
 			availableVariables: []string{"SERVER_URL", "PORT"},
 			object: HTTPRequest{
-				URL:    `${env:SERVER_URL}:${env:PORT}`,
+				URL:    `${env:SERVER_URL}:${var:PORT}`,
 				Method: "GET",
 				Auth: HTTPAuth{
 					Token: "${env:TOKEN}",
@@ -104,8 +104,8 @@ func TestMissingVariableDetection(t *testing.T) {
 					Token: "abc",
 				},
 				Assertions: []Assertion{
-					{Name: "test", Queries: []string{"env:ABC = env:ABC2"}},
-					{Name: "test2", Queries: []string{"env:CDE = env:CDE"}},
+					{Name: "test", Queries: []string{"env:ABC = var:ABC2"}},
+					{Name: "test2", Queries: []string{"env:CDE = var:CDE"}},
 				},
 			},
 			expectedMissingVariables: []string{"ABC", "ABC2", "CDE"},
@@ -127,7 +127,7 @@ func TestMissingVariableDetection(t *testing.T) {
 				},
 				Assertions: []Assertion{
 					{Name: "test", Queries: []string{"env:ABC = env:ABC2"}},
-					{Name: "test2", Queries: []string{"env:CDE = env:CDE"}},
+					{Name: "test2", Queries: []string{"env:CDE = var:CDE"}},
 				},
 			},
 			expectedMissingVariables: []string{"ABC", "ABC2", "CDE"},
@@ -139,11 +139,11 @@ func TestMissingVariableDetection(t *testing.T) {
 				Trigger: trigger.Trigger{
 					Type: trigger.TriggerTypeHTTP,
 					HTTP: &trigger.HTTPRequest{
-						Body: `{"id": ${env:pokemonId}}`,
+						Body: `{"id": ${env:pokemonId},"name": "${var:pokemonName}"}`,
 					},
 				},
 			},
-			expectedMissingVariables: []string{"pokemonId"},
+			expectedMissingVariables: []string{"pokemonId", "pokemonName"},
 		},
 	}
 

diff --git a/server/expression/parser_rules.go b/server/expression/parser_rules.go
@@ -28,6 +28,7 @@ type Term struct {
 	Number       *string       `| @Number `
 	Attribute    *Attribute    `| @Attribute `
 	Environment  *Environment  `| @Environment `
+	Variable     *Variable     `| @Variable `
 	Str          *Str          `| @(QuotedString|SingleQuotedString) )`
 }
 
@@ -57,6 +58,7 @@ var languageLexer = lexer.MustStateful(lexer.Rules{
 		{Name: "Number", Pattern: `([0-9]+(\.[0-9]+)?)`},
 		{Name: "Attribute", Pattern: `attr:[a-zA-Z_0-9][a-zA-Z_0-9.-]*`, Action: nil},
 		{Name: "Environment", Pattern: `env:[a-zA-Z_0-9][a-zA-Z_0-9.]*`, Action: nil},
+		{Name: "Variable", Pattern: `var:[a-zA-Z_0-9][a-zA-Z_0-9.]*`, Action: nil},
 		{Name: "QuotedString", Pattern: `"(\\"|[^"])*"`, Action: nil},
 		{Name: "SingleQuotedString", Pattern: `'(\\'|[^'])*'`, Action: nil},
 
@@ -73,6 +75,7 @@ const (
 	NumberType       TermType = "Number"
 	AttributeType    TermType = "Attribute"
 	EnvironmentType  TermType = "Environment"
+	VariableType     TermType = "Variable"
 	StrType          TermType = "Str"
 )
 
@@ -81,6 +84,10 @@ func (term *Term) Type() TermType {
 		return AttributeType
 	}
 
+	if term.Variable != nil {
+		return VariableType
+	}
+
 	if term.Environment != nil {
 		return EnvironmentType
 	}

diff --git a/server/expression/reflection_test.go b/server/expression/reflection_test.go
@@ -41,6 +41,13 @@ func TestGetTokens(t *testing.T) {
 				{Type: expression.StrType},
 			},
 		},
+		{
+			Statement: `var:url = "http://localhost"`,
+			ExpectedTokens: []expression.Token{
+				{Identifier: "url", Type: expression.VariableType},
+				{Type: expression.StrType},
+			},
+		},
 		{
 			Statement: `"the server is ${env:url}" = "http://localhost"`,
 			ExpectedTokens: []expression.Token{
@@ -49,6 +56,14 @@ func TestGetTokens(t *testing.T) {
 				{Type: expression.StrType},
 			},
 		},
+		{
+			Statement: `"the server is ${var:url}" = "http://localhost"`,
+			ExpectedTokens: []expression.Token{
+				{Type: expression.StrType},
+				{Identifier: "url", Type: expression.VariableType},
+				{Type: expression.StrType},
+			},
+		},
 		{
 			Statement: `"the url has ${env:url | count} characters" = "the url has 22 characters"`,
 			ExpectedTokens: []expression.Token{
@@ -68,6 +83,16 @@ func TestGetTokens(t *testing.T) {
 				{Type: expression.StrType},
 			},
 		},
+		{
+			Statement: `"test ${var:names | get_index var:name_index}" = "John Doe"`,
+			ExpectedTokens: []expression.Token{
+				{Type: expression.StrType},
+				{Identifier: "names", Type: expression.VariableType},
+				{Identifier: "get_index", Type: expression.FunctionCallType},
+				{Identifier: "name_index", Type: expression.VariableType},
+				{Type: expression.StrType},
+			},
+		},
 	}
 
 	for _, testCase := range testCases {

diff --git a/server/expression/tokens.go b/server/expression/tokens.go
@@ -81,6 +81,10 @@ func extractIdentifierFromTerm(term *Term) string {
 		return term.Environment.name
 	}
 
+	if term.Type() == VariableType {
+		return term.Variable.name
+	}
+
 	// all other types don't have names, so return an empty string
 	return ""
 }
diff --git a/server/http/controller.go b/server/http/controller.go
@@ -283,7 +283,7 @@ func (c *controller) DryRunAssertion(ctx context.Context, testID string, runID i
 
 	definition := c.mappers.In.Definition(def.Specs)
 
-	ds := []expression.DataStore{expression.EnvironmentDataStore{
+	ds := []expression.DataStore{expression.VariableDataStore{
 		Values: run.VariableSet.Values,
 	}}
 
@@ -438,7 +438,7 @@ func (c *controller) buildDataStores(ctx context.Context, info openapi.ResolveRe
 			return [][]expression.DataStore{}, err
 		}
 
-		ds = append([]expression.DataStore{expression.EnvironmentDataStore{
+		ds = append([]expression.DataStore{expression.VariableDataStore{
 			Values: environment.Values,
 		}}, ds...)
 	}

diff --git a/server/model/events/events.go b/server/model/events/events.go
@@ -60,7 +60,7 @@ func TriggerResolveStart(testID id.ID, runID int) model.TestRunEvent {
 		Stage:               model.StageTrigger,
 		Type:                "RESOLVE_START",
 		Title:               "Resolving trigger details started",
-		Description:         "The resolution of trigger details based on environment variables has started",
+		Description:         "The resolution of trigger details based on variables has started",
 		CreatedAt:           time.Now(),
 		DataStoreConnection: model.ConnectionResult{},
 		Polling:             model.PollingInfo{},