feat: improve test outputs error handling (#2404)

api/tests.yaml

@@ -177,6 +177,8 @@ components:
                 type: string
               value:
                 type: string
+              error:
+                type: string
 
         metadata:
           type: object

server/executor/assertion_runner.go

@@ -253,7 +253,7 @@ func (e *defaultAssertionRunner) validateOutputResolution(ctx context.Context, r
 			return nil
 		}
 
-		anotherErr := e.eventEmitter.Emit(ctx, events.TestOutputGenerationWarning(request.Test.ID, request.Run.ID, outputName))
+		anotherErr := e.eventEmitter.Emit(ctx, events.TestOutputGenerationWarning(request.Test.ID, request.Run.ID, outputModel.Error, outputName))
 		if anotherErr != nil {
 			log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestOutputGenerationWarning event: %s\n", request.Test.ID, request.Run.ID, anotherErr.Error())
 		}

server/executor/outputs_processor.go

@@ -59,13 +59,29 @@ func outputProcessor(ctx context.Context, outputs maps.Ordered[string, model.Out
 	}
 
 	err = parsed.ForEach(func(key string, out parsedOutput) error {
+		if out.err != nil {
+			res, err = res.Add(key, model.RunOutput{
+				Value:    "",
+				SpanID:   "",
+				Name:     key,
+				Resolved: false,
+				Error:    out.err,
+			})
+			if err != nil {
+				return fmt.Errorf(`cannot process output "%s": %w`, key, err)
+			}
+
+			return nil
+		}
+
 		spans := out.selector.Filter(tr)
 
 		stores := append([]expression.DataStore{expression.MetaAttributesDataStore{SelectedSpans: spans}}, ds...)
 
 		value := ""
 		spanId := ""
 		resolved := false
+		var outputError error = nil
 		spans.
 			ForEach(func(_ int, span model.Span) bool {
 				value = extractAttr(span, stores, out.expr)
@@ -77,13 +93,15 @@ func outputProcessor(ctx context.Context, outputs maps.Ordered[string, model.Out
 			OrEmpty(func() {
 				value = extractAttr(model.Span{}, stores, out.expr)
 				resolved = false
+				outputError = fmt.Errorf(`cannot find matching spans for output "%s"`, key)
 			})
 
 		res, err = res.Add(key, model.RunOutput{
 			Value:    value,
 			SpanID:   spanId,
 			Name:     key,
 			Resolved: resolved,
+			Error:    outputError,
 		})
 		if err != nil {
 			return fmt.Errorf(`cannot process output "%s": %w`, key, err)
@@ -112,25 +130,31 @@ func extractAttr(span model.Span, ds []expression.DataStore, expr expression.Exp
 type parsedOutput struct {
 	selector selectors.Selector
 	expr     expression.Expr
+	err      error
 }
 
 func parseOutputs(outputs maps.Ordered[string, model.Output]) (maps.Ordered[string, parsedOutput], error) {
 	var parsed maps.Ordered[string, parsedOutput]
 
 	parseErr := outputs.ForEach(func(key string, out model.Output) error {
+		var selector selectors.Selector
+		var expr expression.Expr
+		var outputErr error = nil
+
 		expr, err := expression.Parse(out.Value)
 		if err != nil {
-			return fmt.Errorf(`cannot parse output "%s" value "%s": %w`, key, out.Value, err)
+			outputErr = fmt.Errorf(`cannot parse output "%s" value "%s": %w`, key, out.Value, err)
 		}
 
-		selector, err := selectors.New(string(out.Selector))
+		selector, err = selectors.New(string(out.Selector))
 		if err != nil {
-			return fmt.Errorf(`cannot parse output "%s" selector "%s": %w`, key, string(out.Selector), err)
+			outputErr = fmt.Errorf(`cannot parse output "%s" selector "%s": %w`, key, string(out.Selector), err)
 		}
 
 		parsed, _ = parsed.Add(key, parsedOutput{
 			selector: selector,
 			expr:     expr,
+			err:      outputErr,
 		})
 		return nil
 	})

server/http/mappings/tests.go

@@ -325,6 +325,7 @@ func (m OpenAPI) RunOutputs(in maps.Ordered[string, model.RunOutput]) []openapi.
 			Name:   key,
 			Value:  val.Value,
 			SpanId: val.SpanID,
+			Error:  errToString(val.Error),
 		})
 		return nil
 	})
@@ -496,6 +497,7 @@ func (m Model) RunOutputs(in []openapi.TestRunOutputsInner) maps.Ordered[string,
 			Value:  output.Value,
 			Name:   output.Name,
 			SpanID: output.SpanId,
+			Error:  fmt.Errorf(output.Error),
 		})
 	}
 

server/model/events/events.go

@@ -307,14 +307,14 @@ func TraceStoppedInfo(testID id.ID, runID int) model.TestRunEvent {
 	}
 }
 
-func TestOutputGenerationWarning(testID id.ID, runID int, output string) model.TestRunEvent {
+func TestOutputGenerationWarning(testID id.ID, runID int, err error, output string) model.TestRunEvent {
 	return model.TestRunEvent{
 		TestID:              testID,
 		RunID:               runID,
 		Stage:               model.StageTest,
 		Type:                "OUTPUT_GENERATION_WARNING",
 		Title:               fmt.Sprintf(`Output '%s' not generated`, output),
-		Description:         fmt.Sprintf(`The value for the output '%s' could not be generated`, output),
+		Description:         fmt.Sprintf(`The output '%s' returned an error. Error: %s`, output, err.Error()),
 		CreatedAt:           time.Now(),
 		DataStoreConnection: model.ConnectionResult{},
 		Polling:             model.PollingInfo{},

server/model/json.go

@@ -10,6 +10,45 @@ import (
 	"go.opentelemetry.io/otel/trace"
 )
 
+func (ro RunOutput) MarshalJSON() ([]byte, error) {
+	return json.Marshal(&struct {
+		Name     string
+		Value    string
+		SpanID   string
+		Resolved bool
+		Error    string
+	}{
+		Name:     ro.Name,
+		Value:    ro.Value,
+		SpanID:   ro.SpanID,
+		Resolved: ro.Resolved,
+		Error:    errToString(ro.Error),
+	})
+}
+
+func (ro *RunOutput) UnmarshalJSON(data []byte) error {
+	aux := struct {
+		Name     string
+		Value    string
+		SpanID   string
+		Resolved bool
+		Error    string
+	}{}
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+
+	ro.Name = aux.Name
+	ro.Value = aux.Value
+	ro.SpanID = aux.SpanID
+	ro.Resolved = aux.Resolved
+	if err := stringToErr(aux.Error); err != nil {
+		ro.Error = err
+	}
+
+	return nil
+}
+
 func (sar SpanAssertionResult) MarshalJSON() ([]byte, error) {
 	sid := ""
 	if sar.SpanID != nil {

server/model/tests.go

@@ -123,6 +123,7 @@ type (
 		Value    string
 		SpanID   string
 		Resolved bool
+		Error    error
 	}
 
 	AssertionResult struct {

server/openapi/model_test_run_outputs_inner.go

@@ -15,6 +15,8 @@ type TestRunOutputsInner struct {
 	SpanId string `json:"spanId,omitempty"`
 
 	Value string `json:"value,omitempty"`
+
+	Error string `json:"error,omitempty"`
 }
 
 // AssertTestRunOutputsInnerRequired checks if the required fields are not zero-ed

web/src/components/TestOutput/TestOutput.styled.ts

@@ -1,4 +1,4 @@
-import {MoreOutlined} from '@ant-design/icons';
+import {InfoCircleFilled, MoreOutlined} from '@ant-design/icons';
 import {Typography} from 'antd';
 import styled from 'styled-components';
 
@@ -60,3 +60,7 @@ export const ActionsContainer = styled.div`
   display: flex;
   justify-content: flex-end;
 `;
+
+export const IconWarning = styled(InfoCircleFilled)`
+  color: ${({theme}) => theme.color.warningYellow};
+`;

web/src/components/TestOutput/TestOutput.tsx

@@ -1,4 +1,4 @@
-import {Tag} from 'antd';
+import {Tag, Tooltip} from 'antd';
 import {useCallback} from 'react';
 import AttributeValue from 'components/AttributeValue';
 import {useTestOutput} from 'providers/TestOutput/TestOutput.provider';
@@ -18,7 +18,7 @@ interface IProps {
 
 const TestOutput = ({
   index,
-  output: {id, name, isDeleted, isDraft, spanId, selector, value, valueRun, valueRunDraft},
+  output: {id, name, isDeleted, isDraft, spanId, selector, value, valueRun, valueRunDraft, error},
   output,
   onEdit,
   onDelete,
@@ -75,6 +75,11 @@ const TestOutput = ({
               <AttributeValue value={valueRunDraft} />
             </>
           )}
+          {error && (
+            <Tooltip title={error}>
+              <S.IconWarning />
+            </Tooltip>
+          )}
         </S.Entry>
       </S.Row>
     </S.Container>

web/src/models/TestOutput.model.ts

@@ -11,6 +11,7 @@ type TestOutput = {
   valueRunDraft: string;
   id: number;
   spanId: string;
+  error: string;
 };
 
 function TestOutput({name = '', selector = {}, value = ''}: TRawTestOutput, id = -1): TestOutput {
@@ -24,6 +25,7 @@ function TestOutput({name = '', selector = {}, value = ''}: TRawTestOutput, id =
     valueRun: '',
     valueRunDraft: '',
     spanId: '',
+    error: '',
   };
 }
 

web/src/models/TestRunOutput.model.ts

@@ -4,14 +4,16 @@ export type TRawTestRunOutput = {
   name?: string;
   value?: string;
   spanId?: string;
+  error?: string;
 };
 type TestRunOutput = Model<TRawTestRunOutput, {}>;
 
-const TestRunOutput = ({name = '', value = '', spanId = ''}: TRawTestRunOutput): TestRunOutput => {
+const TestRunOutput = ({name = '', value = '', spanId = '', error = ''}: TRawTestRunOutput): TestRunOutput => {
   return {
     name,
     value,
     spanId,
+    error,
   };
 };
 

web/src/redux/testOutputs/slice.ts

@@ -48,6 +48,7 @@ const testOutputsSlice = createSlice({
         ...output,
         valueRun: runOutputs[index]?.value ?? '',
         spanId: runOutputs[index]?.spanId ?? '',
+        error: runOutputs[index]?.error ?? '',
       }));
     },
   },