diff --git a/assertion.go b/assertion.go index 75d9e451..957970f6 100644 --- a/assertion.go +++ b/assertion.go @@ -28,7 +28,7 @@ type AssertionApplied struct { IsOK bool `json:"isOK" yml:"-"` } -func applyAssertions(ctx context.Context, r interface{}, tc TestCase, stepNumber int, rangedIndex int, step TestStep, defaultAssertions *StepAssertions) AssertionsApplied { +func applyAssertions(ctx context.Context, vars *H, tc *TestCase, stepNumber int, rangedIndex int, step TestStep, defaultAssertions *StepAssertions) AssertionsApplied { var sa StepAssertions var errors []Failure var systemerr, systemout string @@ -46,12 +46,11 @@ func applyAssertions(ctx context.Context, r interface{}, tc TestCase, stepNumber sa = *defaultAssertions } - executorResult := GetExecutorResult(r) - isOK := true + localVars := *vars assertions := []AssertionApplied{} for _, assertion := range sa.Assertions { - errs := check(ctx, tc, stepNumber, rangedIndex, assertion, executorResult) + errs := check(ctx, *tc, stepNumber, rangedIndex, assertion, localVars) isAssertionOK := true if errs != nil { errors = append(errors, *errs) @@ -64,12 +63,12 @@ func applyAssertions(ctx context.Context, r interface{}, tc TestCase, stepNumber }) } - if _, ok := executorResult["result.systemerr"]; ok { - systemerr = fmt.Sprintf("%v", executorResult["result.systemerr"]) + if _, ok := localVars["result.systemerr"]; ok { + systemerr = fmt.Sprintf("%v", localVars["result.systemerr"]) } - if _, ok := executorResult["result.systemout"]; ok { - systemout = fmt.Sprintf("%v", executorResult["result.systemout"]) + if _, ok := localVars["result.systemout"]; ok { + systemout = fmt.Sprintf("%v", localVars["result.systemout"]) } return AssertionsApplied{ diff --git a/assertion_test.go b/assertion_test.go index 4f1cd865..8fd9cd26 100644 --- a/assertion_test.go +++ b/assertion_test.go @@ -1,6 +1,8 @@ package venom import ( + "context" + "github.com/stretchr/testify/assert" "reflect" "testing" ) @@ -22,3 +24,40 @@ func Test_splitAssertion(t *testing.T) { } } } + +func TestCheckBranchWithOR(t *testing.T) { + tc := TestCase{} + vars := map[string]interface{}{} + vars["result.statuscode"] = 501 + vars["is_feature_supported"] = "false" + branch := map[string]interface{}{} + + firstSetOfAssertions := []interface{}{`is_feature_supported ShouldEqual true`, `result.statuscode ShouldEqual 200`} + secondSetOfAssertions := []interface{}{`is_feature_supported ShouldEqual false`, `result.statuscode ShouldEqual 501`} + + branch["or"] = []interface{}{ + map[string]interface{}{"and": firstSetOfAssertions}, + map[string]interface{}{"and": secondSetOfAssertions}, + } + + failure := checkBranch(context.Background(), tc, 0, 0, branch, vars) + assert.Nil(t, failure) +} +func TestCheckBranchWithORFailing(t *testing.T) { + tc := TestCase{} + vars := map[string]interface{}{} + vars["result.statuscode"] = 400 + vars["is_feature_supported"] = "false" + branch := map[string]interface{}{} + + firstSetOfAssertions := []interface{}{`result.statuscode ShouldEqual 200`} + secondSetOfAssertions := []interface{}{`result.statuscode ShouldEqual 501`} + + branch["or"] = []interface{}{ + map[string]interface{}{"and": firstSetOfAssertions}, + map[string]interface{}{"and": secondSetOfAssertions}, + } + + failure := checkBranch(context.Background(), tc, 0, 0, branch, vars) + assert.NotNil(t, failure) +} diff --git a/assertions/assertions.go b/assertions/assertions.go index 7169007a..d9b9b393 100644 --- a/assertions/assertions.go +++ b/assertions/assertions.go @@ -1,6 +1,7 @@ package assertions import ( + "bytes" "encoding/json" "fmt" "math" @@ -81,9 +82,21 @@ func ShouldBeArray(actual interface{}, expected ...interface{}) error { if err := need(0, expected); err != nil { return err } + t := []interface{}{} _, err := cast.ToSliceE(actual) if err != nil { + strValue := fmt.Sprintf("%v", actual) + err2 := json.Unmarshal([]byte(strValue), &t) + if err2 != nil { + return fmt.Errorf("expected: %v to be an array but was not", actual) + } + x := bytes.TrimLeft([]byte(strValue), " \t\r\n") + isArray := len(x) > 0 && x[0] == '[' + if isArray { + return nil + } return fmt.Errorf("expected: %v to be an array but was not", actual) + } return nil } @@ -93,8 +106,19 @@ func ShouldBeMap(actual interface{}, expected ...interface{}) error { return err } _, err := cast.ToStringMapE(actual) + t := map[string]interface{}{} if err != nil { - return fmt.Errorf("expected: %v to be a map but was not", actual) + strValue := fmt.Sprintf("%v", actual) + err2 := json.Unmarshal([]byte(strValue), &t) + if err2 != nil { + return fmt.Errorf("expected: %v to be a map but was not", actual) + } + x := bytes.TrimLeft([]byte(strValue), " \t\r\n") + isObject := len(x) > 0 && x[0] == '{' + if isObject { + return nil + } + return fmt.Errorf("expected: %v to be an map but was not", actual) } return nil } @@ -239,10 +263,12 @@ func ShouldBeNil(actual interface{}, expected ...interface{}) error { // ShouldNotExist receives a single parameter and ensures that it is nil, blank or zero value func ShouldNotExist(actual interface{}, expected ...interface{}) error { - if ShouldBeNil(actual) != nil || - ShouldBeBlank(actual) != nil || - ShouldBeZeroValue(actual) != nil { - return fmt.Errorf("expected not exist but it was") + if ShouldBeNil(actual) != nil { + if ShouldBeBlank(actual) != nil { + if ShouldBeZeroValue(actual) != nil { + return fmt.Errorf("expected not exist but it was") + } + } } return nil } diff --git a/assertions/assertions_test.go b/assertions/assertions_test.go index 7cb4aab0..6690f430 100644 --- a/assertions/assertions_test.go +++ b/assertions/assertions_test.go @@ -838,6 +838,55 @@ func TestShouldBeEmpty(t *testing.T) { }) } } +func TestShouldNotExist(t *testing.T) { + type args struct { + actual interface{} + expected []interface{} + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "is nil", + args: args{ + actual: nil, + }, + wantErr: false, + }, + { + name: "is empty", + args: args{ + actual: "", + }, + wantErr: false, + }, + { + name: "is zero value", + args: args{ + actual: 0, + }, + wantErr: false, + }, + { + name: "is_not_empty", + args: args{ + actual: map[string]interface{}{"a": ""}, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := ShouldNotExist(tt.args.actual, tt.args.expected...); (err != nil) != tt.wantErr { + t.Errorf("ShouldNotExist() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } + +} func TestShouldNotBeEmpty(t *testing.T) { type args struct { @@ -1622,3 +1671,76 @@ func TestShouldJSONEqual(t *testing.T) { }) } } + +func TestShouldBeArray(t *testing.T) { + type args struct { + actual interface{} + expected []interface{} + } + + tests := []struct { + name string + args args + wantErr bool + }{ + // Objects and arrays + { + name: "json array", + args: args{ + actual: `[{"amount":10.0},{"amount":20.0}]`, + expected: []interface{}{}, + }, + }, + { + name: "array", + args: args{ + actual: []interface{}{`1`, `2`}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := ShouldBeArray(tt.args.actual, tt.args.expected...); (err != nil) != tt.wantErr { + t.Errorf("ShouldBeArray() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestShouldBeMap(t *testing.T) { + type args struct { + actual interface{} + expected []interface{} + } + aMap := map[string]interface{}{} + aMap["key"] = "value" + aMap["another"] = 123 + + tests := []struct { + name string + args args + wantErr bool + }{ + // Objects and arrays + { + name: "json map", + args: args{ + actual: `{"amount":10.0}`, + expected: []interface{}{}, + }, + }, + { + name: "map", + args: args{ + actual: aMap, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := ShouldBeMap(tt.args.actual, tt.args.expected...); (err != nil) != tt.wantErr { + t.Errorf("ShouldBeArray() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/cmd/venom/root/root.go b/cmd/venom/root/root.go index 36d90914..c34c4540 100644 --- a/cmd/venom/root/root.go +++ b/cmd/venom/root/root.go @@ -18,7 +18,7 @@ func New() *cobra.Command { return rootCmd } -//AddCommands adds child commands to the root command rootCmd. +// AddCommands adds child commands to the root command rootCmd. func addCommands(cmd *cobra.Command) { cmd.AddCommand(run.Cmd) cmd.AddCommand(version.Cmd) diff --git a/cmd/venom/root/root_test.go b/cmd/venom/root/root_test.go new file mode 100644 index 00000000..6a195eeb --- /dev/null +++ b/cmd/venom/root/root_test.go @@ -0,0 +1,16 @@ +package root + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestRunCmd(t *testing.T) { + var validArgs []string + + validArgs = append(validArgs, "run", "../../../tests/assertions") + + rootCmd := New() + rootCmd.SetArgs(validArgs) + assert.Equal(t, 3, len(rootCmd.Commands())) +} diff --git a/cmd/venom/run/cmd.go b/cmd/venom/run/cmd.go index 81b060ed..2f6b5f06 100644 --- a/cmd/venom/run/cmd.go +++ b/cmd/venom/run/cmd.go @@ -60,12 +60,12 @@ func initArgs(cmd *cobra.Command) { // command line flags overrides the configuration file. // Configuration file overrides the environment variables. if _, err := initFromEnv(os.Environ()); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) venom.OSExit(2) } if err := initFromConfigFile(); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) venom.OSExit(2) } cmd.LocalFlags().VisitAll(initFromCommandArguments) @@ -132,7 +132,13 @@ func initFromConfigFile() error { if err != nil { return err } - defer fi.Close() + defer func(fi *os.File) { + err := fi.Close() + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) + venom.OSExit(2) + } + }(fi) return initFromReaderConfigFile(fi) } @@ -145,7 +151,13 @@ func initFromConfigFile() error { if err != nil { return err } - defer fi.Close() + defer func(fi *os.File) { + err := fi.Close() + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) + venom.OSExit(2) + } + }(fi) return initFromReaderConfigFile(fi) } return nil @@ -158,6 +170,7 @@ type ConfigFileData struct { StopOnFailure *bool `json:"stop_on_failure,omitempty" yaml:"stop_on_failure,omitempty"` HtmlReport *bool `json:"html_report,omitempty" yaml:"html_report,omitempty"` Variables *[]string `json:"variables,omitempty" yaml:"variables,omitempty"` + Secrets *[]string `json:"secrets,omitempty" yaml:"secrets,omitempty"` VariablesFiles *[]string `json:"variables_files,omitempty" yaml:"variables_files,omitempty"` Verbosity *int `json:"verbosity,omitempty" yaml:"verbosity,omitempty"` } @@ -195,6 +208,11 @@ func initFromReaderConfigFile(reader io.Reader) error { variables = mergeVariables(varFromFile, variables) } } + if configFileData.Secrets != nil { + for _, secret := range *configFileData.Secrets { + secrets = append(secrets, secret) + } + } if configFileData.VariablesFiles != nil { for _, varFile := range *configFileData.VariablesFiles { if !isInArray(varFile, varFiles) { @@ -314,7 +332,7 @@ var Cmd = &cobra.Command{ Run a single testsuite and load all variables from a file: venom run mytestfile.yml --var-from-file variables.yaml Run all testsuites containing in files ending with *.yml or *.yaml with verbosity: VENOM_VERBOSE=2 venom run - Notice that variables initialized with -var-from-file argument can be overrided with -var argument + Notice that variables initialized with -var-from-file argument can be overridden with -var argument More info: https://github.com/ovh/venom`, Long: `run integration tests`, @@ -341,80 +359,75 @@ var Cmd = &cobra.Command{ v.Verbose = verbose if err := v.InitLogger(); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) venom.OSExit(2) } if v.Verbose == 3 { fCPU, err := os.Create(filepath.Join(v.OutputDir, "pprof_cpu_profile.prof")) if err != nil { - fmt.Fprintf(os.Stderr, "error while create profile file %v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "error while create profile file %v\n", err) venom.OSExit(2) } fMem, err := os.Create(filepath.Join(v.OutputDir, "pprof_mem_profile.prof")) if err != nil { - fmt.Fprintf(os.Stderr, "error while create profile file %v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "error while create profile file %v\n", err) venom.OSExit(2) } if fCPU != nil && fMem != nil { - pprof.StartCPUProfile(fCPU) //nolint - p := pprof.Lookup("heap") - defer p.WriteTo(fMem, 1) //nolint + err := pprof.StartCPUProfile(fCPU) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "error while starting cpu profiling %v\n", err) + venom.OSExit(2) + } //nolint defer pprof.StopCPUProfile() + p := pprof.Lookup("heap") + defer func(p *pprof.Profile, w io.Writer, debug int) { + err := p.WriteTo(w, debug) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) + venom.OSExit(2) + } + }(p, fMem, 1) //nolint } } if verbose >= 2 { displayArg(context.Background()) } - var readers = []io.Reader{} - for _, f := range varFiles { - if f == "" { - continue - } - fi, err := os.Open(f) - if err != nil { - fmt.Fprintf(os.Stderr, "unable to open var-from-file %s: %v\n", f, err) - venom.OSExit(2) - } - defer fi.Close() - readers = append(readers, fi) - } - - mapvars, err := readInitialVariables(context.Background(), variables, readers, os.Environ()) + mapvars, err := readInitialVariables(context.Background(), variables, varFiles) if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) venom.OSExit(2) } v.AddVariables(mapvars) if err := v.Parse(context.Background(), path); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) venom.OSExit(2) } if err := v.Process(context.Background(), path); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) venom.OSExit(2) } if err := v.OutputResult(); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) venom.OSExit(2) } if v.Tests.Status == venom.StatusPass { - fmt.Fprintf(os.Stdout, "final status: %v\n", venom.Green(v.Tests.Status)) - venom.OSExit(0) + _, _ = fmt.Fprintf(os.Stdout, "final status: %v\n", venom.Green(v.Tests.Status)) + } else { + _, _ = fmt.Fprintf(os.Stdout, "final status: %v\n", venom.Red(v.Tests.Status)) + venom.OSExit(2) } - fmt.Fprintf(os.Stdout, "final status: %v\n", venom.Red(v.Tests.Status)) - venom.OSExit(2) - return nil }, } -func readInitialVariables(ctx context.Context, argsVars []string, argVarsFiles []io.Reader, environ []string) (map[string]interface{}, error) { +func readInitialVariables(ctx context.Context, argsVars []string, argVarsFiles []string) (map[string]interface{}, error) { var cast = func(vS string) interface{} { var v interface{} _ = yaml.Unmarshal([]byte(vS), &v) //nolint @@ -425,7 +438,7 @@ func readInitialVariables(ctx context.Context, argsVars []string, argVarsFiles [ for _, r := range argVarsFiles { var tmpResult = map[string]interface{}{} - btes, err := io.ReadAll(r) + btes, err := os.ReadFile(r) if err != nil { return nil, err } diff --git a/cmd/venom/run/cmd_test.go b/cmd/venom/run/cmd_test.go index c23dac6c..ed085821 100644 --- a/cmd/venom/run/cmd_test.go +++ b/cmd/venom/run/cmd_test.go @@ -2,21 +2,36 @@ package run import ( "context" - "io" - "strings" - "testing" - "github.com/ovh/venom" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "os" + "path/filepath" + "testing" ) func Test_readInitialVariables(t *testing.T) { venom.InitTestLogger(t) type args struct { argsVars []string - argVarsFiles []io.Reader + argVarsFiles []string env []string } + location := filepath.Join(os.TempDir(), "test.yml") + defer os.Remove(location) + payload := ` +a: 1 +b: B +c: + - 1 + - 2 + - 3` + + err := os.WriteFile(location, []byte(payload), 777) + if err != nil { + t.Error(err) + } tests := []struct { name string args args @@ -46,15 +61,7 @@ func Test_readInitialVariables(t *testing.T) { { name: "from readers", args: args{ - argVarsFiles: []io.Reader{ - strings.NewReader(` -a: 1 -b: B -c: - - 1 - - 2 - - 3`), - }, + argVarsFiles: []string{location}, }, want: map[string]interface{}{ "a": 1.0, @@ -65,7 +72,7 @@ c: } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := readInitialVariables(context.TODO(), tt.args.argsVars, tt.args.argVarsFiles, tt.args.env) + got, err := readInitialVariables(context.TODO(), tt.args.argsVars, tt.args.argVarsFiles) if (err != nil) != tt.wantErr { t.Errorf("readInitialVariables() error = %v, wantErr %v", err, tt.wantErr) return @@ -105,3 +112,18 @@ func Test_initFromEnv(t *testing.T) { } } } + +func TestRunCmd(t *testing.T) { + var validArgs []string + + validArgs = append(validArgs, "run", "../../../tests/assertions") + + rootCmd := &cobra.Command{ + Use: "venom", + Short: "Venom aim to create, manage and run your integration tests with efficiency", + } + rootCmd.SetArgs(validArgs) + rootCmd.AddCommand(Cmd) + err := rootCmd.Execute() + assert.NoError(t, err) +} diff --git a/cmd/venom/version/version.go b/cmd/venom/version/cmd.go similarity index 100% rename from cmd/venom/version/version.go rename to cmd/venom/version/cmd.go diff --git a/cmd/venom/version/cmd_test.go b/cmd/venom/version/cmd_test.go new file mode 100644 index 00000000..b20124dc --- /dev/null +++ b/cmd/venom/version/cmd_test.go @@ -0,0 +1,22 @@ +package version + +import ( + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestVersionCmd(t *testing.T) { + var validArgs []string + + validArgs = append(validArgs, "version") + + rootCmd := &cobra.Command{ + Use: "venom", + Short: "Venom aim to create, manage and run your integration tests with efficiency", + } + rootCmd.SetArgs(validArgs) + rootCmd.AddCommand(Cmd) + err := rootCmd.Execute() + assert.NoError(t, err) +} diff --git a/process.go b/process.go index 15e33d32..7c0aa9bd 100644 --- a/process.go +++ b/process.go @@ -182,10 +182,11 @@ func (v *Venom) Process(ctx context.Context, path []string) error { v.Tests.TestSuites[i].Start = time.Now() // ##### RUN Test Suite Here - if err := v.runTestSuite(ctx, &v.Tests.TestSuites[i]); err != nil { + cleanTs, err := v.runTestSuite(ctx, &v.Tests.TestSuites[i]) + if err != nil { return err } - + v.Tests.TestSuites[i] = *cleanTs v.Tests.TestSuites[i].End = time.Now() v.Tests.TestSuites[i].Duration = v.Tests.TestSuites[i].End.Sub(v.Tests.TestSuites[i].Start).Seconds() } diff --git a/process_files.go b/process_files.go index 96bd91dd..96176e4b 100644 --- a/process_files.go +++ b/process_files.go @@ -94,7 +94,7 @@ func (v *Venom) readFiles(ctx context.Context, filesPath []string) (err error) { if err != nil { return errors.Wrapf(err, "unable to parse variable %q", k) } - varCloned.Add(k, valueInterpolated) + varsFromPartial[k] = valueInterpolated } } @@ -142,7 +142,6 @@ func (v *Venom) readFiles(ctx context.Context, filesPath []string) (err error) { ts.ShortName = strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath)) // a.yml ts.Filename = filepath.Base(filePath) - ts.Vars = varCloned ts.Vars.Add("venom.testsuite.workdir", ts.WorkDir) ts.Vars.Add("venom.testsuite.name", ts.Name) diff --git a/process_testcase.go b/process_testcase.go index 4cb08825..f8ebd392 100644 --- a/process_testcase.go +++ b/process_testcase.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + "os" + "reflect" "regexp" "strconv" "strings" @@ -132,24 +134,29 @@ func (v *Venom) parseTestCase(ts *TestSuite, tc *TestCase) ([]string, []string, return vars, extractedVars, nil } -func (v *Venom) runTestCase(ctx context.Context, ts *TestSuite, tc *TestCase) { +func (v *Venom) runTestCase(ctx context.Context, ts *TestSuite, tc *TestCase) H { ctx = context.WithValue(ctx, ContextKey("testcase"), tc.Name) - - tc.TestSuiteVars = ts.Vars.Clone() - tc.Vars = ts.Vars.Clone() - tc.Vars.Add("venom.testcase", tc.Name) - tc.Vars.AddAll(ts.ComputedVars) - tc.computedVars = H{} - - ctx = v.processSecrets(ctx, ts, tc) - Info(ctx, "Starting testcase") - - defer Info(ctx, "Ending testcase") - // ##### RUN Test Steps Here - v.runTestSteps(ctx, tc, nil) + ctx = v.processSecrets(ctx, ts, tc) + computedVars := v.runTestSteps(ctx, tc, nil) + Info(ctx, "Ending testcase") + if ts.ComputedVars == nil { + return computedVars + } + return GetOnlyNewVariables(&ts.ComputedVars, computedVars) } +func GetOnlyNewVariables(allVariables *H, computedVars H) H { + cleanVars := H{} + local := *allVariables + for k, _ := range computedVars { + _, exists := local[k] + if !exists { + cleanVars.Add(k, computedVars[k]) + } + } + return cleanVars +} func (v *Venom) processSecrets(ctx context.Context, ts *TestSuite, tc *TestCase) context.Context { computedSecrets := []string{} for k, v := range tc.Vars { @@ -162,42 +169,47 @@ func (v *Venom) processSecrets(ctx context.Context, ts *TestSuite, tc *TestCase) return context.WithValue(ctx, ContextKey("secrets"), computedSecrets) } -func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepResult) { - results, err := testConditionalStatement(ctx, tc, tc.Skip, tc.Vars, "skipping testcase %q: %v") +func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepResult) H { + failures, err := testConditionalStatement(ctx, tc, tc.Skip, tc.Vars, "skipping testcase %q: %v") if err != nil { Error(ctx, "unable to evaluate \"skip\" assertions: %v", err) testStepResult := TestStepResult{} testStepResult.appendError(err) tc.TestStepResults = append(tc.TestStepResults, testStepResult) - return + return nil } - if len(results) > 0 { + if len(failures) > 0 { tc.Status = StatusSkip - for _, s := range results { + for _, s := range failures { tc.Skipped = append(tc.Skipped, Skipped{Value: s}) Warn(ctx, s) } - return + return nil } var knowExecutors = map[string]struct{}{} var previousStepVars = H{} + previousStepVars.AddAll(tc.Vars) + onlyNewVars := H{} fromUserExecutor := tsIn != nil for stepNumber, rawStep := range tc.RawTestSteps { - stepVars := tc.Vars.Clone() + stepVars := H{} stepVars.AddAll(previousStepVars) - stepVars.AddAllWithPrefix(tc.Name, tc.computedVars) + stepVars.Add("venom.testcase", tc.Name) stepVars.Add("venom.teststep.number", stepNumber) ranged, err := parseRanged(ctx, rawStep, stepVars) if err != nil { Error(ctx, "unable to parse \"range\" attribute: %v", err) - tsIn.appendError(err) - return + if tsIn != nil { + tsIn.appendError(err) + } + return nil } for rangedIndex, rangedData := range ranged.Items { + stepVars.AddAll(previousStepVars) tc.TestStepResults = append(tc.TestStepResults, TestStepResult{}) tsResult := &tc.TestStepResults[len(tc.TestStepResults)-1] @@ -212,7 +224,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe if err != nil { Error(ctx, "unable to dump testcase vars: %v", err) tsResult.appendError(err) - return + return nil } for k, v := range vars { @@ -220,7 +232,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe if err != nil { tsResult.appendError(err) Error(ctx, "unable to interpolate variable %q: %v", k, err) - return + return nil } vars[k] = content } @@ -238,12 +250,16 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe } var content string + payloadBytes, _ := json.Marshal(rawStep) + content = string(payloadBytes) for i := 0; i < 10; i++ { - content, err = interpolate.Do(string(rawStep), vars) + content, err = interpolate.Do(content, vars) + //content, err = interpolate.Do(strings.ReplaceAll(string(payloadBytes), "\\\"", "'"), vars) if err != nil { tsResult.appendError(err) Error(ctx, "unable to interpolate step: %v", err) - return + tsResult.Status = StatusFail + break } if !strings.Contains(content, "{{") { break @@ -267,7 +283,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe if err := yaml.Unmarshal([]byte(content), &step); err != nil { tsResult.appendError(err) Error(ctx, "unable to parse step #%d: %v", stepNumber, err) - return + return nil } data2, err := yaml.JSONToYAML([]byte(content)) @@ -280,11 +296,10 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe tsResult.Number = stepNumber tsResult.RangedIndex = rangedIndex tsResult.RangedEnable = ranged.Enabled - tsResult.InputVars = vars + tsResult.InputVars = stepVars tc.testSteps = append(tc.testSteps, step) var e ExecutorRunner - Info(ctx, "variables before execution %v", stepVars) ctx, e, err = v.GetExecutorRunner(ctx, step, stepVars) if err != nil { tsResult.appendError(err) @@ -315,7 +330,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe v.setTestStepName(tsResult, e, step, &ranged, &rangedData, rangedIndex, printStepName) // ##### RUN Test Step Here - skip, err := parseSkip(ctx, tc, tsResult, rawStep, stepNumber) + skip, err := parseSkip(ctx, tc, &stepVars, tsResult, rawStep, stepNumber) if err != nil { tsResult.appendError(err) tsResult.Status = StatusFail @@ -324,7 +339,14 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe } else { tsResult.Start = time.Now() tsResult.Status = StatusRun - v.RunTestStep(ctx, e, tc, tsResult, stepNumber, rangedIndex, step) + _, vars := v.RunTestStep(ctx, e, tc, tsResult, stepNumber, rangedIndex, step, &previousStepVars) + if vars != nil { + previousStepVars.AddAll(vars) + if tsResult.ComputedVars == nil { + tsResult.ComputedVars = H{} + } + tsResult.ComputedVars.AddAll(vars) + } if len(tsResult.Errors) > 0 || !tsResult.AssertionsApplied.OK { tsResult.Status = StatusFail } else { @@ -335,6 +357,18 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe tsResult.Duration = tsResult.End.Sub(tsResult.Start).Seconds() tc.testSteps = append(tc.testSteps, step) + + assign, _, err := processVariableAssignments(ctx, tc.Name, &previousStepVars, rawStep) + if err != nil { + Error(ctx, "unable to process variable assignments: %v", err) + tsResult.Status = StatusFail + tsResult.appendError(err) + } + if assign != nil { + tsResult.ComputedVars.AddAll(assign) + onlyNewVars.AddAll(assign) + previousStepVars.AddAll(assign) + } } var isRequired bool @@ -350,27 +384,15 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe failure := newFailure(ctx, *tc, stepNumber, rangedIndex, "", fmt.Errorf("At least one required assertion failed, skipping remaining steps")) tsResult.appendFailure(*failure) v.printTestStepResult(tc, tsResult, tsIn, stepNumber, true) - return + return nil } v.printTestStepResult(tc, tsResult, tsIn, stepNumber, false) continue } v.printTestStepResult(tc, tsResult, tsIn, stepNumber, false) - - allVars := tc.Vars.Clone() - allVars.AddAll(tsResult.ComputedVars.Clone()) - - assign, _, err := processVariableAssignments(ctx, tc.Name, allVars, rawStep) - if err != nil { - tsResult.appendError(err) - Error(ctx, "unable to process variable assignments: %v", err) - break - } - - tc.computedVars.AddAll(assign) - previousStepVars.AddAll(assign) } } + return onlyNewVars } // Set test step name (defaults to executor name, excepted if it got a "name" attribute. in range, also print key) @@ -429,7 +451,7 @@ func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *Test } // Parse and format skip conditional -func parseSkip(ctx context.Context, tc *TestCase, ts *TestStepResult, rawStep []byte, stepNumber int) (bool, error) { +func parseSkip(ctx context.Context, tc *TestCase, vars *H, ts *TestStepResult, rawStep []byte, stepNumber int) (bool, error) { // Load "skip" attribute from step var assertions struct { Skip []string `yaml:"skip"` @@ -439,14 +461,16 @@ func parseSkip(ctx context.Context, tc *TestCase, ts *TestStepResult, rawStep [] } // Evaluate skip assertions - if len(assertions.Skip) > 0 { - results, err := testConditionalStatement(ctx, tc, assertions.Skip, tc.Vars, fmt.Sprintf("skipping testcase %%q step #%d: %%v", stepNumber)) + if assertions.Skip != nil && len(assertions.Skip) > 0 { + failures, err := testConditionalStatement(ctx, tc, assertions.Skip, *vars, fmt.Sprintf("skipping testcase %%q step #%d: %%v", stepNumber)) if err != nil { Error(ctx, "unable to evaluate \"skip\" assertions: %v", err) return false, err } - if len(results) > 0 { - for _, s := range results { + + if len(failures) > 0 { + Info(ctx, fmt.Sprintf("Skip as there are %v failures", len(failures))) + for _, s := range failures { ts.Skipped = append(ts.Skipped, Skipped{Value: s}) Warn(ctx, s) } @@ -462,6 +486,9 @@ func parseRanged(ctx context.Context, rawStep []byte, stepVars H) (Range, error) //Load "range" attribute and perform actions depending on its typing var ranged Range if err := json.Unmarshal(rawStep, &ranged); err != nil { + ranged.Enabled = false + ranged.Items = []RangeData{} + ranged.Items = append(ranged.Items, RangeData{}) return ranged, fmt.Errorf("unable to parse range expression: %v", err) } @@ -469,6 +496,8 @@ func parseRanged(ctx context.Context, rawStep []byte, stepVars H) (Range, error) //Nil means this is not a ranged data, append an empty item to force at least one iteration and exit case nil: + ranged.Enabled = false + ranged.Items = []RangeData{} ranged.Items = append(ranged.Items, RangeData{}) return ranged, nil @@ -555,7 +584,46 @@ func parseRanged(ctx context.Context, rawStep []byte, stepVars H) (Range, error) return ranged, nil } -func processVariableAssignments(ctx context.Context, tcName string, tcVars H, rawStep json.RawMessage) (H, bool, error) { +func processJsonBlob(key string, value string) (map[string]interface{}, error) { + result := make(map[string]interface{}) + var outJSON interface{} + if err := JSONUnmarshal([]byte(value), &outJSON); err == nil { + result[key+"json"] = outJSON + initialDump, err := DumpStringPreserveCase(outJSON) + if err != nil { + return nil, errors.Wrapf(err, "unable to compute result") + } + // Now we have to dump this object, but the key will change if this is a array or not + if reflect.ValueOf(outJSON).Kind() == reflect.Slice { + prefix := key + "json" + splitPrefix := strings.Split(prefix, ".") + prefix += "." + splitPrefix[len(splitPrefix)-1] + for ko, vo := range initialDump { + result[prefix+ko] = vo + } + } else { + outJSONDump := map[string]interface{}{} + for ko, vo := range initialDump { + if !strings.Contains(ko, "__Type__") && !strings.Contains(ko, "__Len__") { + outJSONDump[key+"json."+ko] = vo + } else { + outJSONDump[ko] = vo + } + } + for ko, vo := range outJSONDump { + result[ko] = vo + } + } + } + //make it compatible with the one before (ie all lowercase) + for k, v := range result { + result[strings.ToLower(k)] = v + } + + return result, nil +} + +func processVariableAssignments(ctx context.Context, tcName string, tcVars *H, rawStep json.RawMessage) (H, bool, error) { var stepAssignment AssignStep var result = make(H) if err := yaml.Unmarshal(rawStep, &stepAssignment); err != nil { @@ -567,20 +635,53 @@ func processVariableAssignments(ctx context.Context, tcName string, tcVars H, ra return nil, false, nil } + localVars := *tcVars var tcVarsKeys []string - for k := range tcVars { + for k := range localVars { tcVarsKeys = append(tcVarsKeys, k) } for varname, assignment := range stepAssignment.Assignments { Debug(ctx, "Processing %s assignment", varname) - varValue, has := tcVars[assignment.From] + // + jsonUpdates := os.Getenv("VENOM_NO_JSON_EXPANSION") + if jsonUpdates == "ON" { + if strings.Contains(assignment.From, "json") { + _, has := localVars[assignment.From] + if !has { + key := getKeyForLookup(assignment.From) + varValue, has := localVars[key] + if !has { + if assignment.Default == nil { + err := fmt.Errorf("%s reference not found and tried to create json for %s", assignment.From, key) + Error(ctx, "%v", err) + return nil, true, err + } + localVars[key] = assignment.Default + result.Add(varname, assignment.Default) + } else { + Debug(ctx, "process json %v", key) + varjson, err := processJsonBlob(key, fmt.Sprintf("%v", varValue)) + if err != nil { + err := fmt.Errorf("%s could not parse json for %s", assignment.From, key) + Error(ctx, "%v", err) + return nil, true, err + } + localVars.AddAll(varjson) + } + } + + } + } + varValue, has := localVars[assignment.From] + if !has { - varValue, has = tcVars[tcName+"."+assignment.From] + varValue, has = localVars[tcName+"."+assignment.From] if !has { + if assignment.Default == nil { err := fmt.Errorf("%s reference not found in %s", assignment.From, strings.Join(tcVarsKeys, "\n")) - Info(ctx, "%v", err) + Error(ctx, "%v", err) return nil, true, err } varValue = assignment.Default @@ -613,3 +714,19 @@ func processVariableAssignments(ctx context.Context, tcName string, tcVars H, ra } return result, true, nil } + +// getKeyForLookup we need the first json key in order to process the json blob +func getKeyForLookup(originalKey string) string { + parts := strings.Split(originalKey, ".") + keyparts := []string{} + for _, part := range parts { + if strings.HasSuffix(part, "json") { + keyparts = append(keyparts, strings.Replace(part, "json", "", 1)) + break + } else { + keyparts = append(keyparts, part) + } + } + key := strings.Join(keyparts, ".") + return key +} diff --git a/process_testcase_test.go b/process_testcase_test.go index 0f7c1d57..1342e910 100644 --- a/process_testcase_test.go +++ b/process_testcase_test.go @@ -26,7 +26,7 @@ func TestProcessVariableAssignments(t *testing.T) { tcVars := H{"here.some.value": "this is the \nvalue"} - result, is, err := processVariableAssignments(context.TODO(), "", tcVars, b) + result, is, err := processVariableAssignments(context.TODO(), "", &tcVars, b) assert.True(t, is) assert.NoError(t, err) assert.NotNil(t, result) @@ -38,9 +38,60 @@ func TestProcessVariableAssignments(t *testing.T) { script: echo 'foo' `) assert.NoError(t, yaml.Unmarshal(b, &wrongStepIn)) - result, is, err = processVariableAssignments(context.TODO(), "", tcVars, b) + result, is, err = processVariableAssignments(context.TODO(), "", &tcVars, b) assert.False(t, is) assert.NoError(t, err) assert.Nil(t, result) assert.Empty(t, result) } + +func TestProcessJsonBlobWithObject(t *testing.T) { + InitTestLogger(t) + items, err := processJsonBlob("test", "{\"key\":123,\"another\":\"one\"}") + assert.NoError(t, err) + assert.NotNil(t, items) + assert.Contains(t, items, "testjson.key") + assert.Contains(t, items, "testjson.another") + assert.Contains(t, items, "testjson") + assert.Contains(t, items, "__Type__") + assert.Contains(t, items, "__Len__") +} + +func TestProcessJsonBlobWithArray(t *testing.T) { + InitTestLogger(t) + items, err := processJsonBlob("test", "{\"key\":123,\"anArray\":[\"one\",\"two\"]}") + assert.NoError(t, err) + assert.NotNil(t, items) + assert.Contains(t, items, "testjson.key") + assert.Contains(t, items, "testjson.anArray") + assert.Contains(t, items, "testjson") + assert.Equal(t, items["anArray.__Type__"], "Array") + assert.Equal(t, items["anArray.__Len__"], "2") + assert.Contains(t, items, "testjson.anArray.anArray0") + assert.Equal(t, items["testjson.anArray.anArray0"], "one") + assert.Contains(t, items, "testjson.anArray.anArray1") + assert.Equal(t, items["testjson.anArray.anArray1"], "two") + assert.Contains(t, items, "__Type__") + assert.Equal(t, items["__Type__"], "Map") + assert.Contains(t, items, "__Len__") +} + +func TestGetKeyForLookup(t *testing.T) { + InitTestLogger(t) + assert.Equal(t, "test", getKeyForLookup("testjson.key")) + assert.Equal(t, "test", getKeyForLookup("testjson.anArray.anArray0")) +} + +func TestProcessRange(t *testing.T) { + InitTestLogger(t) + rawStep := []byte("{\"account_id\":\"{{.value}}\",\"name\":\"Account validation\",\"range\":\"{{.account_ids}}\",\"type\":\"account_validation\"}") + vars := H{} + vars.Add("account_ids", []string{`1`, `2`}) + ranged, err := parseRanged(context.Background(), rawStep, vars) + assert.NoError(t, err) + assert.NotNil(t, ranged) + assert.NotNil(t, ranged.Items) + assert.Nil(t, ranged.RawContent) + assert.True(t, ranged.Enabled) + assert.Equal(t, 2, len(ranged.Items)) +} diff --git a/process_teststep.go b/process_teststep.go index 2273cf0c..010a5328 100644 --- a/process_teststep.go +++ b/process_teststep.go @@ -19,12 +19,12 @@ type dumpFile struct { } // RunTestStep executes a venom testcase is a venom context -func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult *TestStepResult, stepNumber int, rangedIndex int, step TestStep) { +func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult *TestStepResult, stepNumber int, rangedIndex int, step TestStep, vars *H) (interface{}, H) { ctx = context.WithValue(ctx, ContextKey("executor"), e.Name()) var assertRes AssertionsApplied var result interface{} - + newVars := H{} for tsResult.Retries = 0; tsResult.Retries <= e.Retry() && !assertRes.OK; tsResult.Retries++ { if tsResult.Retries > 1 && !assertRes.OK { Debug(ctx, "Sleep %d, it's %d attempt", e.Delay(), tsResult.Retries) @@ -32,7 +32,7 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, } var err error - result, err = v.runTestStepExecutor(ctx, e, tc, tsResult, step) + result, err = v.runTestStepExecutor(ctx, e, tc, tsResult, step, vars) if err != nil { // we save the failure only if it's the last attempt if tsResult.Retries == e.Retry() { @@ -44,13 +44,18 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, Debug(ctx, "result of runTestStepExecutor: %+v", HideSensitive(ctx, result)) mapResult := GetExecutorResult(result) + tsResult.ComputedVars.AddAll(H(mapResult)) mapResultString, _ := DumpString(result) + for k, value := range mapResultString { + tsResult.ComputedVars.Add(k, value) + newVars.Add(k, value) + } if v.Verbose >= 2 { fdump := dumpFile{ Result: result, TestStep: step, - Variables: AllVarsFromCtx(ctx), + Variables: *vars, } output, err := json.MarshalIndent(fdump, "", " ") if err != nil { @@ -61,17 +66,28 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, if oDir == "" { oDir = "." } - filename := path.Join(oDir, fmt.Sprintf("%s.%s.step.%d.%d.dump.json", slug.Make(StringVarFromCtx(ctx, "venom.testsuite.shortName")), slug.Make(tc.Name), stepNumber, rangedIndex)) + format := "%s.%s.step.%d.%d.dump.json" + name := fmt.Sprintf(format, slug.Make(StringVarFromCtx(ctx, "venom.testsuite.shortName")), slug.Make(tc.Name), stepNumber, rangedIndex) + flag, exists := os.LookupEnv("VENOM_LOGS_WITH_TIMESTAMP") + if exists && flag == "ON" { + format = "%s.%s.%s.step.%d.%d.dump.json" + name = fmt.Sprintf(format, slug.Make(StringVarFromCtx(ctx, "venom.testsuite.shortName")), time.Now().UTC().Format("15.04.05.000"), slug.Make(tc.Name), stepNumber, rangedIndex) + } + filename := path.Join(oDir, name) if err := os.WriteFile(filename, []byte(output), 0644); err != nil { Error(ctx, "Error while creating file %s: %v", filename, err) - return + return result, tsResult.ComputedVars } tc.computedVerbose = append(tc.computedVerbose, fmt.Sprintf("writing %s", filename)) } - + allvars, _ := DumpStringPreserveCase(tsResult.ComputedVars) + inputVars, _ := DumpStringPreserveCase(tsResult.InputVars) + for k, value := range inputVars { + allvars[k] = value + } for ninfo, i := range e.Info() { - info, err := interpolate.Do(i, mapResultString) + info, err := interpolate.Do(i, allvars) if err != nil { Error(ctx, "unable to parse %q: %v", i, err) continue @@ -92,35 +108,41 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, } } Info(ctx, info) + flag, exists := os.LookupEnv("VENOM_LOGS_SHOW_INFO") + if exists && flag == "ON" { + v.Println(fmt.Sprintf("\n\t\t\t\t %s", Cyan(info))) + } tsResult.ComputedInfo = append(tsResult.ComputedInfo, info) } - + varsForAssertions := H{} + varsForAssertions.AddAll(*vars) if result == nil { - Debug(ctx, "empty testcase, applying assertions on variables: %v", AllVarsFromCtx(ctx)) - assertRes = applyAssertions(ctx, AllVarsFromCtx(ctx), *tc, stepNumber, rangedIndex, step, nil) + assertRes = applyAssertions(ctx, &varsForAssertions, tc, stepNumber, rangedIndex, step, nil) } else { + varsForAssertions.AddAll(mapResult) if h, ok := e.(executorWithDefaultAssertions); ok { - assertRes = applyAssertions(ctx, result, *tc, stepNumber, rangedIndex, step, h.GetDefaultAssertions()) + assertRes = applyAssertions(ctx, &varsForAssertions, tc, stepNumber, rangedIndex, step, h.GetDefaultAssertions()) } else { - assertRes = applyAssertions(ctx, result, *tc, stepNumber, rangedIndex, step, nil) + assertRes = applyAssertions(ctx, &varsForAssertions, tc, stepNumber, rangedIndex, step, nil) } } tsResult.AssertionsApplied = assertRes - tsResult.ComputedVars.AddAll(H(mapResult)) if assertRes.OK { break } - failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tsResult.ComputedVars, "") - if err != nil { - tsResult.appendError(fmt.Errorf("Error while evaluating retry condition: %v", err)) - break - } - if len(failures) > 0 { - failure := newFailure(ctx, *tc, stepNumber, rangedIndex, "", fmt.Errorf("retry conditions not fulfilled, skipping %d remaining retries", e.Retry()-tsResult.Retries)) - tsResult.Errors = append(tsResult.Errors, *failure) - break + if len(e.RetryIf()) > 0 { + failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tsResult.ComputedVars, "") + if err != nil { + tsResult.appendError(fmt.Errorf("Error while evaluating retry condition: %v", err)) + break + } + if len(failures) > 0 { + failure := newFailure(ctx, *tc, stepNumber, rangedIndex, "", fmt.Errorf("retry conditions not fulfilled, skipping %d remaining retries", e.Retry()-tsResult.Retries)) + tsResult.Errors = append(tsResult.Errors, *failure) + break + } } } @@ -134,14 +156,16 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult.Systemerr += assertRes.systemerr + "\n" tsResult.Systemout += assertRes.systemout + "\n" + + return result, newVars } -func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *TestCase, ts *TestStepResult, step TestStep) (interface{}, error) { +func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *TestCase, ts *TestStepResult, step TestStep, vars *H) (interface{}, error) { ctx = context.WithValue(ctx, ContextKey("executor"), e.Name()) if e.Timeout() == 0 { if e.Type() == "user" { - return v.RunUserExecutor(ctx, e, tc, ts, step) + return v.RunUserExecutor(ctx, e, tc, ts, step, vars) } return e.Run(ctx, step) } @@ -155,7 +179,7 @@ func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *T var err error var result interface{} if e.Type() == "user" { - result, err = v.RunUserExecutor(ctx, e, tc, ts, step) + result, err = v.RunUserExecutor(ctx, e, tc, ts, step, vars) } else { result, err = e.Run(ctx, step) } diff --git a/process_testsuite.go b/process_testsuite.go index 8f141245..c14a7959 100644 --- a/process_testsuite.go +++ b/process_testsuite.go @@ -12,7 +12,7 @@ import ( "github.com/pkg/errors" ) -func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) error { +func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) (*TestSuite, error) { if v.Verbose == 3 { var filename, filenameCPU, filenameMem string if v.OutputDir != "" { @@ -23,7 +23,7 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) error { fCPU, errCPU := os.Create(filenameCPU) fMem, errMem := os.Create(filenameMem) if errCPU != nil || errMem != nil { - return fmt.Errorf("error while create profile file CPU:%v MEM:%v", errCPU, errMem) + return nil, fmt.Errorf("error while create profile file CPU:%v MEM:%v", errCPU, errMem) } else { pprof.StartCPUProfile(fCPU) p := pprof.Lookup("heap") @@ -33,19 +33,19 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) error { } // Initialize the testsuite variables and compute a first interpolation over them - ts.Vars.AddAll(v.variables.Clone()) + ts.Vars.AddAll(v.variables) vars, _ := DumpStringPreserveCase(ts.Vars) for k, v := range vars { computedV, err := interpolate.Do(fmt.Sprintf("%v", v), vars) if err != nil { - return errors.Wrapf(err, "error while computing variable %s=%q", k, v) + return nil, errors.Wrapf(err, "error while computing variable %s=%q", k, v) } ts.Vars.Add(k, computedV) } exePath, err := os.Executable() if err != nil { - return errors.Wrapf(err, "failed to get executable path") + return nil, errors.Wrapf(err, "failed to get executable path") } else { ts.Vars.Add("venom.executable", exePath) } @@ -71,7 +71,8 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) error { } // ##### RUN Test Cases Here v.runTestCases(ctx, ts) - + ts.End = time.Now() + ts.Duration = ts.End.Sub(ts.Start).Seconds() var isFailed bool var nSkip int for _, tc := range ts.TestCases { @@ -96,14 +97,21 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) error { ts.Status = StatusPass v.Tests.NbTestsuitesPass++ } - return nil + clean := v.CleanUpSecrets(*ts) + errOutput := v.GenerateOutputForTestSuite(&clean) + if errOutput != nil { + Error(ctx, "could not generate output for testsuite") + return ts, nil + } + return &clean, nil } func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) { verboseReport := v.Verbose >= 1 v.Println(" • %s (%s)", ts.Name, ts.Filepath) - + previousVariables := H{} + previousVariables.AddAll(ts.Vars) for i := range ts.TestCases { tc := &ts.TestCases[i] tc.IsEvaluated = true @@ -119,7 +127,10 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) { v.Print("\n") } // ##### RUN Test Case Here - v.runTestCase(ctx, ts, tc) + tc.Vars.AddAll(previousVariables) + tc.Vars.Add("venom.testcase", tc.Name) + computedVariables := v.runTestCase(ctx, ts, tc) + previousVariables.AddAllWithPrefix(tc.Name, computedVariables) tc.End = time.Now() tc.Duration = tc.End.Sub(tc.Start).Seconds() } diff --git a/read_partial.go b/read_partial.go index 6a09e8e3..ae11891c 100644 --- a/read_partial.go +++ b/read_partial.go @@ -3,6 +3,7 @@ package venom import ( "bufio" "context" + "reflect" "strings" "unicode" "unicode/utf8" @@ -22,8 +23,11 @@ func getUserExecutorInputYML(ctx context.Context, btesIn []byte) (H, error) { return nil, err } } - for k, v := range tmpResult { - result[k] = v + tmp, ok := tmpResult["foo"] + if ok { + if reflect.ValueOf(tmp).Kind() == reflect.Map { + result = tmp.(map[string]interface{}) + } } return result, nil @@ -44,6 +48,21 @@ func getVarFromPartialYML(ctx context.Context, btesIn []byte) (H, error) { return partial.Vars, nil } +func getExecutorName(btes []byte) (string, error) { + content := readPartialYML(btes, "executor") + type partialType struct { + Executor string `yaml:"executor" json:"executor"` + } + partial := &partialType{} + if len(content) > 0 { + if err := yaml.Unmarshal([]byte(content), &partial); err != nil { + Error(context.Background(), "file content: %s", string(btes)) + return "", errors.Wrapf(err, "error while unmarshal - see venom.log") + } + } + return partial.Executor, nil +} + // readPartialYML extract a yml part from a given string func readPartialYML(btes []byte, attribute string) string { var result []string diff --git a/tests/assertions/ShouldBeLessThan.yml b/tests/assertions/ShouldBeLessThan.yml index 0afbc481..3070cd49 100644 --- a/tests/assertions/ShouldBeLessThan.yml +++ b/tests/assertions/ShouldBeLessThan.yml @@ -8,10 +8,8 @@ testcases: - name: bodyjson assert comparator test steps: - - type: http - method: GET - url: https://eu.api.ovh.com/1.0/sms/rates/destinations?billingCountry=fr&country=fr - headers: - Content-type: application/json + - type: exec + script : | + echo '{"credit": 0.5}' assertions: - - result.bodyjson.credit ShouldBeLessThan 2 \ No newline at end of file + - result.systemoutjson.credit ShouldBeLessThan 2 \ No newline at end of file diff --git a/tests/skip/skip.yml b/tests/skip/skip.yml new file mode 100644 index 00000000..120a25a6 --- /dev/null +++ b/tests/skip/skip.yml @@ -0,0 +1,42 @@ +name : "Testing skip" +testcases: + - name : "skip something" + steps: + - info: + - Set a variable to 1 + - script : "echo '1'" + vars : + out: + from : result.systemout + - name: "Skip this as out is 1 and we are checking if its equal with 2" + script: | + exit 1 + skip: + - out ShouldEqual 2 + - script: "echo 'false'" + vars: + outAsBool: + from: result.systemout + - name: "skip as outAsBool is False" + script: | + exit 1 + skip: + - outAsBool ShouldBeTrue + - name : "producer" + steps: + - info: + - Set a variable to 1 + - script : "echo '1'" + vars : + out: + from : result.systemout + - name: "consumer" + skip: + - producer.out ShouldEqual 1 + step: + - info: + - "Fail should be skipped as we are checking that the producer.out is equal with 1" + script: | + exit 1 + + diff --git a/types.go b/types.go index 05b20c45..06f81e5d 100644 --- a/types.go +++ b/types.go @@ -178,20 +178,20 @@ type TestCase struct { } type TestStepResult struct { - Name string `json:"name"` - Errors []Failure `json:"errors"` - Skipped []Skipped `json:"skipped" yaml:"skipped"` - Status Status `json:"status" yaml:"status"` - Raw interface{} `json:"raw" yaml:"raw"` - Interpolated interface{} `json:"interpolated" yaml:"interpolated"` - Number int `json:"number" yaml:"number"` - RangedIndex int `json:"rangedIndex" yaml:"rangedIndex"` - RangedEnable bool `json:"rangedEnable" yaml:"rangedEnable"` - InputVars map[string]string `json:"inputVars" yaml:"-"` - ComputedVars H `json:"computedVars" yaml:"-"` - ComputedInfo []string `json:"computedInfos" yaml:"-"` - AssertionsApplied AssertionsApplied `json:"assertionsApplied" yaml:"-"` - Retries int `json:"retries" yaml:"retries"` + Name string `json:"name"` + Errors []Failure `json:"errors"` + Skipped []Skipped `json:"skipped" yaml:"skipped"` + Status Status `json:"status" yaml:"status"` + Raw interface{} `json:"raw" yaml:"raw"` + Interpolated interface{} `json:"interpolated" yaml:"interpolated"` + Number int `json:"number" yaml:"number"` + RangedIndex int `json:"rangedIndex" yaml:"rangedIndex"` + RangedEnable bool `json:"rangedEnable" yaml:"rangedEnable"` + InputVars map[string]interface{} `json:"inputVars" yaml:"-"` + ComputedVars H `json:"computedVars" yaml:"-"` + ComputedInfo []string `json:"computedInfos" yaml:"-"` + AssertionsApplied AssertionsApplied `json:"assertionsApplied" yaml:"-"` + Retries int `json:"retries" yaml:"retries"` Systemout string `json:"systemout"` Systemerr string `json:"systemerr"` diff --git a/types_executor.go b/types_executor.go index 87cddb3c..b57c145d 100644 --- a/types_executor.go +++ b/types_executor.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "fmt" - "reflect" + "os" "strings" "github.com/gosimple/slug" @@ -164,6 +164,26 @@ func GetExecutorResult(r interface{}) map[string]interface{} { if err != nil { panic(err) } + jsonUpdates := os.Getenv("VENOM_NO_JSON_EXPANSION") + if jsonUpdates == "ON" { + for key, value := range d { + switch z := value.(type) { + case string: + m, e := processJsonBlob(key, z) + if e != nil { + panic(e) + } + if len(m) > 0 { + for s, i := range m { + if !strings.Contains(strings.ToUpper(s), "__Type__") && !strings.Contains(strings.ToUpper(s), "__Len__") { + d[s] = i + } + } + } + } + } + } + return d } @@ -200,10 +220,11 @@ func (ux UserExecutor) ZeroValueResult() interface{} { return result } -func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn *TestCase, tsIn *TestStepResult, step TestStep) (interface{}, error) { - vrs := tcIn.TestSuiteVars.Clone() +func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn *TestCase, tsIn *TestStepResult, step TestStep, vars *H) (interface{}, error) { + vrs := H{} + vrs.AddAll(*vars) uxIn := runner.GetExecutor().(UserExecutor) - + inputOnlyVrs := H{} for k, va := range uxIn.Input { if strings.HasPrefix(k, "input.") { // do not reinject input.vars from parent user executor if exists @@ -211,10 +232,13 @@ func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn } else if !strings.HasPrefix(k, "venom") { if vl, ok := step[k]; ok && vl != "" { // value from step vrs.AddWithPrefix("input", k, vl) + inputOnlyVrs.AddWithPrefix("input", k, vl) } else { // default value from executor vrs.AddWithPrefix("input", k, va) + inputOnlyVrs.AddWithPrefix("input", k, va) } } else { + inputOnlyVrs.Add(k, va) vrs.Add(k, va) } } @@ -224,31 +248,47 @@ func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn return nil, errors.Wrapf(err, "unable to reload executor") } ux := exe.GetExecutor().(UserExecutor) - + testStepResult := &TestStepResult{Name: slug.Make(ux.Executor)} tc := &TestCase{ TestCaseInput: TestCaseInput{ Name: ux.Executor, RawTestSteps: ux.RawTestSteps, - Vars: vrs, + Vars: inputOnlyVrs, }, + originalName: ux.Executor, TestSuiteVars: tcIn.TestSuiteVars, IsExecutor: true, TestStepResults: make([]TestStepResult, 0), } + tc.Vars.AddAll(vrs) - tc.originalName = tc.Name - tc.Name = slug.Make(tc.Name) + tc.Name = ux.Executor tc.Vars.Add("venom.testcase", tc.Name) tc.Vars.Add("venom.executor.filename", ux.Filename) tc.Vars.Add("venom.executor.name", ux.Executor) + + if tc.TestSuiteVars == nil { + tc.TestSuiteVars = H{} + } + tc.TestSuiteVars.AddAll(vrs) tc.computedVars = H{} - Debug(ctx, "running user executor %v", tc.Name) - Debug(ctx, "with vars: %v", vrs) + Debug(ctx, "running user executor %v", ux.Executor) + + newVars := v.runTestSteps(ctx, tc, tsIn) - v.runTestSteps(ctx, tc, tsIn) + Debug(ctx, "finished running steps of user executor %v", ux.Executor) - computedVars, err := DumpString(tc.computedVars) + if newVars != nil { + testStepResult.ComputedVars.AddAll(newVars) + vrs.AddAll(newVars) + } + computedVars, err := DumpStringPreserveCase(vrs) + otherVars, _ := DumpString(vrs) + //adding the DumpString in order to preserve functionality + for k, v := range otherVars { + computedVars[k] = v + } if err != nil { return nil, errors.Wrapf(err, "unable to dump testcase computedVars") } @@ -299,32 +339,20 @@ func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn return nil, errors.Wrapf(err, "unable to compute result") } - for k, v := range result { - switch z := v.(type) { - case string: - var outJSON interface{} - if err := JSONUnmarshal([]byte(z), &outJSON); err == nil { - result[k+"json"] = outJSON - // Now we have to dump this object, but the key will change if this is a array or not - if reflect.ValueOf(outJSON).Kind() == reflect.Slice { - prefix := k + "json" - splitPrefix := strings.Split(prefix, ".") - prefix += "." + splitPrefix[len(splitPrefix)-1] - outJSONDump, err := Dump(outJSON) - if err != nil { - return nil, errors.Wrapf(err, "unable to compute result") - } - for ko, vo := range outJSONDump { - result[prefix+ko] = vo - } - } else { - outJSONDump, err := DumpWithPrefix(outJSON, k+"json") - if err != nil { - return nil, errors.Wrapf(err, "unable to compute result") - } - for ko, vo := range outJSONDump { - result[ko] = vo - } + jsonUpdates := os.Getenv("VENOM_NO_JSON_EXPANSION") + if jsonUpdates == "ON" { + //ignore the json + Debug(ctx, "%s", "ignore the json expansion") + } else { + for k, value := range result { + switch z := value.(type) { + case string: + items, err := processJsonBlob(k, z) + if err != nil { + return nil, err + } + for key, nv := range items { + result[key] = nv } } } diff --git a/venom.go b/venom.go index 6b48a1d6..39d94bf7 100644 --- a/venom.go +++ b/venom.go @@ -15,7 +15,6 @@ import ( "github.com/confluentinc/bincover" "github.com/fatih/color" - "github.com/ovh/cds/sdk/interpolate" "github.com/pkg/errors" "github.com/rockbears/yaml" "github.com/spf13/cast" @@ -216,13 +215,19 @@ func (v *Venom) getUserExecutorFilesPath(vars map[string]string) (filePaths []st } func (v *Venom) registerUserExecutors(ctx context.Context, name string, vars map[string]string) error { + _, ok := v.executorsUser[name] + if ok { + return nil + } + if v.executorFileCache != nil && len(v.executorFileCache) != 0 { + return errors.Errorf("Could not find executor with name %v ", name) + } executorsPath, err := v.getUserExecutorFilesPath(vars) if err != nil { return err } for _, f := range executorsPath { - Info(ctx, "Reading %v", f) btes, ok := v.executorFileCache[f] if !ok { btes, err = os.ReadFile(f) @@ -231,6 +236,11 @@ func (v *Venom) registerUserExecutors(ctx context.Context, name string, vars map } v.executorFileCache[f] = btes } + executorName, _ := getExecutorName(btes) + if len(executorName) == 0 { + fileName := filepath.Base(f) + executorName = strings.TrimSuffix(fileName, filepath.Ext(fileName)) + } varsFromInput, err := getUserExecutorInputYML(ctx, btes) if err != nil { @@ -238,42 +248,16 @@ func (v *Venom) registerUserExecutors(ctx context.Context, name string, vars map } // varsFromInput contains the default vars from the executor - var varsFromInputMap map[string]string - if len(varsFromInput) > 0 { - varsFromInputMap, err = DumpStringPreserveCase(varsFromInput) - if err != nil { - return errors.Wrapf(err, "unable to parse variables") - } - } - - varsComputed := map[string]string{} - for k, v := range vars { - varsComputed[k] = v - } - for k, v := range varsFromInputMap { - // we only take vars from varsFromInputMap if it's not already exist in vars from teststep vars - if _, ok := vars[k]; !ok { - varsComputed[k] = v - } - } - content, err := interpolate.Do(string(btes), varsComputed) - if err != nil { - return err - } ux := UserExecutor{Filename: f} - if err := yaml.Unmarshal([]byte(content), &ux); err != nil { - return errors.Wrapf(err, "unable to parse file %q with content %v", f, content) - } - - Debug(ctx, "User executor %q revolved with content %v", f, content) - - for k, vr := range varsComputed { - ux.Input.Add(k, vr) + if err := yaml.Unmarshal(btes, &ux); err != nil { + return errors.Wrapf(err, "unable to parse file %q", f) } - v.RegisterExecutorUser(ux.Executor, ux) + ux.Input.AddAll(varsFromInput) + ux.Executor = executorName + v.RegisterExecutorUser(executorName, ux) } return nil } diff --git a/venom_output.go b/venom_output.go index 47214120..7e71636b 100644 --- a/venom_output.go +++ b/venom_output.go @@ -6,6 +6,7 @@ import ( "encoding/json" "encoding/xml" "fmt" + "log" "os" "path" "path/filepath" @@ -55,23 +56,13 @@ func (v *Venom) CleanUpSecrets(testSuite TestSuite) TestSuite { // OutputResult output result to sdtout, files... func (v *Venom) OutputResult() error { - if v.OutputDir == "" { - return nil - } - cleanedTs := []TestSuite{} - for i := range v.Tests.TestSuites { - tcFiltered := []TestCase{} - for _, tc := range v.Tests.TestSuites[i].TestCases { - if tc.IsEvaluated { - tcFiltered = append(tcFiltered, tc) - } + if v.HtmlReport { + if v.OutputDir == "" { + return nil } - v.Tests.TestSuites[i].TestCases = tcFiltered - ts := v.CleanUpSecrets(v.Tests.TestSuites[i]) - cleanedTs = append(cleanedTs, ts) testsResult := &Tests{ - TestSuites: []TestSuite{ts}, + TestSuites: v.Tests.TestSuites, Status: v.Tests.Status, NbTestsuitesFail: v.Tests.NbTestsuitesFail, NbTestsuitesPass: v.Tests.NbTestsuitesPass, @@ -81,66 +72,79 @@ func (v *Venom) OutputResult() error { End: v.Tests.End, } - var data []byte - var err error - - switch v.OutputFormat { - case "json": - data, err = json.MarshalIndent(testsResult, "", " ") - if err != nil { - return errors.Wrapf(err, "Error: cannot format output json (%s)", err) - } - case "tap": - data, err = outputTapFormat(*testsResult) - if err != nil { - return errors.Wrapf(err, "Error: cannot format output tap (%s)", err) - } - case "yml", "yaml": - data, err = yaml.Marshal(testsResult) - if err != nil { - return errors.Wrapf(err, "Error: cannot format output yaml (%s)", err) - } - case "xml": - data, err = outputXMLFormat(*testsResult) - if err != nil { - return errors.Wrapf(err, "Error: cannot format output xml (%s)", err) - } - case "html": - return errors.New("Error: you have to use the --html-report flag") + data, err := outputHTML(testsResult) + if err != nil { + return errors.Wrapf(err, "Error: cannot format output html") } - - fname := strings.TrimSuffix(ts.Filepath, filepath.Ext(ts.Filepath)) - fname = strings.ReplaceAll(fname, "/", "_") - filename := path.Join(v.OutputDir, "test_results_"+fname+"."+v.OutputFormat) + var filename = filepath.Join(v.OutputDir, computeOutputFilename("test_results.html")) + v.PrintFunc("Writing html file %s\n", filename) if err := os.WriteFile(filename, data, 0600); err != nil { - return fmt.Errorf("Error while creating file %s: %v", filename, err) + return errors.Wrapf(err, "Error while creating file %s", filename) } - v.PrintFunc("Writing file %s\n", filename) } - if v.HtmlReport { - testsResult := &Tests{ - TestSuites: cleanedTs, - Status: v.Tests.Status, - NbTestsuitesFail: v.Tests.NbTestsuitesFail, - NbTestsuitesPass: v.Tests.NbTestsuitesPass, - NbTestsuitesSkip: v.Tests.NbTestsuitesSkip, - Duration: v.Tests.Duration, - Start: v.Tests.Start, - End: v.Tests.End, + return nil +} + +func (v *Venom) GenerateOutputForTestSuite(ts *TestSuite) error { + if v.OutputDir == "" { + return nil + } + + tcFiltered := []TestCase{} + for _, tc := range ts.TestCases { + if tc.IsEvaluated { + tcFiltered = append(tcFiltered, tc) } + } + ts.TestCases = tcFiltered + + testsResult := &Tests{ + TestSuites: []TestSuite{*ts}, + Status: ts.Status, + NbTestsuitesFail: ts.NbTestcasesFail, + NbTestsuitesPass: ts.NbTestcasesPass, + NbTestsuitesSkip: ts.NbTestcasesSkip, + Duration: ts.Duration, + Start: ts.Start, + End: ts.End, + } - data, err := outputHTML(testsResult) + var data []byte + var err error + + switch v.OutputFormat { + case "json": + data, err = json.MarshalIndent(testsResult, "", " ") if err != nil { - return errors.Wrapf(err, "Error: cannot format output html") + log.Fatalf("Error: cannot format output json (%s)", err) } - var filename = filepath.Join(v.OutputDir, computeOutputFilename("test_results.html")) - v.PrintFunc("Writing html file %s\n", filename) - if err := os.WriteFile(filename, data, 0600); err != nil { - return errors.Wrapf(err, "Error while creating file %s", filename) + case "tap": + data, err = outputTapFormat(*testsResult) + if err != nil { + log.Fatalf("Error: cannot format output tap (%s)", err) + } + case "yml", "yaml": + data, err = yaml.Marshal(testsResult) + if err != nil { + log.Fatalf("Error: cannot format output yaml (%s)", err) + } + case "xml": + data, err = outputXMLFormat(*testsResult) + if err != nil { + log.Fatalf("Error: cannot format output xml (%s)", err) } } + fname := strings.TrimSuffix(ts.Filepath, filepath.Ext(ts.Filepath)) + fname = strings.ReplaceAll(fname, "/", "_") + filename := path.Join(v.OutputDir, "partial_test_results_"+fname+"."+v.OutputFormat) + if err := os.WriteFile(filename, data, 0600); err != nil { + return fmt.Errorf("error while creating file %s: %v", filename, err) + } + if _, err := v.PrintFunc("Writing file %s\n", filename); err != nil { + return fmt.Errorf("error while writing in file %s: %v", filename, err) + } return nil }