From 9470a6c8e3f3b39af14db3ce9f76a3f3b9abf010 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Thu, 1 Jun 2023 15:45:02 +0200 Subject: [PATCH] feat: improve verbose flag behavior (#675) * venom: improve verbose flag behavior Signed-off-by: Yvonnick Esnault --- README.md | 6 +++--- cmd/venom/main.go | 8 ++++++-- cmd/venom/run/cmd.go | 19 +++++++++++-------- process.go | 14 ++++++++++++-- process_files.go | 16 +++++----------- process_files_test.go | 2 -- process_testsuite.go | 11 ++++++----- venom.go | 5 ++--- venom_output.go | 24 +++++++++++------------- 9 files changed, 56 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index f4785083..9ea5a72f 100644 --- a/README.md +++ b/README.md @@ -149,9 +149,9 @@ Examples: Run a single testsuite and specify a variable: venom run mytestfile.yml --var="foo=bar" Run a single testsuite and load all variables from a file: venom run mytestfile.yml --var-from-file variables.yaml Run all testsuites containing in files ending with *.yml or *.yaml with verbosity: VENOM_VERBOSE=2 venom run - + Notice that variables initialized with -var-from-file argument can be overrided with -var argument - + More info: https://github.com/ovh/venom Flags: @@ -163,7 +163,7 @@ Flags: --stop-on-failure Stop running Test Suite on first Test Case failure --var stringArray --var cds='cds -f config.json' --var cds2='cds -f config.json' --var-from-file strings --var-from-file filename.yaml --var-from-file filename2.yaml: yaml, must contains a dictionary - -v, --verbose count verbose. -vv to very verbose and -vvv to very verbose with CPU Profiling + -v, --verbose count verbose. -v (INFO level in venom.log file), -vv to very verbose (DEBUG level) and -vvv to very verbose with CPU Profiling ``` ## Run test suites in a specific order diff --git a/cmd/venom/main.go b/cmd/venom/main.go index b2fd18f4..e6fe126e 100644 --- a/cmd/venom/main.go +++ b/cmd/venom/main.go @@ -1,12 +1,16 @@ package main import ( + "fmt" + "os" + + "github.com/ovh/venom" "github.com/ovh/venom/cmd/venom/root" - log "github.com/sirupsen/logrus" ) func main() { if err := root.New().Execute(); err != nil { - log.Fatalf("Err:%s", err) + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + venom.OSExit(2) } } diff --git a/cmd/venom/run/cmd.go b/cmd/venom/run/cmd.go index d0da8838..812eae51 100644 --- a/cmd/venom/run/cmd.go +++ b/cmd/venom/run/cmd.go @@ -11,8 +11,8 @@ import ( "strings" "github.com/mitchellh/go-homedir" + "github.com/pkg/errors" "github.com/rockbears/yaml" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -48,7 +48,7 @@ func init() { formatFlag = Cmd.Flags().String("format", "xml", "--format:json, tap, xml, yaml") stopOnFailureFlag = Cmd.Flags().Bool("stop-on-failure", false, "Stop running Test Suite on first Test Case failure") htmlReportFlag = Cmd.Flags().Bool("html-report", false, "Generate HTML Report") - verboseFlag = Cmd.Flags().CountP("verbose", "v", "verbose. -vv to very verbose and -vvv to very verbose with CPU Profiling") + verboseFlag = Cmd.Flags().CountP("verbose", "v", "verbose. -v (INFO level in venom.log file), -vv to very verbose (DEBUG level) and -vvv to very verbose with CPU Profiling") varFilesFlag = Cmd.Flags().StringSlice("var-from-file", []string{""}, "--var-from-file filename.yaml --var-from-file filename2.yaml: yaml, must contains a dictionary") variablesFlag = Cmd.Flags().StringArray("var", nil, "--var cds='cds -f config.json' --var cds2='cds -f config.json'") outputDirFlag = Cmd.PersistentFlags().String("output-dir", "", "Output Directory: create tests results file inside this directory") @@ -347,11 +347,13 @@ var Cmd = &cobra.Command{ if v.Verbose == 3 { fCPU, err := os.Create(filepath.Join(v.OutputDir, "pprof_cpu_profile.prof")) if err != nil { - log.Errorf("error while create profile file %v", err) + fmt.Fprintf(os.Stderr, "error while create profile file %v\n", err) + venom.OSExit(2) } fMem, err := os.Create(filepath.Join(v.OutputDir, "pprof_mem_profile.prof")) if err != nil { - log.Errorf("error while create profile file %v", err) + fmt.Fprintf(os.Stderr, "error while create profile file %v\n", err) + venom.OSExit(2) } if fCPU != nil && fMem != nil { pprof.StartCPUProfile(fCPU) //nolint @@ -371,7 +373,8 @@ var Cmd = &cobra.Command{ } fi, err := os.Open(f) if err != nil { - return fmt.Errorf("unable to open var-from-file %s: %v", f, err) + fmt.Fprintf(os.Stderr, "unable to open var-from-file %s: %v\n", f, err) + venom.OSExit(2) } defer fi.Close() readers = append(readers, fi) @@ -428,11 +431,11 @@ func readInitialVariables(ctx context.Context, argsVars []string, argVarsFiles [ stemp, err := interpolate.Do(string(btes), nil) if err != nil { - return nil, fmt.Errorf("unable to interpolate file %v", err) + return nil, errors.Wrap(err, "unable to interpolate file") } if err := yaml.Unmarshal([]byte(stemp), &tmpResult); err != nil { - return nil, err + return nil, errors.Wrap(err, "unable to unmarshal file") } for k, v := range tmpResult { @@ -451,7 +454,7 @@ func readInitialVariables(ctx context.Context, argsVars []string, argVarsFiles [ } stemp, err := interpolate.Do(tuple[1], nil) if err != nil { - return nil, fmt.Errorf("unable to interpolate arg %s: %v", arg, err) + return nil, errors.Wrapf(err, "unable to interpolate arg %s", arg) } result[tuple[0]] = cast(stemp) venom.Debug(ctx, "Adding variable from vars arg %s=%s", tuple[0], result[tuple[0]]) diff --git a/process.go b/process.go index 00801895..11876ebf 100644 --- a/process.go +++ b/process.go @@ -17,7 +17,15 @@ import ( // InitLogger initializes venom logger func (v *Venom) InitLogger() error { v.Tests.TestSuites = []TestSuite{} - logrus.SetLevel(logrus.DebugLevel) + + switch v.Verbose { + case 1: + logrus.SetLevel(logrus.InfoLevel) + case 2: + logrus.SetLevel(logrus.DebugLevel) + default: + logrus.SetLevel(logrus.WarnLevel) + } if v.OutputDir != "" { if err := os.MkdirAll(v.OutputDir, os.FileMode(0755)); err != nil { @@ -172,7 +180,9 @@ func (v *Venom) Process(ctx context.Context, path []string) error { v.Tests.TestSuites[i].Start = time.Now() // ##### RUN Test Suite Here - v.runTestSuite(ctx, &v.Tests.TestSuites[i]) + if err := v.runTestSuite(ctx, &v.Tests.TestSuites[i]); err != nil { + return err + } v.Tests.TestSuites[i].End = time.Now() v.Tests.TestSuites[i].Duration = v.Tests.TestSuites[i].End.Sub(v.Tests.TestSuites[i].Start).Seconds() diff --git a/process_files.go b/process_files.go index 55653c1b..47eaa084 100644 --- a/process_files.go +++ b/process_files.go @@ -8,16 +8,16 @@ import ( "strings" "time" - "github.com/rockbears/yaml" - "github.com/mattn/go-zglob" + "github.com/rockbears/yaml" "github.com/ovh/cds/sdk/interpolate" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" ) -func getFilesPath(path []string) (filePaths []string, err error) { +func getFilesPath(path []string) ([]string, error) { + filePaths := make([]string, 0) + for _, p := range path { p = strings.TrimSpace(p) @@ -31,7 +31,6 @@ func getFilesPath(path []string) (filePaths []string, err error) { fpaths, err := zglob.Glob(p) if err != nil { - log.Errorf("error reading files on path %q err:%v", path, err) return nil, errors.Wrapf(err, "error reading files on path %q", path) } @@ -61,14 +60,9 @@ func uniq(stringSlice []string) []string { return list } -type partialTestSuite struct { - Name string `json:"name" yaml:"name"` - Vars H `yaml:"vars" json:"vars"` -} - func (v *Venom) readFiles(ctx context.Context, filesPath []string) (err error) { for _, filePath := range filesPath { - log.Info("Reading ", filePath) + Info(ctx, "Reading %v", filePath) btes, err := os.ReadFile(filePath) if err != nil { return errors.Wrapf(err, "unable to read file %q", filePath) diff --git a/process_files_test.go b/process_files_test.go index 02d87244..a2187704 100644 --- a/process_files_test.go +++ b/process_files_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - log "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) @@ -35,7 +34,6 @@ func randomString(n int) string { func Test_getFilesPath(t *testing.T) { InitTestLogger(t) rand.Seed(time.Now().UnixNano()) - log.SetLevel(log.DebugLevel) tests := []struct { init func(t *testing.T) ([]string, error) diff --git a/process_testsuite.go b/process_testsuite.go index c070bd2f..793dcc58 100644 --- a/process_testsuite.go +++ b/process_testsuite.go @@ -9,10 +9,10 @@ import ( "github.com/gosimple/slug" "github.com/ovh/cds/sdk/interpolate" - log "github.com/sirupsen/logrus" + "github.com/pkg/errors" ) -func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) { +func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) error { if v.Verbose == 3 { var filename, filenameCPU, filenameMem string if v.OutputDir != "" { @@ -23,7 +23,7 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) { fCPU, errCPU := os.Create(filenameCPU) fMem, errMem := os.Create(filenameMem) if errCPU != nil || errMem != nil { - log.Errorf("error while create profile file CPU:%v MEM:%v", errCPU, errMem) + return fmt.Errorf("error while create profile file CPU:%v MEM:%v", errCPU, errMem) } else { pprof.StartCPUProfile(fCPU) p := pprof.Lookup("heap") @@ -38,14 +38,14 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) { for k, v := range vars { computedV, err := interpolate.Do(fmt.Sprintf("%v", v), vars) if err != nil { - log.Errorf("error while computing variable %s=%q: %v", k, v, err) + return errors.Wrapf(err, "error while computing variable %s=%q", k, v) } ts.Vars.Add(k, computedV) } exePath, err := os.Executable() if err != nil { - log.Errorf("failed to get executable path: %v", err) + return errors.Wrapf(err, "failed to get executable path") } else { ts.Vars.Add("venom.executable", exePath) } @@ -93,6 +93,7 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) { ts.Status = StatusPass v.Tests.NbTestsuitesPass++ } + return nil } func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) { diff --git a/venom.go b/venom.go index 36707f29..17948e25 100644 --- a/venom.go +++ b/venom.go @@ -18,7 +18,6 @@ import ( "github.com/ovh/cds/sdk/interpolate" "github.com/pkg/errors" "github.com/rockbears/yaml" - log "github.com/sirupsen/logrus" "github.com/spf13/cast" ) @@ -215,7 +214,7 @@ func (v *Venom) registerUserExecutors(ctx context.Context, name string, vars map } for _, f := range executorsPath { - log.Info("Reading ", f) + Info(ctx, "Reading %v", f) btes, ok := v.executorFileCache[f] if !ok { btes, err = os.ReadFile(f) @@ -260,7 +259,7 @@ func (v *Venom) registerUserExecutors(ctx context.Context, name string, vars map return errors.Wrapf(err, "unable to parse file %q with content %v", f, content) } - log.Debugf("User executor %q revolved with content %v", f, content) + Debug(ctx, "User executor %q revolved with content %v", f, content) for k, vr := range varsComputed { ux.Input.Add(k, vr) diff --git a/venom_output.go b/venom_output.go index a10e9eb7..eef98cae 100644 --- a/venom_output.go +++ b/venom_output.go @@ -12,7 +12,7 @@ import ( "github.com/fatih/color" tap "github.com/mndrix/tap-go" - log "github.com/sirupsen/logrus" + "github.com/pkg/errors" yaml "gopkg.in/yaml.v2" ) @@ -56,25 +56,25 @@ func (v *Venom) OutputResult() error { case "json": data, err = json.MarshalIndent(testsResult, "", " ") if err != nil { - log.Fatalf("Error: cannot format output json (%s)", err) + return errors.Wrapf(err, "Error: cannot format output json (%s)", err) } case "tap": data, err = outputTapFormat(*testsResult) if err != nil { - log.Fatalf("Error: cannot format output tap (%s)", err) + return errors.Wrapf(err, "Error: cannot format output tap (%s)", err) } case "yml", "yaml": data, err = yaml.Marshal(testsResult) if err != nil { - log.Fatalf("Error: cannot format output yaml (%s)", err) + return errors.Wrapf(err, "Error: cannot format output yaml (%s)", err) } case "xml": data, err = outputXMLFormat(*testsResult) if err != nil { - log.Fatalf("Error: cannot format output xml (%s)", err) + return errors.Wrapf(err, "Error: cannot format output xml (%s)", err) } case "html": - log.Fatalf("Error: you have to use the --html-report flag") + return errors.New("Error: you have to use the --html-report flag") } fname := strings.TrimSuffix(v.Tests.TestSuites[i].Filepath, filepath.Ext(v.Tests.TestSuites[i].Filepath)) @@ -100,12 +100,12 @@ func (v *Venom) OutputResult() error { data, err := outputHTML(testsResult) if err != nil { - log.Fatalf("Error: cannot format output html (%s)", err) + return errors.Wrapf(err, "Error: cannot format output html") } var filename = filepath.Join(v.OutputDir, computeOutputFilename("test_results.html")) v.PrintFunc("Writing html file %s\n", filename) if err := os.WriteFile(filename, data, 0600); err != nil { - return fmt.Errorf("Error while creating file %s: %v", filename, err) + return errors.Wrapf(err, "Error while creating file %s", filename) } } @@ -135,7 +135,6 @@ func outputTapFormat(tests Tests) ([]byte, error) { continue } } - tapValue.Pass(name) } } @@ -145,7 +144,6 @@ func outputTapFormat(tests Tests) ([]byte, error) { } func outputXMLFormat(tests Tests) ([]byte, error) { - testsXML := TestsXML{} for _, ts := range tests.TestSuites { @@ -191,9 +189,9 @@ func outputXMLFormat(tests Tests) ([]byte, error) { testsXML.TestSuites = append(testsXML.TestSuites, tsXML) } - dataxml, errm := xml.MarshalIndent(testsXML, "", " ") - if errm != nil { - log.Fatalf("Error: cannot format xml output: %s", errm) + dataxml, err := xml.MarshalIndent(testsXML, "", " ") + if err != nil { + errors.Wrapf(err, "Error: cannot format xml output") } data := append([]byte(``), dataxml...)