feat: improve verbose flag behavior (#675)

* venom: improve verbose flag behavior

Signed-off-by: Yvonnick Esnault <yvonnick.esnault@corp.ovh.com>

README.md

@@ -149,9 +149,9 @@ Examples:
   Run a single testsuite and specify a variable: venom run mytestfile.yml --var="foo=bar"
   Run a single testsuite and load all variables from a file: venom run mytestfile.yml --var-from-file variables.yaml
   Run all testsuites containing in files ending with *.yml or *.yaml with verbosity: VENOM_VERBOSE=2 venom run
-  
+
   Notice that variables initialized with -var-from-file argument can be overrided with -var argument
-  
+
   More info: https://github.com/ovh/venom
 
 Flags:
@@ -163,7 +163,7 @@ Flags:
       --stop-on-failure         Stop running Test Suite on first Test Case failure
       --var stringArray         --var cds='cds -f config.json' --var cds2='cds -f config.json'
       --var-from-file strings   --var-from-file filename.yaml --var-from-file filename2.yaml: yaml, must contains a dictionary
-  -v, --verbose count           verbose. -vv to very verbose and -vvv to very verbose with CPU Profiling
+  -v, --verbose count           verbose. -v (INFO level in venom.log file), -vv to very verbose (DEBUG level) and -vvv to very verbose with CPU Profiling
 ```
 
 ## Run test suites in a specific order

cmd/venom/main.go

@@ -1,12 +1,16 @@
 package main
 
 import (
+	"fmt"
+	"os"
+
+	"github.com/ovh/venom"
 	"github.com/ovh/venom/cmd/venom/root"
-	log "github.com/sirupsen/logrus"
 )
 
 func main() {
 	if err := root.New().Execute(); err != nil {
-		log.Fatalf("Err:%s", err)
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		venom.OSExit(2)
 	}
 }

cmd/venom/run/cmd.go

@@ -11,8 +11,8 @@ import (
 	"strings"
 
 	"github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
 	"github.com/rockbears/yaml"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
@@ -48,7 +48,7 @@ func init() {
 	formatFlag = Cmd.Flags().String("format", "xml", "--format:json, tap, xml, yaml")
 	stopOnFailureFlag = Cmd.Flags().Bool("stop-on-failure", false, "Stop running Test Suite on first Test Case failure")
 	htmlReportFlag = Cmd.Flags().Bool("html-report", false, "Generate HTML Report")
-	verboseFlag = Cmd.Flags().CountP("verbose", "v", "verbose. -vv to very verbose and -vvv to very verbose with CPU Profiling")
+	verboseFlag = Cmd.Flags().CountP("verbose", "v", "verbose. -v (INFO level in venom.log file), -vv to very verbose (DEBUG level) and -vvv to very verbose with CPU Profiling")
 	varFilesFlag = Cmd.Flags().StringSlice("var-from-file", []string{""}, "--var-from-file filename.yaml --var-from-file filename2.yaml: yaml, must contains a dictionary")
 	variablesFlag = Cmd.Flags().StringArray("var", nil, "--var cds='cds -f config.json' --var cds2='cds -f config.json'")
 	outputDirFlag = Cmd.PersistentFlags().String("output-dir", "", "Output Directory: create tests results file inside this directory")
@@ -347,11 +347,13 @@ var Cmd = &cobra.Command{
 		if v.Verbose == 3 {
 			fCPU, err := os.Create(filepath.Join(v.OutputDir, "pprof_cpu_profile.prof"))
 			if err != nil {
-				log.Errorf("error while create profile file %v", err)
+				fmt.Fprintf(os.Stderr, "error while create profile file %v\n", err)
+				venom.OSExit(2)
 			}
 			fMem, err := os.Create(filepath.Join(v.OutputDir, "pprof_mem_profile.prof"))
 			if err != nil {
-				log.Errorf("error while create profile file %v", err)
+				fmt.Fprintf(os.Stderr, "error while create profile file %v\n", err)
+				venom.OSExit(2)
 			}
 			if fCPU != nil && fMem != nil {
 				pprof.StartCPUProfile(fCPU) //nolint
@@ -371,7 +373,8 @@ var Cmd = &cobra.Command{
 			}
 			fi, err := os.Open(f)
 			if err != nil {
-				return fmt.Errorf("unable to open var-from-file %s: %v", f, err)
+				fmt.Fprintf(os.Stderr, "unable to open var-from-file %s: %v\n", f, err)
+				venom.OSExit(2)
 			}
 			defer fi.Close()
 			readers = append(readers, fi)
@@ -428,11 +431,11 @@ func readInitialVariables(ctx context.Context, argsVars []string, argVarsFiles [
 
 		stemp, err := interpolate.Do(string(btes), nil)
 		if err != nil {
-			return nil, fmt.Errorf("unable to interpolate file %v", err)
+			return nil, errors.Wrap(err, "unable to interpolate file")
 		}
 
 		if err := yaml.Unmarshal([]byte(stemp), &tmpResult); err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "unable to unmarshal file")
 		}
 
 		for k, v := range tmpResult {
@@ -451,7 +454,7 @@ func readInitialVariables(ctx context.Context, argsVars []string, argVarsFiles [
 		}
 		stemp, err := interpolate.Do(tuple[1], nil)
 		if err != nil {
-			return nil, fmt.Errorf("unable to interpolate arg %s: %v", arg, err)
+			return nil, errors.Wrapf(err, "unable to interpolate arg %s", arg)
 		}
 		result[tuple[0]] = cast(stemp)
 		venom.Debug(ctx, "Adding variable from vars arg %s=%s", tuple[0], result[tuple[0]])

process.go

@@ -17,7 +17,15 @@ import (
 // InitLogger initializes venom logger
 func (v *Venom) InitLogger() error {
 	v.Tests.TestSuites = []TestSuite{}
-	logrus.SetLevel(logrus.DebugLevel)
+
+	switch v.Verbose {
+	case 1:
+		logrus.SetLevel(logrus.InfoLevel)
+	case 2:
+		logrus.SetLevel(logrus.DebugLevel)
+	default:
+		logrus.SetLevel(logrus.WarnLevel)
+	}
 
 	if v.OutputDir != "" {
 		if err := os.MkdirAll(v.OutputDir, os.FileMode(0755)); err != nil {
@@ -172,7 +180,9 @@ func (v *Venom) Process(ctx context.Context, path []string) error {
 
 		v.Tests.TestSuites[i].Start = time.Now()
 		// ##### RUN Test Suite Here
-		v.runTestSuite(ctx, &v.Tests.TestSuites[i])
+		if err := v.runTestSuite(ctx, &v.Tests.TestSuites[i]); err != nil {
+			return err
+		}
 
 		v.Tests.TestSuites[i].End = time.Now()
 		v.Tests.TestSuites[i].Duration = v.Tests.TestSuites[i].End.Sub(v.Tests.TestSuites[i].Start).Seconds()

process_files.go

@@ -8,16 +8,16 @@ import (
 	"strings"
 	"time"
 
-	"github.com/rockbears/yaml"
-
 	"github.com/mattn/go-zglob"
+	"github.com/rockbears/yaml"
 
 	"github.com/ovh/cds/sdk/interpolate"
 	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
 )
 
-func getFilesPath(path []string) (filePaths []string, err error) {
+func getFilesPath(path []string) ([]string, error) {
+	filePaths := make([]string, 0)
+
 	for _, p := range path {
 		p = strings.TrimSpace(p)
 
@@ -31,7 +31,6 @@ func getFilesPath(path []string) (filePaths []string, err error) {
 
 		fpaths, err := zglob.Glob(p)
 		if err != nil {
-			log.Errorf("error reading files on path %q err:%v", path, err)
 			return nil, errors.Wrapf(err, "error reading files on path %q", path)
 		}
 
@@ -61,14 +60,9 @@ func uniq(stringSlice []string) []string {
 	return list
 }
 
-type partialTestSuite struct {
-	Name string `json:"name" yaml:"name"`
-	Vars H      `yaml:"vars" json:"vars"`
-}
-
 func (v *Venom) readFiles(ctx context.Context, filesPath []string) (err error) {
 	for _, filePath := range filesPath {
-		log.Info("Reading ", filePath)
+		Info(ctx, "Reading %v", filePath)
 		btes, err := os.ReadFile(filePath)
 		if err != nil {
 			return errors.Wrapf(err, "unable to read file %q", filePath)

process_files_test.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 	"time"
 
-	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 )
 
@@ -35,7 +34,6 @@ func randomString(n int) string {
 func Test_getFilesPath(t *testing.T) {
 	InitTestLogger(t)
 	rand.Seed(time.Now().UnixNano())
-	log.SetLevel(log.DebugLevel)
 
 	tests := []struct {
 		init    func(t *testing.T) ([]string, error)

process_testsuite.go

@@ -9,10 +9,10 @@ import (
 
 	"github.com/gosimple/slug"
 	"github.com/ovh/cds/sdk/interpolate"
-	log "github.com/sirupsen/logrus"
+	"github.com/pkg/errors"
 )
 
-func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) {
+func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) error {
 	if v.Verbose == 3 {
 		var filename, filenameCPU, filenameMem string
 		if v.OutputDir != "" {
@@ -23,7 +23,7 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) {
 		fCPU, errCPU := os.Create(filenameCPU)
 		fMem, errMem := os.Create(filenameMem)
 		if errCPU != nil || errMem != nil {
-			log.Errorf("error while create profile file CPU:%v MEM:%v", errCPU, errMem)
+			return fmt.Errorf("error while create profile file CPU:%v MEM:%v", errCPU, errMem)
 		} else {
 			pprof.StartCPUProfile(fCPU)
 			p := pprof.Lookup("heap")
@@ -38,14 +38,14 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) {
 	for k, v := range vars {
 		computedV, err := interpolate.Do(fmt.Sprintf("%v", v), vars)
 		if err != nil {
-			log.Errorf("error while computing variable %s=%q: %v", k, v, err)
+			return errors.Wrapf(err, "error while computing variable %s=%q", k, v)
 		}
 		ts.Vars.Add(k, computedV)
 	}
 
 	exePath, err := os.Executable()
 	if err != nil {
-		log.Errorf("failed to get executable path: %v", err)
+		return errors.Wrapf(err, "failed to get executable path")
 	} else {
 		ts.Vars.Add("venom.executable", exePath)
 	}
@@ -93,6 +93,7 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) {
 		ts.Status = StatusPass
 		v.Tests.NbTestsuitesPass++
 	}
+	return nil
 }
 
 func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) {

venom.go

@@ -18,7 +18,6 @@ import (
 	"github.com/ovh/cds/sdk/interpolate"
 	"github.com/pkg/errors"
 	"github.com/rockbears/yaml"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cast"
 )
 
@@ -215,7 +214,7 @@ func (v *Venom) registerUserExecutors(ctx context.Context, name string, vars map
 	}
 
 	for _, f := range executorsPath {
-		log.Info("Reading ", f)
+		Info(ctx, "Reading %v", f)
 		btes, ok := v.executorFileCache[f]
 		if !ok {
 			btes, err = os.ReadFile(f)
@@ -260,7 +259,7 @@ func (v *Venom) registerUserExecutors(ctx context.Context, name string, vars map
 			return errors.Wrapf(err, "unable to parse file %q with content %v", f, content)
 		}
 
-		log.Debugf("User executor %q revolved with content %v", f, content)
+		Debug(ctx, "User executor %q revolved with content %v", f, content)
 
 		for k, vr := range varsComputed {
 			ux.Input.Add(k, vr)

venom_output.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/fatih/color"
 	tap "github.com/mndrix/tap-go"
-	log "github.com/sirupsen/logrus"
+	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -56,25 +56,25 @@ func (v *Venom) OutputResult() error {
 		case "json":
 			data, err = json.MarshalIndent(testsResult, "", "  ")
 			if err != nil {
-				log.Fatalf("Error: cannot format output json (%s)", err)
+				return errors.Wrapf(err, "Error: cannot format output json (%s)", err)
 			}
 		case "tap":
 			data, err = outputTapFormat(*testsResult)
 			if err != nil {
-				log.Fatalf("Error: cannot format output tap (%s)", err)
+				return errors.Wrapf(err, "Error: cannot format output tap (%s)", err)
 			}
 		case "yml", "yaml":
 			data, err = yaml.Marshal(testsResult)
 			if err != nil {
-				log.Fatalf("Error: cannot format output yaml (%s)", err)
+				return errors.Wrapf(err, "Error: cannot format output yaml (%s)", err)
 			}
 		case "xml":
 			data, err = outputXMLFormat(*testsResult)
 			if err != nil {
-				log.Fatalf("Error: cannot format output xml (%s)", err)
+				return errors.Wrapf(err, "Error: cannot format output xml (%s)", err)
 			}
 		case "html":
-			log.Fatalf("Error: you have to use the --html-report flag")
+			return errors.New("Error: you have to use the --html-report flag")
 		}
 
 		fname := strings.TrimSuffix(v.Tests.TestSuites[i].Filepath, filepath.Ext(v.Tests.TestSuites[i].Filepath))
@@ -100,12 +100,12 @@ func (v *Venom) OutputResult() error {
 
 		data, err := outputHTML(testsResult)
 		if err != nil {
-			log.Fatalf("Error: cannot format output html (%s)", err)
+			return errors.Wrapf(err, "Error: cannot format output html")
 		}
 		var filename = filepath.Join(v.OutputDir, computeOutputFilename("test_results.html"))
 		v.PrintFunc("Writing html file %s\n", filename)
 		if err := os.WriteFile(filename, data, 0600); err != nil {
-			return fmt.Errorf("Error while creating file %s: %v", filename, err)
+			return errors.Wrapf(err, "Error while creating file %s", filename)
 		}
 	}
 
@@ -135,7 +135,6 @@ func outputTapFormat(tests Tests) ([]byte, error) {
 					continue
 				}
 			}
-
 			tapValue.Pass(name)
 		}
 	}
@@ -145,7 +144,6 @@ func outputTapFormat(tests Tests) ([]byte, error) {
 }
 
 func outputXMLFormat(tests Tests) ([]byte, error) {
-
 	testsXML := TestsXML{}
 
 	for _, ts := range tests.TestSuites {
@@ -191,9 +189,9 @@ func outputXMLFormat(tests Tests) ([]byte, error) {
 		testsXML.TestSuites = append(testsXML.TestSuites, tsXML)
 	}
 
-	dataxml, errm := xml.MarshalIndent(testsXML, "", "  ")
-	if errm != nil {
-		log.Fatalf("Error: cannot format xml output: %s", errm)
+	dataxml, err := xml.MarshalIndent(testsXML, "", "  ")
+	if err != nil {
+		errors.Wrapf(err, "Error: cannot format xml output")
 	}
 	data := append([]byte(`<?xml version="1.0" encoding="utf-8"?>`), dataxml...)