feat: add retry / delay on teststep

README.md

@@ -5,7 +5,6 @@ It can also output xUnit results files.
 
 <img src="./venom.gif" alt="Venom Demonstration" width="80%">
 
-
 ## Commmand Line
 
 Install with:
@@ -69,6 +68,16 @@ testcases:
     - result.statuscode ShouldEqual 200
     - result.timeseconds ShouldBeLessThan 1
 
+- name: Test with retries and delay in seconds between each try
+  steps:
+  - type: http
+    method: GET
+    url: https://eu.api.ovh.com/1.0/
+    retry: 3
+    delay: 2
+    assertions:
+    - result.statuscode ShouldEqual 200
+
 ```
 
 ## RUN Venom locally on CDS Integration Tests

assertion.go

@@ -25,29 +25,35 @@ func (t *testingT) Error(args ...interface{}) {
 	}
 }
 
-func applyChecks(executorResult ExecutorResult, tc *TestCase, step TestStep, defaultAssertions *StepAssertions, l *log.Entry) {
+// applyChecks apply checks on result, return true if all assertions are OK, false otherwise
+func applyChecks(executorResult ExecutorResult, step TestStep, defaultAssertions *StepAssertions, l *log.Entry) (bool, []Failure, []Failure) {
 
 	var sa StepAssertions
+	var errors []Failure
+	var failures []Failure
+
 	if err := mapstructure.Decode(step, &sa); err != nil {
-		log.Errorf("error decoding assertions: %s", err)
-		return
+		return false, []Failure{{Value: fmt.Sprintf("error decoding assertions: %s", err)}}, failures
 	}
 
 	if len(sa.Assertions) == 0 && defaultAssertions != nil {
 		sa = *defaultAssertions
 	}
 
+	isOK := true
 	for _, assertion := range sa.Assertions {
 		errs, fails := check(assertion, executorResult, l)
 		if errs != nil {
-			tc.Errors = append(tc.Errors, *errs)
+			errors = append(errors, *errs)
+			isOK = false
 		}
 		if fails != nil {
-			tc.Failures = append(tc.Failures, *fails)
+			failures = append(failures, *fails)
+			isOK = false
 		}
 	}
 
-	return
+	return isOK, errors, failures
 }
 
 func check(assertion string, executorResult ExecutorResult, l *log.Entry) (*Failure, *Failure) {

process.go

@@ -14,15 +14,6 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-const (
-	// DetailsLow prints only summary results
-	DetailsLow = "low"
-	// DetailsMedium prints progress bar and summary
-	DetailsMedium = "medium"
-	// DetailsHigh prints progress bar and details
-	DetailsHigh = "high"
-)
-
 var aliases map[string]string
 var bars map[string]*pb.ProgressBar
 var mutex = &sync.Mutex{}
@@ -233,31 +224,58 @@ func runTestCase(ts *TestSuite, tc *TestCase, l *log.Entry, detailsLevel string)
 	l.Infof("start")
 	for _, step := range tc.TestSteps {
 
-		t, err := getExecutor(step)
+		e, err := getExecutorWrap(step)
 		if err != nil {
 			tc.Errors = append(tc.Errors, Failure{Value: err.Error()})
 			break
 		}
 
-		result, err := t.Run(l, aliases, step)
+		runTestStep(e, tc, step, l, detailsLevel)
+
+		if detailsLevel != DetailsLow {
+			bars[ts.Package].Increment()
+		}
+		if len(tc.Failures) > 0 {
+			break
+		}
+	}
+	l.Infof("end")
+}
+
+func runTestStep(e *executorWrap, tc *TestCase, step TestStep, l *log.Entry, detailsLevel string) {
+
+	var isOK bool
+	var errors []Failure
+	var failures []Failure
+
+	var retry int
+	for retry = 0; retry < e.retry && !isOK; retry++ {
+		if retry > 1 && !isOK {
+			log.Debugf("Sleep %d, it's %d attempt", e.delay, retry)
+			time.Sleep(time.Duration(e.delay) * time.Second)
+		}
+
+		result, err := e.executor.Run(l, aliases, step)
 		if err != nil {
 			tc.Failures = append(tc.Failures, Failure{Value: err.Error()})
+			continue
 		}
 
 		log.Debugf("result:%+v", result)
 
-		if h, ok := t.(executorWithDefaultAssertions); ok {
-			applyChecks(result, tc, step, h.GetDefaultAssertions(), l)
+		if h, ok := e.executor.(executorWithDefaultAssertions); ok {
+			isOK, errors, failures = applyChecks(result, step, h.GetDefaultAssertions(), l)
 		} else {
-			applyChecks(result, tc, step, nil, l)
+			isOK, errors, failures = applyChecks(result, step, nil, l)
 		}
-
-		if detailsLevel != DetailsLow {
-			bars[ts.Package].Increment()
-		}
-		if len(tc.Failures) > 0 {
+		if isOK {
 			break
 		}
 	}
-	l.Infof("end")
+	tc.Errors = append(tc.Errors, errors...)
+	tc.Failures = append(tc.Failures, failures...)
+	if retry > 0 && (len(failures) > 0 || len(errors) > 0) {
+		tc.Failures = append(tc.Failures, Failure{Value: fmt.Sprintf("It's a failure after %d attempt(s)", retry+1)})
+	}
+
 }

tests/MyTestSuiteHTTP.yml

@@ -5,6 +5,8 @@ testcases:
   - type: http
     method: GET
     url: https://eu.api.ovh.com/1.0/
+    retry: 3
+    delay: 2
     assertions:
     - result.body ShouldContainSubstring /dedicated/server
     - result.body ShouldContainSubstring /ipLoadbalancing

xunit.go → types.go

@@ -2,8 +2,49 @@ package venom
 
 import (
 	"encoding/xml"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+const (
+	// DetailsLow prints only summary results
+	DetailsLow = "low"
+	// DetailsMedium prints progress bar and summary
+	DetailsMedium = "medium"
+	// DetailsHigh prints progress bar and details
+	DetailsHigh = "high"
 )
 
+// Aliases contains list of aliases
+type Aliases map[string]string
+
+// ExecutorResult represents an executor result on a test step
+type ExecutorResult map[string]string
+
+// StepAssertions contains step assertions
+type StepAssertions struct {
+	Assertions []string `json:"assertions,omitempty" yaml:"assertions,omitempty"`
+}
+
+// Executor execute a testStep.
+type Executor interface {
+	// Run run a Test Step
+	Run(*log.Entry, Aliases, TestStep) (ExecutorResult, error)
+}
+
+// executorWrap contains an executor implementation and some attributes
+type executorWrap struct {
+	executor Executor
+	retry    int // nb retry a test case if it is in failure.
+	delay    int // delay between two retries
+}
+
+// executorWithDefaultAssertions execute a testStep.
+type executorWithDefaultAssertions interface {
+	// GetDefaultAssertion returns default assertions
+	GetDefaultAssertions() *StepAssertions
+}
+
 // Tests contains all informations about tests in a pipeline build
 type Tests struct {
 	XMLName      xml.Name    `xml:"testsuites" json:"-" yaml:"-"`

venom.go

@@ -2,35 +2,8 @@ package venom
 
 import (
 	"fmt"
-
-	log "github.com/Sirupsen/logrus"
 )
 
-type (
-	// Aliases contains list of aliases
-	Aliases map[string]string
-
-	// ExecutorResult represents an executor result on a test step
-	ExecutorResult map[string]string
-)
-
-// StepAssertions contains step assertions
-type StepAssertions struct {
-	Assertions []string `json:"assertions,omitempty" yaml:"assertions,omitempty"`
-}
-
-// Executor execute a testStep.
-type Executor interface {
-	// Run run a Test Step
-	Run(*log.Entry, Aliases, TestStep) (ExecutorResult, error)
-}
-
-// executorWithDefaultAssertions execute a testStep.
-type executorWithDefaultAssertions interface {
-	// GetDefaultAssertion returns default assertions
-	GetDefaultAssertions() *StepAssertions
-}
-
 var (
 	executors = map[string]Executor{}
 )
@@ -40,21 +13,50 @@ func RegisterExecutor(name string, e Executor) {
 	executors[name] = e
 }
 
-// getExecutor initializes a test by name
-func getExecutor(t map[string]interface{}) (Executor, error) {
+// getExecutorWrap initializes a test by name
+// no type -> exec is default
+func getExecutorWrap(t map[string]interface{}) (*executorWrap, error) {
 
 	var name string
-	itype, ok := t["type"]
-	if ok {
+	var retry, delay int
+
+	if itype, ok := t["type"]; ok {
 		name = fmt.Sprintf("%s", itype)
 	}
+
 	if name == "" {
 		name = "exec"
 	}
 
-	e, ok := executors[name]
-	if !ok {
-		return nil, fmt.Errorf("type '%s' is not implemented", name)
+	retry, errRetry := getAttrInt(t, "retry")
+	if errRetry != nil {
+		return nil, errRetry
+	}
+	delay, errDelay := getAttrInt(t, "delay")
+	if errDelay != nil {
+		return nil, errDelay
+	}
+
+	if e, ok := executors[name]; ok {
+		ew := &executorWrap{
+			executor: e,
+			retry:    retry,
+			delay:    delay,
+		}
+		return ew, nil
+	}
+
+	return nil, fmt.Errorf("type '%s' is not implemented", name)
+}
+
+func getAttrInt(t map[string]interface{}, name string) (int, error) {
+	var out int
+	if i, ok := t["retry"]; ok {
+		var ok bool
+		out, ok = i.(int)
+		if !ok {
+			return -1, fmt.Errorf("attribute %s '%s' is not an integer", name, i)
+		}
 	}
-	return e, nil
+	return out, nil
 }