make e2e output summary with numeric information & add 'subtractor' f…

…unction to func.go (#118)

internal/components/verifier/funcs.go

@@ -51,6 +51,9 @@ var customFuncMap = map[string]any{
 
 	// Regex:
 	"regexp": regexpMatch,
+
+	// Calculation:
+	"subtractor": subtractor,
 }
 
 func base64encode(s string) string {
@@ -93,3 +96,10 @@ func regexpMatch(s, pattern string) string {
 	}
 	return s
 }
+
+func subtractor(total int, nums ...int) int {
+	for _, num := range nums {
+		total -= num
+	}
+	return total
+}

pkg/output/output.go

@@ -33,9 +33,12 @@ var (
 )
 
 type YamlCaseResult struct {
-	Passed  []string
-	Failed  []string
-	Skipped []string
+	Passed       []string
+	Failed       []string
+	Skipped      []string
+	PassedCount  int `yaml:"passedCount"`
+	FailedCount  int `yaml:"failedCount"`
+	SkippedCount int `yaml:"skippedCount"`
 }
 
 func HasFormat() bool {
@@ -63,6 +66,10 @@ func printResultInYAML(caseRes []*CaseResult) {
 		}
 	}
 
-	yamlData, _ := yaml.Marshal(yamlCaseResult)
-	fmt.Println(string(yamlData))
+	yamlCaseResult.PassedCount = len(yamlCaseResult.Passed)
+	yamlCaseResult.FailedCount = len(yamlCaseResult.Failed)
+	yamlCaseResult.SkippedCount = len(yamlCaseResult.Skipped)
+
+	yamlCaseResultData, _ := yaml.Marshal(yamlCaseResult)
+	fmt.Println(string(yamlCaseResultData))
 }

test/e2e/concurrency/fail-fast/expected.yaml

@@ -26,4 +26,7 @@ failed:
 skipped:
 {{range .skipped}}
 - {{.}}
-{{end}}
+{{end}}
+passedCount: {{le .passedCount 4}}
+failedCount: {{le .failedCount 5}}
+skippedCount: {{.skippedCount}}

test/e2e/concurrency/fail-fast/internal/verify.yaml

@@ -46,79 +46,14 @@ verify:
       query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
       expected: ./expected.yaml
     - name: failed-case-6
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json" && sleep 5'
       expected: ./expected.yaml
-    - name: passed-case-7
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+    - name: failed-case-7
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json" && sleep 5'
       expected: ./expected.yaml
-    - name: passed-case-8
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+    - name: failed-case-8
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json" && sleep 5'
       expected: ./expected.yaml
     - name: failed-case-9
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-10
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-
-    - name: passed-case-11
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-12
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-13
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-14
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-15
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-16
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-17
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-18
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-19
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-20
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-
-    - name: passed-case-21
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-22
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-23
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-24
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-25
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-26
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-27
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-28
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-29
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-30
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json" && sleep 5'
       expected: ./expected.yaml

test/e2e/concurrency/non-fail-fast/expected.yaml

@@ -23,32 +23,14 @@ passed:
 - passed-case-5
 - passed-case-7
 - passed-case-8
-- passed-case-10
-- passed-case-11
-- passed-case-12
-- passed-case-14
-- passed-case-15
-- passed-case-17
-- passed-case-18
-- passed-case-20
-- passed-case-21
-- passed-case-22
-- passed-case-24
-- passed-case-25
-- passed-case-27
-- passed-case-28
-- passed-case-30
 {{- end}}
 failed:
 {{- contains .failed }}
 - failed-case-3
 - failed-case-6
 - failed-case-9
-- failed-case-13
-- failed-case-16
-- failed-case-19
-- failed-case-23
-- failed-case-26
-- failed-case-29
 {{- end }}
-skipped: []
+skipped: []
+passedCount: 6
+failedCount: 3
+skippedCount: 0

test/e2e/concurrency/non-fail-fast/internal/verify.yaml

@@ -57,68 +57,3 @@ verify:
     - name: failed-case-9
       query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
       expected: ./expected.yaml
-    - name: passed-case-10
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-
-    - name: passed-case-11
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-12
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-13
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-14
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-15
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-16
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-17
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-18
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-19
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-20
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-
-    - name: passed-case-21
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-22
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-23
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-24
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-25
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-26
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-27
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-28
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: failed-case-29
-      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
-      expected: ./expected.yaml
-    - name: passed-case-30
-      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
-      expected: ./expected.yaml

test/e2e/e2e.yaml

@@ -47,11 +47,11 @@ verify:
       query: './bin/linux/e2e verify -c  ./test/e2e/non-concurrency/non-fail-fast/internal/verify.yaml --summary-only -o yaml'
       expected: ./non-concurrency/non-fail-fast/expected.yaml
 
-#    - name: currency & fail-fast mode
-#      query: './bin/linux/e2e verify -c  ./test/e2e/concurrency/fail-fast/internal/verify.yaml --summary-only -o yaml'
-#      expected: ./concurrency/fail-fast/expected.yaml
-#
-#    - name: currency & non-fail-fast mode
-#      query: './bin/linux/e2e verify -c  ./test/e2e/concurrency/non-fail-fast/internal/verify.yaml --summary-only -o yaml'
-#      expected: ./concurrency/non-fail-fast/expected.yaml
+    - name: currency & fail-fast mode
+      query: './bin/linux/e2e verify -c  ./test/e2e/concurrency/fail-fast/internal/verify.yaml --summary-only -o yaml'
+      expected: ./concurrency/fail-fast/expected.yaml
+
+    - name: currency & non-fail-fast mode
+      query: './bin/linux/e2e verify -c  ./test/e2e/concurrency/non-fail-fast/internal/verify.yaml --summary-only -o yaml'
+      expected: ./concurrency/non-fail-fast/expected.yaml
 

test/e2e/non-concurrency/fail-fast/expected.yaml

@@ -27,3 +27,6 @@ skipped:
   - case-7
   - case-8
   - case-9
+passedCount: 5
+failedCount: 1
+skippedCount: 3

test/e2e/non-concurrency/non-fail-fast/expected.yaml

@@ -27,5 +27,8 @@ failed:
   - case-8
   - case-9
 skipped: []
+passedCount: 6
+failedCount: 3
+skippedCount: 0