diff --git a/internal/components/verifier/funcs.go b/internal/components/verifier/funcs.go index 59155e8..dc1fb09 100644 --- a/internal/components/verifier/funcs.go +++ b/internal/components/verifier/funcs.go @@ -51,6 +51,9 @@ var customFuncMap = map[string]any{ // Regex: "regexp": regexpMatch, + + // Calculation: + "subtractor": subtractor, } func base64encode(s string) string { @@ -93,3 +96,10 @@ func regexpMatch(s, pattern string) string { } return s } + +func subtractor(total int, nums ...int) int { + for _, num := range nums { + total -= num + } + return total +} diff --git a/pkg/output/output.go b/pkg/output/output.go index ba5afad..1897fa4 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -33,9 +33,12 @@ var ( ) type YamlCaseResult struct { - Passed []string - Failed []string - Skipped []string + Passed []string + Failed []string + Skipped []string + PassedCount int `yaml:"passedCount"` + FailedCount int `yaml:"failedCount"` + SkippedCount int `yaml:"skippedCount"` } func HasFormat() bool { @@ -63,6 +66,10 @@ func printResultInYAML(caseRes []*CaseResult) { } } - yamlData, _ := yaml.Marshal(yamlCaseResult) - fmt.Println(string(yamlData)) + yamlCaseResult.PassedCount = len(yamlCaseResult.Passed) + yamlCaseResult.FailedCount = len(yamlCaseResult.Failed) + yamlCaseResult.SkippedCount = len(yamlCaseResult.Skipped) + + yamlCaseResultData, _ := yaml.Marshal(yamlCaseResult) + fmt.Println(string(yamlCaseResultData)) } diff --git a/test/e2e/concurrency/fail-fast/expected.yaml b/test/e2e/concurrency/fail-fast/expected.yaml index bb9394e..8074300 100644 --- a/test/e2e/concurrency/fail-fast/expected.yaml +++ b/test/e2e/concurrency/fail-fast/expected.yaml @@ -26,4 +26,7 @@ failed: skipped: {{range .skipped}} - {{.}} -{{end}} \ No newline at end of file +{{end}} +passedCount: {{le .passedCount 4}} +failedCount: {{le .failedCount 5}} +skippedCount: {{.skippedCount}} \ No newline at end of file diff --git a/test/e2e/concurrency/fail-fast/internal/verify.yaml b/test/e2e/concurrency/fail-fast/internal/verify.yaml index 4bd9efa..0a2a906 100644 --- a/test/e2e/concurrency/fail-fast/internal/verify.yaml +++ b/test/e2e/concurrency/fail-fast/internal/verify.yaml @@ -46,79 +46,14 @@ verify: query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' expected: ./expected.yaml - name: failed-case-6 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' + query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json" && sleep 5' expected: ./expected.yaml - - name: passed-case-7 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' + - name: failed-case-7 + query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json" && sleep 5' expected: ./expected.yaml - - name: passed-case-8 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' + - name: failed-case-8 + query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json" && sleep 5' expected: ./expected.yaml - name: failed-case-9 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-10 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - - name: passed-case-11 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-12 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-13 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-14 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-15 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-16 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-17 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-18 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-19 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-20 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - - name: passed-case-21 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-22 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-23 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-24 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-25 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-26 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-27 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-28 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-29 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-30 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' + query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json" && sleep 5' expected: ./expected.yaml \ No newline at end of file diff --git a/test/e2e/concurrency/non-fail-fast/expected.yaml b/test/e2e/concurrency/non-fail-fast/expected.yaml index 42a6951..7cbe4ef 100644 --- a/test/e2e/concurrency/non-fail-fast/expected.yaml +++ b/test/e2e/concurrency/non-fail-fast/expected.yaml @@ -23,32 +23,14 @@ passed: - passed-case-5 - passed-case-7 - passed-case-8 -- passed-case-10 -- passed-case-11 -- passed-case-12 -- passed-case-14 -- passed-case-15 -- passed-case-17 -- passed-case-18 -- passed-case-20 -- passed-case-21 -- passed-case-22 -- passed-case-24 -- passed-case-25 -- passed-case-27 -- passed-case-28 -- passed-case-30 {{- end}} failed: {{- contains .failed }} - failed-case-3 - failed-case-6 - failed-case-9 -- failed-case-13 -- failed-case-16 -- failed-case-19 -- failed-case-23 -- failed-case-26 -- failed-case-29 {{- end }} -skipped: [] \ No newline at end of file +skipped: [] +passedCount: 6 +failedCount: 3 +skippedCount: 0 diff --git a/test/e2e/concurrency/non-fail-fast/internal/verify.yaml b/test/e2e/concurrency/non-fail-fast/internal/verify.yaml index 88a4a9a..9e89c3d 100644 --- a/test/e2e/concurrency/non-fail-fast/internal/verify.yaml +++ b/test/e2e/concurrency/non-fail-fast/internal/verify.yaml @@ -57,68 +57,3 @@ verify: - name: failed-case-9 query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' expected: ./expected.yaml - - name: passed-case-10 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - - name: passed-case-11 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-12 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-13 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-14 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-15 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-16 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-17 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-18 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-19 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-20 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - - name: passed-case-21 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-22 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-23 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-24 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-25 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-26 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-27 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-28 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: failed-case-29 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: passed-case-30 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml \ No newline at end of file diff --git a/test/e2e/e2e.yaml b/test/e2e/e2e.yaml index 244400a..501b6f5 100644 --- a/test/e2e/e2e.yaml +++ b/test/e2e/e2e.yaml @@ -47,11 +47,11 @@ verify: query: './bin/linux/e2e verify -c ./test/e2e/non-concurrency/non-fail-fast/internal/verify.yaml --summary-only -o yaml' expected: ./non-concurrency/non-fail-fast/expected.yaml -# - name: currency & fail-fast mode -# query: './bin/linux/e2e verify -c ./test/e2e/concurrency/fail-fast/internal/verify.yaml --summary-only -o yaml' -# expected: ./concurrency/fail-fast/expected.yaml -# -# - name: currency & non-fail-fast mode -# query: './bin/linux/e2e verify -c ./test/e2e/concurrency/non-fail-fast/internal/verify.yaml --summary-only -o yaml' -# expected: ./concurrency/non-fail-fast/expected.yaml + - name: currency & fail-fast mode + query: './bin/linux/e2e verify -c ./test/e2e/concurrency/fail-fast/internal/verify.yaml --summary-only -o yaml' + expected: ./concurrency/fail-fast/expected.yaml + + - name: currency & non-fail-fast mode + query: './bin/linux/e2e verify -c ./test/e2e/concurrency/non-fail-fast/internal/verify.yaml --summary-only -o yaml' + expected: ./concurrency/non-fail-fast/expected.yaml diff --git a/test/e2e/fail-fast/expected.yaml b/test/e2e/fail-fast/expected.yaml deleted file mode 100644 index d7ea0a9..0000000 --- a/test/e2e/fail-fast/expected.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -passed: -- case-1 -- case-2 -failed: -- case-3 -skipped: [] \ No newline at end of file diff --git a/test/e2e/fail-fast/internal/expected.yaml b/test/e2e/fail-fast/internal/expected.yaml deleted file mode 100644 index 1d9d4bc..0000000 --- a/test/e2e/fail-fast/internal/expected.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file is used to show how to write configuration files and can be used to test. - -args: - case: success -headers: - Accept: application/json - Host: 127.0.0.1:8080 - User-Agent: curl/7.81.0 -origin: 172.18.0.1 -url: http://127.0.0.1:8080/get?case=success \ No newline at end of file diff --git a/test/e2e/fail-fast/internal/verify.yaml b/test/e2e/fail-fast/internal/verify.yaml deleted file mode 100644 index 40b7210..0000000 --- a/test/e2e/fail-fast/internal/verify.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file is used to show how to write configuration files and can be used to test. - -verify: - # verify with retry strategy - retry: - # max retry count - count: 5 - # the interval between two attempts, e.g. 10s, 1m. - interval: 1s - - # when a case fails, whether to stop verifying other cases. This property defaults to true. - fail-fast: true - # Whether to verify cases concurrently. This property defaults to false. - concurrency: false - - - cases: - - name: case-1 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: case-2 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: case-3 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml \ No newline at end of file diff --git a/test/e2e/non-concurrency/fail-fast/expected.yaml b/test/e2e/non-concurrency/fail-fast/expected.yaml index 617744e..0645529 100644 --- a/test/e2e/non-concurrency/fail-fast/expected.yaml +++ b/test/e2e/non-concurrency/fail-fast/expected.yaml @@ -27,3 +27,6 @@ skipped: - case-7 - case-8 - case-9 +passedCount: 5 +failedCount: 1 +skippedCount: 3 diff --git a/test/e2e/non-concurrency/non-fail-fast/expected.yaml b/test/e2e/non-concurrency/non-fail-fast/expected.yaml index 9f7f8f0..171bfdb 100644 --- a/test/e2e/non-concurrency/non-fail-fast/expected.yaml +++ b/test/e2e/non-concurrency/non-fail-fast/expected.yaml @@ -27,5 +27,8 @@ failed: - case-8 - case-9 skipped: [] +passedCount: 6 +failedCount: 3 +skippedCount: 0 diff --git a/test/e2e/non-fail-fast/expected.yaml b/test/e2e/non-fail-fast/expected.yaml deleted file mode 100644 index a90d042..0000000 --- a/test/e2e/non-fail-fast/expected.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Apache Software Foundation (ASF) licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -passed: -- case-1 -- case-2 -- case-4 -failed: -- case-3 -skipped: [] \ No newline at end of file diff --git a/test/e2e/non-fail-fast/internal/expected.yaml b/test/e2e/non-fail-fast/internal/expected.yaml deleted file mode 100644 index 1d9d4bc..0000000 --- a/test/e2e/non-fail-fast/internal/expected.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file is used to show how to write configuration files and can be used to test. - -args: - case: success -headers: - Accept: application/json - Host: 127.0.0.1:8080 - User-Agent: curl/7.81.0 -origin: 172.18.0.1 -url: http://127.0.0.1:8080/get?case=success \ No newline at end of file diff --git a/test/e2e/non-fail-fast/internal/verify.yaml b/test/e2e/non-fail-fast/internal/verify.yaml deleted file mode 100644 index 41e951e..0000000 --- a/test/e2e/non-fail-fast/internal/verify.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file is used to show how to write configuration files and can be used to test. - -verify: - # verify with retry strategy - retry: - # max retry count - count: 5 - # the interval between two attempts, e.g. 10s, 1m. - interval: 1s - - # when a case fails, whether to stop verifying other cases. This property defaults to true. - fail-fast: false - # Whether to verify cases concurrently. This property defaults to false. - concurrency: false - - - cases: - - name: case-1 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: case-2 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml - - name: case-3 - query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"' - expected: ./expected.yaml - - name: case-4 - query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"' - expected: ./expected.yaml \ No newline at end of file