Refactor test setup for greater flexibility.

qaware · Oct 4, 2023 · 60b0a87 · 60b0a87
1 parent 497179b
commit 60b0a87
Show file tree

Hide file tree

Showing 9 changed files with 184 additions and 105 deletions.
diff --git a/EXAMPLES.md b/EXAMPLES.md
@@ -26,9 +26,9 @@ $ docker run -v "$PWD/test/proto:/proto" --network host qaware/protocurl \
    -u http://localhost:8080/happy-day/verify \
    -d "includeReason: true"
 
-=========================== Request Text     =========================== >>>
+=========================== POST Request Text    =========================== >>>
 includeReason: true
-=========================== Response Text    =========================== <<<
+=========================== POST Response Text    =========================== <<<
 isHappyDay: true
 reason: "Thursday is a Happy Day! ⭐"
 formattedDate: "Thu, 01 Jan 1970 00:00:00 GMT"
@@ -41,9 +41,9 @@ $ docker run -v "$PWD/test/proto:/proto" --network host qaware/protocurl \
   -f happyday.proto -i happyday.HappyDayRequest -o happyday.HappyDayResponse \
   -u http://localhost:8080/happy-day/verify -d ""
 
-=========================== Request Text     =========================== >>>
+=========================== POST Request Text    =========================== >>>
 
-=========================== Response Text    =========================== <<<
+=========================== POST Response Text    =========================== <<<
 isHappyDay: true
 formattedDate: "Thu, 01 Jan 1970 00:00:00 GMT"
 ```
@@ -56,11 +56,11 @@ $ docker run -v "$PWD/test/proto:/proto" --network host qaware/protocurl \
   -u http://localhost:8080/happy-day/verify \
   -d "date: { seconds: 1648044939}"
 
-=========================== Request Text     =========================== >>>
+=========================== POST Request Text    =========================== >>>
 date: {
   seconds: 1648044939
 }
-=========================== Response Text    =========================== <<<
+=========================== POST Response Text    =========================== <<<
 formattedDate: "Wed, 23 Mar 2022 14:15:39 GMT"
 ```
 
@@ -96,9 +96,9 @@ $ docker run -v "$PWD/test/proto:/proto" --network host qaware/protocurl \
   -u http://localhost:8080/happy-day/verify \
   -d "{ \"date\": \"2022-03-23T14:15:39Z\" }"
 
-=========================== Request JSON     =========================== >>>
+=========================== POST Request JSON    =========================== >>>
 {"date":"2022-03-23T14:15:39Z"}
-=========================== Response JSON    =========================== <<<
+=========================== POST Response JSON    =========================== <<<
 {"formattedDate":"Wed, 23 Mar 2022 14:15:39 GMT"}
 ```
 
@@ -110,9 +110,9 @@ $ docker run -v "$PWD/test/proto:/proto" --network host qaware/protocurl \
   -u http://localhost:8080/happy-day/verify --out=json:pretty \
   -d "{ \"date\": \"2022-03-23T14:15:39Z\" }"
 
-=========================== Request JSON     =========================== >>>
+=========================== POST Request JSON    =========================== >>>
 {"date":"2022-03-23T14:15:39Z"}
-=========================== Response JSON    =========================== <<<
+=========================== POST Response JSON    =========================== <<<
 {
   "formattedDate": "Wed, 23 Mar 2022 14:15:39 GMT"
 }
@@ -157,11 +157,11 @@ $ docker run -v "$PWD/test/proto:/proto" --network host qaware/protocurl \
   -d "date: { seconds: 1648044939}" \
   --curl -n -H 'Content-Type: application/octet-stream'
 
-=========================== Request Text     =========================== >>>
+=========================== POST Request Text    =========================== >>>
 date: {
   seconds: 1648044939
 }
-=========================== Response Text    =========================== <<<
+=========================== POST Response Text    =========================== <<<
 formattedDate: "Wed, 23 Mar 2022 14:15:39 GMT"
 ```
 
@@ -185,6 +185,7 @@ Invoked with following default & parsed arguments:
   "RequestType": "..HappyDayRequest",
   "ResponseType": "..HappyDayResponse",
   "Url": "http://localhost:8080/happy-day/verify",
+  "Method": "POST",
   "DataText": "date: { seconds: 1648044939}",
   "InTextType": "text",
   "OutTextType": "text",
@@ -409,11 +410,11 @@ Searching for message with base name: HappyDayRequest
 Resolved message package-paths for name HappyDayRequest: [happyday.HappyDayRequest]
 Searching for message with base name: HappyDayRequest
 Resolved message package-paths for name HappyDayRequest: [happyday.HappyDayRequest]
-=========================== Request Text     =========================== >>>
+=========================== POST Request Text    =========================== >>>
 date: {
   seconds: 1648044939
 }
-=========================== Request Binary   =========================== >>>
+=========================== POST Request Binary =========================== >>>
 00000000  0a 06 08 8b d7 ec 91 06                           |........|
 Found curl: /usr/bin/curl
 Invoking curl http request.
@@ -431,19 +432,19 @@ Total curl args:
   -H
   Content-Type: application/x-protobuf
   http://localhost:8080/happy-day/verify
-=========================== Response Headers =========================== <<<
+=========================== POST Response Headers =========================== <<<
 HTTP/1.1 200 OK
 Content-Type: application/x-protobuf
-Date: Fri, 17 Mar 2023 23:19:03 GMT
+Date: Tue, 03 Oct 2023 09:29:50 GMT
 Connection: keep-alive
 Keep-Alive: timeout=5
 Content-Length: 35
-=========================== Response Binary  =========================== <<<
+=========================== POST Response Binary  =========================== <<<
 00000000  08 00 1a 1d 57 65 64 2c  20 32 33 20 4d 61 72 20  |....Wed, 23 Mar |
 00000010  32 30 32 32 20 31 34 3a  31 35 3a 33 39 20 47 4d  |2022 14:15:39 GM|
 00000020  54 22 00                                          |T".|
 Searching for message with base name: HappyDayResponse
 Resolved message package-paths for name HappyDayResponse: [happyday.HappyDayResponse]
-=========================== Response Text    =========================== <<<
+=========================== POST Response Text    =========================== <<<
 formattedDate: "Wed, 23 Mar 2022 14:15:39 GMT"
 ```
diff --git a/README.md b/README.md
@@ -103,9 +103,9 @@ Then protocurl will
 and produce the following output:
 
 ```
-=========================== Request Text     =========================== >>>
+=========================== POST Request Text    =========================== >>>
 includeReason: true
-=========================== Response Text    =========================== <<<
+=========================== POST Response Text    =========================== <<<
 isHappyDay: true
 reason: "Thursday is a Happy Day! ⭐"
 formattedDate: "Thu, 01 Jan 1970 00:00:00 GMT"

diff --git a/TESTS.md b/TESTS.md
@@ -33,7 +33,7 @@ from `test/suite/testcases.json` against the testserver. Each testcase is of the
     "<These are split into an array to make it easier to write them in the JSON file.>",
     "<All of these array elements will be concatenated with spaces.>"
   ],
-  "runAgainWithArg": "<--some-arg>",
+  "rerunwithArgForEachElement": ["<some-arg-for-one-scenario>", "<some-arg-for-another-scenario>"],
   "afterTestBash": "<bash statements>"
 }
 ```
@@ -44,9 +44,11 @@ This happens via `test/suite/run-testcases.sh` - which is dynamically created fr
 of the form
 
 ```
-testSingleRequest '<filename>' '<args concatenated with spaces>' '<bash statements>' '--some-arg'
+testSingleSpec '<filename>' '<args concatenated with spaces>' '<bash statements>' 'some-arg-for-one-scenario' 'some-arg-for-another-scenario'
 ```
 
+todo. update this documentation
+
 During the execution of each line in this script, the output will be written into `test/results/$FILENAME-out.txt` -
 which will be compared via `diff` to `test/results/$FILENAME-expected.txt`. If both match, then the result is accepted.
 

diff --git a/doc/generated.usage.txt b/doc/generated.usage.txt
@@ -28,6 +28,7 @@ Flags:
   -h, --help                      help for protocurl
       --in string                 Specifies, in which format the input -d should be interpreted in. 'text' (default) uses the Protobuf text format and 'json' uses JSON. The type is inferred as JSON if the first token is a '{'.
   -F, --infer-files               Infer the correct files containing the relevant protobuf messages. All proto files in the proto directory provided by -I will be used. If no -f <file> is provided, this -F is set and the files are inferred.
+  -X, --method string             HTTP request method. POST (default) and GET supported. Other methods are passed on on to curl optimistically. (default "POST")
       --no-curl                   Forces the use of the built-in internal http request instead of curl.
   -n, --no-default-headers        Default headers (e.g. "Content-Type") will not be passed to curl. Assumes --curl. Use "-n -H 'Content-Type: FooBar'" to override the default content type.
       --out string                Produces the output in the specified format. 'text' (default) produces Protobuf text format. 'json' produces dense JSON and 'json:pretty' produces pretty-printed JSON. The produced JSON always uses the original Protobuf field names instead of lowerCamelCasing them.

diff --git a/src/protocurl.go b/src/protocurl.go
@@ -108,6 +108,7 @@ func encodeToBinary(requestType string, text string, registry *protoregistry.Fil
 	)
 
 	if !CurrentConfig.ShowOutputOnly {
+		// todo. remove one space here for aligned output
 		fmt.Printf("%s %s Request %s    %s %s\n%s\n",
 			VISUAL_SEPARATOR, CurrentConfig.Method, displayIn(CurrentConfig.InTextType), VISUAL_SEPARATOR,
 			SEND, reconstructedRequestText)

diff --git a/test/results/other-days-are-happy-days-moved-protofiles-tmp-expected.txt b/test/results/other-days-are-happy-days-moved-protofiles-tmp-expected.txt
diff --git a/test/results/wednesday-is-not-a-happy-day-no-curl--X_GET-expected.txt b/test/results/wednesday-is-not-a-happy-day-no-curl--X_GET-expected.txt
@@ -0,0 +1,12 @@
+######### STDOUT #########
+=========================== GET Request Text    =========================== >>>
+date: {
+  seconds: 1648044939
+  nanos: 152000000
+}
+includeReason: true
+=========================== GET Response Text    =========================== <<<
+reason: "Tough luck on Wednesday... 😕"
+formattedDate: "Wed, 23 Mar 2022 14:15:39 GMT"
+######### STDERR #########
+######### EXIT 0 #########
diff --git a/test/suite/test.sh b/test/suite/test.sh
@@ -21,67 +21,71 @@ testSingleRequest() {
   FILENAME="$1"
   ARGS="$2"
   BEFORE_TEST_BASH="$3"
-  RUN_AGAIN_WITH_ARG="$4"
-  AFTER_TEST_BASH="$5"
+  AFTER_TEST_BASH="$4"
 
   if [[ "$FILENAME" == "response-type-arg-overidden-decode-raw" && "$(uname)" == *"MINGW"* ]]; then
     echo "🚧🚧🚧 SKIPPED 🚧🚧🚧 - Skipping response-type-arg-overidden-decode-raw on Windows due to special circumstances."
     return 0
   fi
 
-  if [[ "$RUN_AGAIN_WITH_ARG" != "" ]]; then
-    NEW_ARGS="$RUN_AGAIN_WITH_ARG $ARGS"
-    NEW_FILENAME="${FILENAME}-${RUN_AGAIN_WITH_ARG#--}"
-    testSingleRequest "$FILENAME" "$ARGS" "$BEFORE_TEST_BASH" "" "$AFTER_TEST_BASH"
-    testSingleRequest "$NEW_FILENAME" "$NEW_ARGS" "$BEFORE_TEST_BASH" "" "$AFTER_TEST_BASH"
-  else
+  EXPECTED="test/results/$FILENAME-expected.txt"
+  OUT="test/results/$FILENAME-out.txt"
+  OUT_ERR="test/results/$FILENAME-out-err-tmp.txt"
+  touch "$EXPECTED"
+  rm -f "$OUT" || true
+  rm -f "$OUT_ERR" || true
+  docker rm -f "$FILENAME" >/dev/null 2>&1 || true # stop any previously running container for this testcase
+  EXIT_CODE="?"                                    # default exit code, if process aborts abnormaly
+
+  echo "######### STDOUT #########" >"$OUT"
+
+  set +e
+
+  if [[ "$BEFORE_TEST_BASH" == "" ]]; then BEFORE_TEST_BASH="true"; fi
+  if [[ "$AFTER_TEST_BASH" == "" ]]; then AFTER_TEST_BASH="true"; fi
 
-    EXPECTED="test/results/$FILENAME-expected.txt"
-    OUT="test/results/$FILENAME-out.txt"
-    OUT_ERR="test/results/$FILENAME-out-err-tmp.txt"
-    touch "$EXPECTED"
-    rm -f "$OUT" || true
-    rm -f "$OUT_ERR" || true
-    echo "######### STDOUT #########" >"$OUT"
-    EXIT_CODE="?"
-
-    set +e
-
-    if [[ "$BEFORE_TEST_BASH" == "" && "$AFTER_TEST_BASH" == "" ]]; then
-      eval "$RUN_CLIENT --name $FILENAME $PROTOCURL_IMAGE $ARGS" 2>"$OUT_ERR" >>"$OUT"
-      EXIT_CODE="$?"
-    else
-      if [[ "$BEFORE_TEST_BASH" == "" ]]; then BEFORE_TEST_BASH="true"; fi
-      if [[ "$AFTER_TEST_BASH" == "" ]]; then AFTER_TEST_BASH="true"; fi
-      ARGS="$(echo "$ARGS" | sed 's/"/\\"/g')" # escape before usage inside quoted context
-      eval "$RUN_CLIENT --entrypoint bash \
-        --name $FILENAME $PROTOCURL_IMAGE \
-        -c \"$BEFORE_TEST_BASH && ./bin/protocurl $ARGS && $AFTER_TEST_BASH\"" \
-        2>"$OUT_ERR" >>"$OUT"
-      EXIT_CODE="$?"
-    fi
-    echo "######### STDERR #########" >>"$OUT"
-    cat "$OUT_ERR" >>"$OUT"
-
-    echo "######### EXIT $EXIT_CODE #########" >>"$OUT"
-
-    meaningfulDiff "$EXPECTED" "$OUT" >/dev/null
-
-    if [[ "$?" != 0 ]]; then
-      export TESTS_SUCCESS="false"
-      echo "❌❌❌ FAILURE ❌❌❌ - $FILENAME"
-      echo "=== Found difference between expected and actual output (ignoring $NORMALISED_ASPECTS) ==="
-      meaningfulDiff "$EXPECTED" "$OUT" | sed 's/^/  /'
-      echo "The actual output was saved into $OUT for inspection."
-    else
-      echo "✨✨✨ SUCCESS ✨✨✨ - $FILENAME"
-    fi
-
-    set -e
-
-    rm -f "$OUT_ERR" || true
+  eval "$RUN_CLIENT --entrypoint bash \
+    --name $FILENAME $PROTOCURL_IMAGE \
+    -c '$BEFORE_TEST_BASH && ./bin/protocurl $ARGS && $AFTER_TEST_BASH'" \
+    2>"$OUT_ERR" >>"$OUT"
+  EXIT_CODE="$?"
 
+  echo "######### STDERR #########" >>"$OUT"
+  cat "$OUT_ERR" >>"$OUT"
+
+  echo "######### EXIT $EXIT_CODE #########" >>"$OUT"
+
+  meaningfulDiff "$EXPECTED" "$OUT" >/dev/null
+
+  if [[ "$?" != 0 ]]; then
+    export TESTS_SUCCESS="false"
+    echo "❌❌❌ FAILURE ❌❌❌ - $FILENAME"
+    echo "=== Found difference between expected and actual output (ignoring $NORMALISED_ASPECTS) ==="
+    meaningfulDiff "$EXPECTED" "$OUT" | sed 's/^/  /'
+    echo "The actual output was saved into $OUT for inspection."
+  else
+    echo "✨✨✨ SUCCESS ✨✨✨ - $FILENAME"
   fi
+
+  set -e
+
+  rm -f "$OUT_ERR" || true
+}
+
+# A spec consists of potentially many single requests.
+testSingleSpec() {
+  FILENAME="$1"
+  ARGS="$2"
+  BEFORE_TEST_BASH="$3"
+  AFTER_TEST_BASH="$4"
+
+  testSingleRequest "$FILENAME" "$ARGS" "$BEFORE_TEST_BASH" "$AFTER_TEST_BASH"
+  shift 4
+  for extra_arg in "$@"; do
+    NEW_FILENAME="${FILENAME}-${extra_arg#--}"
+    NEW_FILENAME="$(echo "$NEW_FILENAME" | sed 's/ /_/g')" # sanitise filename
+    testSingleRequest "$NEW_FILENAME" "$extra_arg $ARGS" "$BEFORE_TEST_BASH" "$AFTER_TEST_BASH"
+  done
 }
 
 runAllTests() {
@@ -90,9 +94,10 @@ runAllTests() {
 
   # Convert each element in the JSON to the corresponding call of the testSingleRequest function.
   # Simply look at the produced run-testcases.sh file to see what it looks like.
-  CONVERT_TESTCASE_TO_SINGLE_TEST_INVOCATION=".[] | \"testSingleRequest \(.filename|@sh) \(.args|join(\" \")|@sh) \(.beforeTestBash // \"\"|@sh) \(.runAgainWithArg // \"\"|@sh) \(.afterTestBash // \"\"|@sh)\""
+  CONVERT_TESTCASE_TO_SINGLE_TEST_INVOCATION=".[] | \"testSingleSpec \(.filename|@sh) \(.args|join(\" \")|@sh) \(.beforeTestBash // \"\"|@sh) \(.afterTestBash // \"\"|@sh) \((.rerunwithArgForEachElement // [])|@sh)\""
   cat test/suite/testcases.json | jq -r "$CONVERT_TESTCASE_TO_SINGLE_TEST_INVOCATION" >./test/suite/run-testcases.sh
 
+  export -f testSingleSpec
   export -f testSingleRequest
   chmod +x ./test/suite/run-testcases.sh
   source ./test/suite/run-testcases.sh