ci: clean and simplify tests

Makefile

@@ -45,11 +45,16 @@ test-go:
 	testground plan import --from ./plans/example
 	$(call eachmod,go test -p 1 -v $(GOTFLAGS) ./...)
 
-test-integration: test-integ-cluster-k8s test-integ-local-docker test-integ-local-exec
+test-integration: test-integ-cluster-k8s test-integ-local-exec test-integ-local-docker test-integ-examples
 
+# Note that in these tests, we override the SYNC_SERVICE env var,
+# so that testground connects to the instance exposed with `make kind-cluster`
 test-integ-cluster-k8s:
-	./integration_tests/01_k8s_kind_placebo_ok.sh
-	./integration_tests/02_k8s_kind_placebo_stall.sh
+	SYNC_SERVICE_HOST=localhost ./integration_tests/01_k8s_kind_placebo_ok.sh
+	SYNC_SERVICE_HOST=localhost ./integration_tests/02_k8s_kind_placebo_stall.sh
+
+test-integ-local-exec:
+	./integration_tests/03_exec_go_placebo_ok.sh
 
 test-integ-local-docker:
 	./integration_tests/04_docker_placebo_ok.sh
@@ -60,15 +65,12 @@ test-integ-local-docker:
 	./integration_tests/09_docker_splitbrain_accept.sh
 	./integration_tests/10_docker_splitbrain_reject.sh
 	./integration_tests/11_docker_splitbrain_drop.sh
-	./integration_tests/12_docker_example-js_pingpong.sh
 	./integration_tests/13_docker_builder_configuration.sh
 	./integration_tests/13_02_docker_builder_configuration.sh
 
-test-integ-local-exec:
-	./integration_tests/03_exec_go_placebo_ok.sh
-
 test-integ-examples:
 	./integration_tests/example_01_rust.sh
+	./integration_tests/example_02_js_pingpong.sh
 
 kind-cluster:
 	kind create cluster --wait 90s

integration_tests/01_k8s_kind_placebo_ok.sh

@@ -1,33 +1,37 @@
 #!/bin/bash
-
-export SYNC_SERVICE_HOST=localhost
 my_dir="$(dirname "$0")"
 source "$my_dir/header.sh"
 
-testground plan import --from plans/placebo
-testground build single --builder docker:go --plan placebo --wait | tee build.out
+testground plan import --from ./plans --name testground
+
+pushd $TEMPDIR
+
+testground build single \
+    --plan testground/placebo \
+    --builder docker:go \
+    --wait | tee build.out
 export ARTIFACT=$(awk -F\" '/generated build artifact/ {print $8}' build.out)
 docker tag $ARTIFACT testplan:placebo
 
-# The placebo:ok does not require a sidecar.
+# The placebo:stall does not require a sidecar.
 # To prevent kind from attempting to download the image from DockerHub, build and load the image before executing it.
 # The plan is renamed as `testplan:placebo` because kind will check DockerHub if the tag is `latest`.
 kind load docker-image testplan:placebo
 
-pushd $TEMPDIR
 testground healthcheck --runner local:docker --fix
-testground run single --runner cluster:k8s --builder docker:go --use-build testplan:placebo --instances 1 --plan placebo --testcase ok --collect --wait | tee run.out
-RUNID=$(awk '/finished run with ID/ { print $9 }' run.out)
-echo "checking run $RUNID"
-file $RUNID.tgz
-LENGTH=${#RUNID}
-test $LENGTH -eq 20
-tar -xzvvf $RUNID.tgz
-SIZEOUT=$(cat ./"$RUNID"/single/0/run.out | wc -c)
-echo "run.out is $SIZEOUT bytes."
-SIZEERR=$(cat ./"$RUNID"/single/0/run.err | wc -c)
-test $SIZEOUT -gt 0 && test $SIZEERR -eq 0
+testground run single \
+    --plan=testground/placebo \
+    --testcase=stall \
+    --builder=docker:go \
+    --use-build testplan:placebo \
+    --runner=cluster:k8s \
+    --instances=2 \
+    --collect \
+    --wait | tee run.out
+
+assert_run_output_is_correct run.out
+
 popd
 
 echo "terminating remaining containers"
-testground terminate --builder docker:go
+testground terminate --runner local:docker

integration_tests/02_k8s_kind_placebo_stall.sh

@@ -1,11 +1,15 @@
 #!/bin/bash
-
-export SYNC_SERVICE_HOST=localhost
 my_dir="$(dirname "$0")"
 source "$my_dir/header.sh"
 
-testground plan import --from plans/placebo
-testground build single --builder docker:go --plan placebo --wait | tee build.out
+testground plan import --from ./plans --name testground
+
+pushd $TEMPDIR
+
+testground build single \
+    --plan testground/placebo \
+    --builder docker:go \
+    --wait | tee build.out
 export ARTIFACT=$(awk -F\" '/generated build artifact/ {print $8}' build.out)
 docker tag $ARTIFACT testplan:placebo
 
@@ -15,13 +19,24 @@ docker tag $ARTIFACT testplan:placebo
 kind load docker-image testplan:placebo
 
 testground healthcheck --runner local:docker --fix
-testground run single --runner cluster:k8s --builder docker:go --use-build testplan:placebo --instances 2 --plan placebo --testcase stall --wait &
+testground run single \
+    --plan=testground/placebo \
+    --testcase=stall \
+    --builder=docker:go \
+    --use-build testplan:placebo \
+    --runner=cluster:k8s \
+    --instances=2 \
+    --collect \
+    --wait&
+
 sleep 20
 BEFORE=$(kubectl get pods | grep placebo | grep Running | wc -l)
 testground terminate --runner=cluster:k8s
 sleep 10
 AFTER=$(kubectl get pods | grep placebo | grep Running | wc -l)
 test $BEFORE -gt $AFTER
 
+popd
+
 echo "terminating remaining containers"
-testground terminate --builder docker:go
+testground terminate --runner local:docker

integration_tests/03_exec_go_placebo_ok.sh

@@ -1,22 +1,23 @@
 #!/bin/bash
-
 my_dir="$(dirname "$0")"
 source "$my_dir/header.sh"
 
+testground plan import --from ./plans --name testground
+
 pushd $TEMPDIR
-testground plan import --from plans/placebo
-testground run single --runner local:exec --builder exec:go --instances 2 --plan placebo --testcase ok --collect --wait | tee run.out
-RUNID=$(awk '/finished run with ID/ { print $9 }' run.out)
-echo "checking run $RUNID"
-file $RUNID.tgz
-LENGTH=${#RUNID}
-test $LENGTH -eq 20
-tar -xzvvf $RUNID.tgz
-SIZEOUT=$(cat ./"$RUNID"/single/0/run.out | wc -c)
-echo "run.out is $SIZEOUT bytes."
-SIZEERR=$(cat ./"$RUNID"/single/0/run.err | wc -c)
-test $SIZEOUT -gt 0 && test $SIZEERR -eq 0
+
+testground run single \
+    --plan=testground/placebo \
+    --testcase=ok \
+    --builder=docker:go \
+    --runner=local:exec \
+    --instances=2 \
+    --collect \
+    --wait | tee run.out
+
+assert_run_output_is_correct run.out
+
 popd
 
 echo "terminating remaining containers"
-testground terminate --runner local:exec
+testground terminate --runner local:docker

integration_tests/04_docker_placebo_ok.sh

@@ -1,28 +1,33 @@
 #!/bin/bash
-
 my_dir="$(dirname "$0")"
 source "$my_dir/header.sh"
 
-testground plan import --from plans/placebo
-testground build single --builder docker:go --plan placebo --wait | tee build.out
+testground plan import --from ./plans --name testground
+
+pushd $TEMPDIR
+
+testground build single \
+    --plan testground/placebo \
+    --builder docker:go \
+    --wait | tee build.out
 export ARTIFACT=$(awk -F\" '/generated build artifact/ {print $8}' build.out)
 docker tag $ARTIFACT testplan:placebo
 
-pushd $TEMPDIR
 testground healthcheck --runner local:docker --fix
-testground run single --runner local:docker --builder docker:go --use-build testplan:placebo --instances 1 --plan placebo --testcase ok --collect --wait | tee run.out
-RUNID=$(awk '/finished run with ID/ { print $9 }' run.out)
-echo "checking run $RUNID"
-file $RUNID.tgz
-LENGTH=${#RUNID}
-test $LENGTH -eq 20
-tar -xzvvf $RUNID.tgz
-SIZEOUT=$(cat ./"$RUNID"/single/0/run.out | wc -c)
-echo "run.out is $SIZEOUT bytes."
-SIZEERR=$(cat ./"$RUNID"/single/0/run.err | wc -c)
-test $SIZEOUT -gt 0 && test $SIZEERR -eq 0
+
+testground run single \
+    --plan=testground/placebo \
+    --testcase=ok \
+    --builder=docker:go \
+    --use-build testplan:placebo \
+    --runner=local:docker \
+    --instances=1 \
+    --collect \
+    --wait | tee run.out
+
+assert_run_output_is_correct run.out
+
 popd
 
 echo "terminating remaining containers"
-testground terminate --runner local:docker
-testground terminate --builder docker:go
+testground terminate --runner local:docker

integration_tests/05_docker_placebo_stall.sh

@@ -1,22 +1,38 @@
 #!/bin/bash
-
 my_dir="$(dirname "$0")"
 source "$my_dir/header.sh"
 
-testground plan import --from plans/placebo
-testground build single --builder docker:go --plan placebo --wait | tee build.out
+testground plan import --from ./plans --name testground
+
+pushd $TEMPDIR
+
+testground build single \
+    --plan testground/placebo \
+    --builder docker:go \
+    --wait | tee build.out
 export ARTIFACT=$(awk -F\" '/generated build artifact/ {print $8}' build.out)
 docker tag $ARTIFACT testplan:placebo
 
 testground healthcheck --runner local:docker --fix
-testground run single --runner local:docker --builder docker:go --use-build testplan:placebo --instances 2 --plan placebo --testcase stall --wait &
+
+testground run single \
+    --plan=testground/placebo \
+    --testcase=stall \
+    --builder=docker:go \
+    --use-build testplan:placebo \
+    --runner=local:docker \
+    --instances=2 \
+    --collect \
+    --wait&
+
 sleep 20
 BEFORE=$(docker ps | grep placebo | wc -l)
 testground terminate --runner=local:docker
 sleep 10 
 AFTER=$(docker ps | grep placebo | wc -l)
 test $BEFORE -gt $AFTER
 
+popd
+
 echo "terminating remaining containers"
-testground terminate --runner local:docker
-testground terminate --builder docker:go
+testground terminate --runner local:docker

integration_tests/06_docker_network_ping-pong.sh

@@ -1,30 +1,33 @@
 #!/bin/bash
-
 my_dir="$(dirname "$0")"
 source "$my_dir/header.sh"
 
-testground plan import --from plans/network
-testground build single --builder docker:go --plan network --wait | tee build.out
+testground plan import --from ./plans --name testground
+
+pushd $TEMPDIR
+
+testground build single \
+    --plan testground/network \
+    --builder docker:go \
+    --wait | tee build.out
 export ARTIFACT=$(awk -F\" '/generated build artifact/ {print $8}' build.out)
 docker tag $ARTIFACT testplan:network
 
-pushd $TEMPDIR
 testground healthcheck --runner local:docker --fix
-testground run single --runner local:docker --builder docker:go --use-build testplan:network --instances 2 --plan network --testcase ping-pong --collect --wait | tee stdout.out
-RUNID=$(awk '/finished run with ID/ { print $9 }' stdout.out)
-echo "checking run $RUNID"
-file $RUNID.tgz
-tar -xzvvf $RUNID.tgz
-SIZEOUT=$(cat ./"$RUNID"/single/0/run.out | wc -c)
-echo "run.out is $SIZEOUT bytes."
-SIZEERR=$(cat ./"$RUNID"/single/0/run.err | wc -c)
-test $SIZEOUT -gt 0 && test $SIZEERR -eq 0
-pushd $RUNID
-OUTCOMEOK=$(find . | grep run.out | xargs awk '{print $0, FILENAME}' | grep "\"success_event\"" | wc -l)
-test $OUTCOMEOK -eq 2
-popd
+
+testground run single \
+    --plan=testground/network \
+    --testcase=ping-pong \
+    --builder=docker:go \
+    --use-build testplan:network \
+    --runner=local:docker \
+    --instances=2 \
+    --collect \
+    --wait | tee run.out
+
+assert_run_output_is_correct run.out
+
 popd
 
 echo "terminating remaining containers"
-testground terminate --runner local:docker
-testground terminate --builder docker:go
+testground terminate --runner local:docker

integration_tests/07_docker_network_traffic-allowed.sh

@@ -1,28 +1,33 @@
 #!/bin/bash
-
 my_dir="$(dirname "$0")"
 source "$my_dir/header.sh"
 
-testground plan import --from plans/network
-testground build single --builder docker:go --plan network --wait | tee build.out
+testground plan import --from ./plans --name testground
+
+pushd $TEMPDIR
+
+testground build single \
+    --plan testground/network \
+    --builder docker:go \
+    --wait | tee build.out
 export ARTIFACT=$(awk -F\" '/generated build artifact/ {print $8}' build.out)
 docker tag $ARTIFACT testplan:network
 
-pushd $TEMPDIR
 testground healthcheck --runner local:docker --fix
-testground run single --runner local:docker --builder docker:go --use-build testplan:network --instances 1 --plan network --testcase traffic-allowed --collect --wait | tee stdout.out
-RUNID=$(awk '/finished run with ID/ { print $9 }' stdout.out)
-echo "checking run $RUNID"
-file $RUNID.tgz
-LENGTH=${#RUNID}
-test $LENGTH -eq 20
-tar -xzvvf $RUNID.tgz
-SIZEOUT=$(cat ./"$RUNID"/single/0/run.out | wc -c)
-echo "run.out is $SIZEOUT bytes."
-SIZEERR=$(cat ./"$RUNID"/single/0/run.err | wc -c)
-test $SIZEOUT -gt 0 && test $SIZEERR -eq 0
+
+testground run single \
+    --plan=testground/network \
+    --testcase=traffic-allowed \
+    --builder=docker:go \
+    --use-build testplan:network \
+    --runner=local:docker \
+    --instances=1 \
+    --collect \
+    --wait | tee run.out
+
+assert_run_output_is_correct run.out
+
 popd
 
 echo "terminating remaining containers"
-testground terminate --runner local:docker
-testground terminate --builder docker:go
+testground terminate --runner local:docker