tests/int: fix flaky kill tests

tests/integration/delete.bats

@@ -54,12 +54,13 @@ function test_runc_delete_host_pidns() {
 	# not have own PID ns, its init is no special and the container
 	# will still be up and running.
 	kill -9 "$init_pid"
+	wait_pids_gone 10 0.2 "$init_pid"
 
 	# Get the list of all container processes.
-	pids=$(cat "$cgpath"/cgroup.procs)
-	echo "pids: $pids"
+	mapfile -t pids < <(cat "$cgpath"/cgroup.procs)
+	echo "pids:" "${pids[@]}"
 	# Sanity check -- make sure all processes exist.
-	for p in $pids; do
+	for p in "${pids[@]}"; do
 		kill -0 "$p"
 	done
 
@@ -70,10 +71,15 @@ function test_runc_delete_host_pidns() {
 	runc state test_busybox
 	[ "$status" -ne 0 ] # "Container does not exist"
 
-	# Make sure all processes are gone.
-	pids=$(cat "$cgpath"/cgroup.procs) || true # OK if cgroup is gone
-	echo "pids: $pids"
-	[ -z "$pids" ]
+	# Wait and check that all the processes are gone.
+	wait_pids_gone 10 0.2 "${pids[@]}"
+
+	# Make sure cgroup.procs is empty.
+	mapfile -t pids < <(cat "$cgpath"/cgroup.procs || true)
+	if [ ${#pids[@]} -gt 0 ]; then
+		echo "expected empty cgroup.procs, got:" "${pids[@]}" 1>&2
+		return 1
+	fi
 }
 
 @test "runc delete" {

tests/integration/helpers.bash

@@ -581,6 +581,36 @@ function testcontainer() {
 	[[ "${output}" == *"$2"* ]]
 }
 
+# Check that all the listed processes are gone. Use after kill/stop etc.
+function wait_pids_gone() {
+	if [ $# -lt 3 ]; then
+		echo "Usage: wait_pids_gone ITERATIONS SLEEP PID [PID ...]"
+		return 1
+	fi
+	local iter=$1
+	shift
+	local sleep=$1
+	shift
+	local pids=("$@")
+
+	while true; do
+		for i in "${!pids[@]}"; do
+			# Check if the pid is there; if not, remove it from the list.
+			kill -0 "${pids[i]}" 2>/dev/null || unset "pids[i]"
+		done
+		[ ${#pids[@]} -eq 0 ] && return 0
+		# Rebuild pids array to avoid sparse array issues.
+		pids=("${pids[@]}")
+
+		((--iter > 0)) || break
+
+		sleep "$sleep"
+	done
+
+	echo "Expected all PIDs to be gone, but some are still there:" "${pids[@]}" 1>&2
+	return 1
+}
+
 function setup_recvtty() {
 	[ ! -v ROOT ] && return 1 # must not be called without ROOT set
 	local dir="$ROOT/tty"

tests/integration/kill.bats

@@ -46,24 +46,33 @@ test_host_pidns_kill() {
 		# kills the container; see "kill KILL [host pidns + init gone]"
 		# below).
 		kill -9 "$init_pid"
+		wait_pids_gone 10 0.2 "$init_pid"
 	fi
 
 	# Get the list of all container processes.
-	pids=$(cat "$cgpath"/cgroup.procs)
-	echo "pids: $pids"
+	mapfile -t pids < <(cat "$cgpath"/cgroup.procs)
+	echo "pids:" "${pids[@]}"
 	# Sanity check -- make sure all processes exist.
-	for p in $pids; do
+	for p in "${pids[@]}"; do
 		kill -0 "$p"
 	done
 
 	runc kill test_busybox KILL
 	[ "$status" -eq 0 ]
-	wait_for_container 10 1 test_busybox stopped
-
-	# Make sure all processes are gone.
-	pids=$(cat "$cgpath"/cgroup.procs) || true # OK if cgroup is gone
-	echo "pids: $pids"
-	[ -z "$pids" ]
+	# Wait and check that all processes are gone.
+	wait_pids_gone 10 0.2 "${pids[@]}"
+
+	# Make sure the container is in stopped state. Note if KILL_INIT
+	# is set, container was already stopped by killing its $init_pid
+	# and so this check is NOP/redundant.
+	testcontainer test_busybox stopped
+
+	# Make sure cgroup.procs is empty.
+	mapfile -t pids < <(cat "$cgpath"/cgroup.procs || true)
+	if [ ${#pids[@]} -gt 0 ]; then
+		echo "expected empty cgroup.procs, got:" "${pids[@]}" 1>&2
+		return 1
+	fi
 }
 
 @test "kill detached busybox" {