testsuite: update job-list consistency checks

Problem: The job-list module has eventual consistency for the jobs
that are submitted to the job-manager queue.  A number of helper
functions have been written to check for this consistency before
running tests.  These functions are no longer necessary due to the
new `flux job list-ids --wait-state` option.

Use the flux job list-ids --wait-state option when appropriate.  Remove
no longer necessary code.

t/job-list/job-list-helper.sh

@@ -1,10 +1,6 @@
 #!/bin/sh
 #
 
-# job-list test helper functions
-
-JOB_LIST_WAIT_ITERS=50
-
 # Return the expected jobids list in a given state:
 #   "all", "pending", "running", "inactive", "active",
 #   "completed", "canceled", "failed", "timeout"
@@ -20,32 +16,3 @@ job_list_state_ids() {
 job_list_state_count() {
     job_list_state_ids "$@" | wc -l
 }
-
-# the job-list module has eventual consistency with the jobs stored in
-# the job-manager's queue.  To ensure no raciness in tests, we spin
-# until all of the pending jobs have reached SCHED state, running jobs
-# have reached RUN state, and inactive jobs have reached INACTIVE
-# state.
-#
-# job ids for jobs in these states are expected to be in pending.ids,
-# running.ids, and inactive.ids respectively.
-
-job_list_wait_states() {
-        pending=$(job_list_state_count pending)
-        running=$(job_list_state_count running)
-        inactive=$(job_list_state_count inactive)
-        local i=0
-        while ( [ "$(flux job list --states=sched | wc -l)" != "$pending" ] \
-                || [ "$(flux job list --states=run | wc -l)" != "$running" ] \
-                || [ "$(flux job list --states=inactive | wc -l)" != "$inactive" ]) \
-               && [ $i -lt ${JOB_LIST_WAIT_ITERS} ]
-        do
-                sleep 0.1
-                i=$((i + 1))
-        done
-        if [ "$i" -eq "${JOB_LIST_WAIT_ITERS}" ]
-        then
-            return 1
-        fi
-        return 0
-}

t/t2260-job-list.t

@@ -18,21 +18,10 @@ fj_wait_event() {
 	flux job wait-event --timeout=20 "$@"
 }
 
+# To avoid raciness in tests, need to ensure that job state we care about
+# testing against has been reached in the job-list module.
 wait_jobid_state() {
-	local jobid=$(flux job id $1)
-	local state=$2
-	local i=0
-	while ! flux job list --states=${state} | grep $jobid > /dev/null \
-		   && [ $i -lt 50 ]
-	do
-		sleep 0.1
-		i=$((i + 1))
-	done
-	if [ "$i" -eq "50" ]
-	then
-		return 1
-	fi
-	return 0
+	flux job list-ids --wait-state=$2 $1 > /dev/null
 }
 
 # submit a whole bunch of jobs for job list testing
@@ -147,9 +136,13 @@ test_expect_success 'submit jobs for job list testing' '
 	cat active.ids > all.ids &&
 	cat inactive.ids >> all.ids &&
 	#
-	#  Synchronize all expected states
+	#  The job-list module has eventual consistency with the jobs stored in
+	#  the job-manager queue.  To ensure no raciness in tests, ensure
+	#  jobs above have reached expected states in job-list before continuing.
 	#
-	job_list_wait_states
+	flux job list-ids --wait-state=sched $(job_list_state_ids pending) &&
+	flux job list-ids --wait-state=run $(job_list_state_ids running) &&
+	flux job list-ids --wait-state=inactive $(job_list_state_ids inactive)
 '
 
 # Note: "running" = "run" | "cleanup", we also test just "run" state
@@ -772,7 +765,7 @@ test_expect_success 'flux job list job state timing outputs valid (job inactive)
 test_expect_success 'flux job list job state timing outputs valid (job running)' '
 	jobid=$(flux submit sleep 60 | flux job id) &&
 	fj_wait_event $jobid start >/dev/null &&
-	wait_jobid_state $jobid running &&
+	wait_jobid_state $jobid run &&
 	obj=$(flux job list -s running | grep $jobid) &&
 	echo $obj | jq -e ".t_submit < .t_depend" &&
 	echo $obj | jq -e ".t_depend < .t_run" &&

t/t2261-job-list-update.t

@@ -81,9 +81,13 @@ test_expect_success 'submit jobs for job list testing' '
         cat active.ids > all.ids &&
         cat inactive.ids >> all.ids &&
         #
-        #  Synchronize all expected states
+        #  The job-list module has eventual consistency with the jobs stored in
+        #  the job-manager queue.  To ensure no raciness in tests, ensure
+        #  jobs above have reached expected states in job-list before continuing.
         #
-        job_list_wait_states
+        flux job list-ids --wait-state=sched $(job_list_state_ids pending) &&
+        flux job list-ids --wait-state=run $(job_list_state_ids running) &&
+        flux job list-ids --wait-state=inactive $(job_list_state_ids inactive)
 '
 
 # Note: "running" = "run" | "cleanup", we also test just "run" state

t/t2800-jobs-cmd.t

@@ -23,32 +23,6 @@ runpty="${SHARNESS_TEST_SRCDIR}/scripts/runpty.py --line-buffer -f asciicast"
 #   - running jobs - by start time (most recent first)
 #   - inactive jobs - by completion time (most recent first)
 #
-# the job-list module has eventual consistency with the jobs stored in
-# the job-manager's queue.  To ensure no raciness in tests, we spin
-# until all of the pending jobs have reached SCHED state, running jobs
-# have reached RUN state, and inactive jobs have reached INACTIVE
-# state.
-#
-
-wait_states() {
-	sched=$(job_list_state_count sched)
-	run=$(job_list_state_count run)
-	inactive=$(job_list_state_count inactive)
-	local i=0
-	printf >&2 "Waiting for sched=$sched run=$run inactive=$inactive\n"
-	while ( [ "$(flux jobs -n --filter=sched | wc -l)" != "$sched" ] \
-	     || [ "$(flux jobs -n --filter=run | wc -l)" != "$run" ] \
-	     || [ "$(flux jobs -n --filter=inactive | wc -l)" != "$inactive" ]) \
-	&& [ $i -lt 50 ]
-	do
-		sleep 0.1
-		i=$((i + 1))
-	done
-	if [ "$i" -eq "50" ]; then
-		return 1
-	fi
-	return 0
-}
 
 export FLUX_PYCLI_LOGLEVEL=10
 
@@ -159,8 +133,13 @@ test_expect_success 'submit jobs for job list testing' '
 	tac inactiveids > inactive.ids &&
 	cat inactive.ids active.ids >> all.ids &&
 	#
-	#  Synchronize all expected states
-	wait_states
+	#  The job-list module has eventual consistency with the jobs stored in
+	#  the job-manager queue.  To ensure no raciness in tests, ensure
+	#  jobs above have reached expected states in job-list before continuing.
+	#
+	flux job list-ids --wait-state=sched $(job_list_state_ids sched) &&
+	flux job list-ids --wait-state=run $(job_list_state_ids run) &&
+	flux job list-ids --wait-state=inactive $(job_list_state_ids inactive)
 '
 
 #

t/t2801-top-cmd.t

@@ -14,22 +14,10 @@ export FLUX_URI_RESOLVE_LOCAL=t
 
 # To ensure no raciness in tests below, ensure the job-list
 # module knows about submitted jobs in desired states
-JOB_WAIT_ITERS=100
 job_list_wait_state() {
 	id=$1
-	expected=$2
-	local i=0
-	while [ "$(flux jobs -no {state} ${id})" != "${expected}" ] \
-	       && [ $i -lt ${JOB_WAIT_ITERS} ]
-	do
-		sleep 0.1
-		i=$((i + 1))
-	done
-	if [ "$i" -eq "${JOB_WAIT_ITERS}" ]
-	then
-	    return 1
-	fi
-	return 0
+	state=$2
+	flux job list-ids --wait-state=$2 $1 > /dev/null
 }
 
 test_expect_success 'flux-top -h prints custom usage' '