Expose a pending pods summary in scheudler's dummper output

pkg/scheduler/internal/cache/debugger/comparer.go

@@ -54,7 +54,7 @@ func (c *CacheComparer) Compare() error {
 
 	dump := c.Cache.Dump()
 
-	pendingPods := c.PodQueue.PendingPods()
+	pendingPods, _ := c.PodQueue.PendingPods()
 
 	if missed, redundant := c.CompareNodes(nodes, dump.Nodes); len(missed)+len(redundant) != 0 {
 		klog.InfoS("Cache mismatch", "missedNodes", missed, "redundantNodes", redundant)

pkg/scheduler/internal/cache/debugger/dumper.go

@@ -54,12 +54,12 @@ func (d *CacheDumper) dumpNodes() {
 
 // dumpSchedulingQueue writes pods in the scheduling queue to the scheduler logs.
 func (d *CacheDumper) dumpSchedulingQueue() {
-	pendingPods := d.podQueue.PendingPods()
+	pendingPods, s := d.podQueue.PendingPods()
 	var podData strings.Builder
 	for _, p := range pendingPods {
 		podData.WriteString(printPod(p))
 	}
-	klog.InfoS("Dump of scheduling queue", "pods", podData.String())
+	klog.InfoS("Dump of scheduling queue", "summary", s, "pods", podData.String())
 }
 
 // printNodeInfo writes parts of NodeInfo to a string.

pkg/scheduler/internal/queue/scheduling_queue.go

@@ -102,7 +102,7 @@ type SchedulingQueue interface {
 	MoveAllToActiveOrBackoffQueue(event framework.ClusterEvent, preCheck PreEnqueueCheck)
 	AssignedPodAdded(pod *v1.Pod)
 	AssignedPodUpdated(pod *v1.Pod)
-	PendingPods() []*v1.Pod
+	PendingPods() ([]*v1.Pod, string)
 	// Close closes the SchedulingQueue so that the goroutine which is
 	// waiting to pop items can exit gracefully.
 	Close()
@@ -678,9 +678,12 @@ func (p *PriorityQueue) getUnschedulablePodsWithMatchingAffinityTerm(pod *v1.Pod
 	return podsToMove
 }
 
-// PendingPods returns all the pending pods in the queue. This function is
-// used for debugging purposes in the scheduler cache dumper and comparer.
-func (p *PriorityQueue) PendingPods() []*v1.Pod {
+var pendingPodsSummary = "activeQ:%v; backoffQ:%v; unschedulablePods:%v"
+
+// PendingPods returns all the pending pods in the queue; accompanied by a debugging string
+// recording showing the number of pods in each queue respectively.
+// This function is used for debugging purposes in the scheduler cache dumper and comparer.
+func (p *PriorityQueue) PendingPods() ([]*v1.Pod, string) {
 	p.lock.RLock()
 	defer p.lock.RUnlock()
 	var result []*v1.Pod
@@ -693,7 +696,7 @@ func (p *PriorityQueue) PendingPods() []*v1.Pod {
 	for _, pInfo := range p.unschedulablePods.podInfoMap {
 		result = append(result, pInfo.Pod)
 	}
-	return result
+	return result, fmt.Sprintf(pendingPodsSummary, p.activeQ.Len(), p.podBackoffQ.Len(), len(p.unschedulablePods.podInfoMap))
 }
 
 // Close closes the priority queue.

pkg/scheduler/internal/queue/scheduling_queue_test.go

@@ -745,13 +745,21 @@ func TestPriorityQueue_PendingPods(t *testing.T) {
 	q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(highPriorityPodInfo.Pod), q.SchedulingCycle())
 
 	expectedSet := makeSet([]*v1.Pod{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod})
-	if !reflect.DeepEqual(expectedSet, makeSet(q.PendingPods())) {
+	gotPods, gotSummary := q.PendingPods()
+	if !reflect.DeepEqual(expectedSet, makeSet(gotPods)) {
 		t.Error("Unexpected list of pending Pods.")
 	}
+	if wantSummary := fmt.Sprintf(pendingPodsSummary, 1, 0, 2); wantSummary != gotSummary {
+		t.Errorf("Unexpected pending pods summary: want %v, but got %v.", wantSummary, gotSummary)
+	}
 	// Move all to active queue. We should still see the same set of pods.
 	q.MoveAllToActiveOrBackoffQueue(TestEvent, nil)
-	if !reflect.DeepEqual(expectedSet, makeSet(q.PendingPods())) {
-		t.Error("Unexpected list of pending Pods...")
+	gotPods, gotSummary = q.PendingPods()
+	if !reflect.DeepEqual(expectedSet, makeSet(gotPods)) {
+		t.Error("Unexpected list of pending Pods.")
+	}
+	if wantSummary := fmt.Sprintf(pendingPodsSummary, 1, 2, 0); wantSummary != gotSummary {
+		t.Errorf("Unexpected pending pods summary: want %v, but got %v.", wantSummary, gotSummary)
 	}
 }
 

pkg/scheduler/scheduler_test.go

@@ -418,7 +418,7 @@ func TestFailureHandler_PodAlreadyBound(t *testing.T) {
 // getPodFromPriorityQueue is the function used in the TestDefaultErrorFunc test to get
 // the specific pod from the given priority queue. It returns the found pod in the priority queue.
 func getPodFromPriorityQueue(queue *internalqueue.PriorityQueue, pod *v1.Pod) *v1.Pod {
-	podList := queue.PendingPods()
+	podList, _ := queue.PendingPods()
 	if len(podList) == 0 {
 		return nil
 	}

test/integration/scheduler/filters/filters_test.go

@@ -1713,9 +1713,9 @@ func TestUnschedulablePodBecomesSchedulable(t *testing.T) {
 				t.Errorf("Pod %v was not scheduled: %v", pod.Name, err)
 			}
 			// Make sure pending queue is empty.
-			pendingPods := len(testCtx.Scheduler.SchedulingQueue.PendingPods())
-			if pendingPods != 0 {
-				t.Errorf("pending pods queue is not empty, size is: %d", pendingPods)
+			pendingPods, s := testCtx.Scheduler.SchedulingQueue.PendingPods()
+			if len(pendingPods) != 0 {
+				t.Errorf("pending pods queue is not empty, size is: %d, summary is: %s", len(pendingPods), s)
 			}
 		})
 	}

test/integration/scheduler/queue_test.go

@@ -88,7 +88,8 @@ func TestCoreResourceEnqueue(t *testing.T) {
 
 	// Wait for the three pods to be present in the scheduling queue.
 	if err := wait.Poll(time.Millisecond*200, wait.ForeverTestTimeout, func() (bool, error) {
-		return len(testCtx.Scheduler.SchedulingQueue.PendingPods()) == 3, nil
+		pendingPods, _ := testCtx.Scheduler.SchedulingQueue.PendingPods()
+		return len(pendingPods) == 3, nil
 	}); err != nil {
 		t.Fatal(err)
 	}
@@ -263,7 +264,8 @@ func TestCustomResourceEnqueue(t *testing.T) {
 
 	// Wait for the testing Pod to be present in the scheduling queue.
 	if err := wait.Poll(time.Millisecond*200, wait.ForeverTestTimeout, func() (bool, error) {
-		return len(testCtx.Scheduler.SchedulingQueue.PendingPods()) == 1, nil
+		pendingPods, _ := testCtx.Scheduler.SchedulingQueue.PendingPods()
+		return len(pendingPods) == 1, nil
 	}); err != nil {
 		t.Fatal(err)
 	}