koord-scheduler: strictly check whether ReservationInfo is available or terminating

pkg/scheduler/frameworkext/reservation_info.go

@@ -23,6 +23,7 @@ import (
 	quotav1 "k8s.io/apiserver/pkg/quota/v1"
 	corev1helpers "k8s.io/component-helpers/scheduling/corev1"
 	"k8s.io/klog/v2"
+	k8spodutil "k8s.io/kubernetes/pkg/api/v1/pod"
 	"k8s.io/kubernetes/pkg/api/v1/resource"
 
 	apiext "github.com/koordinator-sh/koordinator/apis/extension"
@@ -168,12 +169,16 @@ func (ri *ReservationInfo) IsAvailable() bool {
 	if ri.Reservation != nil {
 		return reservationutil.IsReservationAvailable(ri.Reservation)
 	}
-	if ri.Pod != nil {
+	if ri.Pod != nil && ri.Pod.Status.Phase == corev1.PodRunning && k8spodutil.IsPodReady(ri.Pod) {
 		return true
 	}
 	return false
 }
 
+func (ri *ReservationInfo) IsTerminating() bool {
+	return !ri.GetObject().GetDeletionTimestamp().IsZero()
+}
+
 func (ri *ReservationInfo) Clone() *ReservationInfo {
 	resourceNames := make([]corev1.ResourceName, 0, len(ri.ResourceNames))
 	for _, v := range ri.ResourceNames {

pkg/scheduler/frameworkext/reservation_info_test.go

@@ -0,0 +1,257 @@
+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package frameworkext
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	schedulingv1alpha1 "github.com/koordinator-sh/koordinator/apis/scheduling/v1alpha1"
+)
+
+func TestIsAvailable(t *testing.T) {
+	tests := []struct {
+		name string
+		obj  metav1.Object
+		want bool
+	}{
+		{
+			name: "normal reservation",
+			obj: &schedulingv1alpha1.Reservation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-r",
+					UID:  "123456",
+				},
+				Spec: schedulingv1alpha1.ReservationSpec{
+					Template: &corev1.PodTemplateSpec{},
+				},
+				Status: schedulingv1alpha1.ReservationStatus{
+					Phase:    schedulingv1alpha1.ReservationAvailable,
+					NodeName: "test-node",
+				},
+			},
+			want: true,
+		},
+		{
+			name: "failed reservation",
+			obj: &schedulingv1alpha1.Reservation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-r",
+					UID:  "123456",
+				},
+				Spec: schedulingv1alpha1.ReservationSpec{
+					Template: &corev1.PodTemplateSpec{},
+				},
+				Status: schedulingv1alpha1.ReservationStatus{
+					Phase:    schedulingv1alpha1.ReservationFailed,
+					NodeName: "test-node",
+				},
+			},
+			want: false,
+		},
+		{
+			name: "ready and running pod",
+			obj: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-p",
+					Namespace: "default",
+					UID:       "123456",
+				},
+				Spec: corev1.PodSpec{
+					NodeName: "test-node",
+				},
+				Status: corev1.PodStatus{
+					Phase: corev1.PodRunning,
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionTrue,
+						},
+					},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "not ready and running pod",
+			obj: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-p",
+					Namespace: "default",
+					UID:       "123456",
+				},
+				Spec: corev1.PodSpec{
+					NodeName: "test-node",
+				},
+				Status: corev1.PodStatus{
+					Phase: corev1.PodRunning,
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionFalse,
+						},
+					},
+				},
+			},
+			want: false,
+		},
+		{
+			name: "not ready and not running pod",
+			obj: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-p",
+					Namespace: "default",
+					UID:       "123456",
+				},
+				Spec: corev1.PodSpec{
+					NodeName: "test-node",
+				},
+				Status: corev1.PodStatus{
+					Phase: corev1.PodPending,
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionFalse,
+						},
+					},
+				},
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var rInfo *ReservationInfo
+			switch obj := tt.obj.(type) {
+			case *schedulingv1alpha1.Reservation:
+				rInfo = NewReservationInfo(obj)
+			case *corev1.Pod:
+				rInfo = NewReservationInfoFromPod(obj)
+			}
+			assert.NotNil(t, rInfo)
+			assert.Equal(t, tt.want, rInfo.IsAvailable())
+		})
+	}
+}
+
+func TestIsTerminating(t *testing.T) {
+	tests := []struct {
+		name string
+		obj  metav1.Object
+		want bool
+	}{
+		{
+			name: "normal reservation",
+			obj: &schedulingv1alpha1.Reservation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-r",
+					UID:  "123456",
+				},
+				Spec: schedulingv1alpha1.ReservationSpec{
+					Template: &corev1.PodTemplateSpec{},
+				},
+				Status: schedulingv1alpha1.ReservationStatus{
+					Phase:    schedulingv1alpha1.ReservationAvailable,
+					NodeName: "test-node",
+				},
+			},
+			want: false,
+		},
+		{
+			name: "deleting reservation",
+			obj: &schedulingv1alpha1.Reservation{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "test-r",
+					UID:               "123456",
+					DeletionTimestamp: &metav1.Time{Time: time.Now()},
+				},
+				Spec: schedulingv1alpha1.ReservationSpec{
+					Template: &corev1.PodTemplateSpec{},
+				},
+				Status: schedulingv1alpha1.ReservationStatus{
+					Phase:    schedulingv1alpha1.ReservationAvailable,
+					NodeName: "test-node",
+				},
+			},
+			want: true,
+		},
+		{
+			name: "ready and running pod",
+			obj: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-p",
+					Namespace: "default",
+					UID:       "123456",
+				},
+				Spec: corev1.PodSpec{
+					NodeName: "test-node",
+				},
+				Status: corev1.PodStatus{
+					Phase: corev1.PodRunning,
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionTrue,
+						},
+					},
+				},
+			},
+			want: false,
+		},
+		{
+			name: "ready, running and deleting pod",
+			obj: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "test-p",
+					Namespace:         "default",
+					UID:               "123456",
+					DeletionTimestamp: &metav1.Time{Time: time.Now()},
+				},
+				Spec: corev1.PodSpec{
+					NodeName: "test-node",
+				},
+				Status: corev1.PodStatus{
+					Phase: corev1.PodRunning,
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionTrue,
+						},
+					},
+				},
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var rInfo *ReservationInfo
+			switch obj := tt.obj.(type) {
+			case *schedulingv1alpha1.Reservation:
+				rInfo = NewReservationInfo(obj)
+			case *corev1.Pod:
+				rInfo = NewReservationInfoFromPod(obj)
+			}
+			assert.NotNil(t, rInfo)
+			assert.Equal(t, tt.want, rInfo.IsTerminating())
+		})
+	}
+}

pkg/scheduler/plugins/reservation/cache.go

@@ -17,6 +17,7 @@ limitations under the License.
 package reservation
 
 import (
+	"fmt"
 	"sync"
 
 	corev1 "k8s.io/api/core/v1"
@@ -129,22 +130,27 @@ func (cache *reservationCache) deleteReservationOperatingPod(pod *corev1.Pod) {
 	cache.deleteReservationOnNode(pod.Spec.NodeName, pod.UID)
 }
 
-func (cache *reservationCache) assumePod(reservationUID types.UID, pod *corev1.Pod) {
-	cache.addPod(reservationUID, pod)
+func (cache *reservationCache) assumePod(reservationUID types.UID, pod *corev1.Pod) error {
+	return cache.addPod(reservationUID, pod)
 }
 
 func (cache *reservationCache) forgetPod(reservationUID types.UID, pod *corev1.Pod) {
 	cache.deletePod(reservationUID, pod)
 }
 
-func (cache *reservationCache) addPod(reservationUID types.UID, pod *corev1.Pod) {
+func (cache *reservationCache) addPod(reservationUID types.UID, pod *corev1.Pod) error {
 	cache.lock.Lock()
 	defer cache.lock.Unlock()
 
 	rInfo := cache.reservationInfos[reservationUID]
-	if rInfo != nil {
-		rInfo.AddAssignedPod(pod)
+	if rInfo == nil {
+		return fmt.Errorf("cannot find target reservation")
 	}
+	if rInfo.IsTerminating() {
+		return fmt.Errorf("target reservation is terminating")
+	}
+	rInfo.AddAssignedPod(pod)
+	return nil
 }
 
 func (cache *reservationCache) updatePod(reservationUID types.UID, oldPod, newPod *corev1.Pod) {

pkg/scheduler/plugins/reservation/plugin.go

@@ -420,15 +420,15 @@ func (pl *Plugin) Reserve(ctx context.Context, cycleState *framework.CycleState,
 		return nil
 	}
 
-	// NOTE: Having entered the Reserve stage means that the Pod scheduling is successful,
-	// even though the associated Reservation may have expired, but in fact the real impact
-	// will not be encountered until the next round of scheduling.
-	assumed := nominatedReservation.Clone()
-	pl.reservationCache.assumePod(assumed.UID(), pod)
+	err := pl.reservationCache.assumePod(nominatedReservation.UID(), pod)
+	if err != nil {
+		klog.ErrorS(err, "Failed to assume pod in reservationCache", "pod", klog.KObj(pod), "reservation", klog.KObj(nominatedReservation))
+		return framework.AsStatus(err)
+	}
 
 	state := getStateData(cycleState)
-	state.assumed = assumed
-	klog.V(4).InfoS("Reserve pod to node with reservations", "pod", klog.KObj(pod), "node", nodeName, "assumed", klog.KObj(assumed))
+	state.assumed = nominatedReservation.Clone()
+	klog.V(4).InfoS("Reserve pod to node with reservations", "pod", klog.KObj(pod), "node", nodeName, "assumed", klog.KObj(nominatedReservation))
 	return nil
 }
 

pkg/scheduler/plugins/reservation/service_test.go

@@ -139,6 +139,15 @@ func TestQueryNodeReservations(t *testing.T) {
 				},
 			},
 		},
+		Status: corev1.PodStatus{
+			Phase: corev1.PodRunning,
+			Conditions: []corev1.PodCondition{
+				{
+					Type:   corev1.PodReady,
+					Status: corev1.ConditionTrue,
+				},
+			},
+		},
 	}
 
 	pl.reservationCache.updateReservationOperatingPod(operatingPod, &corev1.ObjectReference{

pkg/scheduler/plugins/reservation/transformer.go

@@ -98,7 +98,7 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState *
 				continue
 			}
 
-			if !isReservedPod && matchReservation(pod, node, rInfo, reservationAffinity) {
+			if !isReservedPod && !rInfo.IsTerminating() && matchReservation(pod, node, rInfo, reservationAffinity) {
 				matched = append(matched, rInfo)
 
 			} else if len(rInfo.AssignedPods) > 0 {