Merge branch 'master' into fix-some-typo

README.md

@@ -126,6 +126,20 @@ make TAG=latest generate-yaml
 kubectl create -f _output/release/volcano-monitoring-latest.yaml
 ```
 
+## Kubernetes compatibility
+
+|                        | Kubernetes 1.17 | Kubernetes 1.18 | Kubernetes 1.19 | Kubernetes 1.20 | Kubernetes 1.21 | Kubernetes 1.22 | Kubernetes 1.23 | Kubernetes 1.24 | Kubernetes 1.25 |
+|------------------------|-----------------|-----------------|-----------------|-----------------|-----------------|-----------------|-----------------|-----------------|-----------------|
+| Volcano v1.6          | ✓               | ✓               | ✓               | ✓               | ✓               | ✓               | ✓               | -               | -               |
+| Volcano v1.7          | -               | -               | ✓               | ✓               | ✓               | ✓               | ✓               |✓               |✓               |
+| Volcano HEAD (master) | -               | -               | ✓               | ✓               | ✓               | ✓               | ✓               |✓               |✓               |
+
+Key:
+* `✓` Volcano and the Kubernetes version are exactly compatible.
+* `+` Volcano has features or API objects that may not be present in the Kubernetes version.
+* `-` The Kubernetes version has features or API objects that Volcano can't use.
+
+
 ## Meeting
 
 Community weekly meeting for Asia: 15:00 - 16:00 (UTC+8) Friday. ([Convert to your timezone.](https://www.thetimezoneconverter.com/?t=10%3A00&tz=GMT%2B8&))

hack/verify-golangci-lint.sh

@@ -43,7 +43,22 @@ function check_golangci-lint() {
 function golangci-lint_run() {
   echo "begin run golangci-lint"
   cd ${KUBE_ROOT}
-  golangci-lint run -v
+  ret=0
+  golangci-lint run -v || ret=$?
+  if [ $ret -eq 0 ]; then
+    echo "SUCCESS: golangci-lint verified."
+  else
+    echo "FAILED: golangci-lint stale."
+    echo
+    echo "Please review the above warnings. You can test via './hack/verify-golangci-lint.sh' or 'make lint'."
+    echo "If the above warnings do not make sense, you can exempt this warning with a comment"
+    echo " (if your reviewer is okay with it)."
+    echo "In general please prefer to fix the error, we have already disabled specific lints"
+    echo " that the project chooses to ignore."
+    echo "See: https://golangci-lint.run/usage/false-positives/"
+    echo
+    exit 1
+  fi
 }
 
 set +e

pkg/controllers/podgroup/pg_controller_handler.go

@@ -28,8 +28,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
-	batchv1alpha1 "volcano.sh/apis/pkg/apis/batch/v1alpha1"
 
+	batchv1alpha1 "volcano.sh/apis/pkg/apis/batch/v1alpha1"
 	"volcano.sh/apis/pkg/apis/helpers"
 	scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
 	"volcano.sh/volcano/pkg/controllers/util"

pkg/scheduler/actions/allocate/allocate.go

@@ -98,8 +98,8 @@ func (alloc *Action) Execute(ssn *framework.Session) {
 	allNodes := ssn.NodeList
 	predicateFn := func(task *api.TaskInfo, node *api.NodeInfo) error {
 		// Check for Resource Predicate
-		if !task.InitResreq.LessEqual(node.FutureIdle(), api.Zero) {
-			return api.NewFitError(task, node, api.NodeResourceFitFailed)
+		if ok, reason := task.InitResreq.LessEqualWithReason(node.FutureIdle(), api.Zero); !ok {
+			return api.NewFitError(task, node, reason)
 		}
 
 		return ssn.PredicateFn(task, node)
@@ -247,7 +247,6 @@ func (alloc *Action) Execute(ssn *framework.Session) {
 				stmt.Discard()
 			}
 		}
-
 	}
 }
 

pkg/scheduler/api/devices/nvidia/gpushare/device_info.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
+
 	"volcano.sh/volcano/pkg/scheduler/plugins/util/nodelock"
 )
 
@@ -133,7 +134,6 @@ func (gs *GPUDevices) Release(kubeClient kubernetes.Interface, pod *v1.Pod) erro
 	_, err := kubeClient.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
 	if err != nil {
 		return errors.Errorf("patch pod %s failed with patch %s: %v", pod.Name, patch, err)
-
 	}
 
 	for _, id := range ids {
@@ -189,7 +189,6 @@ func (gs *GPUDevices) Allocate(kubeClient kubernetes.Interface, pod *v1.Pod) err
 		pod, err := kubeClient.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
 		if err != nil {
 			return errors.Errorf("patch pod %s failed with patch %s: %v", pod.Name, patch, err)
-
 		}
 		dev, ok := gs.Device[id]
 		if !ok {
@@ -202,7 +201,6 @@ func (gs *GPUDevices) Allocate(kubeClient kubernetes.Interface, pod *v1.Pod) err
 		ids := predicateGPUbyNumber(pod, gs)
 		if len(ids) == 0 {
 			return errors.Errorf("the node %s can't place the pod %s in ns %s", pod.Spec.NodeName, pod.Name, pod.Namespace)
-
 		}
 		patch := AddGPUIndexPatch(ids)
 		pod, err := kubeClient.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})

pkg/scheduler/api/job_info_test.go

@@ -231,7 +231,7 @@ func TestTaskSchedulingReason(t *testing.T) {
 				t3.UID: "Pod ns1/task-3 can possibly be assigned to node1",
 				t4.UID: "Pod ns1/task-4 can possibly be assigned to node2",
 				t5.UID: "Pod ns1/task-5 can possibly be assigned to node3",
-				t6.UID: "all nodes are unavailable: 1 node(s) pod number exceeded, 2 node(s) resource fit failed.",
+				t6.UID: "0/3 nodes are unavailable: 1 node(s) pod number exceeded, 2 node(s) resource fit failed.",
 			},
 		},
 	}

pkg/scheduler/api/resource_info.go

@@ -389,6 +389,38 @@ func (r *Resource) LessEqual(rr *Resource, defaultValue DimensionDefaultValue) b
 	return true
 }
 
+// LessEqualWithReason returns true, "" only on condition that all dimensions of resources in r are less than or equal with that of rr,
+// Otherwise returns false and err string ,which show which resource is insufficient.
+// @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"
+// this function is the same as LessEqual , and it will be merged to LessEqual in the future
+func (r *Resource) LessEqualWithReason(rr *Resource, defaultValue DimensionDefaultValue) (bool, string) {
+	lessEqualFunc := func(l, r, diff float64) bool {
+		if l < r || math.Abs(l-r) < diff {
+			return true
+		}
+		return false
+	}
+
+	if !lessEqualFunc(r.MilliCPU, rr.MilliCPU, minResource) {
+		return false, "Insufficient cpu"
+	}
+	if !lessEqualFunc(r.Memory, rr.Memory, minResource) {
+		return false, "Insufficient memory"
+	}
+
+	for resourceName, leftValue := range r.ScalarResources {
+		rightValue, ok := rr.ScalarResources[resourceName]
+		if !ok && defaultValue == Infinity {
+			continue
+		}
+
+		if !lessEqualFunc(leftValue, rightValue, minResource) {
+			return false, "Insufficient " + string(resourceName)
+		}
+	}
+	return true, ""
+}
+
 // LessPartly returns true if there exists any dimension whose resource amount in r is less than that in rr.
 // Otherwise returns false.
 // @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"

pkg/scheduler/api/resource_info_test.go

@@ -1227,3 +1227,143 @@ func TestMinDimensionResourceInfinity(t *testing.T) {
 		}
 	}
 }
+
+func TestResource_LessEqualResource(t *testing.T) {
+	testsForDefaultZero := []struct {
+		resource1 *Resource
+		resource2 *Resource
+		expected  string
+	}{
+		{
+			resource1: &Resource{},
+			resource2: &Resource{},
+			expected:  "",
+		},
+		{
+			resource1: &Resource{},
+			resource2: &Resource{
+				MilliCPU:        4000,
+				Memory:          2000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 1000, "hugepages-test": 2000},
+			},
+			expected: "",
+		},
+		{
+			resource1: &Resource{
+				MilliCPU:        4000,
+				Memory:          2000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 1000, "hugepages-test": 2000},
+			},
+			resource2: &Resource{},
+			expected:  "Insufficient cpu",
+		},
+		{
+			resource1: &Resource{
+				MilliCPU:        4000,
+				Memory:          4000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 1000, "hugepages-test": 2000},
+			},
+			resource2: &Resource{
+				MilliCPU:        8000,
+				Memory:          8000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 4000, "hugepages-test": 5000},
+			},
+			expected: "",
+		},
+		{
+			resource1: &Resource{
+				MilliCPU:        4000,
+				Memory:          8000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 1000, "hugepages-test": 2000},
+			},
+			resource2: &Resource{
+				MilliCPU:        8000,
+				Memory:          8000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 4000, "hugepages-test": 5000},
+			},
+			expected: "",
+		},
+		{
+			resource1: &Resource{
+				MilliCPU:        4000,
+				Memory:          4000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 4000, "hugepages-test": 2000},
+			},
+			resource2: &Resource{
+				MilliCPU:        8000,
+				Memory:          8000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 4000, "hugepages-test": 5000},
+			},
+			expected: "",
+		},
+		{
+			resource1: &Resource{
+				MilliCPU:        4000,
+				Memory:          4000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 5000, "hugepages-test": 2000},
+			},
+			resource2: &Resource{
+				MilliCPU:        8000,
+				Memory:          8000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 4000, "hugepages-test": 5000},
+			},
+			expected: "Insufficient scalar.test/scalar1",
+		},
+		{
+			resource1: &Resource{
+				MilliCPU:        9000,
+				Memory:          4000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 1000, "hugepages-test": 2000},
+			},
+			resource2: &Resource{
+				MilliCPU:        8000,
+				Memory:          8000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 4000, "hugepages-test": 5000},
+			},
+			expected: "Insufficient cpu",
+		},
+	}
+
+	testsForDefaultInfinity := []struct {
+		resource1 *Resource
+		resource2 *Resource
+		expected  string
+	}{
+		{
+			resource1: &Resource{},
+			resource2: &Resource{},
+			expected:  "",
+		},
+		{
+			resource1: &Resource{},
+			resource2: &Resource{
+				MilliCPU:        4000,
+				Memory:          2000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 1000, "hugepages-test": 2000},
+			},
+			expected: "",
+		},
+		{
+			resource1: &Resource{
+				MilliCPU:        4000,
+				Memory:          2000,
+				ScalarResources: map[v1.ResourceName]float64{"scalar.test/scalar1": 1000, "hugepages-test": 2000},
+			},
+			resource2: &Resource{},
+			expected:  "Insufficient cpu",
+		},
+	}
+
+	for _, test := range testsForDefaultZero {
+		_, reason := test.resource1.LessEqualWithReason(test.resource2, Zero)
+		if !reflect.DeepEqual(test.expected, reason) {
+			t.Errorf("expected: %#v, got: %#v", test.expected, reason)
+		}
+	}
+	for caseID, test := range testsForDefaultInfinity {
+		_, reason := test.resource1.LessEqualWithReason(test.resource2, Infinity)
+		if !reflect.DeepEqual(test.expected, reason) {
+			t.Errorf("caseID %d expected: %#v, got: %#v", caseID, test.expected, reason)
+		}
+	}
+}

pkg/scheduler/api/unschedule_info.go

@@ -66,7 +66,7 @@ func (f *FitErrors) SetNodeError(nodeName string, err error) {
 // Error returns the final error message
 func (f *FitErrors) Error() string {
 	if f.err == "" {
-		f.err = AllNodeUnavailableMsg
+		f.err = fmt.Sprintf("0/%v", len(f.nodes)) + " nodes are unavailable"
 	}
 	if len(f.nodes) == 0 {
 		return f.err

pkg/scheduler/cache/cache.go

@@ -24,7 +24,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-	"volcano.sh/volcano/pkg/scheduler/metrics/source"
 
 	v1 "k8s.io/api/core/v1"
 	schedulingv1 "k8s.io/api/scheduling/v1"
@@ -59,10 +58,12 @@ import (
 	vcinformer "volcano.sh/apis/pkg/client/informers/externalversions"
 	cpuinformerv1 "volcano.sh/apis/pkg/client/informers/externalversions/nodeinfo/v1alpha1"
 	vcinformerv1 "volcano.sh/apis/pkg/client/informers/externalversions/scheduling/v1beta1"
+
 	"volcano.sh/volcano/cmd/scheduler/app/options"
 	schedulingapi "volcano.sh/volcano/pkg/scheduler/api"
 	volumescheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding"
 	"volcano.sh/volcano/pkg/scheduler/metrics"
+	"volcano.sh/volcano/pkg/scheduler/metrics/source"
 	commonutil "volcano.sh/volcano/pkg/util"
 )
 
@@ -592,6 +593,13 @@ func newSchedulerCache(config *rest.Config, schedulerNames []string, defaultQueu
 						}
 					}
 					return true
+				case cache.DeletedFinalStateUnknown:
+					if _, ok := v.Obj.(*v1.Pod); ok {
+						// The carried object may be stale, always pass to clean up stale obj in event handlers.
+						return true
+					}
+					klog.Errorf("Cannot convert object %T to *v1.Pod", v.Obj)
+					return false
 				default:
 					return false
 				}
@@ -1284,8 +1292,8 @@ func (sc *SchedulerCache) GetMetricsData() {
 				klog.Errorf("Error getting node metrics: %v\n", err)
 				continue
 			}
-			klog.V(4).Infof("node: %v, CpuUsageAvg: %v, MemUsageAvg: %v, period:%v", node, nodeMetrics.Cpu, nodeMetrics.Memory, period)
-			nodeUsageMap[node].CPUUsageAvg[period] = nodeMetrics.Cpu
+			klog.V(4).Infof("node: %v, CpuUsageAvg: %v, MemUsageAvg: %v, period:%v", node, nodeMetrics.CPU, nodeMetrics.Memory, period)
+			nodeUsageMap[node].CPUUsageAvg[period] = nodeMetrics.CPU
 			nodeUsageMap[node].MEMUsageAvg[period] = nodeMetrics.Memory
 		}
 	}

pkg/scheduler/cache/dumper.go

@@ -23,6 +23,7 @@ import (
 	"syscall"
 
 	"k8s.io/klog/v2"
+
 	"volcano.sh/volcano/pkg/scheduler/api"
 )
 

pkg/scheduler/metrics/source/metrics_client.go

@@ -22,7 +22,7 @@ import (
 )
 
 type NodeMetrics struct {
-	Cpu    float64
+	CPU    float64
 	Memory float64
 }