diff --git a/cluster-autoscaler/cluster_autoscaler.go b/cluster-autoscaler/cluster_autoscaler.go index b829156cfa..0188af189f 100644 --- a/cluster-autoscaler/cluster_autoscaler.go +++ b/cluster-autoscaler/cluster_autoscaler.go @@ -252,7 +252,6 @@ func run(_ <-chan struct{}) { loopStart := time.Now() updateLastTime("main") - // TODO: remove once switched to all nodes. readyNodes, err := readyNodeLister.List() if err != nil { glog.Errorf("Failed to list ready nodes: %v", err) @@ -316,12 +315,6 @@ func run(_ <-chan struct{}) { continue } - // TODO: remove once all of the unready node handling elements are in place. - if err := CheckGroupsAndNodes(readyNodes, autoscalingContext.CloudProvider); err != nil { - glog.Warningf("Cluster is not ready for autoscaling: %v", err) - continue - } - allUnschedulablePods, err := unschedulablePodLister.List() if err != nil { glog.Errorf("Failed to list unscheduled pods: %v", err) @@ -405,7 +398,7 @@ func run(_ <-chan struct{}) { glog.V(4).Infof("Calculating unneeded nodes") scaleDown.CleanUp(time.Now()) - err := scaleDown.UpdateUnneededNodes(readyNodes, allScheduled, time.Now()) + err := scaleDown.UpdateUnneededNodes(allNodes, allScheduled, time.Now()) if err != nil { glog.Warningf("Failed to scale down: %v", err) continue @@ -424,7 +417,7 @@ func run(_ <-chan struct{}) { scaleDownStart := time.Now() updateLastTime("scaledown") - result, err := scaleDown.TryToScaleDown(readyNodes, allScheduled) + result, err := scaleDown.TryToScaleDown(allNodes, allScheduled) updateDuration("scaledown", scaleDownStart) // TODO: revisit result handling diff --git a/cluster-autoscaler/clusterstate/clusterstate.go b/cluster-autoscaler/clusterstate/clusterstate.go index e7ab085f3e..0ecf0146ec 100644 --- a/cluster-autoscaler/clusterstate/clusterstate.go +++ b/cluster-autoscaler/clusterstate/clusterstate.go @@ -17,13 +17,13 @@ limitations under the License. package clusterstate import ( - "fmt" "reflect" "sync" "time" "k8s.io/contrib/cluster-autoscaler/cloudprovider" "k8s.io/contrib/cluster-autoscaler/utils/deletetaint" + kube_util "k8s.io/contrib/cluster-autoscaler/utils/kubernetes" apiv1 "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/util/sets" @@ -322,7 +322,7 @@ func (csr *ClusterStateRegistry) updateReadinessStats(currentTime time.Time) { for _, node := range csr.nodes { nodeGroup, errNg := csr.cloudProvider.NodeGroupForNode(node) - ready, _, errReady := GetReadinessState(node) + ready, _, errReady := kube_util.GetReadinessState(node) // Node is most likely not autoscaled, however check the errors. if reflect.ValueOf(nodeGroup).IsNil() { @@ -399,19 +399,6 @@ func (csr *ClusterStateRegistry) GetUnregisteredNodes() []UnregisteredNode { return result } -// GetReadinessState gets readiness state for the node -func GetReadinessState(node *apiv1.Node) (isNodeReady bool, lastTransitionTime time.Time, err error) { - for _, condition := range node.Status.Conditions { - if condition.Type == apiv1.NodeReady { - if condition.Status == apiv1.ConditionTrue { - return true, condition.LastTransitionTime.Time, nil - } - return false, condition.LastTransitionTime.Time, nil - } - } - return false, time.Time{}, fmt.Errorf("NodeReady condition for %s not found", node.Name) -} - func isNodeNotStarted(node *apiv1.Node) bool { for _, condition := range node.Status.Conditions { if condition.Type == apiv1.NodeReady && diff --git a/cluster-autoscaler/estimator/binpacking_estimator_test.go b/cluster-autoscaler/estimator/binpacking_estimator_test.go index f708840e3e..2676fc9b17 100644 --- a/cluster-autoscaler/estimator/binpacking_estimator_test.go +++ b/cluster-autoscaler/estimator/binpacking_estimator_test.go @@ -18,8 +18,10 @@ package estimator import ( "testing" + "time" "k8s.io/contrib/cluster-autoscaler/simulator" + . "k8s.io/contrib/cluster-autoscaler/utils/test" "k8s.io/kubernetes/pkg/api/resource" apiv1 "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache" @@ -48,6 +50,7 @@ func TestBinpackingEstimate(t *testing.T) { }, } node.Status.Allocatable = node.Status.Capacity + SetNodeReadyState(node, true, time.Time{}) nodeInfo := schedulercache.NewNodeInfo() nodeInfo.SetNode(node) @@ -76,6 +79,7 @@ func TestBinpackingEstimateComingNodes(t *testing.T) { }, } node.Status.Allocatable = node.Status.Capacity + SetNodeReadyState(node, true, time.Time{}) nodeInfo := schedulercache.NewNodeInfo() nodeInfo.SetNode(node) @@ -109,6 +113,7 @@ func TestBinpackingEstimateWithPorts(t *testing.T) { }, } node.Status.Allocatable = node.Status.Capacity + SetNodeReadyState(node, true, time.Time{}) nodeInfo := schedulercache.NewNodeInfo() nodeInfo.SetNode(node) diff --git a/cluster-autoscaler/expander/waste/waste_test.go b/cluster-autoscaler/expander/waste/waste_test.go index c5401c93e2..0e04109850 100644 --- a/cluster-autoscaler/expander/waste/waste_test.go +++ b/cluster-autoscaler/expander/waste/waste_test.go @@ -18,6 +18,9 @@ package waste import ( "testing" + "time" + + . "k8s.io/contrib/cluster-autoscaler/utils/test" "github.com/stretchr/testify/assert" "k8s.io/contrib/cluster-autoscaler/expander" @@ -51,6 +54,7 @@ func makeNodeInfo(cpu int64, memory int64, pods int64) *schedulercache.NodeInfo }, } node.Status.Allocatable = node.Status.Capacity + SetNodeReadyState(node, true, time.Time{}) nodeInfo := schedulercache.NewNodeInfo() nodeInfo.SetNode(node) diff --git a/cluster-autoscaler/scale_down.go b/cluster-autoscaler/scale_down.go index 569f1f45fb..1898b6f9a6 100644 --- a/cluster-autoscaler/scale_down.go +++ b/cluster-autoscaler/scale_down.go @@ -26,6 +26,7 @@ import ( "k8s.io/contrib/cluster-autoscaler/clusterstate" "k8s.io/contrib/cluster-autoscaler/simulator" "k8s.io/contrib/cluster-autoscaler/utils/deletetaint" + kube_util "k8s.io/contrib/cluster-autoscaler/utils/kubernetes" "k8s.io/kubernetes/pkg/api/errors" apiv1 "k8s.io/kubernetes/pkg/api/v1" @@ -153,7 +154,7 @@ func (sd *ScaleDown) TryToScaleDown(nodes []*apiv1.Node, pods []*apiv1.Pod) (Sca glog.V(2).Infof("%s was unneeded for %s", node.Name, now.Sub(val).String()) - ready, _, _ := clusterstate.GetReadinessState(node) + ready, _, _ := kube_util.GetReadinessState(node) // Check how long the node was underutilized. if ready && !val.Add(sd.context.ScaleDownUnneededTime).Before(now) { diff --git a/cluster-autoscaler/scale_down_test.go b/cluster-autoscaler/scale_down_test.go index e9462d428b..c51446fd43 100644 --- a/cluster-autoscaler/scale_down_test.go +++ b/cluster-autoscaler/scale_down_test.go @@ -63,6 +63,11 @@ func TestFindUnneededNodes(t *testing.T) { n3 := BuildTestNode("n3", 1000, 10) n4 := BuildTestNode("n4", 10000, 10) + SetNodeReadyState(n1, true, time.Time{}) + SetNodeReadyState(n2, true, time.Time{}) + SetNodeReadyState(n3, true, time.Time{}) + SetNodeReadyState(n4, true, time.Time{}) + context := AutoscalingContext{ PredicateChecker: simulator.NewTestPredicateChecker(), ScaleDownUtilizationThreshold: 0.35, @@ -94,6 +99,7 @@ func TestDrainNode(t *testing.T) { p1 := BuildTestPod("p1", 100, 0) p2 := BuildTestPod("p2", 300, 0) n1 := BuildTestNode("n1", 1000, 1000) + SetNodeReadyState(n1, true, time.Time{}) fakeClient.Fake.AddReactor("list", "pods", func(action core.Action) (bool, runtime.Object, error) { return true, &apiv1.PodList{Items: []apiv1.Pod{*p1, *p2}}, nil @@ -204,7 +210,7 @@ func TestScaleDown(t *testing.T) { assert.Equal(t, n1.Name, getStringFromChan(updatedNodes)) } -func TestNoScaleDown(t *testing.T) { +func TestNoScaleDownUnready(t *testing.T) { fakeClient := &fake.Clientset{} n1 := BuildTestNode("n1", 1000, 1000) SetNodeReadyState(n1, false, time.Now().Add(-3*time.Minute)) @@ -249,6 +255,8 @@ func TestNoScaleDown(t *testing.T) { MaxGratefulTerminationSec: 60, ClusterStateRegistry: clusterstate.NewClusterStateRegistry(provider, clusterstate.ClusterStateRegistryConfig{}), } + + // N1 is unready so it requires a bigger unneeded time. scaleDown := NewScaleDown(context) scaleDown.UpdateUnneededNodes([]*apiv1.Node{n1, n2}, []*apiv1.Pod{p2}, time.Now().Add(-5*time.Minute)) result, err := scaleDown.TryToScaleDown([]*apiv1.Node{n1, n2}, []*apiv1.Pod{p2}) @@ -266,6 +274,7 @@ func TestNoScaleDown(t *testing.T) { provider.AddNode("ng1", n1) provider.AddNode("ng1", n2) + // N1 has been unready for 2 hours, ok to delete. context.CloudProvider = provider scaleDown = NewScaleDown(context) scaleDown.UpdateUnneededNodes([]*apiv1.Node{n1, n2}, []*apiv1.Pod{p2}, time.Now().Add(-2*time.Hour)) @@ -275,6 +284,82 @@ func TestNoScaleDown(t *testing.T) { assert.Equal(t, n1.Name, getStringFromChan(deletedNodes)) } +func TestScaleDownNoMove(t *testing.T) { + fakeClient := &fake.Clientset{} + + job := batchv1.Job{ + ObjectMeta: apiv1.ObjectMeta{ + Name: "job", + Namespace: "default", + SelfLink: "/apivs/extensions/v1beta1/namespaces/default/jobs/job", + }, + } + n1 := BuildTestNode("n1", 1000, 1000) + SetNodeReadyState(n1, true, time.Time{}) + + // N2 is unready so no pods can be moved there. + n2 := BuildTestNode("n2", 1000, 1000) + SetNodeReadyState(n2, false, time.Time{}) + + p1 := BuildTestPod("p1", 100, 0) + p1.Annotations = map[string]string{ + "kubernetes.io/created-by": RefJSON(&job), + } + + p2 := BuildTestPod("p2", 800, 0) + p1.Spec.NodeName = "n1" + p2.Spec.NodeName = "n2" + + fakeClient.Fake.AddReactor("list", "pods", func(action core.Action) (bool, runtime.Object, error) { + return true, &apiv1.PodList{Items: []apiv1.Pod{*p1, *p2}}, nil + }) + fakeClient.Fake.AddReactor("get", "pods", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, errors.NewNotFound(apiv1.Resource("pod"), "whatever") + }) + fakeClient.Fake.AddReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) { + getAction := action.(core.GetAction) + switch getAction.GetName() { + case n1.Name: + return true, n1, nil + case n2.Name: + return true, n2, nil + } + return true, nil, fmt.Errorf("Wrong node: %v", getAction.GetName()) + }) + fakeClient.Fake.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) { + t.FailNow() + return false, nil, nil + }) + fakeClient.Fake.AddReactor("update", "nodes", func(action core.Action) (bool, runtime.Object, error) { + t.FailNow() + return false, nil, nil + }) + provider := testprovider.NewTestCloudProvider(nil, func(nodeGroup string, node string) error { + t.FailNow() + return nil + }) + provider.AddNodeGroup("ng1", 1, 10, 2) + provider.AddNode("ng1", n1) + provider.AddNode("ng1", n2) + assert.NotNil(t, provider) + + context := &AutoscalingContext{ + PredicateChecker: simulator.NewTestPredicateChecker(), + CloudProvider: provider, + ClientSet: fakeClient, + Recorder: createEventRecorder(fakeClient), + ScaleDownUtilizationThreshold: 0.5, + ScaleDownUnneededTime: time.Minute, + MaxGratefulTerminationSec: 60, + ClusterStateRegistry: clusterstate.NewClusterStateRegistry(provider, clusterstate.ClusterStateRegistryConfig{}), + } + scaleDown := NewScaleDown(context) + scaleDown.UpdateUnneededNodes([]*apiv1.Node{n1, n2}, []*apiv1.Pod{p1, p2}, time.Now().Add(-5*time.Minute)) + result, err := scaleDown.TryToScaleDown([]*apiv1.Node{n1, n2}, []*apiv1.Pod{p1, p2}) + assert.NoError(t, err) + assert.Equal(t, ScaleDownNoUnneeded, result) +} + func getStringFromChan(c chan string) string { select { case val := <-c: diff --git a/cluster-autoscaler/simulator/cluster_test.go b/cluster-autoscaler/simulator/cluster_test.go index 39ae5d77fc..d1974ff706 100644 --- a/cluster-autoscaler/simulator/cluster_test.go +++ b/cluster-autoscaler/simulator/cluster_test.go @@ -21,7 +21,6 @@ import ( "time" . "k8s.io/contrib/cluster-autoscaler/utils/test" - apiv1 "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache" @@ -35,6 +34,7 @@ func TestUtilization(t *testing.T) { nodeInfo := schedulercache.NewNodeInfo(pod, pod, pod2) node := BuildTestNode("node1", 2000, 2000000) + SetNodeReadyState(node, true, time.Time{}) utilization, err := CalculateUtilization(node, nodeInfo) assert.NoError(t, err) @@ -56,7 +56,9 @@ func TestFindPlaceAllOk(t *testing.T) { "n2": schedulercache.NewNodeInfo(), } node1 := BuildTestNode("n1", 1000, 2000000) + SetNodeReadyState(node1, true, time.Time{}) node2 := BuildTestNode("n2", 1000, 2000000) + SetNodeReadyState(node2, true, time.Time{}) nodeInfos["n1"].SetNode(node1) nodeInfos["n2"].SetNode(node2) @@ -90,7 +92,11 @@ func TestFindPlaceAllBas(t *testing.T) { } nodebad := BuildTestNode("nbad", 1000, 2000000) node1 := BuildTestNode("n1", 1000, 2000000) + SetNodeReadyState(node1, true, time.Time{}) + node2 := BuildTestNode("n2", 1000, 2000000) + SetNodeReadyState(node2, true, time.Time{}) + nodeInfos["n1"].SetNode(node1) nodeInfos["n2"].SetNode(node2) nodeInfos["nbad"].SetNode(nodebad) @@ -120,7 +126,11 @@ func TestFindNone(t *testing.T) { "n2": schedulercache.NewNodeInfo(), } node1 := BuildTestNode("n1", 1000, 2000000) + SetNodeReadyState(node1, true, time.Time{}) + node2 := BuildTestNode("n2", 1000, 2000000) + SetNodeReadyState(node2, true, time.Time{}) + nodeInfos["n1"].SetNode(node1) nodeInfos["n2"].SetNode(node2) @@ -166,6 +176,11 @@ func TestFindEmptyNodes(t *testing.T) { node3 := BuildTestNode("n3", 1000, 2000000) node4 := BuildTestNode("n4", 1000, 2000000) + SetNodeReadyState(node1, true, time.Time{}) + SetNodeReadyState(node2, true, time.Time{}) + SetNodeReadyState(node3, true, time.Time{}) + SetNodeReadyState(node4, true, time.Time{}) + emptyNodes := FindEmptyNodesToRemove([]*apiv1.Node{node1, node2, node3, node4}, []*apiv1.Pod{pod1, pod2}) assert.Equal(t, []*apiv1.Node{node2, node3, node4}, emptyNodes) } diff --git a/cluster-autoscaler/simulator/predicates.go b/cluster-autoscaler/simulator/predicates.go index 8c3f920575..d04c858a72 100644 --- a/cluster-autoscaler/simulator/predicates.go +++ b/cluster-autoscaler/simulator/predicates.go @@ -19,14 +19,16 @@ package simulator import ( "fmt" + kube_util "k8s.io/contrib/cluster-autoscaler/utils/kubernetes" apiv1 "k8s.io/kubernetes/pkg/api/v1" kube_client "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates" - // We need to import provider to intialize default scheduler. - _ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider" "k8s.io/kubernetes/plugin/pkg/scheduler/factory" "k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache" + + // We need to import provider to intialize default scheduler. + _ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider" ) // PredicateChecker checks whether all required predicates are matched for given Pod and Node @@ -42,6 +44,7 @@ func NewPredicateChecker(kubeClient kube_client.Interface) (*PredicateChecker, e } schedulerConfigFactory := factory.NewConfigFactory(kubeClient, "", apiv1.DefaultHardPodAffinitySymmetricWeight, apiv1.DefaultFailureDomains) predicates, err := schedulerConfigFactory.GetPredicates(provider.FitPredicateKeys) + predicates["ready"] = isNodeReadyAndSchedulablePredicate if err != nil { return nil, err } @@ -51,11 +54,21 @@ func NewPredicateChecker(kubeClient kube_client.Interface) (*PredicateChecker, e }, nil } +func isNodeReadyAndSchedulablePredicate(pod *apiv1.Pod, meta interface{}, nodeInfo *schedulercache.NodeInfo) (bool, + []algorithm.PredicateFailureReason, error) { + ready := kube_util.IsNodeReadyAndSchedulable(nodeInfo.Node()) + if !ready { + return false, []algorithm.PredicateFailureReason{predicates.NewFailureReason("node is unready")}, nil + } + return true, []algorithm.PredicateFailureReason{}, nil +} + // NewTestPredicateChecker builds test version of PredicateChecker. func NewTestPredicateChecker() *PredicateChecker { return &PredicateChecker{ predicates: map[string]algorithm.FitPredicate{ "default": predicates.GeneralPredicates, + "ready": isNodeReadyAndSchedulablePredicate, }, } } @@ -78,6 +91,7 @@ func (p *PredicateChecker) FitsAny(pod *apiv1.Pod, nodeInfos map[string]*schedul func (p *PredicateChecker) CheckPredicates(pod *apiv1.Pod, nodeInfo *schedulercache.NodeInfo) error { for _, predicate := range p.predicates { match, failureReason, err := predicate(pod, nil, nodeInfo) + nodename := "unknown" if nodeInfo.Node() != nil { nodename = nodeInfo.Node().Name diff --git a/cluster-autoscaler/simulator/predicates_test.go b/cluster-autoscaler/simulator/predicates_test.go index c82ca52f7e..ebc35b4e08 100644 --- a/cluster-autoscaler/simulator/predicates_test.go +++ b/cluster-autoscaler/simulator/predicates_test.go @@ -18,6 +18,7 @@ package simulator import ( "testing" + "time" . "k8s.io/contrib/cluster-autoscaler/utils/test" "k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache" @@ -39,6 +40,9 @@ func TestPredicates(t *testing.T) { } node1 := BuildTestNode("n1", 1000, 2000000) node2 := BuildTestNode("n2", 1000, 2000000) + SetNodeReadyState(node1, true, time.Time{}) + SetNodeReadyState(node2, true, time.Time{}) + ni1.SetNode(node1) ni2.SetNode(node2) diff --git a/cluster-autoscaler/utils.go b/cluster-autoscaler/utils.go index 6df84b43fe..77eaadc610 100644 --- a/cluster-autoscaler/utils.go +++ b/cluster-autoscaler/utils.go @@ -168,35 +168,6 @@ func createNodeNameToInfoMap(pods []*apiv1.Pod, nodes []*apiv1.Node) map[string] return nodeNameToNodeInfo } -// CheckGroupsAndNodes checks if all node groups have all required nodes. -func CheckGroupsAndNodes(nodes []*apiv1.Node, cloudProvider cloudprovider.CloudProvider) error { - groupCount := make(map[string]int) - for _, node := range nodes { - - group, err := cloudProvider.NodeGroupForNode(node) - if err != nil { - return err - } - if group == nil || reflect.ValueOf(group).IsNil() { - continue - } - id := group.Id() - count, _ := groupCount[id] - groupCount[id] = count + 1 - } - for _, nodeGroup := range cloudProvider.NodeGroups() { - size, err := nodeGroup.TargetSize() - if err != nil { - return err - } - count := groupCount[nodeGroup.Id()] - if size != count { - return fmt.Errorf("wrong number of nodes for node group: %s expected: %d actual: %d", nodeGroup.Id(), size, count) - } - } - return nil -} - // GetNodeInfosForGroups finds NodeInfos for all node groups used to manage the given nodes. It also returns a node group to sample node mapping. // TODO(mwielgus): This returns map keyed by url, while most code (including scheduler) uses node.Name for a key. func GetNodeInfosForGroups(nodes []*apiv1.Node, cloudProvider cloudprovider.CloudProvider, kubeClient kube_client.Interface) (map[string]*schedulercache.NodeInfo, error) { diff --git a/cluster-autoscaler/utils/kubernetes/listers.go b/cluster-autoscaler/utils/kubernetes/listers.go index ee13df7646..81b1d56f9b 100644 --- a/cluster-autoscaler/utils/kubernetes/listers.go +++ b/cluster-autoscaler/utils/kubernetes/listers.go @@ -106,11 +106,9 @@ func (readyNodeLister *ReadyNodeLister) List() ([]*apiv1.Node, error) { } readyNodes := make([]*apiv1.Node, 0, len(nodes.Items)) for i, node := range nodes.Items { - for _, condition := range node.Status.Conditions { - if condition.Type == apiv1.NodeReady && condition.Status == apiv1.ConditionTrue { - readyNodes = append(readyNodes, &nodes.Items[i]) - break - } + if IsNodeReadyAndSchedulable(&node) { + readyNodes = append(readyNodes, &nodes.Items[i]) + break } } return readyNodes, nil diff --git a/cluster-autoscaler/utils/kubernetes/ready.go b/cluster-autoscaler/utils/kubernetes/ready.go new file mode 100644 index 0000000000..40b48cd28f --- /dev/null +++ b/cluster-autoscaler/utils/kubernetes/ready.go @@ -0,0 +1,74 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "fmt" + "time" + + apiv1 "k8s.io/kubernetes/pkg/api/v1" +) + +// IsNodeReadyAndSchedulable returns true if the node is ready and schedulable. +func IsNodeReadyAndSchedulable(node *apiv1.Node) bool { + ready, _, _ := GetReadinessState(node) + if !ready { + return false + } + // Ignore nodes that are marked unschedulable + if node.Spec.Unschedulable { + return false + } + return true +} + +// GetReadinessState gets readiness state for the node +func GetReadinessState(node *apiv1.Node) (isNodeReady bool, lastTransitionTime time.Time, err error) { + canNodeBeReady, readyFound := true, false + lastTransitionTime = time.Time{} + + for _, cond := range node.Status.Conditions { + switch cond.Type { + case apiv1.NodeReady: + readyFound = true + if cond.Status == apiv1.ConditionFalse { + canNodeBeReady = false + } + if lastTransitionTime.Before(cond.LastTransitionTime.Time) { + lastTransitionTime = cond.LastTransitionTime.Time + } + case apiv1.NodeOutOfDisk: + if cond.Status == apiv1.ConditionTrue { + canNodeBeReady = false + } + if lastTransitionTime.Before(cond.LastTransitionTime.Time) { + lastTransitionTime = cond.LastTransitionTime.Time + } + case apiv1.NodeNetworkUnavailable: + if cond.Status == apiv1.ConditionTrue { + canNodeBeReady = false + } + if lastTransitionTime.Before(cond.LastTransitionTime.Time) { + lastTransitionTime = cond.LastTransitionTime.Time + } + } + } + if !readyFound { + return false, time.Time{}, fmt.Errorf("feadiness information not found") + } + return canNodeBeReady, lastTransitionTime, nil +} diff --git a/cluster-autoscaler/utils_test.go b/cluster-autoscaler/utils_test.go index cd813460cc..27f40e5d73 100644 --- a/cluster-autoscaler/utils_test.go +++ b/cluster-autoscaler/utils_test.go @@ -43,6 +43,7 @@ func TestFilterOutSchedulable(t *testing.T) { scheduledPod2.Spec.NodeName = "node1" node := BuildTestNode("node1", 2000, 2000000) + SetNodeReadyState(node, true, time.Time{}) predicateChecker := simulator.NewTestPredicateChecker()