From a8c083b246f2709ec282d2fc611e9fe92935fcbf Mon Sep 17 00:00:00 2001 From: Filip Grzadkowski Date: Mon, 16 May 2016 12:13:47 +0200 Subject: [PATCH 1/2] Fix slicing pods into old pods (to reset pod condition) and new pods (to assume as unschedulable). --- cluster-autoscaler/cluster_autoscaler.go | 12 ++++---- cluster-autoscaler/deploy/deploy.sh | 2 ++ cluster-autoscaler/utils.go | 36 +++++++++++------------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cluster-autoscaler/cluster_autoscaler.go b/cluster-autoscaler/cluster_autoscaler.go index 384bfd3efe..07d858c949 100644 --- a/cluster-autoscaler/cluster_autoscaler.go +++ b/cluster-autoscaler/cluster_autoscaler.go @@ -90,7 +90,7 @@ func main() { continue } - pods, err := unschedulablePodLister.List() + allPods, err := unschedulablePodLister.List() if err != nil { glog.Errorf("Failed to list unscheduled pods: %v", err) continue @@ -99,16 +99,16 @@ func main() { // We need to reset all pods that have been marked as unschedulable not after // the newest node became available for the scheduler. allNodesAvailableTime := GetAllNodesAvailableTime(nodes) - resetOldPods(kubeClient, pods, allNodesAvailableTime) + podsToReset, unschedulablePods := SlicePodsByPodScheduledTime(allPods, allNodesAvailableTime) + ResetPodScheduledCondition(kubeClient, podsToReset) // From now on we only care about unschedulable pods that were marked after the newest // node became available for the scheduler. - pods = filterOldPods(pods, allNodesAvailableTime) - if len(pods) == 0 { + if len(unschedulablePods) == 0 { glog.V(1).Info("No unschedulable pods") continue } - for _, pod := range pods { + for _, pod := range unschedulablePods { glog.V(1).Infof("Pod %s/%s is unschedulable", pod.Namespace, pod.Name) } @@ -144,7 +144,7 @@ func main() { continue } - for _, pod := range pods { + for _, pod := range unschedulablePods { err = predicateChecker.CheckPredicates(pod, nodeInfo) if err == nil { migHelpsSomePods = true diff --git a/cluster-autoscaler/deploy/deploy.sh b/cluster-autoscaler/deploy/deploy.sh index de740a42f5..51a945536a 100755 --- a/cluster-autoscaler/deploy/deploy.sh +++ b/cluster-autoscaler/deploy/deploy.sh @@ -17,6 +17,8 @@ # Usage: # REGISTRY= MIG_LINK= [MIN=1] [MAX=4] [VERSION=v1.1] deploy.sh +set -e + ROOT=$(dirname "${BASH_SOURCE}")/.. if [ -z "$REGISTRY" ]; then diff --git a/cluster-autoscaler/utils.go b/cluster-autoscaler/utils.go index cdfb6464d7..311a6badd2 100644 --- a/cluster-autoscaler/utils.go +++ b/cluster-autoscaler/utils.go @@ -147,40 +147,38 @@ func GetAllNodesAvailableTime(nodes []*kube_api.Node) time.Time { return result.Add(1 * time.Minute) } -// Returns pods for which PodScheduled condition have LastTransitionTime after -// the threshold. +// SlicePodsByPodScheduledTime slices given pod array into those where PodScheduled condition +// have been updated after the thresold and others. // Each pod must be in condition "Scheduled: False; Reason: Unschedulable" -// NOTE: This function must be in sync with resetOldPods. -func filterOldPods(pods []*kube_api.Pod, threshold time.Time) []*kube_api.Pod { - var result []*kube_api.Pod +func SlicePodsByPodScheduledTime(pods []*kube_api.Pod, threshold time.Time) (oldPods []*kube_api.Pod, newPods []*kube_api.Pod) { for _, pod := range pods { _, condition := kube_api.GetPodCondition(&pod.Status, kube_api.PodScheduled) - if condition != nil && condition.LastTransitionTime.After(threshold) { - result = append(result, pod) + if condition != nil { + if condition.LastTransitionTime.After(threshold) { + newPods = append(newPods, pod) + } else { + oldPods = append(oldPods, pod) + } } } - return result + return } -// Resets pod condition PodScheduled to "unknown" for all the pods with LastTransitionTime +// ResetPodScheduledCondition resets pod condition PodScheduled to "unknown" for all the pods with LastTransitionTime // not after the threshold time. -// NOTE: This function must be in sync with resetOldPods. -func resetOldPods(kubeClient *kube_client.Client, pods []*kube_api.Pod, threshold time.Time) { +func ResetPodScheduledCondition(kubeClient *kube_client.Client, pods []*kube_api.Pod) { for _, pod := range pods { - _, condition := kube_api.GetPodCondition(&pod.Status, kube_api.PodScheduled) - if condition != nil && !condition.LastTransitionTime.After(threshold) { - glog.V(4).Infof("Reseting pod condition for %s/%s, last transition: %s", - pod.Namespace, pod.Name, condition.LastTransitionTime.Time.String()) - if err := resetPodScheduledCondition(kubeClient, pod); err != nil { - glog.Errorf("Error during reseting pod condition for %s/%s: %v", pod.Namespace, pod.Name, err) - } + if err := resetPodScheduledConditionForPod(kubeClient, pod); err != nil { + glog.Errorf("Error during reseting pod condition for %s/%s: %v", pod.Namespace, pod.Name, err) } } } -func resetPodScheduledCondition(kubeClient *kube_client.Client, pod *kube_api.Pod) error { +func resetPodScheduledConditionForPod(kubeClient *kube_client.Client, pod *kube_api.Pod) error { _, condition := kube_api.GetPodCondition(&pod.Status, kube_api.PodScheduled) if condition != nil { + glog.V(4).Infof("Reseting pod condition for %s/%s, last transition: %s", + pod.Namespace, pod.Name, condition.LastTransitionTime.Time.String()) condition.Status = kube_api.ConditionUnknown condition.LastTransitionTime = kube_api_unversioned.Now() _, err := kubeClient.Pods(pod.Namespace).UpdateStatus(pod) From 6d4d9aa8ae90edcdc6b24ffb4ce504d34932ea1d Mon Sep 17 00:00:00 2001 From: Filip Grzadkowski Date: Mon, 16 May 2016 13:27:42 +0200 Subject: [PATCH 2/2] Fix makefile so that it doesn't require REGISTRY to be set for every rule --- cluster-autoscaler/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-autoscaler/Makefile b/cluster-autoscaler/Makefile index 5ec1b0c952..045dbf3ece 100644 --- a/cluster-autoscaler/Makefile +++ b/cluster-autoscaler/Makefile @@ -15,9 +15,9 @@ test-unit: clean deps build $(ENVVAR) godep go test --test.short -race ./... $(FLAGS) release: build - ifndef REGISTRY - $(error REGISTRY is undefined) + ERR = $(error REGISTRY is undefined) + $(ERR) endif docker build -t ${REGISTRY}/cluster-autoscaler:${TAG} . gcloud docker push ${REGISTRY}/cluster-autoscaler:${TAG}