Automated cherry pick of #41052

cluster/gce/config-default.sh

@@ -105,7 +105,7 @@ fi
 RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
 
 # Optional: set feature gates
-FEATURE_GATES="${KUBE_FEATURE_GATES:-}"
+FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}"
 
 # Optional: Install cluster DNS.
 ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}"

cluster/gce/config-test.sh

@@ -65,7 +65,7 @@ MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
 RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
 
 # Optional: set feature gates
-FEATURE_GATES="${KUBE_FEATURE_GATES:-}"
+FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}"
 
 TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100}
 

pkg/kubelet/eviction/eviction_manager.go

@@ -31,6 +31,7 @@ import (
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
 	"k8s.io/kubernetes/pkg/util/clock"
+	utilconfig "k8s.io/kubernetes/pkg/util/config"
 	"k8s.io/kubernetes/pkg/util/wait"
 )
 
@@ -103,7 +104,7 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd
 	// the node has memory pressure, admit if not best-effort
 	if hasNodeCondition(m.nodeConditions, api.NodeMemoryPressure) {
 		notBestEffort := qos.BestEffort != qos.GetPodQOS(attrs.Pod)
-		if notBestEffort || kubetypes.IsCriticalPod(attrs.Pod) {
+		if notBestEffort || (kubetypes.IsCriticalPod(attrs.Pod) && utilconfig.DefaultFeatureGate.ExperimentalCriticalPodAnnotation()) {
 			return lifecycle.PodAdmitResult{Admit: true}
 		}
 	}
@@ -248,7 +249,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 	// we kill at most a single pod during each eviction interval
 	for i := range activePods {
 		pod := activePods[i]
-		if kubepod.IsStaticPod(pod) {
+		if utilconfig.DefaultFeatureGate.ExperimentalCriticalPodAnnotation() && kubepod.IsStaticPod(pod) {
 			// The eviction manager doesn't evict static pods. To stop a static
 			// pod, the admin needs to remove the manifest from kubelet's
 			// --config directory.

pkg/kubelet/eviction/eviction_manager_test.go

@@ -28,6 +28,7 @@ import (
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util/clock"
+	utilconfig "k8s.io/kubernetes/pkg/util/config"
 )
 
 // mockPodKiller is used to testing which pod is killed
@@ -159,6 +160,7 @@ func TestMemoryPressure(t *testing.T) {
 		thresholdsFirstObservedAt:    thresholdsObservedAt{},
 	}
 
+	utilconfig.DefaultFeatureGate.Set("ExperimentalCriticalPodAnnotation=True")
 	// create a best effort pod to test admission
 	bestEffortPodToAdmit, _ := podMaker("best-admit", newResourceList("", ""), newResourceList("", ""), "0Gi")
 	burstablePodToAdmit, _ := podMaker("burst-admit", newResourceList("100m", "100Mi"), newResourceList("200m", "200Mi"), "0Gi")

pkg/kubelet/kubelet.go

@@ -2428,20 +2428,26 @@ func (kl *Kubelet) handleMirrorPod(mirrorPod *api.Pod, start time.Time) {
 func (kl *Kubelet) HandlePodAdditions(pods []*api.Pod) {
 	start := kl.clock.Now()
 
-	// Pass critical pods through admission check first.
-	var criticalPods []*api.Pod
-	var nonCriticalPods []*api.Pod
-	for _, p := range pods {
-		if kubetypes.IsCriticalPod(p) {
-			criticalPods = append(criticalPods, p)
-		} else {
-			nonCriticalPods = append(nonCriticalPods, p)
+	if utilconfig.DefaultFeatureGate.ExperimentalCriticalPodAnnotation() {
+		// Pass critical pods through admission check first.
+		var criticalPods []*api.Pod
+		var nonCriticalPods []*api.Pod
+		for _, p := range pods {
+			if kubetypes.IsCriticalPod(p) {
+				criticalPods = append(criticalPods, p)
+			} else {
+				nonCriticalPods = append(nonCriticalPods, p)
+			}
 		}
+		sort.Sort(sliceutils.PodsByCreationTime(criticalPods))
+		sort.Sort(sliceutils.PodsByCreationTime(nonCriticalPods))
+		pods = append(criticalPods, nonCriticalPods...)
+
+	} else {
+		sort.Sort(sliceutils.PodsByCreationTime(pods))
 	}
-	sort.Sort(sliceutils.PodsByCreationTime(criticalPods))
-	sort.Sort(sliceutils.PodsByCreationTime(nonCriticalPods))
 
-	for _, pod := range append(criticalPods, nonCriticalPods...) {
+	for _, pod := range pods {
 
 		if kubepod.IsMirrorPod(pod) {
 			kl.podManager.AddPod(pod)

pkg/kubelet/kubelet_test.go

@@ -64,6 +64,7 @@ import (
 	"k8s.io/kubernetes/pkg/runtime"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util/clock"
+	utilconfig "k8s.io/kubernetes/pkg/util/config"
 	"k8s.io/kubernetes/pkg/util/flowcontrol"
 	"k8s.io/kubernetes/pkg/util/mount"
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
@@ -1999,6 +2000,7 @@ func TestHandlePortConflicts(t *testing.T) {
 
 // Tests that we sort pods based on criticality.
 func TestCriticalPrioritySorting(t *testing.T) {
+	utilconfig.DefaultFeatureGate.Set("ExperimentalCriticalPodAnnotation=True")
 	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
 	kl := testKubelet.kubelet
 	nodes := []api.Node{

pkg/kubelet/qos/policy.go

@@ -19,6 +19,7 @@ package qos
 import (
 	"k8s.io/kubernetes/pkg/api"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
+	utilconfig "k8s.io/kubernetes/pkg/util/config"
 )
 
 const (
@@ -44,7 +45,7 @@ const (
 // and 1000. Containers with higher OOM scores are killed if the system runs out of memory.
 // See https://lwn.net/Articles/391222/ for more information.
 func GetContainerOOMScoreAdjust(pod *api.Pod, container *api.Container, memoryCapacity int64) int {
-	if kubetypes.IsCriticalPod(pod) {
+	if utilconfig.DefaultFeatureGate.ExperimentalCriticalPodAnnotation() && kubetypes.IsCriticalPod(pod) {
 		return CriticalPodOOMAdj
 	}
 

pkg/kubelet/qos/policy_test.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/resource"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
+	utilconfig "k8s.io/kubernetes/pkg/util/config"
 )
 
 const (
@@ -215,6 +216,7 @@ func TestGetContainerOOMScoreAdjust(t *testing.T) {
 			highOOMScoreAdj: -998,
 		},
 	}
+	utilconfig.DefaultFeatureGate.Set("ExperimentalCriticalPodAnnotation=True")
 	for _, test := range oomTests {
 		oomScoreAdj := GetContainerOOMScoreAdjust(test.pod, &test.pod.Spec.Containers[0], test.memoryCapacity)
 		if oomScoreAdj < test.lowOOMScoreAdj || oomScoreAdj > test.highOOMScoreAdj {

pkg/util/config/feature_gate.go

@@ -42,17 +42,21 @@ const (
 	appArmor                  = "AppArmor"
 	dynamicKubeletConfig      = "DynamicKubeletConfig"
 	dynamicVolumeProvisioning = "DynamicVolumeProvisioning"
+	// Ensures guaranteed scheduling of pods marked with a special pod annotation `scheduler.alpha.kubernetes.io/critical-pod`
+	// and also prevents them from being evicted from a node.
+	experimentalCriticalPodAnnotation = "ExperimentalCriticalPodAnnotation"
 )
 
 var (
 	// Default values for recorded features.  Every new feature gate should be
 	// represented here.
 	knownFeatures = map[string]featureSpec{
-		allAlphaGate:              {false, alpha},
-		externalTrafficLocalOnly:  {false, alpha},
-		appArmor:                  {true, beta},
-		dynamicKubeletConfig:      {false, alpha},
-		dynamicVolumeProvisioning: {true, alpha},
+		allAlphaGate:                      {false, alpha},
+		externalTrafficLocalOnly:          {false, alpha},
+		appArmor:                          {true, beta},
+		dynamicKubeletConfig:              {false, alpha},
+		dynamicVolumeProvisioning:         {true, alpha},
+		experimentalCriticalPodAnnotation: {false, alpha},
 	}
 
 	// Special handling for a few gates.
@@ -107,6 +111,10 @@ type FeatureGate interface {
 	// owner: mtaufen
 	// alpha: v1.4
 	DynamicKubeletConfig() bool
+
+	// owner: @vishh
+	// alpha: v1.4
+	ExperimentalCriticalPodAnnotation() bool
 }
 
 // featureGate implements FeatureGate as well as pflag.Value for flag parsing.
@@ -195,6 +203,11 @@ func (f *featureGate) DynamicVolumeProvisioning() bool {
 	return f.lookup(dynamicVolumeProvisioning)
 }
 
+// ExperimentalCriticalPodAnnotation returns true if experimentalCriticalPodAnnotation feature is enabled.
+func (f *featureGate) ExperimentalCriticalPodAnnotation() bool {
+	return f.lookup(experimentalCriticalPodAnnotation)
+}
+
 func (f *featureGate) lookup(key string) bool {
 	defaultValue := f.known[key].enabled
 	if f.enabled != nil {