Merge pull request kubernetes#67786 from fabriziopandini/kubeadm-depr…

…ecate-featureflags

Automatic merge from submit-queue (batch tested with PRs 67776, 67503, 67679, 67786, 67830). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

kubeadm - deprecate feature-gates HighAvailability, SelfHosting, CertsInSecrets

**What this PR does / why we need it**:
As for sig discussion (see meeting notes - August 22 - 2018) we are going to block usage of feature gates HighAvailability, SelfHosting, CertsInSecrets for new clusters and block updates to v1.12 of existing clusters using such features.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` 
Fixes # kubernetes/kubeadm#1058

**Special notes for your reviewer**:
I'm going to open issue to track code cleanup in v1.13 

**Release note**:
```release-note
kubeadm - feature-gates HighAvailability, SelfHosting, CertsInSecrets are now deprecated and can't be used anymore for new clusters. Update of cluster using above feature-gates flag is not supported
```
/sig cluster-lifecycle
/kind feature
/kind cleanup
/assign @timothysc
/cc

cmd/kubeadm/app/cmd/upgrade/common.go

@@ -93,6 +93,14 @@ func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion strin
 		}
 	}
 
+	// Check if feature gate flags used in the cluster are consistent with the set of features currently supported by kubeadm
+	if msg := features.CheckDeprecatedFlags(&features.InitFeatureGates, cfg.FeatureGates); len(msg) > 0 {
+		for _, m := range msg {
+			fmt.Printf("[upgrade/config] %s\n", m)
+		}
+		return nil, fmt.Errorf("[upgrade/config] FATAL. Unable to upgrade a cluster using deprecated feature-gate flags. Please see the release notes")
+	}
+
 	// If the user told us to print this information out; do it!
 	if flags.printConfig {
 		printConfiguration(&cfg.ClusterConfiguration, os.Stdout)

cmd/kubeadm/app/features/features.go

@@ -27,16 +27,16 @@ import (
 )
 
 const (
-	// HighAvailability is alpha in v1.9
+	// HighAvailability is alpha in v1.9 - deprecated in v1.12 (TODO remove in v1.13)
 	HighAvailability = "HighAvailability"
 
 	// CoreDNS is GA in v1.11
 	CoreDNS = "CoreDNS"
 
-	// SelfHosting is alpha in v1.8 and v1.9
+	// SelfHosting is alpha in v1.8 and v1.9 - deprecated in v1.12 (TODO remove in v1.13)
 	SelfHosting = "SelfHosting"
 
-	// StoreCertsInSecrets is alpha in v1.8 and v1.9
+	// StoreCertsInSecrets is alpha in v1.8 and v1.9 - deprecated in v1.12 (TODO remove in v1.13)
 	StoreCertsInSecrets = "StoreCertsInSecrets"
 
 	// DynamicKubeletConfig is beta in v1.11
@@ -46,12 +46,18 @@ const (
 	Auditing = "Auditing"
 )
 
+var selfHostingDeprecationMessage = "featureGates:SelfHosting has been removed in v1.12"
+
+var storeCertsInSecretsDeprecationMessage = "featureGates:StoreCertsInSecrets has been removed in v1.12"
+
+var highAvailabilityMessage = "featureGates:HighAvailability has been removed in v1.12\n" +
+	"\tThis feature has been replaced by the kubeadm join --control-plane workflow."
+
 // InitFeatureGates are the default feature gates for the init command
 var InitFeatureGates = FeatureList{
-	SelfHosting:         {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}},
-	StoreCertsInSecrets: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}},
-	// We don't want to advertise this feature gate exists in v1.9 to avoid confusion as it is not yet working
-	HighAvailability:     {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, HiddenInHelpText: true},
+	SelfHosting:          {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: selfHostingDeprecationMessage},
+	StoreCertsInSecrets:  {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: storeCertsInSecretsDeprecationMessage},
+	HighAvailability:     {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Deprecated}, HiddenInHelpText: true, DeprecationMessage: highAvailabilityMessage},
 	CoreDNS:              {FeatureSpec: utilfeature.FeatureSpec{Default: true, PreRelease: utilfeature.GA}},
 	DynamicKubeletConfig: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}},
 	Auditing:             {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}},
@@ -60,8 +66,9 @@ var InitFeatureGates = FeatureList{
 // Feature represents a feature being gated
 type Feature struct {
 	utilfeature.FeatureSpec
-	MinimumVersion   *version.Version
-	HiddenInHelpText bool
+	MinimumVersion     *version.Version
+	HiddenInHelpText   bool
+	DeprecationMessage string
 }
 
 // FeatureList represents a list of feature gates
@@ -151,10 +158,15 @@ func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) {
 		k := strings.TrimSpace(arr[0])
 		v := strings.TrimSpace(arr[1])
 
-		if !Supports(*f, k) {
+		featureSpec, ok := (*f)[k]
+		if !ok {
 			return nil, fmt.Errorf("unrecognized feature-gate key: %s", k)
 		}
 
+		if featureSpec.PreRelease == utilfeature.Deprecated {
+			return nil, fmt.Errorf("feature-gate key is deprecated: %s", k)
+		}
+
 		boolValue, err := strconv.ParseBool(v)
 		if err != nil {
 			return nil, fmt.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k)
@@ -167,6 +179,29 @@ func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) {
 	return featureGate, nil
 }
 
+// CheckDeprecatedFlags takes a list of existing feature gate flags and validates against the current feature flag set.
+// It used during upgrades for ensuring consistency of feature gates used in an existing cluster, that might
+// be created with a previous version of kubeadm, with the set of features currently supported by kubeadm
+func CheckDeprecatedFlags(f *FeatureList, features map[string]bool) map[string]string {
+	deprecatedMsg := map[string]string{}
+	for k := range features {
+		featureSpec, ok := (*f)[k]
+		if !ok {
+			// This case should never happen, it is implemented only as a sentinel
+			// for removal of flags executed when flags are still in use (always before deprecate, then after one cycle remove)
+			deprecatedMsg[k] = fmt.Sprintf("Unknown feature gate flag: %s", k)
+		}
+
+		if featureSpec.PreRelease == utilfeature.Deprecated {
+			if _, ok := deprecatedMsg[k]; !ok {
+				deprecatedMsg[k] = featureSpec.DeprecationMessage
+			}
+		}
+	}
+
+	return deprecatedMsg
+}
+
 // ResolveFeatureGateDependencies resolve dependencies between feature gates
 func ResolveFeatureGateDependencies(featureGate map[string]bool) {
 

cmd/kubeadm/app/features/features_test.go

@@ -31,6 +31,7 @@ func TestKnownFeatures(t *testing.T) {
 		"feature2": {FeatureSpec: utilfeature.FeatureSpec{Default: true, PreRelease: utilfeature.Alpha}},
 		"feature1": {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}},
 		"feature3": {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.GA}},
+		"hidden":   {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.GA}, HiddenInHelpText: true},
 	}
 
 	r := KnownFeatures(&someFeatures)
@@ -58,8 +59,9 @@ func TestKnownFeatures(t *testing.T) {
 
 func TestNewFeatureGate(t *testing.T) {
 	var someFeatures = FeatureList{
-		"feature1": {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}},
-		"feature2": {FeatureSpec: utilfeature.FeatureSpec{Default: true, PreRelease: utilfeature.Alpha}},
+		"feature1":   {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}},
+		"feature2":   {FeatureSpec: utilfeature.FeatureSpec{Default: true, PreRelease: utilfeature.Alpha}},
+		"deprecated": {FeatureSpec: utilfeature.FeatureSpec{Default: true, PreRelease: utilfeature.Deprecated}},
 	}
 
 	var tests = []struct {
@@ -91,6 +93,10 @@ func TestNewFeatureGate(t *testing.T) {
 			value:         "feature1=true,unknownFeature=false",
 			expectedError: true,
 		},
+		{ //deprecated feature-gate key
+			value:         "deprecated=true",
+			expectedError: true,
+		},
 		{ //one feature
 			value:                "feature1=true",
 			expectedError:        false,
@@ -205,3 +211,32 @@ func TestEnabledDefaults(t *testing.T) {
 		}
 	}
 }
+
+func TestCheckDeprecatedFlags(t *testing.T) {
+	dummyMessage := "dummy message"
+	var someFeatures = FeatureList{
+		"feature1":   {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}},
+		"deprecated": {FeatureSpec: utilfeature.FeatureSpec{Default: true, PreRelease: utilfeature.Deprecated}, DeprecationMessage: dummyMessage},
+	}
+
+	var tests = []struct {
+		features    map[string]bool
+		expectedMsg map[string]string
+	}{
+		{ // feature deprecated
+			features:    map[string]bool{"deprecated": true},
+			expectedMsg: map[string]string{"deprecated": dummyMessage},
+		},
+		{ // valid feature
+			features:    map[string]bool{"feature1": true},
+			expectedMsg: map[string]string{},
+		},
+	}
+
+	for _, test := range tests {
+		msg := CheckDeprecatedFlags(&someFeatures, test.features)
+		if !reflect.DeepEqual(test.expectedMsg, msg) {
+			t.Error("CheckDeprecatedFlags didn't returned expected message")
+		}
+	}
+}