Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

kubeadm: add v1beta4 to scheme; add --allow-experimental-api flag #118866

Merged
merged 2 commits into from Jun 27, 2023
Merged

kubeadm: add v1beta4 to scheme; add --allow-experimental-api flag #118866

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0774f77
kubeadm: add v1beta4 to the kubeadm API scheme
neolit123 Jun 22, 2023
f04484f
kubeadm: add the --allow-experimental-api flag to "config" commands
neolit123 Jun 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 5 additions & 1 deletion cmd/kubeadm/app/apis/kubeadm/scheme/scheme.go
View file
Open in desktop
Expand Up @@ -25,6 +25,7 @@ import (

"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
)

// Scheme is the runtime.Scheme to which all kubeadm api types are registered.
Expand All @@ -42,5 +43,8 @@ func init() {
func AddToScheme(scheme *runtime.Scheme) {
utilruntime.Must(kubeadm.AddToScheme(scheme))
utilruntime.Must(v1beta3.AddToScheme(scheme))
utilruntime.Must(scheme.SetVersionPriority(v1beta3.SchemeGroupVersion))
utilruntime.Must(v1beta4.AddToScheme(scheme))
// TODO: https://github.com/kubernetes/kubeadm/issues/2890
// make v1beta4 highest priority
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is tracked in the tasklist of #118866

Copy link
Member Author

@neolit123 neolit123 Jun 26, 2023 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what happens now is that kubeadm still uses v1beta3 by default, until we adjust the priority here.

utilruntime.Must(scheme.SetVersionPriority(v1beta3.SchemeGroupVersion, v1beta4.SchemeGroupVersion))
}
9 changes: 7 additions & 2 deletions cmd/kubeadm/app/cmd/config.go
View file
Open in desktop
Expand Up @@ -225,6 +225,8 @@ func getDefaultNodeConfigBytes() ([]byte, error) {
// newCmdConfigMigrate returns cobra.Command for "kubeadm config migrate" command
func newCmdConfigMigrate(out io.Writer) *cobra.Command {
var oldCfgPath, newCfgPath string
var allowExperimental bool

cmd := &cobra.Command{
Use: "migrate",
Short: "Read an older version of the kubeadm configuration API types from a file, and output the similar config object for the newer version",
Expand Down Expand Up @@ -252,7 +254,7 @@ func newCmdConfigMigrate(out io.Writer) *cobra.Command {
return err
}

outputBytes, err := configutil.MigrateOldConfig(oldCfgBytes)
outputBytes, err := configutil.MigrateOldConfig(oldCfgBytes, allowExperimental)
if err != nil {
return err
}
Expand All @@ -270,12 +272,14 @@ func newCmdConfigMigrate(out io.Writer) *cobra.Command {
}
cmd.Flags().StringVar(&oldCfgPath, "old-config", "", "Path to the kubeadm config file that is using an old API version and should be converted. This flag is mandatory.")
cmd.Flags().StringVar(&newCfgPath, "new-config", "", "Path to the resulting equivalent kubeadm config file using the new API version. Optional, if not specified output will be sent to STDOUT.")
cmd.Flags().BoolVar(&allowExperimental, options.AllowExperimentalAPI, false, "Allow migration to experimental, unreleased APIs.")
return cmd
}

// newCmdConfigValidate returns cobra.Command for the "kubeadm config validate" command
func newCmdConfigValidate(out io.Writer) *cobra.Command {
var cfgPath string
var allowExperimental bool

cmd := &cobra.Command{
Use: "validate",
Expand All @@ -300,7 +304,7 @@ func newCmdConfigValidate(out io.Writer) *cobra.Command {
return err
}

if err := configutil.ValidateConfig(cfgBytes); err != nil {
if err := configutil.ValidateConfig(cfgBytes, allowExperimental); err != nil {
return err
}
fmt.Fprintln(out, "ok")
Expand All @@ -310,6 +314,7 @@ func newCmdConfigValidate(out io.Writer) *cobra.Command {
Args: cobra.NoArgs,
}
options.AddConfigFlag(cmd.Flags(), &cfgPath)
cmd.Flags().BoolVar(&allowExperimental, options.AllowExperimentalAPI, false, "Allow validation of experimental, unreleased APIs.")
return cmd
}

Expand Down
3 changes: 3 additions & 0 deletions cmd/kubeadm/app/cmd/options/constant.go
View file
Open in desktop
Expand Up @@ -142,4 +142,7 @@ const (

// CleanupTmpDir flag indicates whether reset will cleanup the tmp dir
CleanupTmpDir = "cleanup-tmp-dir"

// AllowExperimentalAPI flag can be used to allow experimental / work in progress APIs
AllowExperimentalAPI = "allow-experimental-api"
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

flag naming is similar to upgrade apply, where we have --allow-foo flags

)
28 changes: 20 additions & 8 deletions cmd/kubeadm/app/util/config/common.go
View file
Open in desktop
Expand Up @@ -37,6 +37,7 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
Expand All @@ -54,7 +55,7 @@ func MarshalKubeadmConfigObject(obj runtime.Object) ([]byte, error) {

// validateSupportedVersion checks if the supplied GroupVersion is not on the lists of old unsupported or deprecated GVs.
// If it is, an error is returned.
func validateSupportedVersion(gv schema.GroupVersion, allowDeprecated bool) error {
func validateSupportedVersion(gv schema.GroupVersion, allowDeprecated, allowExperimental bool) error {
// The support matrix will look something like this now and in the future:
// v1.10 and earlier: v1alpha1
// v1.11: v1alpha1 read-only, writes only v1alpha2 config
Expand All @@ -72,6 +73,13 @@ func validateSupportedVersion(gv schema.GroupVersion, allowDeprecated bool) erro
"kubeadm.k8s.io/v1beta2": "v1.22",
}

// v1.28: v1beta4 is released as experimental
experimentalAPIVersions := map[string]string{
// TODO: https://github.com/kubernetes/kubeadm/issues/2890
// remove this from experimental once v1beta4 is released
"kubeadm.k8s.io/v1beta4": "v1.28",
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

task tracked in #118866

}

// Deprecated API versions are supported by us, but can only be used for migration.
deprecatedAPIVersions := map[string]struct{}{}

Expand All @@ -85,6 +93,10 @@ func validateSupportedVersion(gv schema.GroupVersion, allowDeprecated bool) erro
klog.Warningf("your configuration file uses a deprecated API spec: %q. Please use 'kubeadm config migrate --old-config old.yaml --new-config new.yaml', which will write the new, similar spec using a newer API version.", gv)
}

if _, present := experimentalAPIVersions[gvString]; present && !allowExperimental {
return errors.Errorf("experimental API spec: %q is not allowed. You can use the --%s flag if the command supports it.", gv, options.AllowExperimentalAPI)
}

return nil
}

Expand Down Expand Up @@ -205,7 +217,7 @@ func validateKnownGVKs(gvks []schema.GroupVersionKind) error {

// Skip legacy known GVs so that they don't return errors.
// This makes the function return errors only for GVs that where never known.
if err := validateSupportedVersion(gvk.GroupVersion(), true); err != nil {
if err := validateSupportedVersion(gvk.GroupVersion(), true, true); err != nil {
continue
}

Expand All @@ -229,7 +241,7 @@ func validateKnownGVKs(gvks []schema.GroupVersionKind) error {

// MigrateOldConfig migrates an old configuration from a byte slice into a new one (returned again as a byte slice).
// Only kubeadm kinds are migrated.
func MigrateOldConfig(oldConfig []byte) ([]byte, error) {
func MigrateOldConfig(oldConfig []byte, allowExperimental bool) ([]byte, error) {
newConfig := [][]byte{}

gvkmap, err := kubeadmutil.SplitYAMLDocuments(oldConfig)
Expand All @@ -248,7 +260,7 @@ func MigrateOldConfig(oldConfig []byte) ([]byte, error) {

// Migrate InitConfiguration and ClusterConfiguration if there are any in the config
if kubeadmutil.GroupVersionKindsHasInitConfiguration(gvks...) || kubeadmutil.GroupVersionKindsHasClusterConfiguration(gvks...) {
o, err := documentMapToInitConfiguration(gvkmap, true, true)
o, err := documentMapToInitConfiguration(gvkmap, true, allowExperimental, true)
if err != nil {
return []byte{}, err
}
Expand All @@ -261,7 +273,7 @@ func MigrateOldConfig(oldConfig []byte) ([]byte, error) {

// Migrate JoinConfiguration if there is any
if kubeadmutil.GroupVersionKindsHasJoinConfiguration(gvks...) {
o, err := documentMapToJoinConfiguration(gvkmap, true, true)
o, err := documentMapToJoinConfiguration(gvkmap, true, allowExperimental, true)
if err != nil {
return []byte{}, err
}
Expand All @@ -277,7 +289,7 @@ func MigrateOldConfig(oldConfig []byte) ([]byte, error) {

// ValidateConfig takes a byte slice containing a kubeadm configuration and performs conversion
// to internal types and validation.
func ValidateConfig(oldConfig []byte) error {
func ValidateConfig(oldConfig []byte, allowExperimental bool) error {
gvkmap, err := kubeadmutil.SplitYAMLDocuments(oldConfig)
if err != nil {
return err
Expand All @@ -294,14 +306,14 @@ func ValidateConfig(oldConfig []byte) error {

// Validate InitConfiguration and ClusterConfiguration if there are any in the config
if kubeadmutil.GroupVersionKindsHasInitConfiguration(gvks...) || kubeadmutil.GroupVersionKindsHasClusterConfiguration(gvks...) {
if _, err := documentMapToInitConfiguration(gvkmap, true, true); err != nil {
if _, err := documentMapToInitConfiguration(gvkmap, true, allowExperimental, true); err != nil {
return err
}
}

// Validate JoinConfiguration if there is any
if kubeadmutil.GroupVersionKindsHasJoinConfiguration(gvks...) {
if _, err := documentMapToJoinConfiguration(gvkmap, true, true); err != nil {
if _, err := documentMapToJoinConfiguration(gvkmap, true, allowExperimental, true); err != nil {
return err
}
}
Expand Down
23 changes: 19 additions & 4 deletions cmd/kubeadm/app/util/config/common_test.go
View file
Open in desktop
Expand Up @@ -34,9 +34,10 @@ const KubeadmGroupName = "kubeadm.k8s.io"

func TestValidateSupportedVersion(t *testing.T) {
tests := []struct {
gv schema.GroupVersion
allowDeprecated bool
expectedErr bool
gv schema.GroupVersion
allowDeprecated bool
allowExperimental bool
expectedErr bool
}{
{
gv: schema.GroupVersion{
Expand Down Expand Up @@ -85,11 +86,25 @@ func TestValidateSupportedVersion(t *testing.T) {
Version: "v1",
},
},
{
gv: schema.GroupVersion{
Group: KubeadmGroupName,
Version: "v1beta4",
},
allowExperimental: true,
},
{
gv: schema.GroupVersion{
Group: KubeadmGroupName,
Version: "v1beta4",
},
expectedErr: true,
Comment on lines +90 to +101
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there are no /cmd/kubeadm/test integration tests currently, only unit tests.
integration tests are OK, but effectively it would only be testing if the --allow-experimental-api (BOOL CLI flag) is working.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we need to see what can be done with the migration unit test

restore TestMigrateOldConfigFromFile in common_test.go once v1beta4 is added. See how we can make this test optional if only one API version is available (e.g. in the future only v1beta4 is avaiable and v1beta3 is removed)

kubernetes/kubeadm#2890

},
}

for _, rt := range tests {
t.Run(fmt.Sprintf("%s/allowDeprecated:%t", rt.gv, rt.allowDeprecated), func(t *testing.T) {
err := validateSupportedVersion(rt.gv, rt.allowDeprecated)
err := validateSupportedVersion(rt.gv, rt.allowDeprecated, rt.allowExperimental)
if rt.expectedErr && err == nil {
t.Error("unexpected success")
} else if !rt.expectedErr && err != nil {
Expand Down
6 changes: 3 additions & 3 deletions cmd/kubeadm/app/util/config/initconfiguration.go
View file
Open in desktop
Expand Up @@ -287,17 +287,17 @@ func BytesToInitConfiguration(b []byte) (*kubeadmapi.InitConfiguration, error) {
return nil, err
}

return documentMapToInitConfiguration(gvkmap, false, false)
return documentMapToInitConfiguration(gvkmap, false, false, false)
}

// documentMapToInitConfiguration converts a map of GVKs and YAML documents to defaulted and validated configuration object.
func documentMapToInitConfiguration(gvkmap kubeadmapi.DocumentMap, allowDeprecated, strictErrors bool) (*kubeadmapi.InitConfiguration, error) {
func documentMapToInitConfiguration(gvkmap kubeadmapi.DocumentMap, allowDeprecated, allowExperimental, strictErrors bool) (*kubeadmapi.InitConfiguration, error) {
var initcfg *kubeadmapi.InitConfiguration
var clustercfg *kubeadmapi.ClusterConfiguration

for gvk, fileContent := range gvkmap {
// first, check if this GVK is supported and possibly not deprecated
if err := validateSupportedVersion(gvk.GroupVersion(), allowDeprecated); err != nil {
if err := validateSupportedVersion(gvk.GroupVersion(), allowDeprecated, allowExperimental); err != nil {
return nil, err
}

Expand Down
6 changes: 3 additions & 3 deletions cmd/kubeadm/app/util/config/joinconfiguration.go
View file
Open in desktop
Expand Up @@ -85,12 +85,12 @@ func LoadJoinConfigurationFromFile(cfgPath string) (*kubeadmapi.JoinConfiguratio
return nil, err
}

return documentMapToJoinConfiguration(gvkmap, false, false)
return documentMapToJoinConfiguration(gvkmap, false, false, false)
}

// documentMapToJoinConfiguration takes a map between GVKs and YAML documents (as returned by SplitYAMLDocuments),
// finds a JoinConfiguration, decodes it, dynamically defaults it and then validates it prior to return.
func documentMapToJoinConfiguration(gvkmap kubeadmapi.DocumentMap, allowDeprecated, strictErrors bool) (*kubeadmapi.JoinConfiguration, error) {
func documentMapToJoinConfiguration(gvkmap kubeadmapi.DocumentMap, allowDeprecated, allowExperimental, strictErrors bool) (*kubeadmapi.JoinConfiguration, error) {
joinBytes := []byte{}
for gvk, bytes := range gvkmap {
// not interested in anything other than JoinConfiguration
Expand All @@ -99,7 +99,7 @@ func documentMapToJoinConfiguration(gvkmap kubeadmapi.DocumentMap, allowDeprecat
}

// check if this version is supported and possibly not deprecated
if err := validateSupportedVersion(gvk.GroupVersion(), allowDeprecated); err != nil {
if err := validateSupportedVersion(gvk.GroupVersion(), allowDeprecated, allowExperimental); err != nil {
return nil, err
}

Expand Down