diff --git a/cmd/sealctl/cmd/static_pod.go b/cmd/sealctl/cmd/static_pod.go index aea16d28906..28667374965 100755 --- a/cmd/sealctl/cmd/static_pod.go +++ b/cmd/sealctl/cmd/static_pod.go @@ -33,13 +33,10 @@ func newStaticPodCmd() *cobra.Command { var staticPodCmd = &cobra.Command{ Use: "static-pod", Short: "generator static pod", - //Run: func(cmd *cobra.Command, args []string) { - // - //}, } // check route for host staticPodCmd.AddCommand(newLvscareCmd()) - staticPodCmd.PersistentFlags().StringVar(&staticPodPath, "path", constants.KubernetesEtcStaticPod, "default kubernetes static pod path") + staticPodCmd.PersistentFlags().StringVar(&staticPodPath, "path", "/etc/kubernetes/manifests", "default kubernetes static pod path") return staticPodCmd } diff --git a/cmd/sealos/cmd/cert.go b/cmd/sealos/cmd/cert.go index 65f928e65a3..cd04a9bd100 100644 --- a/cmd/sealos/cmd/cert.go +++ b/cmd/sealos/cmd/cert.go @@ -15,9 +15,9 @@ package cmd import ( + "errors" "fmt" "path" - "strings" "github.com/spf13/cobra" @@ -25,11 +25,12 @@ import ( "github.com/labring/sealos/pkg/clusterfile" "github.com/labring/sealos/pkg/constants" "github.com/labring/sealos/pkg/runtime/kubernetes" + fileutils "github.com/labring/sealos/pkg/utils/file" ) -var altNames string - func newCertCmd() *cobra.Command { + var altNames []string + cmd := &cobra.Command{ Use: "cert", Short: "update Kubernetes API server's cert", @@ -52,26 +53,40 @@ func newCertCmd() *cobra.Command { processor.SyncNewVersionConfig(cluster.Name) clusterPath := constants.Clusterfile(cluster.Name) + pathResolver := constants.NewPathResolver(cluster.Name) + + var kubeadmInitFilepath string + + for _, f := range []string{ + path.Join(pathResolver.ConfigsPath(), "kubeadm-init.yaml"), + path.Join(pathResolver.EtcPath(), "kubeadm-init.yaml"), + } { + if fileutils.IsExist(f) { + kubeadmInitFilepath = f + break + } + } + if kubeadmInitFilepath == "" { + return errors.New("cannot locate the default kubeadm-init.yaml file") + } + cf := clusterfile.NewClusterFile(clusterPath, - clusterfile.WithCustomKubeadmFiles([]string{path.Join(constants.NewData(cluster.Name).EtcPath(), constants.DefaultInitKubeadmFileName)}), + clusterfile.WithCustomKubeadmFiles([]string{kubeadmInitFilepath}), ) if err = cf.Process(); err != nil { return err } + // TODO: using different runtime rt, err := kubernetes.New(cluster, cf.GetKubeadmConfig()) if err != nil { return fmt.Errorf("get default runtime failed, %v", err) } - return rt.UpdateCert(strings.Split(altNames, ",")) - }, - PreRunE: func(cmd *cobra.Command, args []string) error { - if strings.TrimSpace(altNames) == "" { - return fmt.Errorf("this command alt-names param can't empty") - } - return nil + return rt.UpdateCertSANs(altNames) }, } cmd.Flags().StringVarP(&clusterName, "cluster", "c", "default", "name of cluster to applied exec action") - cmd.Flags().StringVar(&altNames, "alt-names", "", "add domain or ip in certs, sealos.io or 10.103.97.2") + cmd.Flags().StringSliceVar(&altNames, "alt-names", []string{}, "add extra Subject Alternative Names for certs, domain or ip, eg. sealos.io or 10.103.97.2") + _ = cmd.MarkFlagRequired("alt-names") + return cmd } diff --git a/cmd/sealos/cmd/root.go b/cmd/sealos/cmd/root.go index 331503585f9..c7e7b952329 100644 --- a/cmd/sealos/cmd/root.go +++ b/cmd/sealos/cmd/root.go @@ -119,7 +119,7 @@ func onBootOnDie() { var rootDirs = []string{ constants.LogPath(), - constants.Workdir(), + constants.WorkDir(), } errExit(file.MkDirs(rootDirs...)) } diff --git a/pkg/apply/gen.go b/pkg/apply/gen.go index e40341e3bfc..e5f606ab07b 100644 --- a/pkg/apply/gen.go +++ b/pkg/apply/gen.go @@ -57,7 +57,7 @@ func NewClusterFromGenArgs(cmd *cobra.Command, args *RunArgs, imageNames []strin if err != nil { return nil, err } - return rt.GetConfig() + return rt.GetRawConfig() } func genImageInfo(imageName string) (*v1beta1.MountImage, error) { diff --git a/pkg/apply/processor/delete.go b/pkg/apply/processor/delete.go index 2efa01ef5b5..c0c6f521448 100644 --- a/pkg/apply/processor/delete.go +++ b/pkg/apply/processor/delete.go @@ -112,7 +112,7 @@ func (d *DeleteProcessor) UnMountImage(cluster *v2.Cluster) error { func (d *DeleteProcessor) CleanFS(cluster *v2.Cluster) error { workDir := constants.ClusterDir(cluster.Name) - dataDir := constants.NewData(cluster.Name).Homedir() + dataDir := constants.NewPathResolver(cluster.Name).Root() return fileutil.CleanFiles(workDir, dataDir) } diff --git a/pkg/apply/processor/interface.go b/pkg/apply/processor/interface.go index 80d4d0dec23..c6e22099e66 100644 --- a/pkg/apply/processor/interface.go +++ b/pkg/apply/processor/interface.go @@ -40,15 +40,16 @@ type Interface interface { Execute(cluster *v2.Cluster) error } +// compatible with older sealos versions func SyncNewVersionConfig(clusterName string) { - d := constants.NewData(clusterName) + d := constants.NewPathResolver(clusterName) if !file.IsExist(d.PkiPath()) { - src, target := path.Join(d.Homedir(), constants.PkiDirName), d.PkiPath() + src, target := path.Join(d.Root(), constants.PkiDirName), d.PkiPath() logger.Info("sync new version copy pki config: %s %s", src, target) _ = file.RecursionCopy(src, target) } if !file.IsExist(d.EtcPath()) { - src, target := path.Join(d.Homedir(), constants.EtcDirName), d.EtcPath() + src, target := path.Join(d.Root(), constants.EtcDirName), d.EtcPath() logger.Info("sync new version copy etc config: %s %s", src, target) _ = file.RecursionCopy(src, target) } @@ -153,7 +154,7 @@ func MirrorRegistry(cluster *v2.Cluster, mounts []v2.MountImage) error { registries := cluster.GetRegistryIPAndPortList() logger.Debug("registry nodes is: %+v", registries) sshClient := ssh.NewSSHByCluster(cluster, true) - syncer := registry.New(constants.NewData(cluster.GetName()), sshClient, mounts) + syncer := registry.New(constants.NewPathResolver(cluster.GetName()), sshClient, mounts) return syncer.Sync(context.Background(), registries...) } diff --git a/pkg/bootstrap/context.go b/pkg/bootstrap/context.go index a7ac6ccdb19..cd9b4f2436e 100644 --- a/pkg/bootstrap/context.go +++ b/pkg/bootstrap/context.go @@ -27,17 +27,17 @@ import ( type Context interface { GetBash() constants.Bash GetCluster() *v2.Cluster - GetData() constants.Data + GetPathResolver() constants.PathResolver GetExecer() ssh.Interface GetRemoter() remote.Interface } type realContext struct { - bash constants.Bash - cluster *v2.Cluster - data constants.Data - execer ssh.Interface - remoter remote.Interface + bash constants.Bash + cluster *v2.Cluster + pathResolver constants.PathResolver + execer ssh.Interface + remoter remote.Interface } func (ctx realContext) GetBash() constants.Bash { @@ -48,8 +48,8 @@ func (ctx realContext) GetCluster() *v2.Cluster { return ctx.cluster } -func (ctx realContext) GetData() constants.Data { - return ctx.data +func (ctx realContext) GetPathResolver() constants.PathResolver { + return ctx.pathResolver } func (ctx realContext) GetExecer() ssh.Interface { @@ -74,10 +74,10 @@ func NewContextFrom(cluster *v2.Cluster) Context { return stringsutil.RenderShellFromEnv(shell, envs) } return &realContext{ - cluster: cluster, - execer: execer, - bash: constants.NewBash(cluster.GetName(), cluster.GetImageLabels(), shellWrapper), - data: constants.NewData(cluster.GetName()), - remoter: remoter, + cluster: cluster, + execer: execer, + bash: constants.NewBash(cluster.GetName(), cluster.GetImageLabels(), shellWrapper), + pathResolver: constants.NewPathResolver(cluster.GetName()), + remoter: remoter, } } diff --git a/pkg/bootstrap/registry.go b/pkg/bootstrap/registry.go index e753919e5f3..e6d63d0632e 100644 --- a/pkg/bootstrap/registry.go +++ b/pkg/bootstrap/registry.go @@ -17,11 +17,10 @@ package bootstrap import ( "fmt" - "github.com/labring/sealos/pkg/registry/helpers" - "k8s.io/apimachinery/pkg/util/sets" "github.com/labring/sealos/pkg/constants" + "github.com/labring/sealos/pkg/registry/helpers" "github.com/labring/sealos/pkg/registry/password" "github.com/labring/sealos/pkg/utils/iputils" "github.com/labring/sealos/pkg/utils/logger" @@ -41,8 +40,8 @@ func (a *registryApplier) Apply(ctx Context, host string) error { if a.upgrade == nil { a.upgrade = password.NewUpgrade(ctx.GetCluster().GetName(), ctx.GetExecer()) } - rc := helpers.GetRegistryInfo(ctx.GetExecer(), ctx.GetData().RootFSPath(), ctx.GetCluster().GetRegistryIPAndPort()) - lnCmd := fmt.Sprintf(constants.DefaultLnFmt, ctx.GetData().RootFSRegistryPath(), rc.Data) + rc := helpers.GetRegistryInfo(ctx.GetExecer(), ctx.GetPathResolver().RootFSPath(), ctx.GetCluster().GetRegistryIPAndPort()) + lnCmd := fmt.Sprintf(constants.DefaultLnFmt, ctx.GetPathResolver().RootFSRegistryPath(), rc.Data) logger.Debug("make soft link: %s", lnCmd) if err := ctx.GetExecer().CmdAsync(host, lnCmd); err != nil { return fmt.Errorf("failed to make link: %v", err) @@ -67,12 +66,12 @@ func (*registryHostApplier) Filter(_ Context, _ string) bool { } func (*registryHostApplier) Undo(ctx Context, host string) error { - rc := helpers.GetRegistryInfo(ctx.GetExecer(), ctx.GetData().RootFSPath(), ctx.GetCluster().GetRegistryIPAndPort()) + rc := helpers.GetRegistryInfo(ctx.GetExecer(), ctx.GetPathResolver().RootFSPath(), ctx.GetCluster().GetRegistryIPAndPort()) return ctx.GetRemoter().HostsDelete(host, rc.Domain) } func (a *registryHostApplier) Apply(ctx Context, host string) error { - rc := helpers.GetRegistryInfo(ctx.GetExecer(), ctx.GetData().RootFSPath(), ctx.GetCluster().GetRegistryIPAndPort()) + rc := helpers.GetRegistryInfo(ctx.GetExecer(), ctx.GetPathResolver().RootFSPath(), ctx.GetCluster().GetRegistryIPAndPort()) if err := ctx.GetRemoter().HostsAdd(host, iputils.GetHostIP(rc.IP), rc.Domain); err != nil { return fmt.Errorf("failed to add hosts: %v", err) diff --git a/pkg/checker/cluster_checker.go b/pkg/checker/cluster_checker.go index fc1ed46b5cf..0d74ec96cd9 100644 --- a/pkg/checker/cluster_checker.go +++ b/pkg/checker/cluster_checker.go @@ -48,7 +48,7 @@ func (n *ClusterChecker) Check(cluster *v2.Cluster, phase string) error { } // checker if all the node is ready - data := constants.NewData(cluster.Name) + data := constants.NewPathResolver(cluster.Name) c, err := kubernetes.NewKubernetesClient(data.AdminFile(), "") if err != nil { return err diff --git a/pkg/checker/crictl_checker.go b/pkg/checker/crictl_checker.go index 3dd70e20c9d..7c2002a9cff 100644 --- a/pkg/checker/crictl_checker.go +++ b/pkg/checker/crictl_checker.go @@ -105,7 +105,7 @@ func (n *CRICtlChecker) Check(cluster *v2.Cluster, phase string) error { } } sshCtx := ssh.NewSSHByCluster(cluster, false) - root := constants.NewData(cluster.Name).RootFSPath() + root := constants.NewPathResolver(cluster.Name).RootFSPath() regInfo := helpers.GetRegistryInfo(sshCtx, root, cluster.GetRegistryIPAndPort()) regStatus, err := n.getRegistryStatus(crictlPath, pauseImage, fmt.Sprintf("%s:%s", regInfo.Domain, regInfo.Port)) diff --git a/pkg/checker/node_checker.go b/pkg/checker/node_checker.go index 8f4609febf3..72734d3b439 100644 --- a/pkg/checker/node_checker.go +++ b/pkg/checker/node_checker.go @@ -49,7 +49,7 @@ func (n *NodeChecker) Check(cluster *v2.Cluster, phase string) error { return nil } // checker if all the node is ready - data := constants.NewData(cluster.Name) + data := constants.NewPathResolver(cluster.Name) c, err := kubernetes.NewKubernetesClient(data.AdminFile(), "") if err != nil { return err diff --git a/pkg/checker/pod_checker.go b/pkg/checker/pod_checker.go index 1f1695b0f6b..fa048f7546d 100644 --- a/pkg/checker/pod_checker.go +++ b/pkg/checker/pod_checker.go @@ -48,7 +48,7 @@ func (n *PodChecker) Check(cluster *v2.Cluster, phase string) error { return nil } // checker if all the node is ready - data := constants.NewData(cluster.Name) + data := constants.NewPathResolver(cluster.Name) c, err := kubernetes.NewKubernetesClient(data.AdminFile(), "") if err != nil { return err diff --git a/pkg/checker/registry_checker.go b/pkg/checker/registry_checker.go index bfcf15617a1..d18a391a997 100644 --- a/pkg/checker/registry_checker.go +++ b/pkg/checker/registry_checker.go @@ -88,7 +88,7 @@ func (n *RegistryChecker) Check(cluster *v2.Cluster, phase string) error { } sshCtx := ssh.NewSSHByCluster(cluster, false) - root := constants.NewData(cluster.Name).RootFSPath() + root := constants.NewPathResolver(cluster.Name).RootFSPath() regInfo := helpers.GetRegistryInfo(sshCtx, root, cluster.GetRegistryIPAndPort()) status.Auth = fmt.Sprintf("%s:%s", regInfo.Username, regInfo.Password) status.RegistryDomain = fmt.Sprintf("%s:%s", regInfo.Domain, regInfo.Port) diff --git a/pkg/checker/svc_checker.go b/pkg/checker/svc_checker.go index 6d48f830cec..6e06d6c928e 100644 --- a/pkg/checker/svc_checker.go +++ b/pkg/checker/svc_checker.go @@ -51,7 +51,7 @@ func (n *SvcChecker) Check(cluster *v2.Cluster, phase string) error { return nil } // checker if all the node is ready - data := constants.NewData(cluster.Name) + data := constants.NewPathResolver(cluster.Name) c, err := kubernetes.NewKubernetesClient(data.AdminFile(), "") if err != nil { return err diff --git a/pkg/clusterfile/util.go b/pkg/clusterfile/util.go index 3a6887220f5..18d49c2b5c0 100644 --- a/pkg/clusterfile/util.go +++ b/pkg/clusterfile/util.go @@ -31,7 +31,7 @@ import ( var ErrClusterNotExist = fmt.Errorf("no cluster exist") func GetDefaultClusterName() (string, error) { - files, err := os.ReadDir(constants.Workdir()) + files, err := os.ReadDir(constants.WorkDir()) if err != nil { return "", err } diff --git a/pkg/config/config.go b/pkg/config/config.go index 5e12c7c366c..5819f5ec168 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -20,11 +20,10 @@ import ( "os" "path/filepath" - "github.com/labring/sealos/pkg/clusterfile" - "github.com/imdario/mergo" "sigs.k8s.io/yaml" + "github.com/labring/sealos/pkg/clusterfile" "github.com/labring/sealos/pkg/constants" "github.com/labring/sealos/pkg/types/v1beta1" "github.com/labring/sealos/pkg/utils/file" @@ -78,7 +77,7 @@ func NewConfiguration(name, rootPath string, configs []v1beta1.Config) Interface func NewDefaultConfiguration(clusterName string) Interface { return &Dumper{ - RootPath: constants.NewData(clusterName).RootFSPath(), + RootPath: constants.NewPathResolver(clusterName).RootFSPath(), } } diff --git a/pkg/constants/bash.go b/pkg/constants/bash.go index f0c94f2161f..b8b5a8400b8 100644 --- a/pkg/constants/bash.go +++ b/pkg/constants/bash.go @@ -40,7 +40,7 @@ type Bash interface { } type bash struct { - data Data + pathResolver PathResolver renderContext map[string]string wrap func(string, string) string } @@ -53,7 +53,7 @@ func (b *bash) getFromRenderContextOrDefault(key string) string { } func (b *bash) WrapBash(host, shell string) string { - return fmt.Sprintf(DefaultBashFmt, b.data.RootFSScriptsPath(), b.wrap(host, shell)) + return fmt.Sprintf(DefaultBashFmt, b.pathResolver.RootFSScriptsPath(), b.wrap(host, shell)) } func (b *bash) CheckBash(host string) string { @@ -77,5 +77,5 @@ func (b *bash) CleanRegistryBash(host string) string { } func NewBash(clusterName string, renderContext map[string]string, shellWrapper func(string, string) string) Bash { - return &bash{data: NewData(clusterName), renderContext: renderContext, wrap: shellWrapper} + return &bash{pathResolver: NewPathResolver(clusterName), renderContext: renderContext, wrap: shellWrapper} } diff --git a/pkg/constants/contants.go b/pkg/constants/consts.go similarity index 95% rename from pkg/constants/contants.go rename to pkg/constants/consts.go index 70c9aa06459..deaecc1c932 100644 --- a/pkg/constants/contants.go +++ b/pkg/constants/consts.go @@ -14,8 +14,8 @@ package constants -import ( - "github.com/containers/storage/pkg/homedir" +const ( + DefaultClusterFileName = "Clusterfile" ) const ( @@ -36,10 +36,6 @@ const ( Cluster = "Cluster" ) -func GetHomeDir() string { - return homedir.Get() -} - var AppName = "sealos" var Contact = ` diff --git a/pkg/constants/kube.go b/pkg/constants/kube.go deleted file mode 100644 index 9121f39a849..00000000000 --- a/pkg/constants/kube.go +++ /dev/null @@ -1,23 +0,0 @@ -/* -Copyright 2022 cuisongliu@qq.com. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package constants - -const ( - KubernetesEtcStaticPod = "/etc/kubernetes/manifests" - KubernetesEtc = "/etc/kubernetes" - KubernetesEtcPKI = "/etc/kubernetes/pki" -) diff --git a/pkg/constants/data.go b/pkg/constants/pathresolver.go similarity index 50% rename from pkg/constants/data.go rename to pkg/constants/pathresolver.go index 969c4c122b9..6b6fa92841d 100644 --- a/pkg/constants/data.go +++ b/pkg/constants/pathresolver.go @@ -33,28 +33,37 @@ var ( ) const ( - DefaultInitKubeadmFileName = "kubeadm-init.yaml" - DefaultJoinMasterKubeadmFileName = "kubeadm-join-master.yaml" - DefaultJoinNodeKubeadmFileName = "kubeadm-join-node.yaml" - DefaultKubeadmTokenFileName = "kubeadm-token.json" - DefaultCertificateKeyFileName = "kubeadm-certificate-key.txt" - DefaultUpdateKubeadmFileName = "kubeadm-update.yaml" - DefaultRootfsKubeadmFileName = "kubeadm.yml" - DefaultRootfsConfigFileName = "config.yml" - DataDirName = "rootfs" - EtcDirName = "etc" - ChartsDirName = "charts" - ManifestsDirName = "manifests" - BinDirName = "bin" - RegistryDirName = "registry" - ImagesDirName = "images" - ImageShimDirName = "shim" - PkiDirName = "pki" - PkiEtcdDirName = "etcd" - ScriptsDirName = "scripts" - StaticsDirName = "statics" + DefaultRootfsConfigFileName = "config.yml" + rootFsDirName = "rootfs" + EtcDirName = "etc" + ChartsDirName = "charts" + ManifestsDirName = "manifests" + BinDirName = "bin" + RegistryDirName = "registry" + ImagesDirName = "images" + ImageShimDirName = "shim" + PkiDirName = "pki" + PkiEtcdDirName = "etcd" + ScriptsDirName = "scripts" + StaticsDirName = "statics" ) +func GetHomeDir() string { + return homedir.Get() +} + +func WorkDir() string { + return DefaultRuntimeRootDir +} + +func ClusterDir(clusterName string) string { + return filepath.Join(DefaultRuntimeRootDir, clusterName) +} + +func Clusterfile(clusterName string) string { + return filepath.Join(DefaultRuntimeRootDir, clusterName, DefaultClusterFileName) +} + func GetRuntimeRootDir(name string) string { if v, ok := os.LookupEnv(strings.ToUpper(name) + "_RUNTIME_ROOT"); ok { return v @@ -82,91 +91,105 @@ func IsRegistryDir(entry fs.DirEntry) bool { return entry.IsDir() && entry.Name() == RegistryDirName } -type Data interface { - Homedir() string +type PathResolver interface { + // remote data dir + Root() string RootFSPath() string RootFSEtcPath() string RootFSStaticsPath() string RootFSScriptsPath() string RootFSRegistryPath() string + RootFSManifestsPath() string + RootFSBinPath() string + RootFSSealctlPath() string + ConfigsPath() string // for storing temporary configs in remote + // for persistent runtime configs in local + RunRoot() string PkiPath() string PkiEtcdPath() string AdminFile() string EtcPath() string TmpPath() string - - RootFSCharsPath() string - RootFSManifestsPath() string - RootFSBinPath() string - RootFSSealctlPath() string } -type data struct { +type defaultPathResolver struct { clusterName string } -func (d *data) RootFSSealctlPath() string { +func (d *defaultPathResolver) RootFSSealctlPath() string { return filepath.Join(d.RootFSPath(), "opt", "sealctl") } -func (d *data) RootFSScriptsPath() string { +func (d *defaultPathResolver) RootFSScriptsPath() string { return filepath.Join(d.RootFSPath(), ScriptsDirName) } -func (d *data) RootFSEtcPath() string { + +func (d *defaultPathResolver) RootFSEtcPath() string { return filepath.Join(d.RootFSPath(), EtcDirName) } -func (d *data) RootFSRegistryPath() string { +func (d *defaultPathResolver) RootFSRegistryPath() string { return filepath.Join(d.RootFSPath(), RegistryDirName) } -func (d *data) RootFSCharsPath() string { +func (d *defaultPathResolver) RootFSCharsPath() string { return filepath.Join(d.RootFSPath(), ChartsDirName) } -func (d *data) RootFSManifestsPath() string { +func (d *defaultPathResolver) RootFSManifestsPath() string { return filepath.Join(d.RootFSPath(), ManifestsDirName) } -func (d *data) RootFSBinPath() string { +func (d *defaultPathResolver) RootFSBinPath() string { return filepath.Join(d.RootFSPath(), BinDirName) } -func (d *data) EtcPath() string { - return filepath.Join(ClusterDir(d.clusterName), EtcDirName) +func (d *defaultPathResolver) ConfigsPath() string { + return filepath.Join(d.Root(), EtcDirName) } -func (d *data) AdminFile() string { - return filepath.Join(d.EtcPath(), "admin.conf") +func (d *defaultPathResolver) RootFSPath() string { + return filepath.Join(d.Root(), rootFsDirName) } -func (d *data) PkiPath() string { - return filepath.Join(ClusterDir(d.clusterName), PkiDirName) +func (d *defaultPathResolver) RootFSStaticsPath() string { + return filepath.Join(d.RootFSPath(), StaticsDirName) } -func (d *data) PkiEtcdPath() string { - return filepath.Join(d.PkiPath(), PkiEtcdDirName) +// data dir +func (d *defaultPathResolver) Root() string { + return filepath.Join(DefaultClusterRootFsDir, "data", d.clusterName) +} + +// only for local +func (d *defaultPathResolver) EtcPath() string { + return filepath.Join(d.RunRoot(), EtcDirName) } -func (d *data) TmpPath() string { - return filepath.Join(ClusterDir(d.clusterName), "tmp") +// $HOME/.$APP_NAME/$CLUSTER_NAME/etc/admin.conf +func (d *defaultPathResolver) AdminFile() string { + return filepath.Join(d.EtcPath(), "admin.conf") } -func (d *data) RootFSPath() string { - return filepath.Join(d.Homedir(), DataDirName) +func (d *defaultPathResolver) PkiPath() string { + return filepath.Join(d.RunRoot(), PkiDirName) } -func (d *data) RootFSStaticsPath() string { - return filepath.Join(d.RootFSPath(), StaticsDirName) +func (d *defaultPathResolver) PkiEtcdPath() string { + return filepath.Join(d.PkiPath(), PkiEtcdDirName) } -func (d *data) Homedir() string { - return filepath.Join(DefaultClusterRootFsDir, "data", d.clusterName) +func (d *defaultPathResolver) TmpPath() string { + return filepath.Join(d.RunRoot(), "tmp") +} + +func (d *defaultPathResolver) RunRoot() string { + return filepath.Join(DefaultRuntimeRootDir, d.clusterName) } -func NewData(clusterName string) Data { - return &data{ +func NewPathResolver(clusterName string) PathResolver { + return &defaultPathResolver{ clusterName: clusterName, } } diff --git a/pkg/constants/worker.go b/pkg/constants/worker.go deleted file mode 100644 index 895469240f6..00000000000 --- a/pkg/constants/worker.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2022 cuisongliu@qq.com. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package constants - -import "path/filepath" - -const ( - DefaultClusterFileName = "Clusterfile" -) - -func Workdir() string { - return DefaultRuntimeRootDir -} - -func ClusterDir(clusterName string) string { - return filepath.Join(Workdir(), clusterName) -} - -func Clusterfile(clusterName string) string { - return filepath.Join(Workdir(), clusterName, DefaultClusterFileName) -} diff --git a/pkg/filesystem/registry/sync.go b/pkg/filesystem/registry/sync.go index 138ba23e351..8a763d398fb 100644 --- a/pkg/filesystem/registry/sync.go +++ b/pkg/filesystem/registry/sync.go @@ -48,7 +48,7 @@ const ( ) type impl struct { - pathResolver PathResolver + pathResolver constants.PathResolver ssh ssh.Interface mounts []v2.MountImage } @@ -129,7 +129,7 @@ func trimPortStr(s string) string { return s } -func getRegistryServeCommand(pathResolver PathResolver, port string) string { +func getRegistryServeCommand(pathResolver constants.PathResolver, port string) string { return fmt.Sprintf("%s registry serve filesystem -p %s --disable-logging=true %s", pathResolver.RootFSSealctlPath(), port, pathResolver.RootFSRegistryPath(), ) @@ -205,12 +205,6 @@ func syncViaHTTP(targets []string) func(context.Context, string) error { } } -type PathResolver interface { - RootFSSealctlPath() string - RootFSRegistryPath() string - RootFSPath() string -} - -func New(pathResolver PathResolver, ssh ssh.Interface, mounts []v2.MountImage) filesystem.RegistrySyncer { +func New(pathResolver constants.PathResolver, ssh ssh.Interface, mounts []v2.MountImage) filesystem.RegistrySyncer { return &impl{pathResolver, ssh, mounts} } diff --git a/pkg/filesystem/rootfs/rootfs_default.go b/pkg/filesystem/rootfs/rootfs_default.go index a2e55076d76..5ff8be634ff 100644 --- a/pkg/filesystem/rootfs/rootfs_default.go +++ b/pkg/filesystem/rootfs/rootfs_default.go @@ -55,7 +55,7 @@ func (f *defaultRootfs) getSSH(cluster *v2.Cluster) ssh.Interface { } func (f *defaultRootfs) mountRootfs(cluster *v2.Cluster, ipList []string) error { - target := constants.NewData(f.getClusterName(cluster)).RootFSPath() + target := constants.NewPathResolver(f.getClusterName(cluster)).RootFSPath() ctx := context.Background() eg, _ := errgroup.WithContext(ctx) envProcessor := env.NewEnvProcessor(cluster) @@ -136,7 +136,7 @@ func (f *defaultRootfs) mountRootfs(cluster *v2.Cluster, ipList []string) error } func (f *defaultRootfs) unmountRootfs(cluster *v2.Cluster, ipList []string) error { - clusterRootfsDir := constants.NewData(f.getClusterName(cluster)).Homedir() + clusterRootfsDir := constants.NewPathResolver(f.getClusterName(cluster)).Root() rmRootfs := fmt.Sprintf("rm -rf %s", clusterRootfsDir) deleteHomeDirCmd := fmt.Sprintf("rm -rf %s", constants.ClusterDir(cluster.Name)) eg, _ := errgroup.WithContext(context.Background()) diff --git a/pkg/guest/guest.go b/pkg/guest/guest.go index 1de911e7252..640704430d8 100644 --- a/pkg/guest/guest.go +++ b/pkg/guest/guest.go @@ -47,7 +47,7 @@ func NewGuestManager() (Interface, error) { func (d *Default) Apply(cluster *v2.Cluster, mounts []v2.MountImage, targetHosts []string) error { kubeConfig := filepath.Join(constants.GetHomeDir(), ".kube", "config") if !fileutil.IsExist(kubeConfig) { - adminFile := constants.NewData(cluster.Name).AdminFile() + adminFile := constants.NewPathResolver(cluster.Name).AdminFile() data, err := fileutil.ReadAll(adminFile) if err != nil { return fmt.Errorf("read admin.conf error in guest: %w", err) diff --git a/pkg/registry/password/apply.go b/pkg/registry/password/apply.go index f86ce1fc93a..130bba80b21 100644 --- a/pkg/registry/password/apply.go +++ b/pkg/registry/password/apply.go @@ -137,7 +137,7 @@ func (r *RegistryPasswdResults) Apply(cluster *v1beta1.Cluster) error { if r.upgrade == nil { r.upgrade = NewUpgrade(cluster.Name, r.execer) } - root := constants.NewData(cluster.Name).RootFSPath() + root := constants.NewPathResolver(cluster.Name).RootFSPath() registry := helpers.GetRegistryInfo(r.execer, root, cluster.GetRegistryIPAndPort()) shim := helpers.GetImageCRIShimInfo(r.execer, r.ImageCRIShimFilePath, cluster.GetMaster0IPAndPort()) if registry == nil || shim == nil { diff --git a/pkg/remote/remote.go b/pkg/remote/remote.go index 97e80e0e8b6..2d286e6eb2e 100644 --- a/pkg/remote/remote.go +++ b/pkg/remote/remote.go @@ -51,7 +51,7 @@ const ( type remote struct { clusterName string // TODO: remove it? - pathResolver constants.Data + pathResolver constants.PathResolver execer ssh.Interface } @@ -164,6 +164,6 @@ func New(clusterName string, sshInterface ssh.Interface) Interface { return &remote{ clusterName: clusterName, execer: sshInterface, - pathResolver: constants.NewData(clusterName), + pathResolver: constants.NewPathResolver(clusterName), } } diff --git a/pkg/runtime/interface.go b/pkg/runtime/interface.go index 74d6740c1f8..3f70b858227 100644 --- a/pkg/runtime/interface.go +++ b/pkg/runtime/interface.go @@ -15,12 +15,20 @@ package runtime type Interface interface { + Ruler Init() error Reset() error ScaleUp(newMasterIPList []string, newNodeIPList []string) error ScaleDown(deleteMastersIPList []string, deleteNodesIPList []string) error - SyncNodeIPVS(mastersIPList, nodeIPList []string) error Upgrade(version string) error - GetConfig() ([]byte, error) - UpdateCert(certs []string) error + GetRawConfig() ([]byte, error) +} + +type Ruler interface { + SyncNodeIPVS(masters, nodes []string) error +} + +type CertManager interface { + Renew() error + UpdateCertSANs(certSANs []string) error } diff --git a/pkg/runtime/kubernetes/update_cert.go b/pkg/runtime/kubernetes/certs.go similarity index 84% rename from pkg/runtime/kubernetes/update_cert.go rename to pkg/runtime/kubernetes/certs.go index e1b1764612e..f3550d24885 100644 --- a/pkg/runtime/kubernetes/update_cert.go +++ b/pkg/runtime/kubernetes/certs.go @@ -25,7 +25,6 @@ import ( "k8s.io/apimachinery/pkg/util/json" "github.com/labring/sealos/pkg/client-go/kubernetes" - "github.com/labring/sealos/pkg/constants" "github.com/labring/sealos/pkg/utils/file" "github.com/labring/sealos/pkg/utils/logger" "github.com/labring/sealos/pkg/utils/yaml" @@ -38,8 +37,8 @@ const ( KubeletConf = "kubelet.conf" ) -func (k *KubeadmRuntime) UpdateCert(certs []string) error { - // set sans to kubeadm config object +func (k *KubeadmRuntime) UpdateCertSANs(certSans []string) error { + // set extra cert SANs for kubeadm configmap object if err := k.CompleteKubeadmConfig(setCGroupDriverAndSocket, setCertificateKey); err != nil { return err } @@ -47,8 +46,8 @@ func (k *KubeadmRuntime) UpdateCert(certs []string) error { if err := k.mergeWithBuiltinKubeadmConfig(); err != nil { return err } - if len(certs) != 0 { - k.setCertSANS(append(k.getCertSANS(), certs...)) + if len(certSans) != 0 { + k.setCertSANs(append(k.getCertSANs(), certSans...)) } return nil } @@ -85,23 +84,24 @@ func (k *KubeadmRuntime) saveNewKubeadmConfig() error { } logger.Debug("current cluster config data: %+v", obj) //set certs to obj interface - err = unstructured.SetNestedStringSlice(obj, k.getCertSANS(), "apiServer", "certSANs") + err = unstructured.SetNestedStringSlice(obj, k.getCertSANs(), "apiServer", "certSANs") if err != nil { return err } - certPath := path.Join(k.getContentData().EtcPath(), constants.DefaultUpdateKubeadmFileName) + certPath := path.Join(k.getContentData().EtcPath(), defaultUpdateKubeadmFileName) return yaml.MarshalYamlToFile(certPath, obj) } func (k *KubeadmRuntime) uploadConfigFromKubeadm() error { logger.Info("start to upload kubeadm config for inCluster ...") - outConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultUpdateKubeadmFileName) - data, err := file.ReadAll(outConfigPath) + in := path.Join(k.getContentData().EtcPath(), defaultUpdateKubeadmFileName) + out := path.Join(k.getContentData().ConfigsPath(), defaultUpdateKubeadmFileName) + data, err := file.ReadAll(in) if err != nil { return err } logger.Debug("current update yaml data is %s", string(data)) - err = k.sshCopy(k.getMaster0IPAndPort(), outConfigPath, outConfigPath) + err = k.sshCopy(k.getMaster0IPAndPort(), in, out) if err != nil { return fmt.Errorf("copy config update kubeadm yaml error: %s", err.Error()) } @@ -115,7 +115,7 @@ func (k *KubeadmRuntime) uploadConfigFromKubeadm() error { return nil } -func (k *KubeadmRuntime) UpdateCertByInit() error { +func (k *KubeadmRuntime) InitCertsAndKubeConfigs() error { logger.Info("start to generate cert and kubeConfig...") for _, ipAndPort := range k.getMasterIPAndPortList() { if err := k.sshCmdAsync(ipAndPort, "rm -rf /etc/kubernetes/admin.conf"); err != nil { @@ -125,23 +125,14 @@ func (k *KubeadmRuntime) UpdateCertByInit() error { if err := k.initCert(); err != nil { return err } - if err := k.CreateKubeConfig(); err != nil { + if err := k.CreateKubeConfigFiles(); err != nil { return fmt.Errorf("failed to generate kubernetes conf: %w", err) } return k.SendJoinMasterKubeConfigs(k.getMasterIPAndPortList()[:1], AdminConf, ControllerConf, SchedulerConf, KubeletConf) } func (k *KubeadmRuntime) initCert() error { - pipeline := []func() error{ - k.GenerateCert, - k.SendNewCertAndKeyToMasters, - } - for _, f := range pipeline { - if err := f(); err != nil { - return fmt.Errorf("failed to generate cert %v", err) - } - } - return nil + return k.runPipelines("init cert", k.GenerateCert, k.SendNewCertAndKeyToMasters) } func (k *KubeadmRuntime) showKubeadmCert() error { diff --git a/pkg/runtime/kubernetes/consts.go b/pkg/runtime/kubernetes/consts.go new file mode 100644 index 00000000000..a5a624f8e53 --- /dev/null +++ b/pkg/runtime/kubernetes/consts.go @@ -0,0 +1,31 @@ +// Copyright © 2023 sealos. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +const ( + defaultInitKubeadmFileName = "kubeadm-init.yaml" + defaultJoinMasterKubeadmFileName = "kubeadm-join-master.yaml" + defaultJoinNodeKubeadmFileName = "kubeadm-join-node.yaml" + defaultKubeadmTokenFileName = "kubeadm-token.json" + defaultCertificateKeyFileName = "kubeadm-certificate-key.txt" + defaultUpdateKubeadmFileName = "kubeadm-update.yaml" + defaultRootfsKubeadmFileName = "kubeadm.yml" +) + +const ( + kubernetesEtcStaticPod = "/etc/kubernetes/manifests" + kubernetesEtc = "/etc/kubernetes" + kubernetesEtcPKI = "/etc/kubernetes/pki" +) diff --git a/pkg/runtime/kubernetes/init.go b/pkg/runtime/kubernetes/init.go index a0a29f1ffda..2343bd5bd31 100644 --- a/pkg/runtime/kubernetes/init.go +++ b/pkg/runtime/kubernetes/init.go @@ -19,18 +19,17 @@ import ( "path" "github.com/labring/sealos/pkg/cert" - "github.com/labring/sealos/pkg/constants" "github.com/labring/sealos/pkg/utils/file" "github.com/labring/sealos/pkg/utils/logger" ) -func (k *KubeadmRuntime) ConfigInitKubeadmToMaster0() error { +func (k *KubeadmRuntime) InitKubeadmConfigToMaster0() error { data, err := k.generateInitConfigs() if err != nil { return fmt.Errorf("generate init config error: %v", err) } - initConfigPath := path.Join(k.getContentData().TmpPath(), constants.DefaultInitKubeadmFileName) - outConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultInitKubeadmFileName) + initConfigPath := path.Join(k.getContentData().TmpPath(), defaultInitKubeadmFileName) + outConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultInitKubeadmFileName) err = file.WriteFile(initConfigPath, data) if err != nil { return fmt.Errorf("failed to write tmp init kubeadm config: %v", err) @@ -53,31 +52,27 @@ func (k *KubeadmRuntime) GenerateCert() error { logger.Debug("GenerateCert param:", k.getContentData().PkiPath(), k.getContentData().PkiEtcdPath(), - k.getCertSANS(), + k.getCertSANs(), k.getMaster0IP(), hostName, k.getServiceCIDR(), k.getDNSDomain()) - err = cert.GenerateCert( + return cert.GenerateCert( k.getContentData().PkiPath(), k.getContentData().PkiEtcdPath(), - k.getCertSANS(), + k.getCertSANs(), k.getMaster0IP(), hostName, k.getServiceCIDR(), k.getDNSDomain(), ) - if err != nil { - return fmt.Errorf("generate certs failed: %v", err) - } - return k.sendNewCertAndKey([]string{k.getMaster0IPAndPort()}) } func (k *KubeadmRuntime) SendNewCertAndKeyToMasters() error { return k.sendNewCertAndKey(k.getMasterIPAndPortList()) } -func (k *KubeadmRuntime) CreateKubeConfig() error { +func (k *KubeadmRuntime) CreateKubeConfigFiles() error { logger.Info("start to create kubeconfig...") hostName, err := k.execHostname(k.getMaster0IPAndPort()) if err != nil { diff --git a/pkg/runtime/kubernetes/kubeadm.go b/pkg/runtime/kubernetes/kubeadm.go index 7c9e5f657fd..83c857d4a8c 100644 --- a/pkg/runtime/kubernetes/kubeadm.go +++ b/pkg/runtime/kubernetes/kubeadm.go @@ -19,6 +19,7 @@ import ( "fmt" "path" "path/filepath" + "sync" "time" "github.com/Masterminds/semver/v3" @@ -27,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/util/json" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "github.com/labring/sealos/pkg/constants" "github.com/labring/sealos/pkg/runtime/types" fileutil "github.com/labring/sealos/pkg/utils/file" "github.com/labring/sealos/pkg/utils/iputils" @@ -60,11 +60,11 @@ func (k *KubeadmRuntime) setKubeadmAPIVersion() { } func (k *KubeadmRuntime) setKubeVersion(version string) { - k.KubeadmConfig.SetKubeVersion(version) + k.kubeadmConfig.SetKubeVersion(version) } func (k *KubeadmRuntime) setAPIVersion(apiVersion string) { - k.KubeadmConfig.SetAPIVersion(apiVersion) + k.kubeadmConfig.SetAPIVersion(apiVersion) } // GetterKubeadmAPIVersion is covert version to kubeadmAPIServerVersion @@ -108,31 +108,41 @@ func (k *KubeadmRuntime) getCGroupDriver(node string) (string, error) { return driver, nil } +var ( + mergeOnce sync.Once + mergeErr error +) + // MergeKubeadmConfig Unsafe, dangerous use of goroutines. func (k *KubeadmRuntime) MergeKubeadmConfig() error { - for _, fn := range []string{ - "", // generate default kubeadm configs - k.getDefaultKubeadmConfig(), // merging from predefined path of file if file exists - } { - if err := k.KubeadmConfig.Merge(fn); err != nil { - return err - } - } - // merge from clusterfile - if k.config.KubeadmConfig != nil { - if err := k.KubeadmConfig.LoadFromClusterfile(k.config.KubeadmConfig); err != nil { - return fmt.Errorf("failed to load kubeadm config from clusterfile: %v", err) - } - } - k.setKubeadmAPIVersion() - k.setFeatureGatesConfiguration() - return k.validateVIP(k.getVip()) + mergeOnce.Do(func() { + mergeErr = func() error { + for _, fn := range []string{ + "", // generate default kubeadm configs + k.getDefaultKubeadmConfig(), // merging from predefined path of file if file exists + } { + if err := k.kubeadmConfig.Merge(fn); err != nil { + return err + } + } + // merge from clusterfile + if k.config.KubeadmConfig != nil { + if err := k.kubeadmConfig.LoadFromClusterfile(k.config.KubeadmConfig); err != nil { + return fmt.Errorf("failed to load kubeadm config from clusterfile: %v", err) + } + } + k.setKubeadmAPIVersion() + k.setFeatureGatesConfiguration() + return k.validateVIP(k.getVip()) + }() + }) + return mergeErr } func (k *KubeadmRuntime) validateVIP(ip string) error { for k, sub := range map[string]string{ - "podSubnet": k.KubeadmConfig.ClusterConfiguration.Networking.PodSubnet, - "serviceSubnet": k.KubeadmConfig.ClusterConfiguration.Networking.ServiceSubnet, + "podSubnet": k.kubeadmConfig.ClusterConfiguration.Networking.PodSubnet, + "serviceSubnet": k.kubeadmConfig.ClusterConfiguration.Networking.ServiceSubnet, } { if contains, err := iputils.Contains(sub, ip); err != nil { return err @@ -144,7 +154,7 @@ func (k *KubeadmRuntime) validateVIP(ip string) error { } func (k *KubeadmRuntime) getDefaultKubeadmConfig() string { - return filepath.Join(k.getContentData().RootFSEtcPath(), constants.DefaultRootfsKubeadmFileName) + return filepath.Join(k.getContentData().RootFSEtcPath(), defaultRootfsKubeadmFileName) } func (k *KubeadmRuntime) getVip() string { @@ -152,10 +162,10 @@ func (k *KubeadmRuntime) getVip() string { } func (k *KubeadmRuntime) getAPIServerPort() int32 { - if k.KubeadmConfig.InitConfiguration.LocalAPIEndpoint.BindPort == 0 { - k.KubeadmConfig.InitConfiguration.LocalAPIEndpoint.BindPort = types.DefaultAPIServerPort + if k.kubeadmConfig.InitConfiguration.LocalAPIEndpoint.BindPort == 0 { + k.kubeadmConfig.InitConfiguration.LocalAPIEndpoint.BindPort = types.DefaultAPIServerPort } - return k.KubeadmConfig.InitConfiguration.LocalAPIEndpoint.BindPort + return k.kubeadmConfig.InitConfiguration.LocalAPIEndpoint.BindPort } func (k *KubeadmRuntime) getVipAndPort() string { @@ -170,8 +180,8 @@ func (k *KubeadmRuntime) getClusterAPIServer() string { return fmt.Sprintf("https://%s:%d", k.getAPIServerDomain(), k.getAPIServerPort()) } -func (k *KubeadmRuntime) getCertSANS() []string { - return k.KubeadmConfig.ClusterConfiguration.APIServer.CertSANs +func (k *KubeadmRuntime) getCertSANs() []string { + return k.kubeadmConfig.ClusterConfiguration.APIServer.CertSANs } func (k *KubeadmRuntime) initCertSANS() { @@ -180,15 +190,15 @@ func (k *KubeadmRuntime) initCertSANS() { certSans = append(certSans, k.getAPIServerDomain()) certSans = append(certSans, k.getVip()) certSans = append(certSans, k.getMasterIPList()...) - certSans = append(certSans, k.getCertSANS()...) - k.setCertSANS(certSans) + certSans = append(certSans, k.getCertSANs()...) + k.setCertSANs(certSans) } -func (k *KubeadmRuntime) setCertSANS(certs []string) { +func (k *KubeadmRuntime) setCertSANs(certs []string) { var certSans []string certSans = append(certSans, certs...) certSans = stringsutil.RemoveDuplicate(certSans) - k.KubeadmConfig.ClusterConfiguration.APIServer.CertSANs = certSans + k.kubeadmConfig.ClusterConfiguration.APIServer.CertSANs = certSans } func (k *KubeadmRuntime) mergeWithBuiltinKubeadmConfig() error { @@ -220,7 +230,7 @@ func (k *KubeadmRuntime) mergeWithBuiltinKubeadmConfig() error { certs = append(certs, certsStruct[i].(string)) } logger.Debug("current cluster certSANs: %+v", certs) - k.setCertSANS(certs) + k.setCertSANs(certs) return k.setNetWorking(obj) } @@ -237,33 +247,32 @@ func (k *KubeadmRuntime) setNetWorking(obj map[string]interface{}) error { return fmt.Errorf("networking %s not exist", key) } } - k.KubeadmConfig.ClusterConfiguration.Networking.ServiceSubnet = networkingMap["serviceSubnet"] - k.KubeadmConfig.ClusterConfiguration.Networking.DNSDomain = networkingMap["dnsDomain"] - k.KubeadmConfig.ClusterConfiguration.Networking.PodSubnet = networkingMap["podSubnet"] + k.kubeadmConfig.ClusterConfiguration.Networking.ServiceSubnet = networkingMap["serviceSubnet"] + k.kubeadmConfig.ClusterConfiguration.Networking.DNSDomain = networkingMap["dnsDomain"] + k.kubeadmConfig.ClusterConfiguration.Networking.PodSubnet = networkingMap["podSubnet"] return nil } func (k *KubeadmRuntime) getServiceCIDR() string { - return k.KubeadmConfig.ClusterConfiguration.Networking.ServiceSubnet + return k.kubeadmConfig.ClusterConfiguration.Networking.ServiceSubnet } func (k *KubeadmRuntime) getDNSDomain() string { - if k.KubeadmConfig.ClusterConfiguration.Networking.DNSDomain == "" { - k.KubeadmConfig.ClusterConfiguration.Networking.DNSDomain = types.DefaultDNSDomain + if k.kubeadmConfig.ClusterConfiguration.Networking.DNSDomain == "" { + k.kubeadmConfig.ClusterConfiguration.Networking.DNSDomain = types.DefaultDNSDomain } - return k.KubeadmConfig.ClusterConfiguration.Networking.DNSDomain + return k.kubeadmConfig.ClusterConfiguration.Networking.DNSDomain } -func (k *KubeadmRuntime) writeTokenFile() error { +func (k *KubeadmRuntime) writeTokenFile(file string) error { if err := setCertificateKey(k); err != nil { return err } - tokenFile := path.Join(k.getContentData().EtcPath(), constants.DefaultKubeadmTokenFileName) data, err := k.execToken(k.getMaster0IPAndPort(), k.getInitCertificateKey()) if err != nil { return err } - if err = fileutil.WriteFile(tokenFile, []byte(data)); err != nil { + if err = fileutil.WriteFile(file, []byte(data)); err != nil { return err } var t types.Token @@ -271,16 +280,16 @@ func (k *KubeadmRuntime) writeTokenFile() error { if err != nil { return err } - k.Token = &t + k.token = &t return nil } func (k *KubeadmRuntime) setKubernetesToken() error { - if k.Token == nil { + if k.token == nil { logger.Info("start to get kubernetes token...") - tokenFile := path.Join(k.getContentData().EtcPath(), constants.DefaultKubeadmTokenFileName) + tokenFile := path.Join(k.getContentData().EtcPath(), defaultKubeadmTokenFileName) if !fileutil.IsExist(tokenFile) { - err := k.writeTokenFile() + err := k.writeTokenFile(tokenFile) if err != nil { return err } @@ -296,124 +305,124 @@ func (k *KubeadmRuntime) setKubernetesToken() error { } now := time.Now() sub := t.Expires.Time.Sub(now) - //only get 3min token + // regenerate token if it expires in 180 seconds if sub <= 180 { - err = k.writeTokenFile() + err = k.writeTokenFile(tokenFile) if err != nil { return err } } else { - k.Token = &t + k.token = &t } } } - k.setJoinToken(k.Token.JoinToken) - k.setTokenCaCertHash(k.Token.DiscoveryTokenCaCertHash) - k.setJoinCertificateKey(k.Token.CertificateKey) + k.setJoinToken(k.token.JoinToken) + k.setTokenCaCertHash(k.token.DiscoveryTokenCaCertHash) + k.setJoinCertificateKey(k.token.CertificateKey) return nil } func (k *KubeadmRuntime) setJoinToken(token string) { - if k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken == nil { - k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken = &kubeadm.BootstrapTokenDiscovery{} + if k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken == nil { + k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken = &kubeadm.BootstrapTokenDiscovery{} } - k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.Token = token + k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.Token = token } func (k *KubeadmRuntime) getJoinToken() string { - if k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken == nil { + if k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken == nil { return "" } - return k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.Token + return k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.Token } func (k *KubeadmRuntime) setTokenCaCertHash(tokenCaCertHash []string) { - if k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken == nil { - k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken = &kubeadm.BootstrapTokenDiscovery{} + if k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken == nil { + k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken = &kubeadm.BootstrapTokenDiscovery{} } - k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.CACertHashes = tokenCaCertHash + k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.CACertHashes = tokenCaCertHash } func (k *KubeadmRuntime) getTokenCaCertHash() []string { - if k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken == nil || len(k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.CACertHashes) == 0 { + if k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken == nil || len(k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.CACertHashes) == 0 { return nil } - return k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.CACertHashes + return k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.CACertHashes } func (k *KubeadmRuntime) setJoinCertificateKey(certificateKey string) { - if k.KubeadmConfig.JoinConfiguration.ControlPlane == nil { - k.KubeadmConfig.JoinConfiguration.ControlPlane = &kubeadm.JoinControlPlane{} + if k.kubeadmConfig.JoinConfiguration.ControlPlane == nil { + k.kubeadmConfig.JoinConfiguration.ControlPlane = &kubeadm.JoinControlPlane{} } - k.KubeadmConfig.JoinConfiguration.ControlPlane.CertificateKey = certificateKey + k.kubeadmConfig.JoinConfiguration.ControlPlane.CertificateKey = certificateKey } func (k *KubeadmRuntime) setInitCertificateKey(certificateKey string) { - k.KubeadmConfig.InitConfiguration.CertificateKey = certificateKey + k.kubeadmConfig.InitConfiguration.CertificateKey = certificateKey } func (k *KubeadmRuntime) getInitCertificateKey() string { - return k.KubeadmConfig.InitConfiguration.CertificateKey + return k.kubeadmConfig.InitConfiguration.CertificateKey } func (k *KubeadmRuntime) getJoinCertificateKey() string { - if k.KubeadmConfig.JoinConfiguration.ControlPlane == nil { + if k.kubeadmConfig.JoinConfiguration.ControlPlane == nil { return "" } - return k.KubeadmConfig.JoinConfiguration.ControlPlane.CertificateKey + return k.kubeadmConfig.JoinConfiguration.ControlPlane.CertificateKey } func (k *KubeadmRuntime) setAPIServerEndpoint(endpoint string) { - k.KubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.APIServerEndpoint = endpoint + k.kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.APIServerEndpoint = endpoint } func (k *KubeadmRuntime) setInitAdvertiseAddress(advertiseAddress string) { - k.KubeadmConfig.InitConfiguration.LocalAPIEndpoint.AdvertiseAddress = advertiseAddress + k.kubeadmConfig.InitConfiguration.LocalAPIEndpoint.AdvertiseAddress = advertiseAddress } func (k *KubeadmRuntime) setJoinAdvertiseAddress(advertiseAddress string) { - if k.KubeadmConfig.JoinConfiguration.ControlPlane == nil { - k.KubeadmConfig.JoinConfiguration.ControlPlane = &kubeadm.JoinControlPlane{} + if k.kubeadmConfig.JoinConfiguration.ControlPlane == nil { + k.kubeadmConfig.JoinConfiguration.ControlPlane = &kubeadm.JoinControlPlane{} } - k.KubeadmConfig.JoinConfiguration.ControlPlane.LocalAPIEndpoint.AdvertiseAddress = advertiseAddress + k.kubeadmConfig.JoinConfiguration.ControlPlane.LocalAPIEndpoint.AdvertiseAddress = advertiseAddress } func (k *KubeadmRuntime) cleanJoinLocalAPIEndPoint() { - k.KubeadmConfig.JoinConfiguration.ControlPlane = nil + k.kubeadmConfig.JoinConfiguration.ControlPlane = nil } func (k *KubeadmRuntime) setControlPlaneEndpoint(endpoint string) { - k.KubeadmConfig.ClusterConfiguration.ControlPlaneEndpoint = endpoint + k.kubeadmConfig.ClusterConfiguration.ControlPlaneEndpoint = endpoint } func (k *KubeadmRuntime) setCgroupDriver(cGroup string) { - k.KubeadmConfig.KubeletConfiguration.CgroupDriver = cGroup + k.kubeadmConfig.KubeletConfiguration.CgroupDriver = cGroup } func (k *KubeadmRuntime) setInitTaints() { - if len(k.Cluster.GetAllIPS()) == 1 && - k.KubeadmConfig.InitConfiguration.NodeRegistration.Taints == nil { + if len(k.cluster.GetAllIPS()) == 1 && + k.kubeadmConfig.InitConfiguration.NodeRegistration.Taints == nil { //set this field to an empty slice avoid to taint control-plane in single host - k.KubeadmConfig.InitConfiguration.NodeRegistration.Taints = make([]v1.Taint, 0) + k.kubeadmConfig.InitConfiguration.NodeRegistration.Taints = make([]v1.Taint, 0) } } func (k *KubeadmRuntime) setExcludeCIDRs() { - k.KubeadmConfig.KubeProxyConfiguration.IPVS.ExcludeCIDRs = append( - k.KubeadmConfig.KubeProxyConfiguration.IPVS.ExcludeCIDRs, fmt.Sprintf("%s/32", k.getVip())) - k.KubeadmConfig.KubeProxyConfiguration.IPVS.ExcludeCIDRs = stringsutil.RemoveDuplicate(k.KubeadmConfig.KubeProxyConfiguration.IPVS.ExcludeCIDRs) + k.kubeadmConfig.KubeProxyConfiguration.IPVS.ExcludeCIDRs = append( + k.kubeadmConfig.KubeProxyConfiguration.IPVS.ExcludeCIDRs, fmt.Sprintf("%s/32", k.getVip())) + k.kubeadmConfig.KubeProxyConfiguration.IPVS.ExcludeCIDRs = stringsutil.RemoveDuplicate(k.kubeadmConfig.KubeProxyConfiguration.IPVS.ExcludeCIDRs) } func (k *KubeadmRuntime) getEtcdDataDir() string { const defaultEtcdDataDir = "/var/lib/etcd" - if k.KubeadmConfig.ClusterConfiguration.Etcd.Local == nil { + if k.kubeadmConfig.ClusterConfiguration.Etcd.Local == nil { return defaultEtcdDataDir } - if k.KubeadmConfig.ClusterConfiguration.Etcd.Local.DataDir == "" { + if k.kubeadmConfig.ClusterConfiguration.Etcd.Local.DataDir == "" { return defaultEtcdDataDir } - return k.KubeadmConfig.ClusterConfiguration.Etcd.Local.DataDir + return k.kubeadmConfig.ClusterConfiguration.Etcd.Local.DataDir } func (k *KubeadmRuntime) getCRISocket(node string) (string, error) { @@ -427,8 +436,8 @@ func (k *KubeadmRuntime) getCRISocket(node string) (string, error) { //nolint:all func (k *KubeadmRuntime) setCRISocket(criSocket string) { - k.KubeadmConfig.JoinConfiguration.NodeRegistration.CRISocket = criSocket - k.KubeadmConfig.InitConfiguration.NodeRegistration.CRISocket = criSocket + k.kubeadmConfig.JoinConfiguration.NodeRegistration.CRISocket = criSocket + k.kubeadmConfig.InitConfiguration.NodeRegistration.CRISocket = criSocket } var setCGroupDriverAndSocket = func(krt *KubeadmRuntime) error { @@ -436,7 +445,7 @@ var setCGroupDriverAndSocket = func(krt *KubeadmRuntime) error { } var setCertificateKey = func(krt *KubeadmRuntime) error { - certificateKeyFile := path.Join(krt.getContentData().EtcPath(), constants.DefaultCertificateKeyFileName) + certificateKeyFile := path.Join(krt.getContentData().EtcPath(), defaultCertificateKeyFileName) var key string if !fileutil.IsExist(certificateKeyFile) { key, _ = rand.CreateCertificateKey() @@ -459,7 +468,10 @@ func (k *KubeadmRuntime) generateInitConfigs() ([]byte, error) { if err := k.CompleteKubeadmConfig(setCGroupDriverAndSocket, setCertificateKey); err != nil { return nil, err } - conversion := k.KubeadmConfig.GetConvertedKubeadmConfig() + conversion, err := k.kubeadmConfig.ToConvertedKubeadmConfig() + if err != nil { + return nil, err + } return yaml.MarshalYamlConfigs(&conversion.InitConfiguration, &conversion.ClusterConfiguration, &conversion.KubeletConfiguration, @@ -477,25 +489,18 @@ func (k *KubeadmRuntime) CompleteKubeadmConfig(fns ...func(*KubeadmRuntime) erro } k.setInitAdvertiseAddress(k.getMaster0IP()) k.setControlPlaneEndpoint(fmt.Sprintf("%s:%d", k.getAPIServerDomain(), k.getAPIServerPort())) - if k.KubeadmConfig.ClusterConfiguration.APIServer.ExtraArgs == nil { - k.KubeadmConfig.ClusterConfiguration.APIServer.ExtraArgs = make(map[string]string) + if k.kubeadmConfig.ClusterConfiguration.APIServer.ExtraArgs == nil { + k.kubeadmConfig.ClusterConfiguration.APIServer.ExtraArgs = make(map[string]string) } k.setExcludeCIDRs() k.initCertSANS() k.setInitTaints() // after all merging done, set default fields - k.finalizeInitConfig() + k.kubeadmConfig.SetDefaults() - if err := k.KubeadmConfig.ConvertKubeadmVersion(); err != nil { - return fmt.Errorf("convert kubeadm version failed: %w", err) - } return nil } -func (k *KubeadmRuntime) finalizeInitConfig() { - k.KubeadmConfig.FinalizeInitConfig() -} - func (k *KubeadmRuntime) generateJoinNodeConfigs(node string) ([]byte, error) { if err := k.MergeKubeadmConfig(); err != nil { return nil, err @@ -505,10 +510,11 @@ func (k *KubeadmRuntime) generateJoinNodeConfigs(node string) ([]byte, error) { } k.cleanJoinLocalAPIEndPoint() k.setAPIServerEndpoint(k.getVipAndPort()) - if err := k.KubeadmConfig.ConvertKubeadmVersion(); err != nil { - return nil, fmt.Errorf("convert kubeadm version failed: %w", err) + + conversion, err := k.kubeadmConfig.ToConvertedKubeadmConfig() + if err != nil { + return nil, err } - conversion := k.KubeadmConfig.GetConvertedKubeadmConfig() return yaml.MarshalYamlConfigs( &conversion.KubeletConfiguration, &conversion.JoinConfiguration) @@ -523,10 +529,11 @@ func (k *KubeadmRuntime) generateJoinMasterConfigs(masterIP string) ([]byte, err } k.setJoinAdvertiseAddress(iputils.GetHostIP(masterIP)) k.setAPIServerEndpoint(fmt.Sprintf("%s:%d", k.getMaster0IP(), k.getAPIServerPort())) - if err := k.KubeadmConfig.ConvertKubeadmVersion(); err != nil { - return nil, fmt.Errorf("convert kubeadm version failed: %w", err) + + conversion, err := k.kubeadmConfig.ToConvertedKubeadmConfig() + if err != nil { + return nil, err } - conversion := k.KubeadmConfig.GetConvertedKubeadmConfig() return yaml.MarshalYamlConfigs(&conversion.JoinConfiguration, &conversion.KubeletConfiguration) } diff --git a/pkg/runtime/kubernetes/kubeadm_cmd.go b/pkg/runtime/kubernetes/kubeadm_cmd.go index 9778d9a7890..855dd46ff2d 100644 --- a/pkg/runtime/kubernetes/kubeadm_cmd.go +++ b/pkg/runtime/kubernetes/kubeadm_cmd.go @@ -23,8 +23,6 @@ import ( "strings" "github.com/Masterminds/semver/v3" - - "github.com/labring/sealos/pkg/constants" ) type CommandType int @@ -52,10 +50,10 @@ const ( ) func (k *KubeadmRuntime) Command(version string, cmdType CommandType) (cmd string) { - initConfigPath := k.initMasterKubeadmConfigFile() - joinMasterConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultJoinMasterKubeadmFileName) - joinNodeConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultJoinNodeKubeadmFileName) - updateClusterConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultUpdateKubeadmFileName) + initConfigPath := k.getInitMasterKubeadmConfigFilePath() + joinMasterConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultJoinMasterKubeadmFileName) + joinNodeConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultJoinNodeKubeadmFileName) + updateClusterConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultUpdateKubeadmFileName) var discoveryTokens []string for _, data := range k.getTokenCaCertHash() { @@ -93,7 +91,3 @@ func (k *KubeadmRuntime) Command(version string, cmdType CommandType) (cmd strin } return fmt.Sprintf("%s%s", cmd, vlogToStr(k.klogLevel)) } - -func (k *KubeadmRuntime) initMasterKubeadmConfigFile() string { - return path.Join(k.getContentData().EtcPath(), constants.DefaultInitKubeadmFileName) -} diff --git a/pkg/runtime/kubernetes/kubeconfig.go b/pkg/runtime/kubernetes/kubeconfig.go index 216defc24fa..2ce46ebdab7 100644 --- a/pkg/runtime/kubernetes/kubeconfig.go +++ b/pkg/runtime/kubernetes/kubeconfig.go @@ -20,10 +20,10 @@ import "path" const copyKubeAdminConfigCommand = `rm -rf $HOME/.kube/config && mkdir -p $HOME/.kube && cp /etc/kubernetes/admin.conf $HOME/.kube/config` -func (k *KubeadmRuntime) copyNodeKubeConfig(hosts []string) error { - srcKubeFile := k.getContentData().AdminFile() - desKubeFile := path.Join(".kube", "config") - return k.sendFileToHosts(hosts, srcKubeFile, desKubeFile) +func (k *KubeadmRuntime) copyKubeConfigFileToNodes(hosts []string) error { + src := k.getContentData().AdminFile() + dst := path.Join(".kube", "config") + return k.sendFileToHosts(hosts, src, dst) } func (k *KubeadmRuntime) copyMasterKubeConfig(host string) error { diff --git a/pkg/runtime/kubernetes/master.go b/pkg/runtime/kubernetes/master.go index 96c06a478e5..7d0a739ebfa 100644 --- a/pkg/runtime/kubernetes/master.go +++ b/pkg/runtime/kubernetes/master.go @@ -19,7 +19,6 @@ import ( "fmt" "path" - "github.com/labring/sealos/pkg/constants" "github.com/labring/sealos/pkg/ssh" "github.com/labring/sealos/pkg/utils/file" "github.com/labring/sealos/pkg/utils/logger" @@ -30,8 +29,8 @@ import ( func (k *KubeadmRuntime) InitMaster0() error { logger.Info("start to init master0...") - - err := k.execHostsAppend(k.getMaster0IPAndPort(), k.getMaster0IP(), k.getAPIServerDomain()) + master0 := k.getMaster0IPAndPort() + err := k.execHostsAppend(master0, k.getMaster0IP(), k.getAPIServerDomain()) if err != nil { return fmt.Errorf("add apiserver domain hosts failed %v", err) } @@ -40,11 +39,11 @@ func (k *KubeadmRuntime) InitMaster0() error { if cmdInit == "" { return fmt.Errorf("get init master command failed, kubernetes version is %s", k.getKubeVersion()) } - err = k.sshCmdAsync(k.getMaster0IPAndPort(), cmdInit) + err = k.sshCmdAsync(master0, cmdInit) if err != nil { return fmt.Errorf("init master0 failed, error: %s. Please clean and reinstall", err.Error()) } - return k.copyMasterKubeConfig(k.getMaster0IPAndPort()) + return k.copyMasterKubeConfig(master0) } // sendJoinCPConfig send join CP masters configuration @@ -67,8 +66,8 @@ func (k *KubeadmRuntime) ConfigJoinMasterKubeadmToMaster(master string) error { if err != nil { return fmt.Errorf("failed to generate join master kubeadm config: %s", err.Error()) } - joinConfigPath := path.Join(k.getContentData().TmpPath(), constants.DefaultJoinMasterKubeadmFileName) - outConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultJoinMasterKubeadmFileName) + joinConfigPath := path.Join(k.getContentData().TmpPath(), defaultJoinMasterKubeadmFileName) + outConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultJoinMasterKubeadmFileName) err = file.WriteFile(joinConfigPath, data) if err != nil { return fmt.Errorf("write config join master kubeadm config error: %s", err.Error()) @@ -90,7 +89,7 @@ func (k *KubeadmRuntime) joinMasters(masters []string) error { return fmt.Errorf("join masters wait for ssh ready time out: %w", err) } - if err = k.CopyStaticFiles(masters); err != nil { + if err = k.copyStaticFiles(masters); err != nil { return err } @@ -107,11 +106,12 @@ func (k *KubeadmRuntime) joinMasters(masters []string) error { if err = k.sendJoinCPConfig(masters); err != nil { return err } + // does it necessary? if err = k.mergeWithBuiltinKubeadmConfig(); err != nil { return err } - cmd := k.Command(k.getKubeVersion(), JoinMaster) - if cmd == "" { + joinCmd := k.Command(k.getKubeVersion(), JoinMaster) + if joinCmd == "" { return fmt.Errorf("get join master command failed, kubernetes version is %s", k.getKubeVersion()) } for _, master := range masters { @@ -127,7 +127,7 @@ func (k *KubeadmRuntime) joinMasters(masters []string) error { return fmt.Errorf("add master0 apiserver domain hosts to %s failed %v", master, err) } - err = k.sshCmdAsync(master, cmd) + err = k.sshCmdAsync(master, joinCmd) if err != nil { return fmt.Errorf("exec kubeadm join in %s failed %v", master, err) } @@ -175,6 +175,7 @@ func (k *KubeadmRuntime) deleteMaster(master string) error { //remove master masterIPs := strings.SliceRemoveStr(k.getMasterIPList(), master) if len(masterIPs) > 0 { + // TODO: do we need draining first? if err := k.RemoveNodeFromK8sClient(master); err != nil { logger.Warn(fmt.Errorf("delete master %s failed %v", master, err)) } diff --git a/pkg/runtime/kubernetes/node.go b/pkg/runtime/kubernetes/node.go index b0bf5a20e29..d6587331e5b 100644 --- a/pkg/runtime/kubernetes/node.go +++ b/pkg/runtime/kubernetes/node.go @@ -46,7 +46,7 @@ func (k *KubeadmRuntime) joinNodes(newNodesIPList []string) error { eg.Go(func() error { logger.Info("start to join %s as worker", node) k.mu.Lock() - err = k.ConfigJoinNodeKubeadmToNode(node) + err = k.copyKubeadmConfigToNode(node) if err != nil { return fmt.Errorf("failed to copy join node kubeadm config %s %v", node, err) } @@ -65,11 +65,11 @@ func (k *KubeadmRuntime) joinNodes(newNodesIPList []string) error { return fmt.Errorf("run ipvs once failed %v", err) } logger.Info("start join node: %s", node) - cmd := k.Command(k.getKubeVersion(), JoinNode) - if cmd == "" { + joinCmd := k.Command(k.getKubeVersion(), JoinNode) + if joinCmd == "" { return fmt.Errorf("get join node command failed, kubernetes version is %s", k.getKubeVersion()) } - if err = k.sshCmdAsync(node, cmd); err != nil { + if err = k.sshCmdAsync(node, joinCmd); err != nil { return fmt.Errorf("failed to join node %s %v", node, err) } logger.Info("succeeded in joining %s as worker", node) @@ -79,14 +79,14 @@ func (k *KubeadmRuntime) joinNodes(newNodesIPList []string) error { return eg.Wait() } -func (k *KubeadmRuntime) ConfigJoinNodeKubeadmToNode(node string) error { +func (k *KubeadmRuntime) copyKubeadmConfigToNode(node string) error { logger.Info("start to copy kubeadm join config to node: %s", node) data, err := k.generateJoinNodeConfigs(node) if err != nil { return fmt.Errorf("failed to generate join kubeadm config: %v", err) } - joinConfigPath := path.Join(k.getContentData().TmpPath(), constants.DefaultJoinNodeKubeadmFileName) - outConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultJoinNodeKubeadmFileName) + joinConfigPath := path.Join(k.getContentData().TmpPath(), defaultJoinNodeKubeadmFileName) + outConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultJoinNodeKubeadmFileName) err = file.WriteFile(joinConfigPath, data) if err != nil { return fmt.Errorf("write config join kubeadm config error: %s", err.Error()) diff --git a/pkg/runtime/kubernetes/runtime.go b/pkg/runtime/kubernetes/runtime.go index b9064903034..a1929245d5a 100644 --- a/pkg/runtime/kubernetes/runtime.go +++ b/pkg/runtime/kubernetes/runtime.go @@ -33,43 +33,45 @@ import ( type KubeadmRuntime struct { config *types.Config // from argument - Cluster *v2.Cluster + cluster *v2.Cluster - Token *types.Token - KubeadmConfig *types.KubeadmConfig // a deep copy from config.KubeadmConfig or a new one + token *types.Token + kubeadmConfig *types.KubeadmConfig // a deep copy from config.KubeadmConfig or a new one klogLevel int cli kubernetes.Client clusterClient ssh.Interface - pathResolver constants.Data + pathResolver constants.PathResolver remoteUtil remote.Interface mu sync.Mutex } func (k *KubeadmRuntime) Init() error { - pipeline := []func() error{ - k.ConfigInitKubeadmToMaster0, - k.UpdateCertByInit, + return k.runPipelines("init masters", + k.InitKubeadmConfigToMaster0, + k.InitCertsAndKubeConfigs, k.CopyStaticFilesToMasters, k.InitMaster0, - } - - return k.pipeline("init", pipeline) + ) } -func (k *KubeadmRuntime) GetConfig() ([]byte, error) { +func (k *KubeadmRuntime) GetRawConfig() ([]byte, error) { if k.config.KubeadmConfig == nil { return nil, errors.New("please provide a nonnull config") } in := *k.config.KubeadmConfig - k.KubeadmConfig = &in + k.kubeadmConfig = &in if err := k.CompleteKubeadmConfig(); err != nil { return nil, err } - k.Cluster.Status = v2.ClusterStatus{} - conversion := k.KubeadmConfig.GetConvertedKubeadmConfig() - objects := []interface{}{k.Cluster, + k.cluster.Status = v2.ClusterStatus{} + + conversion, err := k.kubeadmConfig.ToConvertedKubeadmConfig() + if err != nil { + return nil, err + } + objects := []interface{}{k.cluster, conversion.InitConfiguration, conversion.ClusterConfiguration, conversion.JoinConfiguration, @@ -100,7 +102,7 @@ func (k *KubeadmRuntime) ScaleUp(newMasterIPList []string, newNodeIPList []strin if err := k.joinNodes(newNodeIPList); err != nil { return err } - return k.copyNodeKubeConfig(newNodeIPList) + return k.copyKubeConfigFileToNodes(newNodeIPList) } return nil } @@ -122,14 +124,14 @@ func (k *KubeadmRuntime) ScaleDown(deleteMastersIPList []string, deleteNodesIPLi func newKubeadmRuntime(cluster *v2.Cluster, kubeadm *types.KubeadmConfig) (*KubeadmRuntime, error) { sshClient := ssh.NewSSHByCluster(cluster, true) k := &KubeadmRuntime{ - Cluster: cluster, + cluster: cluster, config: &types.Config{ KubeadmConfig: kubeadm, APIServerDomain: types.DefaultAPIServerDomain, }, - KubeadmConfig: types.NewKubeadmConfig(), + kubeadmConfig: types.NewKubeadmConfig(), clusterClient: sshClient, - pathResolver: constants.NewData(cluster.GetName()), + pathResolver: constants.NewPathResolver(cluster.GetName()), remoteUtil: remote.New(cluster.GetName(), sshClient), } if err := k.Validate(); err != nil { @@ -146,13 +148,13 @@ func New(cluster *v2.Cluster, kubeadm *types.KubeadmConfig) (*KubeadmRuntime, er } func (k *KubeadmRuntime) Validate() error { - if len(k.Cluster.Spec.Hosts) == 0 { + if len(k.cluster.Spec.Hosts) == 0 { return fmt.Errorf("master hosts cannot be empty") } if k.getMaster0IP() == "" { return fmt.Errorf("master hosts ip cannot be empty") } - if k.getKubeVersionFromImage() == "" && k.Cluster.DeletionTimestamp.IsZero() { + if k.getKubeVersionFromImage() == "" && k.cluster.DeletionTimestamp.IsZero() { return fmt.Errorf("cluster image kubernetes version cannot be empty") } return nil diff --git a/pkg/runtime/kubernetes/runtime_getter.go b/pkg/runtime/kubernetes/runtime_getter.go index d6356ceca04..b40c11c9864 100644 --- a/pkg/runtime/kubernetes/runtime_getter.go +++ b/pkg/runtime/kubernetes/runtime_getter.go @@ -19,6 +19,7 @@ package kubernetes import ( "context" "fmt" + "path" "strings" "golang.org/x/sync/errgroup" @@ -36,12 +37,12 @@ import ( ) func (k *KubeadmRuntime) getKubeVersion() string { - return k.KubeadmConfig.ClusterConfiguration.KubernetesVersion + return k.kubeadmConfig.ClusterConfiguration.KubernetesVersion } // old implementation doesn't consider multiple rootfs images; here get the first rootfs image func (k *KubeadmRuntime) getKubeVersionFromImage() string { - img := k.Cluster.GetRootfsImage() + img := k.cluster.GetRootfsImage() if img == nil || img.Labels == nil { return "" } @@ -49,11 +50,11 @@ func (k *KubeadmRuntime) getKubeVersionFromImage() string { } func (k *KubeadmRuntime) getMaster0IP() string { - return iputils.GetHostIP(k.Cluster.GetMaster0IP()) + return iputils.GetHostIP(k.cluster.GetMaster0IP()) } func (k *KubeadmRuntime) getMasterIPList() []string { - return k.Cluster.GetMasterIPList() + return k.cluster.GetMasterIPList() } func (k *KubeadmRuntime) getMasterIPListAndHTTPSPort() []string { @@ -65,19 +66,19 @@ func (k *KubeadmRuntime) getMasterIPListAndHTTPSPort() []string { } func (k *KubeadmRuntime) getNodeIPList() []string { - return k.Cluster.GetNodeIPList() + return k.cluster.GetNodeIPList() } func (k *KubeadmRuntime) getMasterIPAndPortList() []string { - return k.Cluster.GetMasterIPAndPortList() + return k.cluster.GetMasterIPAndPortList() } func (k *KubeadmRuntime) getNodeIPAndPortList() []string { - return k.Cluster.GetNodeIPAndPortList() + return k.cluster.GetNodeIPAndPortList() } func (k *KubeadmRuntime) getMaster0IPAndPort() string { - return k.Cluster.GetMaster0IPAndPort() + return k.cluster.GetMaster0IPAndPort() } func (k *KubeadmRuntime) getMaster0IPAPIServer() string { @@ -100,7 +101,7 @@ func (k *KubeadmRuntime) getVIPFromImage() string { if vip == "" { vip = DefaultVIP } else { - envsInRootFsImage := k.Cluster.GetRootfsImage().Env + envsInRootFsImage := k.cluster.GetRootfsImage().Env envs := maps.MergeMap(envsInRootFsImage, k.getEnvInterface().Getenv(k.getMaster0IP())) vip = stringsutil.RenderTextFromEnv(vip, envs) } @@ -146,12 +147,11 @@ func (k *KubeadmRuntime) execIPVSPod(ip string, masters []string) error { } func (k *KubeadmRuntime) execToken(ip, certificateKey string) (string, error) { - return k.getRemoteInterface().Token(ip, k.initMasterKubeadmConfigFile(), certificateKey) + return k.getRemoteInterface().Token(ip, k.getInitMasterKubeadmConfigFilePath(), certificateKey) } func (k *KubeadmRuntime) execHostname(ip string) (string, error) { hostname, err := k.getRemoteInterface().Hostname(ip) - // tips: if hostname is upper,kubelet init master0 is not allowed to modify node return strings.ToLower(hostname), err } @@ -164,7 +164,7 @@ func (k *KubeadmRuntime) execCert(ip string) error { if err != nil { return err } - return k.getRemoteInterface().Cert(ip, k.getCertSANS(), iputils.GetHostIP(ip), hostname, k.getServiceCIDR(), k.getDNSDomain()) + return k.getRemoteInterface().Cert(ip, k.getCertSANs(), iputils.GetHostIP(ip), hostname, k.getServiceCIDR(), k.getDNSDomain()) } func (k *KubeadmRuntime) execHostsDelete(ip, domain string) error { @@ -188,7 +188,7 @@ func (k *KubeadmRuntime) sshCopy(host, srcFilePath, dstFilePath string) error { } func (k *KubeadmRuntime) getImageLabels() map[string]string { - return k.Cluster.GetImageLabels() + return k.cluster.GetImageLabels() } func (k *KubeadmRuntime) getSSHInterface() ssh.Interface { @@ -196,14 +196,14 @@ func (k *KubeadmRuntime) getSSHInterface() ssh.Interface { } func (k *KubeadmRuntime) getEnvInterface() env.Interface { - return env.NewEnvProcessor(k.Cluster) + return env.NewEnvProcessor(k.cluster) } func (k *KubeadmRuntime) getRemoteInterface() remote.Interface { return k.remoteUtil } -func (k *KubeadmRuntime) getContentData() constants.Data { +func (k *KubeadmRuntime) getContentData() constants.PathResolver { return k.pathResolver } @@ -226,3 +226,7 @@ func (k *KubeadmRuntime) getKubeExpansion() (kubernetes.Expansion, error) { } return kubernetes.NewKubeExpansion(ki.Kubernetes()), nil } + +func (k *KubeadmRuntime) getInitMasterKubeadmConfigFilePath() string { + return path.Join(k.getContentData().ConfigsPath(), defaultInitKubeadmFileName) +} diff --git a/pkg/runtime/kubernetes/static_files.go b/pkg/runtime/kubernetes/static_files.go index 38585cbe08e..5626f643117 100644 --- a/pkg/runtime/kubernetes/static_files.go +++ b/pkg/runtime/kubernetes/static_files.go @@ -19,13 +19,13 @@ import ( "fmt" "path/filepath" - "github.com/labring/sealos/pkg/utils/logger" - "golang.org/x/sync/errgroup" + + "github.com/labring/sealos/pkg/utils/logger" ) const ( - AuditPolicyYml = "audit-policy.yml" + auditPolicyYml = "audit-policy.yml" copyFileToDirCommand = "mkdir -p %s && cp -f %s %s" ) @@ -39,15 +39,15 @@ type StaticFile struct { var MasterStaticFiles = []*StaticFile{ { DestinationDir: "/etc/kubernetes", - Name: AuditPolicyYml, + Name: auditPolicyYml, }, } func (k *KubeadmRuntime) CopyStaticFilesToMasters() error { - return k.CopyStaticFiles(k.getMasterIPAndPortList()) + return k.copyStaticFiles(k.getMasterIPAndPortList()) } -func (k *KubeadmRuntime) CopyStaticFiles(nodes []string) error { +func (k *KubeadmRuntime) copyStaticFiles(nodes []string) error { logger.Info("start to copy static files to masters") for _, file := range MasterStaticFiles { staticFilePath := filepath.Join(k.getContentData().RootFSStaticsPath(), file.Name) diff --git a/pkg/runtime/kubernetes/upgrade.go b/pkg/runtime/kubernetes/upgrade.go index c8fc962b100..c8dd1ad05d8 100644 --- a/pkg/runtime/kubernetes/upgrade.go +++ b/pkg/runtime/kubernetes/upgrade.go @@ -179,15 +179,15 @@ func (k *KubeadmRuntime) autoUpdateConfig(version string) error { } kk := &KubeadmRuntime{ - KubeadmConfig: defaultKubeadmConfig, + kubeadmConfig: defaultKubeadmConfig, } kk.setKubeVersion(version) kk.setFeatureGatesConfiguration() - if err = kk.KubeadmConfig.ConvertKubeadmVersion(); err != nil { + + conversion, err := kk.kubeadmConfig.ToConvertedKubeadmConfig() + if err != nil { return err } - - conversion := kk.KubeadmConfig.GetConvertedKubeadmConfig() newClusterData, err := yaml.MarshalYamlConfigs(&conversion.ClusterConfiguration) if err != nil { logger.Error("failed to encode ClusterConfiguration: %s", err) diff --git a/pkg/runtime/kubernetes/utils.go b/pkg/runtime/kubernetes/utils.go index cb70db26fb0..022532f44f3 100644 --- a/pkg/runtime/kubernetes/utils.go +++ b/pkg/runtime/kubernetes/utils.go @@ -19,13 +19,13 @@ package kubernetes import ( "context" "fmt" + "path" "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/labring/sealos/pkg/client-go/kubernetes" - "github.com/labring/sealos/pkg/constants" "github.com/labring/sealos/pkg/utils/logger" ) @@ -34,10 +34,10 @@ const ( KUBESCHEDULERCONFIGFILE = "/etc/kubernetes/scheduler.conf" ) -func (k *KubeadmRuntime) pipeline(name string, pipeline []func() error) error { - for _, f := range pipeline { - if err := f(); err != nil { - return fmt.Errorf("failed to %s %v", name, err) +func (k *KubeadmRuntime) runPipelines(phase string, pipelines ...func() error) error { + for i := range pipelines { + if err := pipelines[i](); err != nil { + return fmt.Errorf("failed to %s: %v", phase, err) } } return nil @@ -75,15 +75,15 @@ func (k *KubeadmRuntime) ReplaceKubeConfigV1991V1992(masters []string) bool { return false } -func (k *KubeadmRuntime) sendKubeConfigFile(hosts []string, kubeFile string) error { - absKubeFile := fmt.Sprintf("%s/%s", constants.KubernetesEtc, kubeFile) - sealosKubeFile := fmt.Sprintf("%s/%s", k.getContentData().EtcPath(), kubeFile) - return k.sendFileToHosts(hosts, sealosKubeFile, absKubeFile) +func (k *KubeadmRuntime) sendKubeConfigFile(hosts []string, filename string) error { + dst := path.Join(kubernetesEtc, filename) + src := path.Join(k.getContentData().EtcPath(), filename) + return k.sendFileToHosts(hosts, src, dst) } func (k *KubeadmRuntime) sendNewCertAndKey(hosts []string) error { logger.Info("start to copy etc pki files to masters") - return k.sendFileToHosts(hosts, k.getContentData().PkiPath(), constants.KubernetesEtcPKI) + return k.sendFileToHosts(hosts, k.getContentData().PkiPath(), kubernetesEtcPKI) } func (k *KubeadmRuntime) sendFileToHosts(Hosts []string, src, dst string) error { @@ -124,5 +124,5 @@ func (k *KubeadmRuntime) RemoveNodeFromK8sClient(ip string) error { } func (k *KubeadmRuntime) setFeatureGatesConfiguration() { - k.KubeadmConfig.FinalizeFeatureGatesConfiguration() + k.kubeadmConfig.FinalizeFeatureGatesConfiguration() } diff --git a/pkg/runtime/types/kubeadm_config.go b/pkg/runtime/types/kubeadm_config.go index 9bcd71af05b..a9680c74cb1 100644 --- a/pkg/runtime/types/kubeadm_config.go +++ b/pkg/runtime/types/kubeadm_config.go @@ -39,8 +39,6 @@ type KubeadmConfig struct { kubeadm.JoinConfiguration kubeproxyconfigv1alpha1.KubeProxyConfiguration kubeletconfigv1beta1.KubeletConfiguration - - conversion *ConvertedKubeadmConfig } type ConvertedKubeadmConfig struct { @@ -114,28 +112,23 @@ func (k *KubeadmConfig) Merge(kubeadmYamlPath string) error { return nil } -func (k *KubeadmConfig) GetConvertedKubeadmConfig() *ConvertedKubeadmConfig { - return k.conversion -} - -func (k *KubeadmConfig) ConvertKubeadmVersion() error { - if k.conversion == nil { - k.conversion = &ConvertedKubeadmConfig{} - } +func (k *KubeadmConfig) ToConvertedKubeadmConfig() (*ConvertedKubeadmConfig, error) { + conversion := &ConvertedKubeadmConfig{} var err error + switch k.InitConfiguration.APIVersion { case KubeadmV1beta2: var v1beta2InitConfiguration v1beta2.InitConfiguration var v1beta2ClusterConfiguration v1beta2.ClusterConfiguration var v1beta2JoinConfiguration v1beta2.JoinConfiguration if err = v1beta2.Convert_kubeadm_InitConfiguration_To_v1beta2_InitConfiguration(&k.InitConfiguration, &v1beta2InitConfiguration, nil); err != nil { - return err + return nil, err } if err = v1beta2.Convert_kubeadm_ClusterConfiguration_To_v1beta2_ClusterConfiguration(&k.ClusterConfiguration, &v1beta2ClusterConfiguration, nil); err != nil { - return err + return nil, err } if err = v1beta2.Convert_kubeadm_JoinConfiguration_To_v1beta2_JoinConfiguration(&k.JoinConfiguration, &v1beta2JoinConfiguration, nil); err != nil { - return err + return nil, err } v1beta2InitConfiguration.APIVersion = v1beta2.SchemeGroupVersion.String() @@ -144,22 +137,22 @@ func (k *KubeadmConfig) ConvertKubeadmVersion() error { v1beta2InitConfiguration.Kind = "InitConfiguration" v1beta2ClusterConfiguration.Kind = "ClusterConfiguration" v1beta2JoinConfiguration.Kind = "JoinConfiguration" - k.conversion.InitConfiguration = v1beta2InitConfiguration - k.conversion.ClusterConfiguration = v1beta2ClusterConfiguration - k.conversion.JoinConfiguration = v1beta2JoinConfiguration + conversion.InitConfiguration = v1beta2InitConfiguration + conversion.ClusterConfiguration = v1beta2ClusterConfiguration + conversion.JoinConfiguration = v1beta2JoinConfiguration case KubeadmV1beta3: var v1beta3InitConfiguration v1beta3.InitConfiguration var v1beta3ClusterConfiguration v1beta3.ClusterConfiguration var v1beta3JoinConfiguration v1beta3.JoinConfiguration if err = v1beta3.Convert_kubeadm_InitConfiguration_To_v1beta3_InitConfiguration(&k.InitConfiguration, &v1beta3InitConfiguration, nil); err != nil { - return err + return nil, err } if err = v1beta3.Convert_kubeadm_ClusterConfiguration_To_v1beta3_ClusterConfiguration(&k.ClusterConfiguration, &v1beta3ClusterConfiguration, nil); err != nil { - return err + return nil, err } if err = v1beta3.Convert_kubeadm_JoinConfiguration_To_v1beta3_JoinConfiguration(&k.JoinConfiguration, &v1beta3JoinConfiguration, nil); err != nil { - return err + return nil, err } v1beta3InitConfiguration.APIVersion = v1beta3.SchemeGroupVersion.String() v1beta3ClusterConfiguration.APIVersion = v1beta3.SchemeGroupVersion.String() @@ -167,13 +160,13 @@ func (k *KubeadmConfig) ConvertKubeadmVersion() error { v1beta3InitConfiguration.Kind = "InitConfiguration" v1beta3ClusterConfiguration.Kind = "ClusterConfiguration" v1beta3JoinConfiguration.Kind = "JoinConfiguration" - k.conversion.InitConfiguration = v1beta3InitConfiguration - k.conversion.ClusterConfiguration = v1beta3ClusterConfiguration - k.conversion.JoinConfiguration = v1beta3JoinConfiguration + conversion.InitConfiguration = v1beta3InitConfiguration + conversion.ClusterConfiguration = v1beta3ClusterConfiguration + conversion.JoinConfiguration = v1beta3JoinConfiguration default: - k.conversion.JoinConfiguration = k.JoinConfiguration - k.conversion.InitConfiguration = k.InitConfiguration - k.conversion.ClusterConfiguration = k.ClusterConfiguration + conversion.JoinConfiguration = k.JoinConfiguration + conversion.InitConfiguration = k.InitConfiguration + conversion.ClusterConfiguration = k.ClusterConfiguration } { @@ -183,14 +176,14 @@ func (k *KubeadmConfig) ConvertKubeadmVersion() error { v1alpha1KubeProxy.APIVersion = kubeproxyconfigv1alpha1.SchemeGroupVersion.String() v1beta1Kubelet.Kind = "KubeletConfiguration" v1alpha1KubeProxy.Kind = "KubeProxyConfiguration" - k.conversion.KubeProxyConfiguration = v1alpha1KubeProxy - k.conversion.KubeletConfiguration = v1beta1Kubelet + conversion.KubeProxyConfiguration = v1alpha1KubeProxy + conversion.KubeletConfiguration = v1beta1Kubelet } - return nil + return conversion, nil } -func (k *KubeadmConfig) FinalizeInitConfig() { +func (k *KubeadmConfig) SetDefaults() { for _, obj := range []k8sruntime.Object{ &k.ClusterConfiguration, &k.InitConfiguration, diff --git a/pkg/version/utils/utils.go b/pkg/version/utils/utils.go index 4efd7f9d1d5..a36b9b0ab09 100644 --- a/pkg/version/utils/utils.go +++ b/pkg/version/utils/utils.go @@ -33,8 +33,8 @@ func GetKubernetesVersion(cluster *v2.Cluster) *version.KubernetesVersion { if cluster == nil { cmd = "kubectl version -o yaml" } else { - data := constants.NewData(cluster.Name) - cmd = fmt.Sprintf("kubectl version --kubeconfig %s -o yaml", data.AdminFile()) + cmd = fmt.Sprintf("kubectl version --kubeconfig %s -o yaml", + constants.NewPathResolver(cluster.Name).AdminFile()) } serverVersion, err := exec.RunBashCmd(cmd) if err != nil || serverVersion == "" { diff --git a/test/e2e/suites/checkers/fake.go b/test/e2e/suites/checkers/fake.go index eb79ea55cb5..8eae43ccb09 100644 --- a/test/e2e/suites/checkers/fake.go +++ b/test/e2e/suites/checkers/fake.go @@ -17,6 +17,7 @@ limitations under the License. package checkers import ( + "errors" "fmt" "os" "path/filepath" @@ -132,11 +133,23 @@ func (f *FakeClientGroup) Verify() error { return nil } +func (f *fakeClient) getFilePath(filename string) (string, error) { + for _, f := range []string{ + fmt.Sprintf("/var/lib/sealos/data/%s/etc/%s", f.clusterName, filename), + fmt.Sprintf("/root/.sealos/%s/etc/%s", f.clusterName, filename), + } { + if utils.IsFileExist(f) { + return f, nil + } + } + return "", errors.New("both init file does't exists") +} + func (f *fakeClient) loadInitConfig() error { logger.Info("verify default cluster info") - initFile := fmt.Sprintf("/root/.sealos/%s/etc/kubeadm-init.yaml", f.clusterName) - if !utils.IsFileExist(initFile) { - return fmt.Errorf("file %s not exist", initFile) + initFile, err := f.getFilePath("kubeadm-init.yaml") + if err != nil { + return err } data, err := os.ReadFile(filepath.Clean(initFile)) if err != nil { @@ -164,10 +177,12 @@ func (f *fakeClient) loadInitConfig() error { } return utils.UnmarshalYamlFile(clusterConfig, &f.Cluster) } + func (f *fakeClient) loadUpdateConfig() error { logger.Info("verify default cluster info") - initFile := fmt.Sprintf("/root/.sealos/%s/etc/kubeadm-update.yaml", f.clusterName) - if !utils.IsFileExist(initFile) { + initFile, err := f.getFilePath("kubeadm-update.yaml") + if err != nil { + logger.Error(err) f.UpdateConfiguration = nil return nil }