feat: k3s runtime (#3787)

* style: remove getter functions in pkg/runtime and rename ssh functions names

* style: rename GetKubeadmConfig -> GetRuntimeConfig

Signed-off-by: cuisongliu <cuisongliu@qq.com>

# Conflicts:
#	pkg/guest/guest.go

* feat: k3s runtime implementation

Signed-off-by: cuisongliu <cuisongliu@qq.com>

* Update utils.go

---------

Signed-off-by: cuisongliu <cuisongliu@qq.com>
Co-authored-by: cuisongliu <cuisongliu@qq.com>

cmd/sealos/cmd/cert.go

@@ -71,13 +71,13 @@ func newCertCmd() *cobra.Command {
 			}
 
 			cf := clusterfile.NewClusterFile(clusterPath,
-				clusterfile.WithCustomKubeadmFiles([]string{kubeadmInitFilepath}),
+				clusterfile.WithCustomRuntimeConfigFiles([]string{kubeadmInitFilepath}),
 			)
 			if err = cf.Process(); err != nil {
 				return err
 			}
 			// TODO: using different runtime
-			rt, err := kubernetes.New(cluster, cf.GetKubeadmConfig())
+			rt, err := kubernetes.New(cluster, cf.GetRuntimeConfig())
 			if err != nil {
 				return fmt.Errorf("get default runtime failed, %v", err)
 			}

cmd/sealos/cmd/gen.go

@@ -49,6 +49,7 @@ func newGenCmd() *cobra.Command {
 		Cluster: &apply.Cluster{},
 		SSH:     &apply.SSH{},
 	}
+
 	var out string
 	var genCmd = &cobra.Command{
 		Use:     "gen",
@@ -78,6 +79,7 @@ func newGenCmd() *cobra.Command {
 	setRequireBuildahAnnotation(genCmd)
 	genArgs.RegisterFlags(genCmd.Flags())
 	genCmd.Flags().StringVarP(&out, "output", "o", "", "print output to named file")
+	genCmd.Flags().String("distribution", "kubernetes", "kubernetes distribution")
 	return genCmd
 }
 

controllers/cluster/controllers/cluster_controller.go

@@ -312,7 +312,7 @@ func getSSHclient(infra *infrav1.Infra) ssh.Interface {
 		User:   defaultUser,
 		PkData: infra.Spec.SSH.PkData,
 	}
-	c := ssh.NewSSHClient(s, true)
+	c := ssh.MustNewClient(s, true)
 
 	return c
 }

pkg/apply/gen.go

@@ -23,8 +23,7 @@ import (
 
 	"github.com/labring/sealos/pkg/apply/processor"
 	"github.com/labring/sealos/pkg/buildah"
-	"github.com/labring/sealos/pkg/runtime/kubernetes"
-	"github.com/labring/sealos/pkg/runtime/types"
+	"github.com/labring/sealos/pkg/runtime/factory"
 	"github.com/labring/sealos/pkg/types/v1beta1"
 	"github.com/labring/sealos/pkg/utils/iputils"
 )
@@ -53,7 +52,13 @@ func NewClusterFromGenArgs(cmd *cobra.Command, args *RunArgs, imageNames []strin
 		return nil, fmt.Errorf("input first image %s is not kubernetes image", imageNames)
 	}
 	cluster.Status.Mounts = append(cluster.Status.Mounts, *img)
-	rt, err := kubernetes.New(cluster, types.NewKubeadmConfig())
+
+	distribution, _ := cmd.Flags().GetString("distribution")
+	cfg, err := factory.NewRuntimeConfig(distribution)
+	if err != nil {
+		return nil, err
+	}
+	rt, err := factory.New(distribution, cluster, cfg)
 	if err != nil {
 		return nil, err
 	}

pkg/apply/processor/create.go

@@ -29,7 +29,7 @@ import (
 	"github.com/labring/sealos/pkg/filesystem/rootfs"
 	"github.com/labring/sealos/pkg/guest"
 	"github.com/labring/sealos/pkg/runtime"
-	"github.com/labring/sealos/pkg/runtime/kubernetes"
+	"github.com/labring/sealos/pkg/runtime/factory"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 	"github.com/labring/sealos/pkg/utils/logger"
 	"github.com/labring/sealos/pkg/utils/maps"
@@ -101,7 +101,9 @@ func (c *CreateProcessor) preProcess(cluster *v2.Cluster) error {
 	for i := range cluster.Status.Mounts {
 		cluster.Status.Mounts[i].Env = maps.MergeMap(cluster.Status.Mounts[i].Env, c.ExtraEnvs)
 	}
-	rt, err := kubernetes.New(cluster, c.ClusterFile.GetKubeadmConfig())
+
+	distribution := cluster.GetDistribution()
+	rt, err := factory.New(distribution, cluster, c.ClusterFile.GetRuntimeConfig())
 	if err != nil {
 		return fmt.Errorf("failed to init runtime, %v", err)
 	}

pkg/apply/processor/delete.go

@@ -25,7 +25,7 @@ import (
 	"github.com/labring/sealos/pkg/clusterfile"
 	"github.com/labring/sealos/pkg/constants"
 	"github.com/labring/sealos/pkg/filesystem/rootfs"
-	"github.com/labring/sealos/pkg/runtime/kubernetes"
+	"github.com/labring/sealos/pkg/runtime/factory"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 	fileutil "github.com/labring/sealos/pkg/utils/file"
 	"github.com/labring/sealos/pkg/utils/logger"
@@ -85,7 +85,8 @@ func (d *DeleteProcessor) UndoBootstrap(cluster *v2.Cluster) error {
 }
 
 func (d *DeleteProcessor) Reset(cluster *v2.Cluster) error {
-	rt, err := kubernetes.New(cluster, d.ClusterFile.GetKubeadmConfig())
+	distribution := cluster.GetDistribution()
+	rt, err := factory.New(distribution, cluster, d.ClusterFile.GetRuntimeConfig())
 	if err != nil {
 		return fmt.Errorf("failed to delete runtime, %v", err)
 	}

pkg/apply/processor/install.go

@@ -28,7 +28,7 @@ import (
 	"github.com/labring/sealos/pkg/filesystem/rootfs"
 	"github.com/labring/sealos/pkg/guest"
 	"github.com/labring/sealos/pkg/runtime"
-	"github.com/labring/sealos/pkg/runtime/kubernetes"
+	"github.com/labring/sealos/pkg/runtime/factory"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 	"github.com/labring/sealos/pkg/utils/confirm"
 	"github.com/labring/sealos/pkg/utils/logger"
@@ -170,7 +170,8 @@ func (c *InstallProcessor) PreProcess(cluster *v2.Cluster) error {
 		cluster.SetMountImage(mount)
 		c.NewMounts = append(c.NewMounts, *mount)
 	}
-	rt, err := kubernetes.New(cluster, c.ClusterFile.GetKubeadmConfig())
+	distribution := cluster.GetDistribution()
+	rt, err := factory.New(distribution, cluster, c.ClusterFile.GetRuntimeConfig())
 	if err != nil {
 		return fmt.Errorf("failed to init runtime, %v", err)
 	}

pkg/apply/processor/interface.go

@@ -153,7 +153,7 @@ func ConfirmDeleteNodes() error {
 func MirrorRegistry(cluster *v2.Cluster, mounts []v2.MountImage) error {
 	registries := cluster.GetRegistryIPAndPortList()
 	logger.Debug("registry nodes is: %+v", registries)
-	sshClient := ssh.NewSSHByCluster(cluster, true)
+	sshClient := ssh.NewCacheClientFromCluster(cluster, true)
 	syncer := registry.New(constants.NewPathResolver(cluster.GetName()), sshClient, mounts)
 	return syncer.Sync(context.Background(), registries...)
 }

pkg/apply/processor/scale.go

@@ -29,7 +29,7 @@ import (
 	"github.com/labring/sealos/pkg/filesystem/rootfs"
 	"github.com/labring/sealos/pkg/guest"
 	"github.com/labring/sealos/pkg/runtime"
-	"github.com/labring/sealos/pkg/runtime/kubernetes"
+	"github.com/labring/sealos/pkg/runtime/factory"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 	fileutil "github.com/labring/sealos/pkg/utils/file"
 	"github.com/labring/sealos/pkg/utils/logger"
@@ -202,11 +202,13 @@ func (c *ScaleProcessor) preProcess(cluster *v2.Cluster) error {
 	if err = SyncClusterStatus(cluster, c.Buildah, false); err != nil {
 		return err
 	}
+	distribution := cluster.GetDistribution()
+
 	var rt runtime.Interface
 	if c.IsScaleUp {
-		rt, err = kubernetes.New(cluster, c.ClusterFile.GetKubeadmConfig())
+		rt, err = factory.New(distribution, cluster, c.ClusterFile.GetRuntimeConfig())
 	} else {
-		rt, err = kubernetes.New(c.ClusterFile.GetCluster(), c.ClusterFile.GetKubeadmConfig())
+		rt, err = factory.New(distribution, c.ClusterFile.GetCluster(), c.ClusterFile.GetRuntimeConfig())
 	}
 	if err != nil {
 		return fmt.Errorf("failed to init runtime: %v", err)

pkg/apply/reset.go

@@ -68,7 +68,7 @@ func (r *ClusterArgs) resetArgs(cmd *cobra.Command, args *ResetArgs) error {
 		nodes := stringsutil.SplitRemoveEmpty(args.Cluster.Nodes, ",")
 		r.hosts = []v2.Host{}
 
-		sshClient := ssh.NewSSHByCluster(r.cluster, true)
+		sshClient := ssh.NewCacheClientFromCluster(r.cluster, true)
 		r.setHostWithIpsPort(masters, []string{v2.MASTER, GetHostArch(sshClient, masters[0])})
 		if len(nodes) > 0 {
 			r.setHostWithIpsPort(nodes, []string{v2.NODE, GetHostArch(sshClient, nodes[0])})

pkg/apply/run.go

@@ -122,7 +122,7 @@ func (r *ClusterArgs) runArgs(cmd *cobra.Command, args *RunArgs, imageList []str
 	nodes := stringsutil.SplitRemoveEmpty(args.Cluster.Nodes, ",")
 	r.hosts = []v2.Host{}
 
-	sshClient := ssh.NewSSHByCluster(r.cluster, true)
+	sshClient := ssh.NewCacheClientFromCluster(r.cluster, true)
 	if len(masters) > 0 {
 		host, port := iputils.GetHostIPAndPortOrDefault(masters[0], defaultPort)
 		master0addr := net.JoinHostPort(host, port)

pkg/apply/run_test.go

@@ -167,7 +167,7 @@ func TestNewApplierFromArgs(t *testing.T) {
 					},
 					Spec: v2.ClusterSpec{
 						Hosts: []v2.Host{
-							{IPS: []string{iputils.LocalIP(addr) + ":22"}, Roles: []string{v2.MASTER, GetHostArch(ssh.NewSSHClient(&v2.SSH{}, true), iputils.LocalIP(addr)+":22")}},
+							{IPS: []string{iputils.LocalIP(addr) + ":22"}, Roles: []string{v2.MASTER, GetHostArch(ssh.MustNewClient(&v2.SSH{}, true), iputils.LocalIP(addr)+":22")}},
 						},
 						Image: []string{"labring/kubernetes:v1.24.0"},
 						SSH:   v2.SSH{},

pkg/apply/scale.go

@@ -171,7 +171,7 @@ func verifyAndSetNodes(cmd *cobra.Command, cluster *v2.Cluster, scaleArgs *Scale
 			global := cluster.Spec.SSH.DeepCopy()
 			ssh.OverSSHConfig(global, override)
 
-			sshClient := ssh.NewSSHClient(global, true)
+			sshClient := ssh.MustNewClient(global, true)
 
 			host := &v2.Host{
 				IPS:   addrs,

pkg/apply/utils.go

@@ -148,7 +148,7 @@ func CheckAndInitialize(cluster *v2.Cluster) {
 
 	if len(cluster.Spec.Hosts) == 0 {
 		clusterSSH := cluster.GetSSH()
-		sshClient := ssh.NewSSHClient(&clusterSSH, true)
+		sshClient := ssh.MustNewClient(&clusterSSH, true)
 
 		localIpv4 := iputils.GetLocalIpv4()
 		defaultPort := defaultSSHPort(cluster.Spec.SSH.Port)

pkg/bootstrap/context.go

@@ -61,7 +61,7 @@ func (ctx realContext) GetRemoter() remote.Interface {
 }
 
 func NewContextFrom(cluster *v2.Cluster) Context {
-	execer := ssh.NewSSHByCluster(cluster, true)
+	execer := ssh.NewCacheClientFromCluster(cluster, true)
 	envProcessor := env.NewEnvProcessor(cluster)
 	remoter := remote.New(cluster.GetName(), execer)
 

pkg/checker/crictl_checker.go

@@ -104,7 +104,7 @@ func (n *CRICtlChecker) Check(cluster *v2.Cluster, phase string) error {
 			pauseImage = mountImg.Env["sandboxImage"]
 		}
 	}
-	sshCtx := ssh.NewSSHByCluster(cluster, false)
+	sshCtx := ssh.NewCacheClientFromCluster(cluster, false)
 	root := constants.NewPathResolver(cluster.Name).RootFSPath()
 	regInfo := helpers.GetRegistryInfo(sshCtx, root, cluster.GetRegistryIPAndPort())
 

pkg/checker/host_checker.go

@@ -40,7 +40,7 @@ func (a HostChecker) Check(cluster *v2.Cluster, _ string) error {
 	if len(a.IPs) != 0 {
 		ipList = a.IPs
 	}
-	sshClient := ssh.NewSSHByCluster(cluster, false)
+	sshClient := ssh.NewCacheClientFromCluster(cluster, false)
 	if err := checkHostnameUnique(sshClient, ipList); err != nil {
 		return err
 	}

pkg/checker/registry_checker.go

@@ -87,7 +87,7 @@ func (n *RegistryChecker) Check(cluster *v2.Cluster, phase string) error {
 		}
 	}
 
-	sshCtx := ssh.NewSSHByCluster(cluster, false)
+	sshCtx := ssh.NewCacheClientFromCluster(cluster, false)
 	root := constants.NewPathResolver(cluster.Name).RootFSPath()
 	regInfo := helpers.GetRegistryInfo(sshCtx, root, cluster.GetRegistryIPAndPort())
 	status.Auth = fmt.Sprintf("%s:%s", regInfo.Username, regInfo.Password)

pkg/clusterfile/clusterfile.go

@@ -18,44 +18,45 @@ import (
 	"errors"
 	"sync"
 
-	"github.com/labring/sealos/pkg/runtime/types"
+	"github.com/labring/sealos/pkg/runtime"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 )
 
 var ErrTypeNotFound = errors.New("no corresponding type structure was found")
 
 type ClusterFile struct {
-	path               string
-	customConfigFiles  []string
-	customKubeadmFiles []string
-	customValues       []string
-	customSets         []string
-	customEnvs         []string
-	setDefaults        bool
-	Cluster            *v2.Cluster
-	Configs            []v2.Config
-	KubeConfig         *types.KubeadmConfig
-	//Plugins    []v1.Plugin
+	path                     string
+	customConfigFiles        []string
+	customRuntimeConfigFiles []string
+	customValues             []string
+	customSets               []string
+	customEnvs               []string
+	setDefaults              bool
+
+	cluster       *v2.Cluster
+	configs       []v2.Config
+	runtimeConfig runtime.Config
+
 	once sync.Once
 }
 
 type Interface interface {
 	PreProcessor
 	GetCluster() *v2.Cluster
 	GetConfigs() []v2.Config
-	GetKubeadmConfig() *types.KubeadmConfig
+	GetRuntimeConfig() runtime.Config
 }
 
 func (c *ClusterFile) GetCluster() *v2.Cluster {
-	return c.Cluster
+	return c.cluster
 }
 
 func (c *ClusterFile) GetConfigs() []v2.Config {
-	return c.Configs
+	return c.configs
 }
 
-func (c *ClusterFile) GetKubeadmConfig() *types.KubeadmConfig {
-	return c.KubeConfig
+func (c *ClusterFile) GetRuntimeConfig() runtime.Config {
+	return c.runtimeConfig
 }
 
 type OptionFunc func(*ClusterFile)
@@ -72,9 +73,9 @@ func WithCustomConfigFiles(files []string) OptionFunc {
 	}
 }
 
-func WithCustomKubeadmFiles(files []string) OptionFunc {
+func WithCustomRuntimeConfigFiles(files []string) OptionFunc {
 	return func(c *ClusterFile) {
-		c.customKubeadmFiles = files
+		c.customRuntimeConfigFiles = files
 	}
 }
 

pkg/clusterfile/clusterfile_test.go

@@ -21,7 +21,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 
-	"github.com/labring/sealos/pkg/runtime/types"
+	"github.com/labring/sealos/pkg/runtime"
+	"github.com/labring/sealos/pkg/runtime/kubernetes/types"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 )
 
@@ -121,7 +122,7 @@ func Test_NewClusterFile(t *testing.T) {
 
 func Test_NoClusterFileWithSingleSchedule(t *testing.T) {
 	type args struct {
-		kubeadmConfig *types.KubeadmConfig
+		runtimeConfig runtime.Config
 	}
 	tests := []struct {
 		name    string
@@ -131,7 +132,7 @@ func Test_NoClusterFileWithSingleSchedule(t *testing.T) {
 		{
 			name: "run single with cluster file not exists",
 			args: args{
-				kubeadmConfig: &types.KubeadmConfig{
+				runtimeConfig: &types.KubeadmConfig{
 					InitConfiguration: kubeadm.InitConfiguration{
 						SkipPhases: []string{
 							"mark-control-plane",
@@ -156,7 +157,7 @@ func Test_NoClusterFileWithSingleSchedule(t *testing.T) {
 			if cf.GetConfigs() != nil {
 				t.Error("configs is not nil")
 			}
-			if !reflect.DeepEqual(cf.GetKubeadmConfig(), tt.args.kubeadmConfig) {
+			if !reflect.DeepEqual(cf.GetRuntimeConfig(), tt.args.runtimeConfig) {
 				t.Error("kubeadmConfig not equal")
 			}
 		})
@@ -167,7 +168,7 @@ func Test_NewClusterFileWithSingleSchedule(t *testing.T) {
 	type args struct {
 		cluster            *v2.Cluster
 		config             v2.Config
-		kubeadmConfig      *types.KubeadmConfig
+		runtimeConfig      runtime.Config
 		customEnv          []string
 		sets               []string
 		values             []string
@@ -223,7 +224,7 @@ func Test_NewClusterFileWithSingleSchedule(t *testing.T) {
 						Data: "test\n",
 					},
 				},
-				kubeadmConfig: &types.KubeadmConfig{
+				runtimeConfig: &types.KubeadmConfig{
 					InitConfiguration: kubeadm.InitConfiguration{
 						TypeMeta: metav1.TypeMeta{
 							APIVersion: "kubeadm.k8s.io/v1beta3",
@@ -255,7 +256,7 @@ func Test_NewClusterFileWithSingleSchedule(t *testing.T) {
 				WithCustomSets(tt.args.sets),
 				WithCustomValues(tt.args.values),
 				WithCustomConfigFiles(tt.args.customConfigs),
-				WithCustomKubeadmFiles(tt.args.customKubeadmFiles),
+				WithCustomRuntimeConfigFiles(tt.args.customKubeadmFiles),
 			)
 			err := cf.Process()
 			if (err != nil) != tt.wantErr {
@@ -269,7 +270,7 @@ func Test_NewClusterFileWithSingleSchedule(t *testing.T) {
 			if !reflect.DeepEqual(cf.GetConfigs()[0], tt.args.config) {
 				t.Error("config not equal")
 			}
-			if !reflect.DeepEqual(cf.GetKubeadmConfig(), tt.args.kubeadmConfig) {
+			if !reflect.DeepEqual(cf.GetRuntimeConfig(), tt.args.runtimeConfig) {
 				t.Error("kubeadmConfig not equal")
 			}
 		})