Merge pull request #2255 from will4j/feature/images-transport

feature/support image transport argument to allow export images from and to docker daemon

cmd/kk/cmd/artifact/export.go

@@ -34,6 +34,7 @@ type ArtifactExportOptions struct {
 	Output             string
 	CriSocket          string
 	DownloadCmd        string
+	ImageTransport     string
 	SkipRemoveArtifact bool
 }
 
@@ -80,6 +81,7 @@ func (o *ArtifactExportOptions) Run() error {
 		ManifestFile:       o.ManifestFile,
 		Output:             o.Output,
 		CriSocket:          o.CriSocket,
+		ImageTransport:     o.ImageTransport,
 		Debug:              o.CommonOptions.Verbose,
 		IgnoreErr:          o.CommonOptions.IgnoreErr,
 		SkipRemoveArtifact: o.SkipRemoveArtifact,
@@ -93,6 +95,7 @@ func (o *ArtifactExportOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&o.Output, "output", "o", "", "Path to a output path")
 	cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s",
 		`The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`)
+	cmd.Flags().StringVarP(&o.ImageTransport, "image-transport", "", "", "Image transport to pull from, take values from [docker, docker-daemon]")
 	cmd.Flags().BoolVarP(&o.SkipRemoveArtifact, "skip-remove-artifact", "", false, "Skip remove artifact")
 
 }

cmd/kk/cmd/artifact/images/push.go

@@ -37,6 +37,7 @@ type ArtifactImagesPushOptions struct {
 	CommonOptions *options.CommonOptions
 
 	ImageDirPath   string
+	ImageTransport string
 	Artifact       string
 	ClusterCfgFile string
 }
@@ -89,11 +90,12 @@ func (o *ArtifactImagesPushOptions) Validate(_ []string) error {
 
 func (o *ArtifactImagesPushOptions) Run() error {
 	arg := common.Argument{
-		ImagesDir: o.ImageDirPath,
-		Artifact:  o.Artifact,
-		FilePath:  o.ClusterCfgFile,
-		Debug:     o.CommonOptions.Verbose,
-		IgnoreErr: o.CommonOptions.IgnoreErr,
+		ImagesDir:      o.ImageDirPath,
+		Artifact:       o.Artifact,
+		FilePath:       o.ClusterCfgFile,
+		ImageTransport: o.ImageTransport,
+		Debug:          o.CommonOptions.Verbose,
+		IgnoreErr:      o.CommonOptions.IgnoreErr,
 	}
 	return runPush(arg)
 }
@@ -102,6 +104,7 @@ func (o *ArtifactImagesPushOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&o.ImageDirPath, "images-dir", "", "", "Path to a KubeKey artifact images directory")
 	cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact")
 	cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file")
+	cmd.Flags().StringVarP(&o.ImageTransport, "image-transport", "", "", "Image transport to push to, take values from [docker, docker-daemon]")
 }
 
 func runPush(arg common.Argument) error {
@@ -122,7 +125,8 @@ func newImagesPushPipeline(runtime *common.KubeRuntime) error {
 
 	m := []module.Module{
 		&artifact.UnArchiveModule{Skip: noArtifact},
-		&images.CopyImagesToRegistryModule{ImagePath: runtime.Arg.ImagesDir},
+		&images.CopyImagesToRegistryModule{ImagePath: runtime.Arg.ImagesDir,
+			ImageTransport: runtime.Arg.ImageTransport},
 		&filesystem.ChownWorkDirModule{},
 	}
 

cmd/kk/pkg/common/artifact_runtime.go

@@ -35,6 +35,7 @@ type ArtifactArgument struct {
 	Debug              bool
 	IgnoreErr          bool
 	DownloadCommand    func(path, url string) string
+	ImageTransport     string
 	SkipRemoveArtifact bool
 }
 

cmd/kk/pkg/common/common.go

@@ -99,4 +99,7 @@ const (
 
 	// Artifact pipeline
 	Artifact = "artifact"
+
+	// Image Copy Transports
+	DockerDaemon = "docker-daemon"
 )

cmd/kk/pkg/common/kube_runtime.go

@@ -48,6 +48,7 @@ type Argument struct {
 	FromCluster         bool
 	KubeConfig          string
 	Artifact            string
+	ImageTransport      string
 	InstallPackages     bool
 	ImagesDir           string
 	Namespace           string

cmd/kk/pkg/images/module.go

@@ -49,6 +49,7 @@ func (p *PullModule) Init() {
 
 type CopyImagesToLocalModule struct {
 	common.ArtifactModule
+	ImageTransport string
 }
 
 func (c *CopyImagesToLocalModule) Init() {
@@ -58,7 +59,7 @@ func (c *CopyImagesToLocalModule) Init() {
 	copyImage := &task.LocalTask{
 		Name:   "SaveImages",
 		Desc:   "Copy images to a local OCI path from registries",
-		Action: new(SaveImages),
+		Action: &SaveImages{ImageTransport: c.ImageTransport},
 	}
 
 	c.Tasks = []task.Interface{
@@ -68,8 +69,9 @@ func (c *CopyImagesToLocalModule) Init() {
 
 type CopyImagesToRegistryModule struct {
 	common.KubeModule
-	Skip      bool
-	ImagePath string
+	Skip           bool
+	ImagePath      string
+	ImageTransport string
 }
 
 func (c *CopyImagesToRegistryModule) IsSkip() bool {
@@ -83,13 +85,14 @@ func (c *CopyImagesToRegistryModule) Init() {
 	copyImage := &task.LocalTask{
 		Name:   "CopyImagesToRegistry",
 		Desc:   "Copy images to a private registry from an artifact OCI Path",
-		Action: &CopyImagesToRegistry{ImagesPath: c.ImagePath},
+		Action: &CopyImagesToRegistry{ImagesPath: c.ImagePath, ImageTransport: c.ImageTransport},
 	}
 
 	pushManifest := &task.LocalTask{
-		Name:   "PushManifest",
-		Desc:   "Push multi-arch manifest to private registry",
-		Action: new(PushManifest),
+		Name:    "PushManifest",
+		Desc:    "Push multi-arch manifest to private registry",
+		Prepare: &ShouldPushManifest{ImageTransport: c.ImageTransport},
+		Action:  new(PushManifest),
 	}
 
 	c.Tasks = []task.Interface{

cmd/kk/pkg/images/prepares.go

@@ -0,0 +1,34 @@
+/*
+ Copyright 2024 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package images
+
+import (
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
+)
+
+type ShouldPushManifest struct {
+	common.KubePrepare
+	ImageTransport string
+}
+
+func (m *ShouldPushManifest) PreCheck(_ connector.Runtime) (bool, error) {
+	if m.ImageTransport != common.DockerDaemon {
+		return true, nil
+	}
+	return false, nil
+}

cmd/kk/pkg/images/tasks.go

@@ -155,6 +155,7 @@ func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name s
 
 type SaveImages struct {
 	common.ArtifactAction
+	ImageTransport string
 }
 
 func (s *SaveImages) Execute(runtime connector.Runtime) error {
@@ -175,17 +176,17 @@ func (s *SaveImages) Execute(runtime connector.Runtime) error {
 			auth = v
 		}
 
-		srcName := fmt.Sprintf("docker://%s", image)
+		srcName := formatImageName(s.ImageTransport, image)
 		for _, platform := range s.Manifest.Spec.Arches {
 			arch, variant := ParseArchVariant(platform)
 			// placeholder
 			if variant != "" {
 				variant = "-" + variant
 			}
 			// Ex:
-			// oci:./kubekey/artifact/images:kubesphere:kube-apiserver:v1.21.5-amd64
-			// oci:./kubekey/artifact/images:kubesphere:kube-apiserver:v1.21.5-arm-v7
-			destName := fmt.Sprintf("oci:%s:%s:%s-%s%s", dirName, imageFullName[1], suffixImageName(imageFullName[2:]), arch, variant)
+			// oci:./kubekey/artifact/images:docker.io/kubesphere/kube-apiserver:v1.21.5-amd64
+			// oci:./kubekey/artifact/images:docker.io/kubesphere/kube-apiserver:v1.21.5-arm-v7
+			destName := fmt.Sprintf("oci:%s:%s-%s%s", dirName, image, arch, variant)
 			logger.Log.Infof("Source: %s", srcName)
 			logger.Log.Infof("Destination: %s", destName)
 
@@ -232,7 +233,8 @@ func (s *SaveImages) Execute(runtime connector.Runtime) error {
 
 type CopyImagesToRegistry struct {
 	common.KubeAction
-	ImagesPath string
+	ImagesPath     string
+	ImageTransport string
 }
 
 func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error {
@@ -260,18 +262,21 @@ func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error {
 		ref := m.Annotations.RefName
 
 		// Ex:
-		// calico:cni:v3.20.0-amd64
-		nameArr := strings.Split(ref, ":")
-		if len(nameArr) != 3 {
+		// docker.io/calico/cni:v3.20.0-amd64
+		repoAddr, namespace, imageName, imageTag, err := parseImageFullName(ref)
+		if err != nil {
 			return errors.Errorf("invalid ref name: %s", ref)
 		}
 
+		if c.ImageTransport != common.DockerDaemon {
+			repoAddr = c.KubeConf.Cluster.Registry.PrivateRegistry
+		}
 		image := Image{
-			RepoAddr:          c.KubeConf.Cluster.Registry.PrivateRegistry,
-			Namespace:         nameArr[0],
+			RepoAddr:          repoAddr,
+			Namespace:         namespace,
 			NamespaceOverride: c.KubeConf.Cluster.Registry.NamespaceOverride,
-			Repo:              nameArr[1],
-			Tag:               nameArr[2],
+			Repo:              imageName,
+			Tag:               imageTag,
 		}
 
 		uniqueImage, p := ParseImageWithArchTag(image.ImageName())
@@ -305,7 +310,7 @@ func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error {
 		}
 
 		srcName := fmt.Sprintf("oci:%s:%s", imagesPath, ref)
-		destName := fmt.Sprintf("docker://%s", image.ImageName())
+		destName := formatImageName(c.ImageTransport, uniqueImage)
 		logger.Log.Infof("Source: %s", srcName)
 		logger.Log.Infof("Destination: %s", destName)
 

cmd/kk/pkg/images/utils.go

@@ -26,6 +26,7 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
 )
 
@@ -182,9 +183,32 @@ func validateImageName(imageFullName string) error {
 	return nil
 }
 
-func suffixImageName(imageFullName []string) string {
+func parseImageFullName(imageFullName string) (string, string, string, string, error) {
+	if err := validateImageName(imageFullName); err != nil {
+		return "", "", "", "", err
+	}
+	image := strings.Split(imageFullName, "/")
+	partsNum := len(image)
+	repoAddr := image[0]
+	namespace := concatImageName(image[1 : partsNum-1])
+	nameAndTag := strings.Split(image[partsNum-1], ":")
+	return repoAddr, namespace, nameAndTag[0], nameAndTag[1], nil
+}
+
+func concatImageName(imageFullName []string) string {
 	if len(imageFullName) >= 2 {
 		return strings.Join(imageFullName, "/")
 	}
 	return imageFullName[0]
 }
+
+func formatImageName(transport string, imageFullName string) string {
+	switch transport {
+	case common.DockerDaemon:
+		return fmt.Sprintf("docker-daemon:%s", imageFullName)
+	case common.Docker:
+		fallthrough
+	default:
+		return fmt.Sprintf("docker://%s", imageFullName)
+	}
+}

cmd/kk/pkg/pipelines/artifact_export.go

@@ -34,7 +34,7 @@ import (
 func NewArtifactExportPipeline(runtime *common.ArtifactRuntime) error {
 	m := []module.Module{
 		&confirm.CheckFileExistModule{FileName: runtime.Arg.Output},
-		&images.CopyImagesToLocalModule{},
+		&images.CopyImagesToLocalModule{ImageTransport: runtime.Arg.ImageTransport},
 		&binaries.ArtifactBinariesModule{},
 		&artifact.RepositoryModule{},
 		&artifact.ArchiveModule{},