Merge pull request kcl-lang#63 from zong-zhe/add-kpm-pull

Feat: add 'kpm pull' to pull the kcl package from oci registry.

kpm.go

@@ -29,6 +29,7 @@ func main() {
 		cmd.NewRunCmd(setting),
 		cmd.NewRegCmd(setting),
 		cmd.NewPushCmd(setting),
+		cmd.NewPullCmd(setting),
 	}
 	err = app.Run(os.Args)
 	if err != nil {

pkg/cmd/cmd_pull.go

@@ -0,0 +1,56 @@
+// Copyright 2023 The KCL Authors. All rights reserved.
+
+package cmd
+
+import (
+	"path/filepath"
+
+	"github.com/urfave/cli/v2"
+	"kusionstack.io/kpm/pkg/oci"
+	"kusionstack.io/kpm/pkg/opt"
+	"kusionstack.io/kpm/pkg/reporter"
+	"kusionstack.io/kpm/pkg/settings"
+)
+
+// NewPullCmd new a Command for `kpm pull`.
+func NewPullCmd(settings *settings.Settings) *cli.Command {
+	return &cli.Command{
+		Hidden: false,
+		Name:   "pull",
+		Usage:  "pull kcl package from OCI registry.",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  FLAG_TAG,
+				Usage: "the tag for oci artifact",
+			},
+		},
+		Action: func(c *cli.Context) error {
+			tag := c.String(FLAG_TAG)
+			ociUrl := c.Args().Get(0)
+			localPath := c.Args().Get(1)
+
+			if len(ociUrl) == 0 {
+				reporter.Report("kpm: oci url must be specified.")
+				reporter.ExitWithReport("kpm: run 'kpm pull help' for more information.")
+			}
+
+			ociOpt, err := opt.ParseOciOptionFromOciUrl(ociUrl, tag)
+			if err != nil {
+				return err
+			}
+
+			absPullPath, err := filepath.Abs(localPath)
+			if err != nil {
+				return err
+			}
+
+			err = oci.Pull(absPullPath, ociOpt.Reg, ociOpt.Repo, ociOpt.Tag, settings)
+			if err != nil {
+				return err
+			}
+
+			reporter.Report("kpm: the kcl package tar is pulled successfully.")
+			return nil
+		},
+	}
+}

pkg/cmd/cmd_run.go

@@ -106,6 +106,8 @@ func runTar(tarPath, entryFile string, vendorMode bool, kclArgs string) (string,
 	return compileResult, nil
 }
 
+const KCL_PKG_TAR = "*.tar"
+
 // runOci will compile the kcl package from an OCI reference.
 func runOci(ociRef, version, entryFile string, vendorMode bool, settings *settings.Settings, kclArgs string) (string, error) {
 	ociOpts, err := opt.ParseOciOptionFromString(ociRef, version)
@@ -114,20 +116,30 @@ func runOci(ociRef, version, entryFile string, vendorMode bool, settings *settin
 		return "", err
 	}
 
-	pwd, err := os.Getwd()
-
+	// 1. Create the temporary directory to pull the tar.
+	tmpDir, err := os.MkdirTemp("", "")
 	if err != nil {
 		return "", errors.InternalBug
 	}
+	// clean the temp dir.
+	defer os.RemoveAll(tmpDir)
 
-	localPath := ociOpts.AddStoragePathSuffix(pwd)
-	localTarPath, err := oci.Pull(localPath, ociOpts.Reg, ociOpts.Repo, ociOpts.Tag, settings)
+	localPath := ociOpts.AddStoragePathSuffix(tmpDir)
+
+	// 2. Pull the tar.
+	err = oci.Pull(localPath, ociOpts.Reg, ociOpts.Repo, ociOpts.Tag, settings)
 
 	if err != nil {
 		return "", err
 	}
 
-	return runTar(localTarPath, entryFile, vendorMode, kclArgs)
+	// 3.Get the (*.tar) file path.
+	matches, err := filepath.Glob(filepath.Join(localPath, KCL_PKG_TAR))
+	if err != nil || len(matches) != 1 {
+		return "", errors.FailedPullFromOci
+	}
+
+	return runTar(matches[0], entryFile, vendorMode, kclArgs)
 }
 
 // runPkg will compile current kcl package.

pkg/oci/oci.go

@@ -64,28 +64,26 @@ func Logout(hostname string, setting *settings.Settings) error {
 	return nil
 }
 
-const KCP_PKG_TAR = "*.tar"
-
 // Pull will pull the oci artifacts from oci registry to local path.
-func Pull(localPath, hostName, repoName, tag string, settings *settings.Settings) (string, error) {
+func Pull(localPath, hostName, repoName, tag string, settings *settings.Settings) error {
 	// 0. Create a file store
 	fs, err := file.New(localPath)
 	if err != nil {
-		return "", errors.FailedPullFromOci
+		return errors.FailedPullFromOci
 	}
 	defer fs.Close()
 
 	// 1. Connect to a remote repository
 	ctx := context.Background()
 	repo, err := remote.NewRepository(filepath.Join(hostName, repoName))
 	if err != nil {
-		return "", errors.FailedPullFromOci
+		return errors.FailedPullFromOci
 	}
 
 	// 2. Login
 	credential, err := loadCredential(hostName, settings)
 	if err != nil {
-		return "", errors.FailedPullFromOci
+		return errors.FailedPullFromOci
 	}
 	repo.Client = &remoteauth.Client{
 		Client:     retry.DefaultClient,
@@ -96,16 +94,10 @@ func Pull(localPath, hostName, repoName, tag string, settings *settings.Settings
 	// 3. Copy from the remote repository to the file store
 	_, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions)
 	if err != nil {
-		return "", errors.FailedPullFromOci
-	}
-
-	// 4.Get the (*.tar) file path.
-	matches, err := filepath.Glob(filepath.Join(localPath, KCP_PKG_TAR))
-	if err != nil && len(matches) != 1 {
-		return "", errors.FailedPullFromOci
+		return errors.FailedPullFromOci
 	}
 
-	return matches[0], nil
+	return nil
 }
 
 // Push will push the oci artifacts to oci registry from local path
@@ -123,7 +115,11 @@ func Push(localPath, hostName, repoName, tag string, settings *settings.Settings
 	fileNames := []string{localPath}
 	fileDescriptors := make([]v1.Descriptor, 0, len(fileNames))
 	for _, name := range fileNames {
-		fileDescriptor, err := fs.Add(ctx, name, DEFAULT_OCI_ARTIFACT_TYPE, "")
+		// The file name of the pushed file cannot be a file path,
+		// If the file name is a path, the path will be created during pulling.
+		// During pulling, a file should be downloaded separately,
+		// and a file path is created for each download, which is not good.
+		fileDescriptor, err := fs.Add(ctx, filepath.Base(name), DEFAULT_OCI_ARTIFACT_TYPE, "")
 		if err != nil {
 			return err
 		}

test/e2e/kpm_test.go

@@ -37,8 +37,13 @@ var _ = ginkgo.Describe("Kpm CLI Testing", func() {
 				stdout, stderr, err := ExecKpmWithWorkDir(ts.Input, workspace)
 
 				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
-				gomega.Expect(stdout).To(gomega.MatchRegexp(ts.ExpectStdout))
-				gomega.Expect(stderr).To(gomega.MatchRegexp(ts.ExpectStderr))
+				// Using regular expressions may miss some cases where the string is empty.
+				//
+				// Since the login/logout-related test cases will output time information,
+				// they cannot be compared with method 'Equal',
+				// so 'ContainSubstring' is used to compare the results.
+				gomega.Expect(stdout).To(gomega.ContainSubstring(ts.ExpectStdout))
+				gomega.Expect(stderr).To(gomega.ContainSubstring(ts.ExpectStderr))
 			})
 		}
 	})

test/e2e/test_suites/kpm/no_args/kpm_pull_with_no_args/test_suite.env

@@ -0,0 +1,2 @@
+KPM_HOME=""
+KCLVM_VENDOR_HOME=""

test/e2e/test_suites/kpm/no_args/kpm_pull_with_no_args/test_suite.input

@@ -0,0 +1 @@
+kpm pull

test/e2e/test_suites/kpm/no_args/kpm_pull_with_no_args/test_suite.stderr

@@ -0,0 +1,2 @@
+kpm: oci url must be specified.
+kpm: run 'kpm pull help' for more information.

test/e2e/test_suites/kpm/no_args/kpm_pull_with_no_tag_arg/test_suite.env

@@ -0,0 +1,2 @@
+KPM_HOME=""
+KCLVM_VENDOR_HOME=""

test/e2e/test_suites/kpm/no_args/kpm_pull_with_no_tag_arg/test_suite.input

@@ -0,0 +1 @@
+kpm pull --tag

test/e2e/test_suites/kpm/no_args/kpm_pull_with_no_tag_arg/test_suite.stderr

@@ -0,0 +1 @@
+flag needs an argument: -tag

test/e2e/test_suites/kpm/no_args/kpm_pull_with_no_tag_arg/test_suite.stdout

@@ -0,0 +1,14 @@
+Incorrect Usage: flag needs an argument: -tag
+
+NAME:
+   kpm pull - pull kcl package from OCI registry.
+
+USAGE:
+   kpm pull command [command options] [arguments...]
+
+COMMANDS:
+   help, h  Shows a list of commands or help for one command
+
+OPTIONS:
+   --tag value  the tag for oci artifact
+   --help, -h   show help

test/e2e/test_suites/kpm/no_args/kpm_pull_with_only_tag/test_suite.env

@@ -0,0 +1,2 @@
+KPM_HOME=""
+KCLVM_VENDOR_HOME=""

test/e2e/test_suites/kpm/no_args/kpm_pull_with_only_tag/test_suite.input

@@ -0,0 +1 @@
+kpm pull --tag v0.0.1

test/e2e/test_suites/kpm/no_args/kpm_pull_with_only_tag/test_suite.stderr

@@ -0,0 +1,2 @@
+kpm: oci url must be specified.
+kpm: run 'kpm pull help' for more information.

test/e2e/test_suites/kpm/no_args/login_reg_without_args/test_suite.stdout

@@ -1 +1 @@
-Username:
+Username: 

test/e2e/test_suites/kpm/no_args/login_reg_without_password/test_suite.stdout

@@ -1 +1 @@
-Password:
+Password: