feat: support file multiple match and other optimizations (#96)

* feat: support file multiple match

* feat: support save file with ignoring dir

* feat: compatible with the old version of bscp server

* feat: update go.mod file

client/client.go

@@ -187,7 +187,11 @@ func (c *client) PullFiles(app string, opts ...AppOption) (*Release, error) { //
 			Uid:    c.opts.uid,
 		},
 		Token: c.opts.token,
-		Key:   option.Key,
+		Match: option.Match,
+	}
+	// compatible with the old version of bscp server which can only recognize param req.Key
+	if len(option.Match) > 0 {
+		req.Key = option.Match[0]
 	}
 	// merge labels, if key conflict, app value will overwrite client value
 	req.AppMeta.Labels = util.MergeLabels(c.opts.labels, option.Labels)

client/options.go

@@ -126,8 +126,8 @@ func WithEnableMonitorResourceUsage(enable bool) Option {
 
 // AppOptions options for app pull and watch
 type AppOptions struct {
-	// Key watch config item key
-	Key string
+	// Match matches config items
+	Match []string
 	// Labels instance labels
 	Labels map[string]string
 	// UID instance unique uid
@@ -137,10 +137,10 @@ type AppOptions struct {
 // AppOption setter for app options
 type AppOption func(*AppOptions)
 
-// WithAppKey set watch config item key
-func WithAppKey(key string) AppOption {
+// WithAppMatch set match condition for config items
+func WithAppMatch(match []string) AppOption {
 	return func(o *AppOptions) {
-		o.Key = key
+		o.Match = match
 	}
 }
 

cmd/bscp/get.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/dustin/go-humanize"
@@ -30,7 +31,8 @@ import (
 
 var (
 	outputFormat string
-	download     string
+	downloadDir  string
+	ignoreDir    bool
 )
 
 const (
@@ -125,7 +127,10 @@ func init() {
 		"bscp file cache threshold gigabyte")
 	mustBindPFlag(getFileViper, "file_cache.threshold_gb", getFileCmd.Flags().Lookup("cache-threshold-gb"))
 	getFileCmd.Flags().StringVarP(&outputFormat, "output", "o", "", "output format, One of: json|content")
-	getFileCmd.Flags().StringVarP(&download, "download", "d", "", "file path for saving the downloaded content")
+	getFileCmd.Flags().StringVarP(&downloadDir, "download-dir", "d", "",
+		"the directory for saving the downloaded content")
+	getFileCmd.Flags().BoolVar(&ignoreDir, "ignore-dir", false,
+		"ignore directory hierarchy when downloading files, must be used with -d option")
 
 	// kv 参数
 	getKvCmd.Flags().StringP("app", "a", "", "app name")
@@ -204,13 +209,7 @@ func runGetApp(args []string) error {
 
 // runGetFileList gets file list
 func runGetFileList(bscp client.Client, app string, match []string) error {
-	var opts []client.AppOption
-	if len(match) > 0 {
-		opts = append(opts, client.WithAppKey(match[0]))
-	}
-	opts = append(opts, client.WithAppLabels(conf.Labels))
-
-	release, err := bscp.PullFiles(app, opts...)
+	release, err := getFileRelease(bscp, app, match)
 	if err != nil {
 		return err
 	}
@@ -255,15 +254,19 @@ func runGetFileContents(bscp client.Client, app string, match []string) error {
 	return err
 }
 
-// getFileOutput gets file output
-func getFileOutput(bscp client.Client, app string, match []string) (string, error) {
+// getFileRelease gets file release
+func getFileRelease(bscp client.Client, app string, match []string) (*client.Release, error) {
 	var opts []client.AppOption
 	if len(match) > 0 {
-		opts = append(opts, client.WithAppKey(match[0]))
+		opts = append(opts, client.WithAppMatch(match))
 	}
 	opts = append(opts, client.WithAppLabels(conf.Labels))
+	return bscp.PullFiles(app, opts...)
+}
 
-	release, err := bscp.PullFiles(app, opts...)
+// getFileOutput gets file output
+func getFileOutput(bscp client.Client, app string, match []string) (string, error) {
+	release, err := getFileRelease(bscp, app, match)
 	if err != nil {
 		return "", err
 	}
@@ -281,8 +284,8 @@ func getFileOutput(bscp client.Client, app string, match []string) (string, erro
 
 	output := ""
 	for idx, file := range release.FileItems {
-		output += fmt.Sprintf("***start No.%d***\nfile: %s\ncontentID: %s\nconent: \n%s\n***end No.%d***\n\n",
-			idx+1, path.Join(file.Path, file.Name), file.FileMeta.ContentSpec.Signature, contents[idx], idx+1)
+		output += fmt.Sprintf("***start No.%d***\nfile: %s\nconent: \n%s\n***end No.%d***\n\n",
+			idx+1, path.Join(file.Path, file.Name), contents[idx], idx+1)
 	}
 	return output, nil
 }
@@ -310,11 +313,65 @@ func getfileContents(files []*client.ConfigItemFile) ([][]byte, error) {
 
 // runDownloadFile downloads file
 func runDownloadFile(bscp client.Client, app string, match []string) error {
-	output, err := getFileOutput(bscp, app, match)
+	// check if download directory exists
+	fileInfo, err := os.Stat(downloadDir)
+	if err != nil {
+		return fmt.Errorf("check download directory %s failed, err: %s", downloadDir, err)
+	}
+	if !fileInfo.IsDir() {
+		return fmt.Errorf("check download directory %s failed, err: path exists but is not a directory", downloadDir)
+	}
+
+	release, err := getFileRelease(bscp, app, match)
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(download, []byte(output), 0644)
+	if len(release.FileItems) == 0 {
+		fmt.Println("no matched files to download")
+		return nil
+	}
+
+	dstFiles := make([]string, len(release.FileItems))
+	var dstFile string
+	var existFiles []string
+	for idx, f := range release.FileItems {
+		if ignoreDir {
+			dstFile = path.Join(downloadDir, f.Name)
+			// check if file exists when --ignore-dir is enabled
+			if _, err := os.Stat(dstFile); err == nil {
+				existFiles = append(existFiles, dstFile)
+			}
+		} else {
+			dstFile = path.Join(downloadDir, f.Path, f.Name)
+		}
+		dstFiles[idx] = dstFile
+	}
+	if len(existFiles) > 0 {
+		return fmt.Errorf("the file in %v already exists, "+
+			"you can remove the arg --ignore-dir or delete the existed files or make your other choices", existFiles)
+	}
+
+	// save content to dst file
+	g, _ := errgroup.WithContext(context.Background())
+	g.SetLimit(10)
+	for i, f := range release.FileItems {
+		idx, file := i, f
+		g.Go(func() error {
+			fileDir := filepath.Dir(dstFiles[idx])
+			err := os.MkdirAll(fileDir, os.ModePerm)
+			if err != nil {
+				return err
+			}
+			fmt.Printf("saving to file %s\n", dstFiles[idx])
+			return file.SaveToFile(dstFiles[idx])
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return err
+	}
+
+	fmt.Printf("saved %d files successfully\n", len(dstFiles))
+	return nil
 }
 
 // runGetFile executes the get file command.
@@ -347,7 +404,7 @@ func runGetFile(args []string) error {
 		return err
 	}
 
-	if download != "" {
+	if downloadDir != "" {
 		return runDownloadFile(bscp, conf.App, args)
 	}
 

cmd/bscp/pull.go

@@ -84,7 +84,7 @@ func Pull(cmd *cobra.Command, args []string) {
 	}
 	for _, app := range conf.Apps {
 		opts := []client.AppOption{}
-		opts = append(opts, client.WithAppKey("**"))
+		opts = append(opts, client.WithAppMatch([]string{"**"}))
 		opts = append(opts, client.WithAppLabels(app.Labels))
 		opts = append(opts, client.WithAppUID(app.UID))
 		if err = pullAppFiles(ctx, bscp, conf.TempDir, conf.Biz, app.Name, opts); err != nil {

go.mod

@@ -3,7 +3,7 @@ module github.com/TencentBlueKing/bscp-go
 go 1.20
 
 require (
-	github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240425034551-e53e35b46a7b
+	github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240517115418-9396857e3664
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dustin/go-humanize v1.0.1
 	github.com/fsnotify/fsnotify v1.7.0

go.sum

@@ -9,8 +9,8 @@ github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0k
 github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
 github.com/Tencent/bk-bcs/bcs-common v0.0.0-20240425034551-e53e35b46a7b h1:PSc0MyZSH0vtLveQWLLDW26XoNcjHoMwjm9P6dP/+4M=
 github.com/Tencent/bk-bcs/bcs-common v0.0.0-20240425034551-e53e35b46a7b/go.mod h1:BpINYXjhHwE2FCY6WeNHN5c5t7i2KNDSfCKTb1xM2oI=
-github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240425034551-e53e35b46a7b h1:8Dz2+ksJHuCdOEs9JzRIFF9qb6AJwjDG63x1eCxEAPg=
-github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240425034551-e53e35b46a7b/go.mod h1:TgQtpjNOO2JB6cmglAcLbmtOyYsF0Jxfavas+YUN3xc=
+github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240517115418-9396857e3664 h1:mT3XyViRU9w4GVWeThDQ7avq4xb4eQOpWJnW+rjcfn8=
+github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240517115418-9396857e3664/go.mod h1:TgQtpjNOO2JB6cmglAcLbmtOyYsF0Jxfavas+YUN3xc=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

internal/cache/cache.go

@@ -169,7 +169,7 @@ func (c *Cache) CopyToFile(ci *sfs.ConfigItemMetaV1, filePath string) bool {
 	}
 	defer src.Close()
 
-	dst, err = os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
+	dst, err = os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 	if err != nil {
 		logger.Error("open destination file failed", slog.String("file", filePath), logger.ErrAttr(err))
 		return false

internal/downloader/downloader.go

@@ -170,7 +170,7 @@ func (dl *downloader) Download(fileMeta *pbfs.FileMeta, downloadUri string, file
 				sfs.SecondaryError{SpecificFailedReason: sfs.FilePathNotFound,
 					Err: fmt.Errorf("target file path is empty")})
 		}
-		file, err := os.OpenFile(toFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm)
+		file, err := os.OpenFile(toFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
 		if err != nil {
 			return sfs.WrapPrimaryError(sfs.DownloadFailed,
 				sfs.SecondaryError{SpecificFailedReason: sfs.OpenFileFailed,