From b254852d0551a16ed2d8a1defcec42874f6c92ec Mon Sep 17 00:00:00 2001 From: johnypeng <2651903873@qq.com> Date: Mon, 20 May 2024 14:28:18 +0800 Subject: [PATCH] feat: support file multiple match and other optimizations (#96) * feat: support file multiple match * feat: support save file with ignoring dir * feat: compatible with the old version of bscp server * feat: update go.mod file --- client/client.go | 6 +- client/options.go | 10 ++-- cmd/bscp/get.go | 93 +++++++++++++++++++++++++------ cmd/bscp/pull.go | 2 +- go.mod | 2 +- go.sum | 4 +- internal/cache/cache.go | 2 +- internal/downloader/downloader.go | 2 +- 8 files changed, 91 insertions(+), 30 deletions(-) diff --git a/client/client.go b/client/client.go index e87d25fe8..f75081faa 100644 --- a/client/client.go +++ b/client/client.go @@ -187,7 +187,11 @@ func (c *client) PullFiles(app string, opts ...AppOption) (*Release, error) { // Uid: c.opts.uid, }, Token: c.opts.token, - Key: option.Key, + Match: option.Match, + } + // compatible with the old version of bscp server which can only recognize param req.Key + if len(option.Match) > 0 { + req.Key = option.Match[0] } // merge labels, if key conflict, app value will overwrite client value req.AppMeta.Labels = util.MergeLabels(c.opts.labels, option.Labels) diff --git a/client/options.go b/client/options.go index c814e9d47..f8d508065 100644 --- a/client/options.go +++ b/client/options.go @@ -126,8 +126,8 @@ func WithEnableMonitorResourceUsage(enable bool) Option { // AppOptions options for app pull and watch type AppOptions struct { - // Key watch config item key - Key string + // Match matches config items + Match []string // Labels instance labels Labels map[string]string // UID instance unique uid @@ -137,10 +137,10 @@ type AppOptions struct { // AppOption setter for app options type AppOption func(*AppOptions) -// WithAppKey set watch config item key -func WithAppKey(key string) AppOption { +// WithAppMatch set match condition for config items +func WithAppMatch(match []string) AppOption { return func(o *AppOptions) { - o.Key = key + o.Match = match } } diff --git a/cmd/bscp/get.go b/cmd/bscp/get.go index 3c85faa55..686fd3c64 100644 --- a/cmd/bscp/get.go +++ b/cmd/bscp/get.go @@ -17,6 +17,7 @@ import ( "fmt" "os" "path" + "path/filepath" "strings" "github.com/dustin/go-humanize" @@ -30,7 +31,8 @@ import ( var ( outputFormat string - download string + downloadDir string + ignoreDir bool ) const ( @@ -125,7 +127,10 @@ func init() { "bscp file cache threshold gigabyte") mustBindPFlag(getFileViper, "file_cache.threshold_gb", getFileCmd.Flags().Lookup("cache-threshold-gb")) getFileCmd.Flags().StringVarP(&outputFormat, "output", "o", "", "output format, One of: json|content") - getFileCmd.Flags().StringVarP(&download, "download", "d", "", "file path for saving the downloaded content") + getFileCmd.Flags().StringVarP(&downloadDir, "download-dir", "d", "", + "the directory for saving the downloaded content") + getFileCmd.Flags().BoolVar(&ignoreDir, "ignore-dir", false, + "ignore directory hierarchy when downloading files, must be used with -d option") // kv 参数 getKvCmd.Flags().StringP("app", "a", "", "app name") @@ -204,13 +209,7 @@ func runGetApp(args []string) error { // runGetFileList gets file list func runGetFileList(bscp client.Client, app string, match []string) error { - var opts []client.AppOption - if len(match) > 0 { - opts = append(opts, client.WithAppKey(match[0])) - } - opts = append(opts, client.WithAppLabels(conf.Labels)) - - release, err := bscp.PullFiles(app, opts...) + release, err := getFileRelease(bscp, app, match) if err != nil { return err } @@ -255,15 +254,19 @@ func runGetFileContents(bscp client.Client, app string, match []string) error { return err } -// getFileOutput gets file output -func getFileOutput(bscp client.Client, app string, match []string) (string, error) { +// getFileRelease gets file release +func getFileRelease(bscp client.Client, app string, match []string) (*client.Release, error) { var opts []client.AppOption if len(match) > 0 { - opts = append(opts, client.WithAppKey(match[0])) + opts = append(opts, client.WithAppMatch(match)) } opts = append(opts, client.WithAppLabels(conf.Labels)) + return bscp.PullFiles(app, opts...) +} - release, err := bscp.PullFiles(app, opts...) +// getFileOutput gets file output +func getFileOutput(bscp client.Client, app string, match []string) (string, error) { + release, err := getFileRelease(bscp, app, match) if err != nil { return "", err } @@ -281,8 +284,8 @@ func getFileOutput(bscp client.Client, app string, match []string) (string, erro output := "" for idx, file := range release.FileItems { - output += fmt.Sprintf("***start No.%d***\nfile: %s\ncontentID: %s\nconent: \n%s\n***end No.%d***\n\n", - idx+1, path.Join(file.Path, file.Name), file.FileMeta.ContentSpec.Signature, contents[idx], idx+1) + output += fmt.Sprintf("***start No.%d***\nfile: %s\nconent: \n%s\n***end No.%d***\n\n", + idx+1, path.Join(file.Path, file.Name), contents[idx], idx+1) } return output, nil } @@ -310,11 +313,65 @@ func getfileContents(files []*client.ConfigItemFile) ([][]byte, error) { // runDownloadFile downloads file func runDownloadFile(bscp client.Client, app string, match []string) error { - output, err := getFileOutput(bscp, app, match) + // check if download directory exists + fileInfo, err := os.Stat(downloadDir) + if err != nil { + return fmt.Errorf("check download directory %s failed, err: %s", downloadDir, err) + } + if !fileInfo.IsDir() { + return fmt.Errorf("check download directory %s failed, err: path exists but is not a directory", downloadDir) + } + + release, err := getFileRelease(bscp, app, match) if err != nil { return err } - return os.WriteFile(download, []byte(output), 0644) + if len(release.FileItems) == 0 { + fmt.Println("no matched files to download") + return nil + } + + dstFiles := make([]string, len(release.FileItems)) + var dstFile string + var existFiles []string + for idx, f := range release.FileItems { + if ignoreDir { + dstFile = path.Join(downloadDir, f.Name) + // check if file exists when --ignore-dir is enabled + if _, err := os.Stat(dstFile); err == nil { + existFiles = append(existFiles, dstFile) + } + } else { + dstFile = path.Join(downloadDir, f.Path, f.Name) + } + dstFiles[idx] = dstFile + } + if len(existFiles) > 0 { + return fmt.Errorf("the file in %v already exists, "+ + "you can remove the arg --ignore-dir or delete the existed files or make your other choices", existFiles) + } + + // save content to dst file + g, _ := errgroup.WithContext(context.Background()) + g.SetLimit(10) + for i, f := range release.FileItems { + idx, file := i, f + g.Go(func() error { + fileDir := filepath.Dir(dstFiles[idx]) + err := os.MkdirAll(fileDir, os.ModePerm) + if err != nil { + return err + } + fmt.Printf("saving to file %s\n", dstFiles[idx]) + return file.SaveToFile(dstFiles[idx]) + }) + } + if err := g.Wait(); err != nil { + return err + } + + fmt.Printf("saved %d files successfully\n", len(dstFiles)) + return nil } // runGetFile executes the get file command. @@ -347,7 +404,7 @@ func runGetFile(args []string) error { return err } - if download != "" { + if downloadDir != "" { return runDownloadFile(bscp, conf.App, args) } diff --git a/cmd/bscp/pull.go b/cmd/bscp/pull.go index 76ef6a0d9..2c589d8dd 100644 --- a/cmd/bscp/pull.go +++ b/cmd/bscp/pull.go @@ -84,7 +84,7 @@ func Pull(cmd *cobra.Command, args []string) { } for _, app := range conf.Apps { opts := []client.AppOption{} - opts = append(opts, client.WithAppKey("**")) + opts = append(opts, client.WithAppMatch([]string{"**"})) opts = append(opts, client.WithAppLabels(app.Labels)) opts = append(opts, client.WithAppUID(app.UID)) if err = pullAppFiles(ctx, bscp, conf.TempDir, conf.Biz, app.Name, opts); err != nil { diff --git a/go.mod b/go.mod index bb5e7af03..c261f3439 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/TencentBlueKing/bscp-go go 1.20 require ( - github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240425034551-e53e35b46a7b + github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240517115418-9396857e3664 github.com/denisbrodbeck/machineid v1.0.1 github.com/dustin/go-humanize v1.0.1 github.com/fsnotify/fsnotify v1.7.0 diff --git a/go.sum b/go.sum index 98da49ba0..ad90c3762 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0k github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/Tencent/bk-bcs/bcs-common v0.0.0-20240425034551-e53e35b46a7b h1:PSc0MyZSH0vtLveQWLLDW26XoNcjHoMwjm9P6dP/+4M= github.com/Tencent/bk-bcs/bcs-common v0.0.0-20240425034551-e53e35b46a7b/go.mod h1:BpINYXjhHwE2FCY6WeNHN5c5t7i2KNDSfCKTb1xM2oI= -github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240425034551-e53e35b46a7b h1:8Dz2+ksJHuCdOEs9JzRIFF9qb6AJwjDG63x1eCxEAPg= -github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240425034551-e53e35b46a7b/go.mod h1:TgQtpjNOO2JB6cmglAcLbmtOyYsF0Jxfavas+YUN3xc= +github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240517115418-9396857e3664 h1:mT3XyViRU9w4GVWeThDQ7avq4xb4eQOpWJnW+rjcfn8= +github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp v0.0.0-20240517115418-9396857e3664/go.mod h1:TgQtpjNOO2JB6cmglAcLbmtOyYsF0Jxfavas+YUN3xc= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 9baba0604..f3fd3eec1 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -169,7 +169,7 @@ func (c *Cache) CopyToFile(ci *sfs.ConfigItemMetaV1, filePath string) bool { } defer src.Close() - dst, err = os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + dst, err = os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { logger.Error("open destination file failed", slog.String("file", filePath), logger.ErrAttr(err)) return false diff --git a/internal/downloader/downloader.go b/internal/downloader/downloader.go index bce16e4dc..205a923bb 100644 --- a/internal/downloader/downloader.go +++ b/internal/downloader/downloader.go @@ -170,7 +170,7 @@ func (dl *downloader) Download(fileMeta *pbfs.FileMeta, downloadUri string, file sfs.SecondaryError{SpecificFailedReason: sfs.FilePathNotFound, Err: fmt.Errorf("target file path is empty")}) } - file, err := os.OpenFile(toFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) + file, err := os.OpenFile(toFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return sfs.WrapPrimaryError(sfs.DownloadFailed, sfs.SecondaryError{SpecificFailedReason: sfs.OpenFileFailed,