支持多服务拉取和监听 (#2)

* [新增] 支持多服务拉取和监听
[新增] 配置文件labels采用patch策略,服务labels覆盖并merge实例labels
[优化] 并发下载文件

* golang-lint

* 优化cli参数;优化配置文件更新并发限制

* 优化cli参数;优化配置文件更新并发限制

cli/cmd/common.go

@@ -16,6 +16,11 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
+
+	"github.com/TencentBlueKing/bscp-go/cli/config"
+	"github.com/TencentBlueKing/bscp-go/cli/constant"
+	"github.com/spf13/viper"
 )
 
 var (
@@ -28,11 +33,14 @@ var (
 	token     string
 	tempDir   string
 	validArgs []string
+	conf      = new(config.ClientConfig)
+	// flag values
+	configPath string
 )
 
 var (
-	// important: promise of compatibility
-	// priority: Command Options -> Settings Files -> Environment Variables -> Defaults
+	// !important: promise of compatibility
+	// priority: Config File -> Command Options -> Environment Variables -> Defaults
 
 	// rootEnvs variable definition
 	rootEnvs = map[string]string{
@@ -50,8 +58,25 @@ var (
 	}
 )
 
-// validateArgs validate the common args
-func validateArgs() error {
+// initArgs init the common args
+func initArgs() error {
+
+	if configPath != "" {
+		fmt.Println("use config file: ", configPath)
+		viper.SetConfigFile(configPath)
+		if err := viper.ReadInConfig(); err != nil {
+			return fmt.Errorf("read config file failed, err: %s", err.Error())
+		}
+		if err := viper.Unmarshal(conf); err != nil {
+			return fmt.Errorf("unmarshal config file failed, err: %s", err.Error())
+		}
+		if err := conf.Validate(); err != nil {
+			return fmt.Errorf("validate watch config failed, err: %s", err.Error())
+		}
+		return nil
+	}
+
+	fmt.Println("use command line args or environment variables")
 	if bizID == 0 {
 		return fmt.Errorf("biz id must not be 0")
 	}
@@ -81,9 +106,24 @@ func validateArgs() error {
 	validArgs = append(validArgs, fmt.Sprintf("--token=%s", "***"))
 
 	if tempDir == "" {
-		tempDir = fmt.Sprintf("/data/bscp/%d/%s", bizID, appName)
+		tempDir = constant.DefaultTempDir
 	}
 	validArgs = append(validArgs, fmt.Sprintf("--temp-dir=%s", tempDir))
 
+	fmt.Println("args:", strings.Join(validArgs, " "))
+
+	// construct config
+	conf.Biz = bizID
+	conf.FeedAddrs = strings.Split(feedAddrs, ",")
+	conf.Token = token
+	conf.Labels = labels
+	conf.UID = uid
+	conf.TempDir = tempDir
+
+	apps := []*config.AppConfig{}
+	for _, app := range strings.Split(appName, ",") {
+		apps = append(conf.Apps, &config.AppConfig{Name: app})
+	}
+	conf.Apps = apps
 	return nil
 }

cli/cmd/pull.go

@@ -17,15 +17,17 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"strings"
+	"strconv"
 	"time"
 
 	"bscp.io/pkg/dal/table"
 	"bscp.io/pkg/logs"
 	"github.com/spf13/cobra"
 
+	"github.com/TencentBlueKing/bscp-go/cli/constant"
 	"github.com/TencentBlueKing/bscp-go/cli/util"
 	"github.com/TencentBlueKing/bscp-go/client"
+	"github.com/TencentBlueKing/bscp-go/option"
 	"github.com/TencentBlueKing/bscp-go/pkg/eventmeta"
 )
 
@@ -40,55 +42,63 @@ var (
 
 // Pull executes the pull command.
 func Pull(cmd *cobra.Command, args []string) {
-	if err := validateArgs(); err != nil {
+	if err := initArgs(); err != nil {
 		logs.Errorf(err.Error())
 		os.Exit(1)
 	}
-	fmt.Println("args:", strings.Join(validArgs, " "))
 
-	opts := []client.Option{}
-	opts = append(opts, client.FeedAddrs(strings.Split(feedAddrs, ",")))
-	opts = append(opts, client.BizID(bizID))
-	opts = append(opts, client.Labels(labels))
-	opts = append(opts, client.Token(token))
-	opts = append(opts, client.LogVerbosity(logVerbosity))
-
-	if uid != "" {
-		opts = append(opts, client.UID(uid))
-	}
-	bscp, err := client.New(opts...)
+	bscp, err := client.New(
+		option.FeedAddrs(conf.FeedAddrs),
+		option.BizID(conf.Biz),
+		option.Token(conf.Token),
+		option.Labels(conf.Labels),
+		option.UID(conf.UID),
+		option.LogVerbosity(logVerbosity),
+	)
 	if err != nil {
 		logs.Errorf(err.Error())
 		os.Exit(1)
 	}
+	for _, app := range conf.Apps {
+		opts := []option.AppOption{}
+		opts = append(opts, option.WithKey("**"))
+		opts = append(opts, option.WithLabels(app.Labels))
+		opts = append(opts, option.WithUID(app.UID))
+		if conf.TempDir != "" {
+			tempDir = conf.TempDir
+		}
+		if err = pullAppFiles(bscp, tempDir, conf.Biz, app.Name, opts); err != nil {
+			logs.Errorf(err.Error())
+			os.Exit(1)
+		}
+	}
+}
+
+func pullAppFiles(bscp *client.Client, tempDir string, biz uint32, app string, opts []option.AppOption) error {
 	// 1. prepare app workspace dir
-	if e := os.MkdirAll(tempDir, os.ModePerm); e != nil {
-		logs.Errorf(e.Error())
-		os.Exit(1)
+	appDir := path.Join(tempDir, strconv.Itoa(int(biz)), app)
+	if e := os.MkdirAll(appDir, os.ModePerm); e != nil {
+		return e
 	}
-	releaseID, files, preHook, postHook, err := bscp.PullFiles(appName, "**")
+	releaseID, files, preHook, postHook, err := bscp.PullFiles(app, opts...)
 	if err != nil {
-		logs.Errorf(err.Error())
-		os.Exit(1)
+		return err
 	}
 	// 2. execute pre hook
 	if preHook != nil {
-		if err := util.ExecuteHook(tempDir, preHook, table.PreHook); err != nil {
-			logs.Errorf(err.Error())
-			os.Exit(1)
+		if err := util.ExecuteHook(appDir, preHook, table.PreHook); err != nil {
+			return err
 		}
 	}
 	// 3. download files and save to temp dir
-	filesDir := path.Join(tempDir, "files")
+	filesDir := path.Join(appDir, "files")
 	if err := util.UpdateFiles(filesDir, files); err != nil {
-		logs.Errorf(err.Error())
-		os.Exit(1)
+		return err
 	}
 	// 4. execute post hook
 	if postHook != nil {
-		if err := util.ExecuteHook(tempDir, postHook, table.PostHook); err != nil {
-			logs.Errorf(err.Error())
-			os.Exit(1)
+		if err := util.ExecuteHook(appDir, postHook, table.PostHook); err != nil {
+			return err
 		}
 	}
 	// 5. append metadata to metadata.json
@@ -97,30 +107,35 @@ func Pull(cmd *cobra.Command, args []string) {
 		Status:    eventmeta.EventStatusSuccess,
 		EventTime: time.Now().Format(time.RFC3339),
 	}
-	if err := eventmeta.AppendMetadataToFile(tempDir, metadata); err != nil {
-		logs.Errorf("append metadata to file failed, err: %s", err.Error())
-		os.Exit(1)
+	if err := eventmeta.AppendMetadataToFile(appDir, metadata); err != nil {
+		return err
 	}
+	logs.Infof("pull files success, current releaseID: %d", releaseID)
+	return nil
 }
 
 func init() {
-	// important: promise of compatibility
+	// !important: promise of compatibility
 	PullCmd.Flags().SortFlags = false
 
+	PullCmd.Flags().StringVarP(&feedAddrs, "feed-addrs", "f", "",
+		"feed server address, eg: 'bscp.io:8080,bscp.io:8081'")
 	PullCmd.Flags().Uint32VarP(&bizID, "biz", "b", 0, "biz id")
 	PullCmd.Flags().StringVarP(&appName, "app", "a", "", "app name")
-	PullCmd.Flags().StringVarP(&labelsStr, "labels", "l", "", "labels")
-	PullCmd.Flags().StringVarP(&uid, "uid", "u", "", "uid")
-	PullCmd.Flags().StringVarP(&feedAddrs, "feed-addrs", "f", "", "feed server address, eg: 'bscp.io:8080,bscp.io:8081'")
 	PullCmd.Flags().StringVarP(&token, "token", "t", "", "sdk token")
-	PullCmd.Flags().StringVarP(&tempDir, "temp-dir", "c", "",
-		"app config file temp dir, default: '/data/bscp/{biz_id}/{app_name}")
+	PullCmd.Flags().StringVarP(&labelsStr, "labels", "l", "", "labels")
+	// TODO: set client UID
+	PullCmd.Flags().StringVarP(&tempDir, "temp-dir", "d", "",
+		fmt.Sprintf("bscp temp dir, default: '%s'", constant.DefaultTempDir))
+	PullCmd.Flags().StringVarP(&configPath, "config", "c", "", "config file path")
 
-	for env, flag := range commonEnvs {
-		flag := PullCmd.Flags().Lookup(flag)
+	for env, f := range commonEnvs {
+		flag := PullCmd.Flags().Lookup(f)
 		flag.Usage = fmt.Sprintf("%v [env %v]", flag.Usage, env)
 		if value := os.Getenv(env); value != "" {
-			flag.Value.Set(value)
+			if err := flag.Value.Set(value); err != nil {
+				panic(err)
+			}
 		}
 	}
 }

cli/cmd/root.go

@@ -49,11 +49,13 @@ func init() {
 	rootCmd.AddCommand(VersionCmd)
 	rootCmd.PersistentFlags().UintVarP(&logVerbosity, "verbosity", "v", 0, "log verbosity")
 
-	for env, flag := range rootEnvs {
-		flag := rootCmd.PersistentFlags().Lookup(flag)
+	for env, f := range rootEnvs {
+		flag := rootCmd.PersistentFlags().Lookup(f)
 		flag.Usage = fmt.Sprintf("%v [env %v]", flag.Usage, env)
 		if value := os.Getenv(env); value != "" {
-			flag.Value.Set(value)
+			if err := flag.Value.Set(value); err != nil {
+				panic(err)
+			}
 		}
 	}
 }