diff --git a/CHANGELOG.md b/CHANGELOG.md index 6929920f..0cbfbfc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# 2.3.4 +1. listbucket， listbucket2增加捕捉interrupt信号(CTR-C), 打印marker +2. account在本地记录账号，默认不覆盖, 加了-w强制覆盖选项 +3. listbucket2 增加append 模式（-a)开启, 修复列举几亿空间的时候，列举一半左右程序中断问题 +4. 修复dircache 列表没有输出到文件使用-o选项的时候 +5. 修复qupload, qupload2使用多线程上传导致的部分文件上传失败问题 +6. 加了-L 选项到qshell, 使用当前工作路径作为qshell的配置目录 + # 2.3.3 1. 修复qdownload配置cdn_domain使用了测试域名作为HOST 引起超过10G流量限制的问题 2. listbucket2 max-retry选项只限制出错下载次数，不限制接口返回空的次数 diff --git a/README.md b/README.md index 4dc26f58..beb895b3 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ qshell是利用[七牛文档上公开的API](http://developer.qiniu.com)实现 |版本 |支持平台|链接| |--------|---------|----| -|qshell-v2.3.3 |Mac OSX, Linux, Windows|[下载](http://devtools.qiniu.com/qshell-v2.3.3.zip)| +|qshell-v2.3.4 |Mac OSX, Linux, Windows|[下载](http://devtools.qiniu.com/qshell-v2.3.4.zip)| ## 安装 @@ -69,7 +69,8 @@ $ qshell account $ qshell account -- ``` -可以连续使用qshell account 添加账号ak, sk, name信息，qshell会保存这些账号的信息， 可以使用qshell user命令列举账号信息，在各个账号之间切换, 删除账号等 +可以连续使用qshell account 添加账号ak, sk, name信息，qshell会保存这些账号的信息， 可以使用qshell user命令列举账号信息，在各个账号之间切换, 删除账号等。 +如果使用的2.3.0之前的版本account命令记录的账户信息，需要先使用qshell user clean清楚保存的账户信息，然后使用qshell account命令重新记录账户信息。 2. 添加完账户后，就可以使用qshell上传，下载文件了 @@ -123,6 +124,7 @@ fi |-h|打印命令列表帮助信息，遇到参数忘记的情况下，可以使用该命令| |-v|打印工具版本，反馈问题的时候，请提前告知工具对应版本号| |-C|qshell配置文件, 其配置格式请看下一节| +|-L|使用当前工作路径作为qshell的配置目录| ## 配置文件 diff --git a/cmd/account.go b/cmd/account.go index b79e7336..0d0772c6 100644 --- a/cmd/account.go +++ b/cmd/account.go @@ -7,7 +7,12 @@ import ( "os" ) +var ( + accountOver bool +) + func init() { + cmdAccount.Flags().BoolVarP(&accountOver, "overwrite", "w", false, "overwrite account or not when account exists in local db, by default not overwrite") RootCmd.AddCommand(cmdAccount) } @@ -37,7 +42,7 @@ func Account(cmd *cobra.Command, params []string) { name := params[2] pt, oldPath := iqshell.AccPath(), iqshell.OldAccPath() - sErr := iqshell.SetAccount2(accessKey, secretKey, name, pt, oldPath) + sErr := iqshell.SetAccount2(accessKey, secretKey, name, pt, oldPath, accountOver) if sErr != nil { fmt.Println(sErr) os.Exit(iqshell.STATUS_ERROR) diff --git a/cmd/root.go b/cmd/root.go index 0996ca0a..bbf9f68d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/viper" "os" "os/user" + "path/filepath" "runtime" ) @@ -15,6 +16,7 @@ var ( DebugFlag bool VersionFlag bool cfgFile string + local bool ) const ( @@ -61,8 +63,10 @@ func init() { RootCmd.PersistentFlags().BoolVarP(&DebugFlag, "debug", "d", false, "debug mode") RootCmd.PersistentFlags().BoolVarP(&VersionFlag, "version", "v", false, "show version") RootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "C", "", "config file (default is $HOME/.qshell.json)") + RootCmd.PersistentFlags().BoolVarP(&local, "local", "L", false, "use current directory as config file path") viper.BindPFlag("config", RootCmd.PersistentFlags().Lookup("config")) + viper.BindPFlag("local", RootCmd.PersistentFlags().Lookup("local")) } func initConfig() { @@ -86,6 +90,32 @@ func initConfig() { viper.AddConfigPath(curUser.HomeDir) viper.SetConfigName(".qshell") } + + if local { + dir, gErr := os.Getwd() + if gErr != nil { + fmt.Fprintf(os.Stderr, "get current directory: %v\n", gErr) + os.Exit(1) + } + viper.Set("path.root_path", dir+"/.qshell") + } else { + curUser, gErr := user.Current() + if gErr != nil { + fmt.Fprintf(os.Stderr, "Error: get current user error: %v\n", gErr) + os.Exit(1) + } + viper.Set("path.root_path", curUser.HomeDir+"/.qshell") + } + rootPath := viper.GetString("path.root_path") + + viper.SetDefault("path.acc_db_path", filepath.Join(rootPath, "account.db")) + viper.SetDefault("path.acc_path", filepath.Join(rootPath, "account.json")) + viper.SetDefault("hosts.up_host", "upload.qiniup.com") + viper.SetDefault("hosts.rs_host", storage.DefaultRsHost) + viper.SetDefault("hosts.rsf_host", storage.DefaultRsfHost) + viper.SetDefault("hosts.io_host", "iovip.qbox.me") + viper.SetDefault("hosts.api_host", storage.DefaultAPIHost) + if rErr := viper.ReadInConfig(); rErr != nil { if _, ok := rErr.(viper.ConfigFileNotFoundError); !ok { fmt.Fprintf(os.Stderr, "read config file: %v\n", rErr) diff --git a/cmd/rs.go b/cmd/rs.go index 2ffe5058..8e82b5a4 100644 --- a/cmd/rs.go +++ b/cmd/rs.go @@ -131,6 +131,7 @@ var ( endDate string maxRetry int finalKey string + appendMode bool ) func init() { @@ -144,10 +145,11 @@ func init() { lsBucketCmd2.Flags().StringVarP(&listMarker, "marker", "m", "", "list marker") lsBucketCmd2.Flags().StringVarP(&prefix, "prefix", "p", "", "list by prefix") lsBucketCmd2.Flags().StringVarP(&suffixes, "suffixes", "q", "", "list by key suffixes, separated by comma") - lsBucketCmd2.Flags().IntVarP(&maxRetry, "max-retry", "x", 20, "max retries when error occurred") + lsBucketCmd2.Flags().IntVarP(&maxRetry, "max-retry", "x", -1, "max retries when error occurred") lsBucketCmd2.Flags().StringVarP(&outFile, "out", "o", "", "output file") lsBucketCmd2.Flags().StringVarP(&startDate, "start", "s", "", "start date with format yyyy-mm-dd-hh-MM-ss") lsBucketCmd2.Flags().StringVarP(&endDate, "end", "e", "", "end date with format yyyy-mm-dd-hh-MM-ss") + lsBucketCmd2.Flags().BoolVarP(&appendMode, "append", "a", false, "append to file") moveCmd.Flags().BoolVarP(&mOverwrite, "overwrite", "w", false, "overwrite mode") moveCmd.Flags().StringVarP(&finalKey, "key", "k", "", "filename saved in bucket") @@ -163,9 +165,8 @@ func init() { func DirCache(cmd *cobra.Command, params []string) { var cacheResultFile string cacheRootPath := params[0] - if len(params) == 2 { - cacheResultFile = params[1] - } + + cacheResultFile = outFile if cacheResultFile == "" { cacheResultFile = "stdout" } @@ -176,10 +177,6 @@ func DirCache(cmd *cobra.Command, params []string) { } func ListBucket2(cmd *cobra.Command, params []string) { - if maxRetry <= 0 { - fmt.Fprintf(os.Stderr, "maxRetry must be greater than 0\n") - os.Exit(1) - } bucket := params[0] var dateParser = func(datestr string) (time.Time, error) { @@ -221,7 +218,7 @@ func ListBucket2(cmd *cobra.Command, params []string) { } } bm := iqshell.GetBucketManager() - retErr := bm.ListBucket2(bucket, prefix, listMarker, outFile, "", start, end, sf, maxRetry) + retErr := bm.ListBucket2(bucket, prefix, listMarker, outFile, "", start, end, sf, maxRetry, appendMode) if retErr != nil { os.Exit(iqshell.STATUS_ERROR) } diff --git a/cmd/version.go b/cmd/version.go index c1a25366..cb8d7667 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -6,7 +6,7 @@ import ( "runtime" ) -var version = "v2.3.3" +var version = "v2.3.4" var versionCmd = &cobra.Command{ Use: "version", diff --git a/docs/account.md b/docs/account.md index a5dbcf76..6dd2eb7e 100644 --- a/docs/account.md +++ b/docs/account.md @@ -3,6 +3,9 @@ `account`命令用来设置当前用户的`AccessKey`和`SecretKey`，这对Key主要用在其他的需要授权的命令中，比如`stat`,`delete`,`listbucket`命令中。 该命令设置的信息，经过加密保存在命令执行的目录下的`.qshell/account.json`文件中。 +本地数据库会记录`account`注册的所有, 和的信息， 所以当用`account`注册账户信息时，如果qshell发现本地数据库有同样的名字为 +的账户， 那么默认qshell会返回错误信息报告该名字的账户已经存在，如果要覆盖注册，需要使用强制覆盖选项--overwrite 或者 -w + # 格式 ``` @@ -12,10 +15,13 @@ qshell account 打印当前设置的`AccessKey`, `SecretKey`和`Name` ``` -qshell account +qshell account [--overwrite | -w] ``` -设置当前用户的`AccessKey`, `SecretKey`和`Name` +设置当前用户的`AccessKey`, `SecretKey`和`Name`, Name是用户可以任意取的名字，表示当前在本地记录的账户的名称，和在七牛注册的邮箱信息没有关系 + +# 选项 +-w --overwrite 强制覆盖已经存在的账户 # 参数 @@ -23,7 +29,7 @@ qshell account |--------|--------| |AccessKey|七牛账号对应的AccessKey [获取](https://portal.qiniu.com/user/key)| |SecretKey|七牛账号对应的SecretKey [获取](https://portal.qiniu.com/user/key)| -|Name|账户的名字| +|Name|账户的名字, 可以任意取，和在七牛注册的邮箱信息没有关系， 只是qshell本地用来标示对 | # 示例 diff --git a/docs/cdnrefresh.md b/docs/cdnrefresh.md index 92055ab5..5461fdd3 100644 --- a/docs/cdnrefresh.md +++ b/docs/cdnrefresh.md @@ -13,7 +13,7 @@ qshell cdnrefresh [-i ] 刷新目录的命令格式： ``` -qshell cdnrefresh --dirs +qshell cdnrefresh --dirs -i ``` 注意需要刷新的目录，必须以`/`结尾。如果没有制定输入文件默认从终端读取输入内容 diff --git a/docs/listbucket2.md b/docs/listbucket2.md index 94748444..042ed551 100644 --- a/docs/listbucket2.md +++ b/docs/listbucket2.md @@ -15,7 +15,7 @@ Key\tSize\tHash\tPutTime\tMimeType\tFileType\tEndUser # 格式 ``` -qshell listbucket2 [--prefix | --suffixes ] [--start ] [--max-retry ][--end ] [-o ] +qshell listbucket2 [--prefix | --suffixes ] [--start ] [--max-retry ][--end ] [ [-a] -o ] ``` # 鉴权 @@ -32,8 +32,9 @@ qshell listbucket2 [--prefix | --suffixes ] [--sta | ListBucketResultFile | 获取的文件列表保存在本地的文件名，如果不指定该参数，则会把结果输出到终端，一般可用于获取小规模文件列表测试使用 | Y | | StartDate | 列举整个空间，然后从中筛选出文件上传日期在之后的文件 | Y | | EndDate | 列举整个空间， 然后从中筛选出文件上传日期在之前的文件 | Y | -| RetryCount | 列举整个空间文件出错以后，最大的尝试次数；超过最大尝试次数以后，程序退出，打印出marker | Y | -| suffixes | 列举整个空间文件， 然后从中筛选出文件后缀为在[suffixes1, suffixes2, ...]中的文件 |Y| +| RetryCount | 列举整个空间文件出错以后，最大的尝试次数；超过最大尝试次数以后，程序退出，打印出marker | Y | +| suffixes | 列举整个空间文件， 然后从中筛选出文件后缀为在[suffixes1, suffixes2, ...]中的文件 | Y | +| a | 开启选项o 的append模式， 如果本地保存文件列表的文件已经存在，如果希望像该文件添加内容，使用该选项, 必须和-o选项一起使用 | Y | # 常用使用场景介绍 @@ -43,31 +44,37 @@ qshell listbucket2 [--prefix | --suffixes ] [--sta ``` qshell listbucket2 -o ``` + + (2) 如果本地文件`ListBucketResultFile`已经存在，有上一次列举的内容，如果希望把新的列表添加到该文件中，需要使用选项-a开启-o选项的append 模式 + + ``` + qshell listbucket2 -a -o + ``` - (2) 获取空间所有文件，输出到屏幕上(标准输出) + (3) 获取空间所有文件，输出到屏幕上(标准输出) ``` qshell listbucket2 ``` -（3）获取空间中指定前缀的文件列表 +（4）获取空间中指定前缀的文件列表 ``` qshell listbucket2 [--prefix ] -o ``` - (4) 获取空间中指定前缀的文件列表， 输出到屏幕上 + (5) 获取空间中指定前缀的文件列表， 输出到屏幕上 ``` qshell listbucket2 [--prefix ] ``` - (5) 获取2018-10-30到2018-11-02上传的文件 + (6) 获取2018-10-30到2018-11-02上传的文件 ``` qshell listbucket2 --start 2018-10-30 --end 2018-11-02 ``` - (6) 获取后缀为mp4, html的文件 + (7) 获取后缀为mp4, html的文件 ``` qshell listbucket2 --suffixes mp4,html diff --git a/go.mod b/go.mod index 2b503292..769cb295 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/onsi/gomega v1.4.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/qiniu/api.v7 v7.2.5-0.20181112070011-bc6998c1186a+incompatible + github.com/qiniu/api.v7 v7.2.6-0.20181128092015-8c3e1ca2eb33+incompatible github.com/qiniu/x v7.0.8+incompatible // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/spf13/cobra v0.0.3 diff --git a/go.sum b/go.sum index a669b048..2f8b1c92 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/qiniu/api.v7 v7.2.5-0.20181112070011-bc6998c1186a+incompatible h1:FnBHGjkt7JJ2ebnQU0cRWrjk14ShzTSfvnLqjin2AWI= -github.com/qiniu/api.v7 v7.2.5-0.20181112070011-bc6998c1186a+incompatible/go.mod h1:V8/EzlTgLN6q0s0CJmg/I81ytsvldSF22F7h6MI02+c= +github.com/qiniu/api.v7 v7.2.6-0.20181128092015-8c3e1ca2eb33+incompatible h1:L3WEV1XhimkIAI0u1b2KlUIJyURecprw0EW42gsvYTA= +github.com/qiniu/api.v7 v7.2.6-0.20181128092015-8c3e1ca2eb33+incompatible/go.mod h1:V8/EzlTgLN6q0s0CJmg/I81ytsvldSF22F7h6MI02+c= github.com/qiniu/x v7.0.8+incompatible h1:P4LASsfwJY7SoZ13dwqBwGhZh7HKU8cdFVCUkmz0gZ8= github.com/qiniu/x v7.0.8+incompatible/go.mod h1:KpRKWYG/GaidPQVpoQ2Cvuvtts3gYnoo2PftgdmAiU4= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= diff --git a/iqshell/account.go b/iqshell/account.go index 5d13dac7..50926143 100644 --- a/iqshell/account.go +++ b/iqshell/account.go @@ -99,7 +99,7 @@ func DecryptSecretKey(accessKey, encryptedKey string) (string, error) { return secretKey, nil } -func setdb(acc Account) (err error) { +func setdb(acc Account, accountOver bool) (err error) { accDbPath := AccDBPath() if accDbPath == "" { return fmt.Errorf("empty account db path") @@ -111,6 +111,19 @@ func setdb(acc Account) (err error) { } defer ldb.Close() + if !accountOver { + + exists, hErr := ldb.Has([]byte(acc.Name), nil) + if hErr != nil { + err = hErr + return + } + if exists { + err = fmt.Errorf("Account Name: %s already exist in local db", acc.Name) + return + } + } + ldbWOpt := opt.WriteOptions{ Sync: true, } @@ -127,7 +140,7 @@ func setdb(acc Account) (err error) { return } -func SetAccount2(accessKey, secretKey, name, accPath, oldPath string) (err error) { +func SetAccount2(accessKey, secretKey, name, accPath, oldPath string, accountOver bool) (err error) { acc := Account{ Name: name, AccessKey: accessKey, @@ -139,7 +152,7 @@ func SetAccount2(accessKey, secretKey, name, accPath, oldPath string) (err error return } - err = setdb(acc) + err = setdb(acc, accountOver) return } @@ -151,21 +164,21 @@ func SetAccount(acc Account, accPath, oldPath string) (err error) { } if _, sErr := os.Stat(QShellRootPath); sErr != nil { if mErr := os.MkdirAll(QShellRootPath, 0755); mErr != nil { - err = fmt.Errorf("Mkdir `%s` error, %s", QShellRootPath, mErr) + err = fmt.Errorf("Mkdir `%s` error: %s", QShellRootPath, mErr) return } } accountFh, openErr := os.OpenFile(accPath, os.O_CREATE|os.O_RDWR, 0600) if openErr != nil { - err = fmt.Errorf("Open account file error, %s", openErr) + err = fmt.Errorf("Open account file error: %s", openErr) return } defer accountFh.Close() oldAccountFh, openErr := os.OpenFile(oldPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) if openErr != nil { - err = fmt.Errorf("Open account file error, %s", openErr) + err = fmt.Errorf("Open account file error: %s", openErr) return } defer oldAccountFh.Close() diff --git a/iqshell/config.go b/iqshell/config.go index 0d250e34..50555330 100644 --- a/iqshell/config.go +++ b/iqshell/config.go @@ -1,32 +1,10 @@ package iqshell import ( - "fmt" - "github.com/qiniu/api.v7/storage" "github.com/spf13/viper" - "os" - "os/user" "path/filepath" ) -func init() { - curUser, gErr := user.Current() - if gErr != nil { - fmt.Println("Error: get current user error,", gErr) - os.Exit(STATUS_HALT) - } - rootPath := filepath.Join(curUser.HomeDir, ".qshell") - - viper.SetDefault("path.root_path", filepath.Join(curUser.HomeDir, ".qshell")) - viper.SetDefault("path.acc_db_path", filepath.Join(rootPath, "account.db")) - viper.SetDefault("path.acc_path", filepath.Join(rootPath, "account.json")) - viper.SetDefault("hosts.up_host", "upload.qiniup.com") - viper.SetDefault("hosts.rs_host", storage.DefaultRsHost) - viper.SetDefault("hosts.rsf_host", storage.DefaultRsfHost) - viper.SetDefault("hosts.io_host", "iovip.qbox.me") - viper.SetDefault("hosts.api_host", storage.DefaultAPIHost) -} - func RootPath() string { return viper.GetString("path.root_path") } diff --git a/iqshell/list_bucket.go b/iqshell/list_bucket.go index 7f863160..fdf33070 100644 --- a/iqshell/list_bucket.go +++ b/iqshell/list_bucket.go @@ -2,9 +2,11 @@ package iqshell import ( "bufio" + "context" "fmt" "github.com/astaxie/beego/logs" "os" + "os/signal" "strings" "time" ) @@ -56,14 +58,23 @@ func errorWarning(marker string, err error) { *@return listError */ func (m *BucketManager) ListFiles(bucket, prefix, marker, listResultFile string) (retErr error) { - return m.ListBucket2(bucket, prefix, marker, listResultFile, "", time.Time{}, time.Time{}, nil, 20) + return m.ListBucket2(bucket, prefix, marker, listResultFile, "", time.Time{}, time.Time{}, nil, 20, false) } -func (m *BucketManager) ListBucket2(bucket, prefix, marker, listResultFile, delimiter string, startDate, endDate time.Time, suffixes []string, maxRetry int) (retErr error) { - if maxRetry <= 0 { - retErr = fmt.Errorf("maxRetry must be greater than 0") - return - } +func (m *BucketManager) ListBucket2(bucket, prefix, marker, listResultFile, delimiter string, startDate, endDate time.Time, suffixes []string, maxRetry int, appendMode bool) (retErr error) { + lastMarker := marker + + sigChan := make(chan os.Signal, 1) + ctx, cancel := context.WithCancel(context.Background()) + + signal.Notify(sigChan, os.Interrupt) + + go func() { + // 捕捉Ctrl-C, 退出下面列举的循环 + <-sigChan + cancel() + maxRetry = 0 + }() var listResultFh *os.File @@ -71,7 +82,14 @@ func (m *BucketManager) ListBucket2(bucket, prefix, marker, listResultFile, deli listResultFh = os.Stdout } else { var openErr error - listResultFh, openErr = os.Create(listResultFile) + var mode int + + if appendMode { + mode = os.O_APPEND | os.O_RDWR + } else { + mode = os.O_CREATE | os.O_RDWR | os.O_TRUNC + } + listResultFh, openErr = os.OpenFile(listResultFile, mode, 0666) if openErr != nil { retErr = openErr logs.Error("Failed to open list result file `%s`", listResultFile) @@ -82,20 +100,33 @@ func (m *BucketManager) ListBucket2(bucket, prefix, marker, listResultFile, deli bWriter := bufio.NewWriter(listResultFh) - lastMarker := marker notfilterTime := startDate.IsZero() && endDate.IsZero() notfilterSuffix := len(suffixes) == 0 - for c := 0; c < maxRetry; { - entries, lErr := m.ListBucket(bucket, prefix, delimiter, marker) + var c int + for { + if maxRetry >= 0 && c >= maxRetry { + break + } + entries, lErr := m.ListBucketContext(ctx, bucket, prefix, delimiter, marker) if entries == nil && lErr == nil { // no data - return + if lastMarker == "" { + break + } else { + fmt.Fprintf(os.Stderr, "meet empty body when list not completed\n") + continue + } } if lErr != nil { + retErr = lErr errorWarning(lastMarker, retErr) - c++ + if maxRetry > 0 { + c++ + } + time.Sleep(1) + continue } for listItem := range entries { @@ -144,7 +175,9 @@ func (m *BucketManager) ListBucket2(bucket, prefix, marker, listResultFile, deli if fErr != nil { retErr = fErr errorWarning(lastMarker, retErr) - c++ + if maxRetry > 0 { + c++ + } } if lastMarker == "" { break @@ -152,6 +185,7 @@ func (m *BucketManager) ListBucket2(bucket, prefix, marker, listResultFile, deli marker = lastMarker } } + if lastMarker != "" { fmt.Println("Marker: ", lastMarker) } diff --git a/iqshell/qupload.go b/iqshell/qupload.go index a9bf4634..06799fb1 100644 --- a/iqshell/qupload.go +++ b/iqshell/qupload.go @@ -105,6 +105,7 @@ type UploadConfig struct { CallbackUrls string `json:"callback_urls,omitempty"` CallbackHost string `json:"callback_host,omitempty"` PutPolicy storage.PutPolicy + Plock sync.Mutex } func (cfg *UploadConfig) JobId() string { @@ -282,6 +283,9 @@ func (cfg *UploadConfig) PrepareLogger(storePath, jobId string) { func (cfg *UploadConfig) UploadToken(mac *qbox.Mac, uploadFileKey string) string { + cfg.Plock.Lock() + defer cfg.Plock.Unlock() + cfg.PutPolicy.Scope = cfg.Bucket if cfg.Overwrite { cfg.PutPolicy.Scope = fmt.Sprintf("%s:%s", cfg.Bucket, uploadFileKey) @@ -578,7 +582,6 @@ func QiniuUpload(threadCount int, uploadConfig *UploadConfig, exporter *FileExpo defer upWaitGroup.Done() upToken := uploadConfig.UploadToken(bm.GetMac(), uploadFileKey) - if localFileSize > putThreshold { resumableUploadFile(uploadConfig, ldb, &ldbWOpt, ldbKey, upToken, storePath, localFilePath, uploadFileKey, localFileLastModified, exporter)