feature:(Add interval download function)

* feature:(Add interval download function)
Co-authored-by: cong.zhou <cong.zhou@m.upai.com>

README.md

@@ -121,6 +121,15 @@ docker run --rm upx upx -v
 | --async   | 异步删除，目录可能需要二次删除 |
 | --mtime v | 参考 Linux `find` |
 
+### 下载　`get`
+
+>get 存储路径　本地路径
+
+| options | 说明                          |
+|---------|-----------------------------|
+| --start | 只下载路径字典序大于等于 `start` 的文件或目录 |
+| --end   | 只下载路径字典序小于 `end` 的文件或目录     |
+
 ### 增量同步 `sync`
 
 > sync 本地路径 存储路径

commands.go

@@ -270,19 +270,30 @@ func NewGetCommand() cli.Command {
 			if strings.Contains(base, "*") {
 				mc.Wildcard, upPath = base, dir
 			}
+			if c.String("start") != "" {
+				mc.Start = c.String("start")
+			}
+			if c.String("end") != "" {
+				mc.End = c.String("end")
+			}
 			if c.String("mtime") != "" {
 				err := parseMTime(c.String("mtime"), mc)
 				if err != nil {
 					PrintErrorAndExit("get %s: parse mtime: %v", upPath, err)
 				}
 			}
-			session.Get(upPath, localPath, mc, c.Int("w"))
-
+			if mc.Start != "" || mc.End != "" {
+				session.GetStartBetweenEndFiles(upPath, localPath, mc, c.Int("w"))
+			} else {
+				session.Get(upPath, localPath, mc, c.Int("w"))
+			}
 			return nil
 		},
 		Flags: []cli.Flag{
 			cli.IntFlag{Name: "w", Usage: "max concurrent threads", Value: 5},
 			cli.StringFlag{Name: "mtime", Usage: "file's data was last modified n*24 hours ago, same as linux find command."},
+			cli.StringFlag{Name: "start", Usage: "file download range starting location"},
+			cli.StringFlag{Name: "end", Usage: "file download range ending location"},
 		},
 	}
 }

get_test.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func GetStartBetweenEndFiles(t *testing.T, src, dst, correct, start, end string) {
+	var err error
+	src = AbsPath(src)
+
+	if start != "" && start[0] != '/' {
+		start = filepath.Join(src, start)
+	}
+	if end != "" && end[0] != '/' {
+		end = filepath.Join(src, end)
+	}
+	if dst == "" {
+		_, err = Upx("get", src, "--start="+start, "--end="+end)
+	} else {
+		_, err = Upx("get", src, dst, "--start="+start, "--end="+end)
+	}
+	Nil(t, err)
+}
+
+/*
+测试目录					test1:start=123 end=999    test2:start=111 end=666
+input:						local:					  local:
+	|-- 111						├── 333					  ├── 111
+	|-- 333						├── 666					  ├── 333
+	|-- 777						│ ├── 111				  └── 444
+	|-- 444						│ ├── 333				      └── 666
+	!   `-- 666					│ ├── 666
+	`-- 666						│ └── 777
+		|-- 111					└── 777
+		|-- 333
+		|-- 666
+    	`-- 777
+*/
+func TestGetStartBetweenEndFiles(t *testing.T) {
+	nowpath, _ := os.Getwd()
+	root := strings.Join(strings.Split(ROOT, " "), "-")
+	base := root + "/get/"
+	pwd, err := ioutil.TempDir("", "test")
+	Nil(t, err)
+	localBase := filepath.Join(pwd, "get")
+
+	func() {
+		SetUp()
+		err := os.MkdirAll(localBase, 0755)
+		Nil(t, err)
+	}()
+	defer TearDown()
+
+	err = os.Chdir(localBase)
+	Nil(t, err)
+	Upx("mkdir", base)
+	Upx("cd", base)
+
+	type uploadFiles []struct {
+		name    string
+		file    string
+		dst     string
+		correct string
+	}
+	type uploadDirs []struct {
+		dir     string
+		dst     string
+		correct string
+	}
+	//构造测试目录
+	files := uploadFiles{
+		{name: "111", file: filepath.Join(localBase, "111"), dst: "", correct: filepath.Join(base, "111")},
+		{name: "333", file: filepath.Join(localBase, "333"), dst: "", correct: path.Join(base, "333")},
+		{name: "333", file: "333", dst: path.Join(base, "333"), correct: path.Join(base, "333")},
+		{name: "777", file: "777", dst: base, correct: path.Join(base, "777")},
+		{name: "666", file: "666", dst: base + "/444/", correct: path.Join(base, "444", "666")},
+	}
+	for _, file := range files {
+		CreateFile(file.name)
+		putFile(t, file.file, file.dst, file.correct)
+	}
+	log.Println(122)
+
+	dirs := uploadDirs{
+		{dir: localBase, dst: base + "/666/", correct: base + "/666/"},
+	}
+	for _, dir := range dirs {
+		putDir(t, dir.dir, dir.dst, dir.correct)
+	}
+
+	type list struct {
+		start   string
+		end     string
+		testDir string
+	}
+	type test struct {
+		input list
+		real  []string
+		want  []string
+	}
+	//构造测试
+	tests := []test{
+		{input: list{start: "123", end: "999", testDir: filepath.Join(nowpath, "test1")}, real: localFile("test1", base), want: upFile(t, base, "123", "999")},
+		{input: list{start: "111", end: "666", testDir: filepath.Join(nowpath, "test2")}, real: localFile("test2", base), want: upFile(t, base, "444", "666")},
+	}
+	for _, tc := range tests {
+		input := tc.input
+
+		err = os.MkdirAll(input.testDir, os.ModePerm)
+		if err != nil {
+			log.Println(err)
+		}
+
+		GetStartBetweenEndFiles(t, base, input.testDir, input.testDir, input.start, input.end)
+
+		sort.Strings(tc.real)
+		sort.Strings(tc.want)
+		Equal(t, len(tc.real), len(tc.want))
+
+		for i := 0; i < len(tc.real); i++ {
+			log.Println("compare:", tc.real[i], " ", tc.want[i])
+			Equal(t, tc.real[i], tc.want[i])
+		}
+	}
+}
+
+//递归获取下载到本地的文件
+func localFile(local, up string) []string {
+	var locals []string
+	localLen := len(local)
+	fInfos, _ := ioutil.ReadDir(local + "/")
+	for _, fInfo := range fInfos {
+		fp := filepath.Join(local, fInfo.Name())
+		//使用云存储目录作为前缀方便比较
+		locals = append(locals, up[:len(up)-1]+fp[localLen:])
+		if IsDir(fp) {
+			localFile(fp, up)
+		}
+	}
+	return locals
+}
+
+//递归获取云存储目录文件
+func upFile(t *testing.T, up, start, end string) []string {
+	b, err := Upx("ls", up)
+	Nil(t, err)
+
+	var ups []string
+	output := strings.TrimRight(string(b), "\n")
+	for _, line := range strings.Split(output, "\n") {
+		items := strings.Split(line, " ")
+		fp := filepath.Join(up, items[len(items)-1])
+		ups = append(ups, fp)
+		if items[0][0] == 'd' {
+			upFile(t, fp, start, end)
+		}
+	}
+
+	var upfiles []string
+	for _, file := range ups {
+		if file >= start && file < end {
+			upfiles = append(upfiles, file)
+		}
+	}
+	return upfiles
+}
+
+func IsDir(path string) bool {
+	s, err := os.Stat(path)
+	if err != nil {
+
+		return false
+	}
+
+	return s.IsDir()
+}
+
+func AbsPath(upPath string) (ret string) {
+	if strings.HasPrefix(upPath, "/") {
+		ret = path.Join(upPath)
+	} else {
+		ret = path.Join("/", upPath)
+	}
+	if strings.HasSuffix(upPath, "/") && ret != "/" {
+		ret += "/"
+	}
+	return
+}

match.go

@@ -27,6 +27,9 @@ type MatchConfig struct {
 	Before   time.Time
 	After    time.Time
 
+	Start string
+	End   string
+
 	ItemType int
 }
 

session.go

@@ -396,6 +396,61 @@ func (sess *Session) Get(upPath, localPath string, match *MatchConfig, workers i
 	}
 }
 
+func (sess *Session) GetStartBetweenEndFiles(upPath, localPath string, match *MatchConfig, workers int) {
+	fpath := sess.AbsPath(upPath)
+	isDir, exist := sess.IsUpYunDir(fpath)
+	if !exist {
+		if match.ItemType == DIR {
+			isDir = true
+		} else {
+			PrintErrorAndExit("get: cannot down %s:No such file or directory", fpath)
+		}
+	}
+	if isDir && match != nil && match.Wildcard == "" {
+		if match.ItemType == FILE {
+			PrintErrorAndExit("get: cannot down %s: Is a directory", fpath)
+		}
+	}
+
+	fInfoChan := make(chan *upyun.FileInfo, 1)
+	objectsConfig := &upyun.GetObjectsConfig{
+		Path:        fpath,
+		ObjectsChan: fInfoChan,
+		QuitChan:    make(chan bool, 1),
+	}
+	go func() {
+		err := sess.updriver.List(objectsConfig)
+		if err != nil {
+			PrintErrorAndExit("ls %s: %v", fpath, err)
+		}
+	}()
+
+	startList := match.Start
+	if startList != "" && startList[0] != '/' {
+		startList = filepath.Join(fpath, startList)
+	}
+	endList := match.End
+	if endList != "" && endList[0] != '/' {
+		endList = filepath.Join(fpath, endList)
+	}
+
+	for fInfo := range fInfoChan {
+		fp := filepath.Join(fpath, fInfo.Name)
+		if (fp >= startList || startList == "") && (fp < endList || endList == "") {
+			sess.Get(fp, localPath, match, workers)
+		} else if strings.HasPrefix(startList, fp) {
+			//前缀相同进入下一级文件夹，继续递归判断
+			if fInfo.IsDir {
+				sess.GetStartBetweenEndFiles(fp, localPath+fInfo.Name+"/", match, workers)
+			}
+		}
+		if fp >= endList && endList != "" && fInfo.IsDir {
+			close(objectsConfig.QuitChan)
+			break
+		}
+	}
+}
+
 func (sess *Session) putFileWithProgress(barId int, localPath, upPath string, localInfo os.FileInfo) (int, error) {
 	var err error
 	bar, idx := AddBar(barId, int(localInfo.Size()))