Supports skipping existing files

.gitignore

@@ -8,4 +8,4 @@ info
 od
 TestUploadFolder
 OneDriveUploader
-./*.toml
+*.toml

Readme-zh-CN.md

@@ -9,6 +9,7 @@
 - 支持自定义上传分块大小.
 - 支持多线程上传(多文件同时上传).
 - 支持根据文件大小动态调整重试次数.
+- 支持跳过网盘中已存在的同名文件.
 - 支持通过Telegram Bot实时监控上传进度，方便使用全自动下载脚本时对上传的实时监控
 
 ## 授权
@@ -59,6 +60,8 @@ Usage of OneDriveUploader:
         //使用Telegram机器人实时监控上传，此处需填写机器人的access token，形如123456789:xxxxxxxxx，输入时需使用双引号包裹。当写入内容为“1”时，使用配置文件中的BotKey和UserID作为载入项
   -uid string
         // 使用Telegram机器人实时监控上传，此处需填写接收人的userID，形如123456789
+  -m int
+        // 选择模式，0为替换OneDrive中同名文件，1为跳过，默认为0
   -v int
         // 选择版本，其中0为国际版，1为个人版(家庭版)，默认为0
 ```
@@ -104,6 +107,9 @@ OneDriveUploader -c xxx.json -f "Download" -r "Test"
 # 将同目录下的 Download 文件夹上传到 OneDrive 网盘根目录中, 使用 10 线程
 OneDriveUploader -c xxx.json -t 10 -f "Download" 
 
+# 将同目录下的 Download 文件夹上传到 OneDrive 网盘根目录中, 使用 10 线程，并跳过同名文件
+OneDriveUploader -c xxx.json -t 10 -f "Download" -m 1
+
 # 将同目录下的 Download 文件夹上传到 OneDrive 网盘根目录中, 使用 10 线程，同时设置超时时间为30秒
 OneDriveUploader -c xxx.json -t 10 -f "Download" -to 30
 

Readme.md

@@ -11,6 +11,7 @@ MoeClub wrote a [very good version](https://github.com/MoeClub/OneList/tree/mast
 - Support for customising the upload chunk size.
 - Supports multi-threaded uploads (multiple files at the same time).
 - Support for dynamically adjusting the number of retries according to the file size.
+- Supports skipping the existing files with the same name in the OneDrive.
 - Support for real-time monitoring of upload progress via Telegram Bot, for easy monitoring of uploads when using fully automated download scripts.
 
 
@@ -65,6 +66,8 @@ Usage of OneDriveUploader:
         //Use the Telegram bot to monitor uploads in real time, here you need to fill in the access token of the bot, e.g. 123456789:xxxxxxxxxx, use double quotes to wrap it
   -uid string
         // Use the Telegram bot to monitor uploads in real time, here you need to fill in the recipient's userID, shaped like 123456789
+  -m int
+        // Select the mode, 0 is to replace the file with the same name in onedrive, 1 is to skip, the default is 0
   -v int
         // Select the version, where 0 is the Business version and 1 is the personal (home) version, the default is 0
 ```
@@ -80,8 +83,6 @@ Usage of OneDriveUploader:
     "BlockSize": 10,
     // Maximum single file size. (Currently: 100GB single file limit for Personal Edition (Home Edition); 15GB single file limit for other editions, Microsoft will update to 100GB over time. Unit: GB)
     "SigleFile": 100,
-    // Cache refresh interval.
-    "RefreshInterval": 1500,
     // If this is the Chinese version (CenturyLink), this should be true.
     "MainLand": false,
     //Software language
@@ -112,7 +113,10 @@ OneDriveUploader -c xxx.json -f "Download" -r "Test"
 # Upload the Download folder from the same directory to the root of the OneDrive, using 10 threads
 OneDriveUploader -c xxx.json -t 10 -f "Download" 
 
-# Upload the download folder in the same directory to the root directory of onedrive network disk, use 10 threads, and set the timeout to 30 seconds
+# Upload the download folder in the same directory to the root directory of onedrive, use 10 threads, and skip the file with the same name
+OneDriveUploader -c xxx.json -t 10 -f "Download" -m 1
+
+# Upload the download folder in the same directory to the root directory of onedrive, use 10 threads, and set the timeout to 30 seconds
 OneDriveUploader -c xxx.json -t 10 -f "Download" -to 30
 
 # Upload the Download folder from the same directory to the root of the OneDrive, using 10 threads, while using Telegram Bot to monitor the progress of the upload in real time

api/restore/upload/onedriveRecoverableRestore.go

@@ -110,14 +110,18 @@ func (rs *RestoreService) getUploadSession(userID string, bearerToken string, co
 	if err != nil {
 		return nil, err
 	}
+	//fmt.Printf("%+v", body)
+	//log.Panicf("%+v", uploadSessionPath)
 
 	//Create request instance
-	req, err := rs.NewRequest("POST", uploadSessionPath, getRessumableUploadSessionHeader(bearerToken), body)
+	req, err := rs.NewRequest("PUT", uploadSessionPath, getRessumableUploadSessionHeader(bearerToken), body)
+	//log.Panicf("%+v", req)
 	if err != nil {
 		return nil, err
 	}
 	//Execute the request
 	resp, err := rs.Do(req)
+	// log.Panicf("%+v", resp)
 	if err != nil {
 		//Need to return a generic object from onedrive upload instead of response directly
 		return nil, err

api/restore/upload/onedriveRestore.go

@@ -1,8 +1,10 @@
 package upload
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"github.com/buger/jsonparser"
 	"io/ioutil"
 	"log"
 	"main/fileutil"
@@ -28,6 +30,31 @@ type RestoreService struct {
 	*httpLocal.OneDrive
 }
 
+func (rs *RestoreService) GetDriveItem(userId string, bearerToken string, targetFolder string) (map[string]bool, error) {
+	uploadPath := fmt.Sprintf("/users/%s/drive/root:/%s:/children", userId, targetFolder)
+	req, err := rs.NewRequest("GET", uploadPath, getSimpleUploadHeader(bearerToken), nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := rs.Do(req)
+	if err != nil {
+		//log.Panicln(err)
+		return nil, err
+	}
+	//s, _ := ioutil.ReadAll(resp.Body)
+	buf := new(bytes.Buffer)
+	buf.ReadFrom(resp.Body)
+	a := make(map[string]bool, 0)
+
+	jsonparser.ArrayEach(buf.Bytes(), func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
+		i, _ := jsonparser.GetString(value, "name")
+		//fmt.Println(i)
+		a[i] = true
+	}, "value")
+
+	return a, nil
+}
+
 // SimpleUploadToOriginalLoc allows you to provide the contents of a new file or update the
 // contents of an existing file in a single API call. This method only supports
 // files up to 4MB in size. For larger files use ResembleUpload().
@@ -53,7 +80,7 @@ func (rs *RestoreService) SimpleUploadToOriginalLoc(userId string, bearerToken s
 		if err != nil {
 			log.Panicf(locText("failToStore"), err)
 		}
-		//Handle query parameter for conflict resolution
+		//Handle query parameter for conflict resolution 冲突解决的句柄查询参数
 		//The different values for @microsoft.graph.conflictBehavior= rename|replace|fail
 		q := url.Values{}
 		q.Add("@microsoft.graph.conflictBehavior", conflictOption)

graph/net/http/onedriveClient.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"main/fileutil"
 	"io"
+	"main/fileutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -98,6 +98,7 @@ func (od *OneDrive) NewRequest(method, uri string, requestHeaders map[string]str
 	if err != nil {
 		return nil, fmt.Errorf("Unable to parse the file into Bytes  reason: %v", err)
 	}
+
 	var req *http.Request
 	if isValidUrl(uri) {
 		req, err = http.NewRequest(method, uri, reqBody)
@@ -124,6 +125,7 @@ func (od *OneDrive) NewRequest(method, uri string, requestHeaders map[string]str
 //Execute request
 func (od *OneDrive) Do(req *http.Request) (*http.Response, error) {
 	resp, err := od.Client.Do(req)
+	//log.Panicf("%+v\n%+v", resp, err)
 	if err != nil {
 		return nil, err
 	}

i18n.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"time"
 )
@@ -87,14 +88,17 @@ type Loc struct {
 }
 
 func (loc *Loc) init(locLanguage string) {
-	_, err := os.Stat(fmt.Sprintf("%s.toml", locLanguage))
+	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
+	dropErr(err)
+	dir = filepath.Join(dir, fmt.Sprintf("%s.toml", locLanguage))
+	_, err = os.Stat(dir)
 	if err != nil {
 		resp, err := http.Get(fmt.Sprintf("https://cdn.jsdelivr.net/gh/gaowanliang/OneDriveUploader/i18n/%s.toml", locLanguage))
 		dropErr(err)
 		defer resp.Body.Close()
 		data, err := ioutil.ReadAll(resp.Body)
 		dropErr(err)
-		ioutil.WriteFile(fmt.Sprintf("%s.toml", locLanguage), data, 0666)
+		ioutil.WriteFile(dir, data, 0666)
 	} else {
 		url := "https://cdn.jsdelivr.net/gh/gaowanliang/OneDriveUploader@latest/i18n/"
 		j := pageDownload(url)
@@ -107,16 +111,16 @@ func (loc *Loc) init(locLanguage string) {
 			}
 
 		}
-		oldLanFileTime := GetFileModTime(fmt.Sprintf("%s.toml", locLanguage))
+		oldLanFileTime := GetFileModTime(dir)
 		if newLanFileTime > oldLanFileTime {
-			err := os.RemoveAll(fmt.Sprintf("%s.toml", locLanguage))
+			err = os.RemoveAll(dir)
 			dropErr(err)
 			resp, err := http.Get(fmt.Sprintf("https://cdn.jsdelivr.net/gh/gaowanliang/OneDriveUploader/i18n/%s.toml", locLanguage))
 			dropErr(err)
 			defer resp.Body.Close()
 			data, err := ioutil.ReadAll(resp.Body)
 			dropErr(err)
-			ioutil.WriteFile(fmt.Sprintf("%s.toml", locLanguage), data, 0644)
+			ioutil.WriteFile(dir, data, 0644)
 		}
 	}
 	loc.localize = i18n.NewLocalizer(bundle, locLanguage)

i18n/en.toml

@@ -1,5 +1,5 @@
-completeUpload = "%s upload completed, time consuming %d s,average upload speed %s/s"
-configCreateSuccess = "The registration is successful. A new login file has been created in the running directory:%s"
+completeUpload = "`%s` upload completed, time consuming `%d s`,average upload speed `%s/s`"
+configCreateSuccess = "The registration is successful. A new login file has been created in the running directory:`%s`"
 failToLink = "OneDrive account `%s` \n There was a connection problem when uploading `%s` Retrying. This is the %d times retrying"
 failToLoadFiles = "Failed to Load Files from source :%v"
 failToStore = "Failed to Restore :%v"
@@ -10,3 +10,4 @@ oneDriveUploadTip2 = "OneDrive account `%s` \nUploading `%s`\nSize:`%s` Complete
 startToUpload = "`%s` start uploading, size:`%s`"
 startToUpload1 = "start uploading to onedrive"
 telegramSendError = "Telegram Send Error:%s"
+existSkip = "Already exists, auto skip"

i18n/zh-CN.toml

@@ -1,5 +1,5 @@
-completeUpload = "%s上传完成，耗时 %d 秒，平均上传速度 %s/s"
-configCreateSuccess = "注册成功，已在运行目录下新建登录文件%s"
+completeUpload = "`%s`上传完成，耗时 `%d 秒`，平均上传速度 `%s/s`"
+configCreateSuccess = "注册成功，已在运行目录下新建登录文件`%s`"
 failToLink = "向OneDrive账户 `%s` 上传 `%s` 时出现连接问题，正在重试，当前为第%d次重试"
 failToLoadFiles = "无法从源加载文件 :%v"
 failToStore = "无法还原 :%v"
@@ -10,3 +10,4 @@ oneDriveUploadTip2 = "正在向OneDrive账户 `%s` 上传 `%s`\n大小:`%s` 已
 startToUpload = "`%s` 开始上传，大小: `%s`"
 startToUpload1 = "开始上传至OneDrive"
 telegramSendError = "Telegram 发送错误:%s"
+existSkip = "已存在，自动跳过"

main.go

@@ -62,6 +62,7 @@ func Upload(infoPath string, filePath string, targetFolder string, threads int,
 	} else {
 		restore(restoreSrvc, fileInfoToUpload, threads)
 	}*/
+
 	restore(restoreSrvc, fileInfoToUpload, targetFolder, threads, sendMsg, locText, infoPath)
 	err = os.Chdir(oldDir)
 	if err != nil {
@@ -76,32 +77,61 @@ func changeBlockSize(MB int) {
 func restore(restoreSrvc *upload.RestoreService, filesToRestore map[string]fileutil.FileInfo, targetFolder string, threads int, sendMsg func() (func(text string), string, string), locText func(text string) string, infoPath string) {
 	var wg sync.WaitGroup
 	pool := make(chan struct{}, threads)
+	checkPath := make(map[string]bool, 0)
+	pathFiles := make(map[string]map[string]bool, 0)
+	temps, _botKey, iUserID := sendMsg()
+	var iSendMsg func(string)
+
+	temp := func(text string) {
+		temps(text)
+		if _botKey != "" && iUserID != "" {
+			iSendMsg(text)
+		}
+	}
 	for filePath, fileInfo := range filesToRestore {
 		wg.Add(1)
 		pool <- struct{}{}
+
+		paths, fileName := filepath.Split(filepath.Join(targetFolder, filePath))
+		if mode == 1 {
+			if paths == "" {
+				paths = "/"
+			}
+			paths = strings.ReplaceAll(paths, "\\", "/")
+			if paths[len(paths)-1] == '/' {
+				paths = paths[:len(paths)-1]
+			}
+			if _, ok := checkPath[paths]; !ok {
+				userID, bearerToken := httpLocal.GetMyIDAndBearer(infoPath, thread, block, lang, timeOut, _botKey, _UserID)
+				files, _ := restoreSrvc.GetDriveItem(userID, bearerToken, paths)
+				checkPath[paths] = true
+				pathFiles[paths] = files
+			}
+			// log.Println(checkPath, paths, pathFiles, fileName, filePath)
+		}
+
 		go func(filePath string, fileInfo fileutil.FileInfo) {
 			defer wg.Done()
 			defer func() {
 				<-pool
 			}()
-			temps, botKey, iUserID := sendMsg()
-			var iSendMsg func(string)
-			tip := "`" + filePath + "`" + loc.print("startToUpload1")
-			if botKey != "" && iUserID != "" {
-				iSendMsg = botSend(botKey, iUserID, tip)
-			}
-			temp := func(text string) {
-				temps(text)
-				if botKey != "" && iUserID != "" {
-					iSendMsg(text)
+			if _, ok := pathFiles[paths][fileName]; !ok || mode == 0 {
+				tip := "`" + filePath + "`" + loc.print("startToUpload1")
+				if _botKey != "" && iUserID != "" {
+					iSendMsg = botSend(_botKey, iUserID, tip)
 				}
+				temp(tip)
+				userID, bearerToken := httpLocal.GetMyIDAndBearer(infoPath, thread, block, lang, timeOut, _botKey, _UserID)
+				username := strings.ReplaceAll(filepath.Base(infoPath), ".json", "")
+				restoreSrvc.SimpleUploadToOriginalLoc(userID, bearerToken, "replace", targetFolder, filePath, fileInfo, temp, locText, username)
+			} else {
+				tip := filePath + "已存在，自动跳过"
+				if _botKey != "" && iUserID != "" {
+					iSendMsg = botSend(_botKey, iUserID, tip)
+				}
+				temp(tip)
+				time.Sleep(time.Second * 3)
 			}
-			temp(tip)
-			userID, bearerToken := httpLocal.GetMyIDAndBearer(infoPath, thread, block, lang, timeOut, botKey, _UserID)
-			username := strings.ReplaceAll(filepath.Base(infoPath), ".json", "")
-			restoreSrvc.SimpleUploadToOriginalLoc(userID, bearerToken, "rename", targetFolder, filePath, fileInfo, temp, locText, username)
-
-			//printResp(resp)
 			defer temp("close")
 		}(filePath, fileInfo)
 	}
@@ -218,6 +248,7 @@ var block int
 var botKey string
 var _UserID string
 var thread int
+var mode int
 
 func main() {
 	var codeURL string
@@ -238,6 +269,7 @@ func main() {
 	flag.StringVar(&_UserID, "uid", "", "Use the telegram robot to monitor the upload in real time. Fill in the user ID of the receiver, such as 123456789")
 	flag.StringVar(&targetFolder, "r", "", "Set the directory you want to upload to onedrive")
 	flag.IntVar(&timeOut, "to", 60, "When uploading, the timeout of each block is 60s by default")
+	flag.IntVar(&mode, "m", 0, "Select the mode, 0 is to replace the file with the same name in onedrive, 1 is to skip, the default is 0")
 	flag.StringVar(&lang, "l", "en", "Set the software language, English by default")
 	// 从arguments中解析注册的flag。必须在所有flag都注册好而未访问其值时执行。未注册却使用flag -help时，会返回ErrHelp。
 	flag.Parse()