feat: refactor offline download (#5408 close #4108)

* wip: refactor offline download (#5331) * base tool * working: aria2 * refactor: change type of percentage to float64 * wip: adapt aria2 * wip: use items in offline_download * wip: use tool manager * wip: adapt qBittorrent * chore: fix typo * Squashed commit of the following: commit 4fc0a77 Author: Andy Hsu <i@nn.ci> Date: Fri Oct 20 21:06:25 2023 +0800 fix(baidu_netdisk): upload file > 4GB (close #5392) commit aaffaee Author: gmugu <94156510@qq.com> Date: Thu Oct 19 19:17:53 2023 +0800 perf(webdav): support request with cookies (#5391) commit 8ef8023 Author: NewbieOrange <NewbieOrange@users.noreply.github.com> Date: Thu Oct 19 19:17:09 2023 +0800 fix(aliyundrive_open): upload progress for normal upload (#5398) commit cdfbe6d Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Wed Oct 18 16:27:07 2023 +0800 fix: hash gcid empty file (#5394) commit 94d0287 Author: Andy Hsu <i@nn.ci> Date: Sat Oct 14 13:17:51 2023 +0800 ci: remove `pr-welcome` label when close issue [skip ci] commit 7f73354 Author: itsHenry <2671230065@qq.com> Date: Sat Oct 14 13:12:46 2023 +0800 feat(cloudreve): support thumbnail (#5373 close #5348) * feat(cloudreve): support thumbnail * chore: remove unnecessary code commit b9e192b Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Thu Oct 12 20:57:12 2023 +0800 fix(115): limit request rate (#5367 close #5275) * fix(115):limit request rate * chore(115): fix unit of `limit_rate` --------- Co-authored-by: Andy Hsu <i@nn.ci> commit 69a98ea Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Wed Oct 11 22:01:55 2023 +0800 fix(deps): update module github.com/aliyun/aliyun-oss-go-sdk to v2.2.9+incompatible (#5141) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit 1ebc96a Author: Andy Hsu <i@nn.ci> Date: Tue Oct 10 18:32:00 2023 +0800 fix(wopan): fatal error concurrent map writes (close #5352) commit 66e2324 Author: Andy Hsu <i@nn.ci> Date: Tue Oct 10 18:23:11 2023 +0800 chore(deps): upgrade dependencies commit 7600dc2 Author: Andy Hsu <i@nn.ci> Date: Tue Oct 10 18:13:58 2023 +0800 fix(aliyundrive_open): change default api to raw server (close #5358) commit 8ef89ad Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Tue Oct 10 18:08:27 2023 +0800 fix(baidu_netdisk): hash and `error 2` (#5356) * fix(baidu):hash and error:2 * fix:invalid memory address commit 35d6722 Author: jeffmingup <1960588251@qq.com> Date: Sun Oct 8 19:29:45 2023 +0800 fix(onedrive_app): incorrect api on `_accessToken` (#5346) commit 1a283bb Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Fri Oct 6 16:04:39 2023 +0800 feat(google_drive): add `hash_info`, `ctime`, `thumbnail` (#5334) commit a008f54 Author: nkh0472 <67589323+nkh0472@users.noreply.github.com> Date: Thu Oct 5 13:10:51 2023 +0800 docs: minor language improvements (#5329) [skip ci] * fix: adapt update progress type * Squashed commit of the following: commit 65c5ec0 Author: itsHenry <2671230065@qq.com> Date: Sat Nov 4 13:35:09 2023 +0800 feat(cloudreve): folder size count and switch (#5457 close #5395) commit a632596 Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Mon Oct 30 15:11:20 2023 +0800 fix(deps): update module github.com/charmbracelet/lipgloss to v0.9.1 (#5234) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit 4dff494 Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Mon Oct 30 15:10:36 2023 +0800 fix(deps): update golang.org/x/exp digest to 7918f67 (#5366) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit cc86d6f Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Sun Oct 29 14:45:55 2023 +0800 fix(deps): update module golang.org/x/net to v0.17.0 [security] (#5370) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit c0f9c8e Author: Andy Hsu <i@nn.ci> Date: Thu Oct 26 19:21:09 2023 +0800 feat: add ignore direct link params (close #5434)
alist-org · Nov 6, 2023 · 769281b · 769281b
1 parent 3bbdd4f
commit 769281b
Show file tree

Hide file tree

Showing 48 changed files with 1,238 additions and 258 deletions.
diff --git a/cmd/root.go b/cmd/root.go
@@ -5,6 +5,8 @@ import (
 	"os"
 
 	"github.com/alist-org/alist/v3/cmd/flags"
+	_ "github.com/alist-org/alist/v3/drivers"
+	_ "github.com/alist-org/alist/v3/internal/offline_download"
 	"github.com/spf13/cobra"
 )
 

diff --git a/cmd/server.go b/cmd/server.go
@@ -13,7 +13,6 @@ import (
 	"time"
 
 	"github.com/alist-org/alist/v3/cmd/flags"
-	_ "github.com/alist-org/alist/v3/drivers"
 	"github.com/alist-org/alist/v3/internal/bootstrap"
 	"github.com/alist-org/alist/v3/internal/conf"
 	"github.com/alist-org/alist/v3/pkg/utils"
@@ -35,8 +34,7 @@ the address is defined in config file`,
 			utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart)
 			time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second)
 		}
-		bootstrap.InitAria2()
-		bootstrap.InitQbittorrent()
+		bootstrap.InitOfflineDownloadTools()
 		bootstrap.LoadStorages()
 		if !flags.Debug && !flags.Dev {
 			gin.SetMode(gin.ReleaseMode)

diff --git a/drivers/123/upload.go b/drivers/123/upload.go
@@ -107,7 +107,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi
 			if err != nil {
 				return err
 			}
-			up(j * 100 / chunkCount)
+			up(float64(j) * 100 / float64(chunkCount))
 		}
 	}
 	// complete s3 upload

diff --git a/drivers/189/util.go b/drivers/189/util.go
@@ -380,7 +380,7 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F
 		if err != nil {
 			return err
 		}
-		up(int(i * 100 / count))
+		up(float64(i) * 100 / float64(count))
 	}
 	fileMd5 := hex.EncodeToString(md5Sum.Sum(nil))
 	sliceMd5 := fileMd5

diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go
@@ -513,7 +513,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo
 			if err != nil {
 				return err
 			}
-			up(int(threadG.Success()) * 100 / count)
+			up(float64(threadG.Success()) * 100 / float64(count))
 			return nil
 		})
 	}
@@ -676,7 +676,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode
 					return err
 				}
 
-				up(int(threadG.Success()) * 100 / len(uploadUrls))
+				up(float64(threadG.Success()) * 100 / float64(len(uploadUrls)))
 				uploadProgress.UploadParts[i] = ""
 				return nil
 			})
@@ -812,7 +812,7 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model
 		if _, err := tempFile.Seek(status.GetSize(), io.SeekStart); err != nil {
 			return nil, err
 		}
-		up(int(status.GetSize()/file.GetSize()) * 100)
+		up(float64(status.GetSize()) / float64(file.GetSize()) * 100)
 	}
 
 	return y.OldUploadCommit(ctx, status.FileCommitUrl, status.UploadFileId)

diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go
@@ -7,14 +7,15 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
-	"github.com/alist-org/alist/v3/internal/stream"
 	"io"
 	"math"
 	"math/big"
 	"net/http"
 	"os"
 	"time"
 
+	"github.com/alist-org/alist/v3/internal/stream"
+
 	"github.com/alist-org/alist/v3/drivers/base"
 	"github.com/alist-org/alist/v3/internal/conf"
 	"github.com/alist-org/alist/v3/internal/driver"
@@ -304,7 +305,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil
 		}
 		res.Body.Close()
 		if count > 0 {
-			up(i * 100 / count)
+			up(float64(i) * 100 / float64(count))
 		}
 	}
 	var resp2 base.Json

diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go
@@ -258,7 +258,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m
 				return nil, err
 			}
 			offset += partSize
-			up(i * 100 / count)
+			up(float64(i*100) / float64(count))
 		}
 	} else {
 		log.Debugf("[aliyundrive_open] rapid upload success, file id: %s", createResp.FileId)

diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go
@@ -278,7 +278,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
 			if err != nil {
 				return err
 			}
-			up(int(threadG.Success()) * 100 / len(precreateResp.BlockList))
+			up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList)))
 			precreateResp.BlockList[i] = -1
 			return nil
 		})

diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go
@@ -329,7 +329,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil
 				if err != nil {
 					return err
 				}
-				up(int(threadG.Success()) * 100 / len(precreateResp.BlockList))
+				up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList)))
 				precreateResp.BlockList[i] = -1
 				return nil
 			})

diff --git a/drivers/dropbox/driver.go b/drivers/dropbox/driver.go
@@ -203,7 +203,7 @@ func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt
 		_ = res.Body.Close()
 
 		if count > 0 {
-			up((i + 1) * 100 / count)
+			up(float64(i+1) * 100 / float64(count))
 		}
 
 		offset += byteSize

diff --git a/drivers/mega/driver.go b/drivers/mega/driver.go
@@ -4,11 +4,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/alist-org/alist/v3/pkg/http_range"
-	"github.com/rclone/rclone/lib/readers"
 	"io"
 	"time"
 
+	"github.com/alist-org/alist/v3/pkg/http_range"
+	"github.com/rclone/rclone/lib/readers"
+
 	"github.com/alist-org/alist/v3/internal/driver"
 	"github.com/alist-org/alist/v3/internal/errs"
 	"github.com/alist-org/alist/v3/internal/model"
@@ -169,7 +170,7 @@ func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea
 			if err != nil {
 				return err
 			}
-			up(id * 100 / u.Chunks())
+			up(float64(id) * 100 / float64(u.Chunks()))
 		}
 
 		_, err = u.Finish()

diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go
@@ -308,7 +308,7 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre
 				if resp.StatusCode != http.StatusOK {
 					return fmt.Errorf("upload err,code=%d", resp.StatusCode)
 				}
-				up(100 * int(threadG.Success()) / len(parts))
+				up(100 * float64(threadG.Success()) / float64(len(parts)))
 				initUpdload.PartInfos[i] = ""
 				return nil
 			})

diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go
@@ -203,7 +203,7 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil
 			return errors.New(string(data))
 		}
 		res.Body.Close()
-		up(int(finish * 100 / stream.GetSize()))
+		up(float64(finish) * 100 / float64(stream.GetSize()))
 	}
 	return nil
 }
diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go
@@ -194,7 +194,7 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model.
 			return errors.New(string(data))
 		}
 		res.Body.Close()
-		up(int(finish * 100 / stream.GetSize()))
+		up(float64(finish) * 100 / float64(stream.GetSize()))
 	}
 	return nil
 }
diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go
@@ -209,7 +209,7 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File
 		}
 		md5s = append(md5s, m)
 		partNumber++
-		up(int(100 * (total - left) / total))
+		up(100 * float64(total-left) / float64(total))
 	}
 	err = d.upCommit(pre, md5s)
 	if err != nil {

diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go
@@ -4,13 +4,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"github.com/alist-org/alist/v3/internal/stream"
 	"io"
 	"net/url"
 	stdpath "path"
 	"strings"
 	"time"
 
+	"github.com/alist-org/alist/v3/internal/stream"
+
 	"github.com/alist-org/alist/v3/internal/driver"
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -104,7 +105,7 @@ func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) e
 		},
 		Reader:   io.NopCloser(bytes.NewReader([]byte{})),
 		Mimetype: "application/octet-stream",
-	}, func(int) {})
+	}, func(float64) {})
 }
 
 func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error {

diff --git a/drivers/teambition/util.go b/drivers/teambition/util.go
@@ -189,7 +189,7 @@ func (d *Teambition) chunkUpload(ctx context.Context, file model.FileStreamer, t
 		if err != nil {
 			return nil, err
 		}
-		up(i * 100 / newChunk.Chunks)
+		up(float64(i) * 100 / float64(newChunk.Chunks))
 	}
 	_, err = base.RestyClient.R().SetHeader("Authorization", token).Post(
 		fmt.Sprintf("https://%s.teambition.net/upload/chunk/%s",

diff --git a/drivers/terabox/driver.go b/drivers/terabox/driver.go
@@ -213,7 +213,7 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt
 		}
 		log.Debugln(res.String())
 		if len(precreateResp.BlockList) > 0 {
-			up(i * 100 / len(precreateResp.BlockList))
+			up(float64(i) * 100 / float64(len(precreateResp.BlockList)))
 		}
 	}
 	_, err = d.create(rawPath, stream.GetSize(), 0, precreateResp.Uploadid, block_list_str)

diff --git a/drivers/trainbit/driver.go b/drivers/trainbit/driver.go
@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"math"
 	"net/http"
 	"net/url"
 	"strings"
@@ -128,7 +127,7 @@ func (d *Trainbit) Put(ctx context.Context, dstDir model.Obj, stream model.FileS
 		stream,
 		func(byteNum int) {
 			total += int64(byteNum)
-			up(int(math.Round(float64(total) / float64(stream.GetSize()) * 100)))
+			up(float64(total) / float64(stream.GetSize()) * 100)
 		},
 	}
 	req, err := http.NewRequest(http.MethodPost, endpoint.String(), progressReader)

diff --git a/drivers/wopan/driver.go b/drivers/wopan/driver.go
@@ -159,7 +159,7 @@ func (d *Wopan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre
 		ContentType: stream.GetMimetype(),
 	}, dstDir.GetID(), d.FamilyID, wopan.Upload2COption{
 		OnProgress: func(current, total int64) {
-			up(int(100 * current / total))
+			up(100 * float64(current) / float64(total))
 		},
 	})
 	return err

diff --git a/internal/aria2/monitor.go b/internal/aria2/monitor.go
@@ -2,7 +2,6 @@ package aria2
 
 import (
 	"fmt"
-	"github.com/alist-org/alist/v3/internal/stream"
 	"os"
 	"path"
 	"path/filepath"
@@ -11,6 +10,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/alist-org/alist/v3/internal/stream"
+
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/pkg/task"
@@ -100,7 +101,7 @@ func (m *Monitor) Update() (bool, error) {
 		downloaded = 0
 	}
 	progress := float64(downloaded) / float64(total) * 100
-	m.tsk.SetProgress(int(progress))
+	m.tsk.SetProgress(progress)
 	switch info.Status {
 	case "complete":
 		err := m.Complete()

diff --git a/internal/bootstrap/aria2.go b/internal/bootstrap/aria2.go
diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go
@@ -4,6 +4,7 @@ import (
 	"github.com/alist-org/alist/v3/cmd/flags"
 	"github.com/alist-org/alist/v3/internal/conf"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/offline_download/tool"
 	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/pkg/utils"
 	"github.com/alist-org/alist/v3/pkg/utils/random"
@@ -142,10 +143,6 @@ func InitialSettings() []model.SettingItem {
 		{Key: conf.IgnoreDirectLinkParams, Value: "sign,alist_ts", Type: conf.TypeString, Group: model.GLOBAL},
 		{Key: conf.WebauthnLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC},
 
-		// aria2 settings
-		{Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
-		{Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
-
 		// single settings
 		{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
 		{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,none", Group: model.INDEX},
@@ -168,11 +165,8 @@ func InitialSettings() []model.SettingItem {
 		{Key: conf.SSODefaultDir, Value: "/", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE},
 		{Key: conf.SSODefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.SSO, Flag: model.PRIVATE},
 		{Key: conf.SSOCompatibilityMode, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PUBLIC},
-
-		// qbittorrent settings
-		{Key: conf.QbittorrentUrl, Value: "http://admin:adminadmin@localhost:8080/", Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
-		{Key: conf.QbittorrentSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE},
 	}
+	initialSettingItems = append(initialSettingItems, tool.Tools.Items()...)
 	if flags.Dev {
 		initialSettingItems = append(initialSettingItems, []model.SettingItem{
 			{Key: "test_deprecated", Value: "test_value", Type: conf.TypeString, Flag: model.DEPRECATED},

diff --git a/internal/bootstrap/offline_download.go b/internal/bootstrap/offline_download.go
@@ -0,0 +1,17 @@
+package bootstrap
+
+import (
+	"github.com/alist-org/alist/v3/internal/offline_download/tool"
+	"github.com/alist-org/alist/v3/pkg/utils"
+)
+
+func InitOfflineDownloadTools() {
+	for k, v := range tool.Tools {
+		res, err := v.Init()
+		if err != nil {
+			utils.Log.Warnf("init tool %s failed: %s", k, err)
+		} else {
+			utils.Log.Infof("init tool %s success: %s", k, res)
+		}
+	}
+}
diff --git a/internal/bootstrap/qbittorrent.go b/internal/bootstrap/qbittorrent.go
diff --git a/internal/driver/driver.go b/internal/driver/driver.go
@@ -109,7 +109,7 @@ type PutResult interface {
 	Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error)
 }
 
-type UpdateProgress func(percentage int)
+type UpdateProgress func(percentage float64)
 
 type Progress struct {
 	Total int64
@@ -120,7 +120,7 @@ type Progress struct {
 func (p *Progress) Write(b []byte) (n int, err error) {
 	n = len(b)
 	p.Done += int64(n)
-	p.up(int(float64(p.Done) / float64(p.Total) * 100))
+	p.up(float64(p.Done) / float64(p.Total) * 100)
 	return
 }
 

diff --git a/internal/model/setting.go b/internal/model/setting.go
@@ -6,7 +6,7 @@ const (
 	STYLE
 	PREVIEW
 	GLOBAL
-	ARIA2
+	OFFLINE_DOWNLOAD
 	INDEX
 	SSO
 )