-
Notifications
You must be signed in to change notification settings - Fork 42
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- 增加多线程分片下载文件 - 增加下载文件但是进程退出后，重新下载恢复之前的进度
- Loading branch information
Showing
8 changed files
with
182 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| package partial | ||
|
|
||
| type Chunk struct { | ||
| Start int64 | ||
| End int64 | ||
| Error error | ||
| Buffer []byte | ||
| } | ||
|
|
||
| func cut(chunks []*Chunk, i int) ([]*Chunk, []*Chunk) { | ||
| if len(chunks) <= i { | ||
| return chunks, []*Chunk{} | ||
| } | ||
|
|
||
| return chunks[0:i], chunks[i:] | ||
| } | ||
|
|
||
| func (p *MultiPartialDownloader) retryDownloadChunk(start, end int64) ([]byte, error) { | ||
| var err error | ||
| var buffer []byte | ||
|
|
||
| // 重试三次 | ||
| for t := 0; t < 3; t++ { | ||
| buffer, err = p.DownFunc(p, start, end-1) | ||
| if err == nil { | ||
| break | ||
| } | ||
| } | ||
| return buffer, err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| package partial | ||
|
|
||
| import ( | ||
| "errors" | ||
| "os" | ||
| "sync" | ||
| ) | ||
|
|
||
| func (p *MultiPartialDownloader) Download() error { | ||
| fileinfo, err := os.Stat(p.FilePath) | ||
| if err == nil { | ||
| p.LocalSize = fileinfo.Size() | ||
| } | ||
|
|
||
| // 计算需要下载的块数 | ||
| needDownSize := p.FinalSize - p.LocalSize | ||
| chunkCount := needDownSize / ChunkSize | ||
| if needDownSize%ChunkSize != 0 { | ||
| chunkCount++ | ||
| } | ||
|
|
||
| // 下载任务队列 | ||
| var queue []*Chunk | ||
| for i := 0; i < int(chunkCount); i++ { | ||
| start := p.LocalSize + int64(i)*ChunkSize | ||
| end := p.LocalSize + int64(i+1)*ChunkSize | ||
| if end > p.FinalSize { | ||
| end = p.FinalSize | ||
| } | ||
| queue = append(queue, &Chunk{ | ||
| Start: start, | ||
| End: end, | ||
| }) | ||
| } | ||
|
|
||
| for { | ||
| var chunks []*Chunk | ||
| chunks, queue = cut(queue, p.Works) | ||
| if len(chunks) <= 0 { | ||
| break | ||
| } | ||
|
|
||
| var wg sync.WaitGroup | ||
| for _, chunk := range chunks { | ||
| wg.Add(1) | ||
| go func(c *Chunk) { | ||
| defer wg.Done() | ||
| buffer, err := p.retryDownloadChunk(c.Start, c.End) | ||
| c.Buffer = buffer | ||
| c.Error = err | ||
| }(chunk) | ||
| } | ||
|
|
||
| wg.Wait() | ||
|
|
||
| for _, chunk := range chunks { | ||
| if chunk.Error != nil { | ||
| return chunk.Error | ||
| } | ||
| if len(chunk.Buffer) == 0 { | ||
| return errors.New("chunk buffer download but size is 0") | ||
| } | ||
| p.Writer.Write(chunk.Buffer) | ||
| } | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package partial | ||
|
|
||
| import ( | ||
| "io" | ||
| ) | ||
|
|
||
| const ChunkSize = 1024 * 1024 * 10 | ||
|
|
||
| type MultiPartialDownloader struct { | ||
| FilePath string | ||
| FinalSize int64 | ||
| LocalSize int64 | ||
| Writer io.Writer | ||
| Works int | ||
| DownFunc ChunkDownFunc | ||
| } | ||
|
|
||
| type ChunkDownFunc func(file *MultiPartialDownloader, start, end int64) ([]byte, error) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters