-
Notifications
You must be signed in to change notification settings - Fork 0
/
ranget.go
113 lines (91 loc) · 2.8 KB
/
ranget.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
package main
import (
"net/http"
"github.com/cognusion/go-humanity"
"github.com/cognusion/go-rangetripper"
"github.com/cheggaaa/pb/v3"
"github.com/spf13/pflag"
"fmt"
"io"
"log"
"os"
"time"
)
func main() {
var (
urlString string
outFile string
chunkString string
chunkSize int64
max int
debug bool
trash bool
timeout time.Duration
progressChan <-chan int64
doneChan = make(chan interface{})
bar *pb.ProgressBar
barTmpl = `{{ counters . }} {{ bar . }} {{ percent . }} {{ rtime . }} {{ speed . "%s/s"}}`
timingOut = log.New(io.Discard, "[TIMING]", 0)
debugOut = log.New(io.Discard, "[DEBUG] ", 0)
)
pflag.StringVar(&urlString, "url", "", "What to fetch")
pflag.StringVar(&outFile, "out", "./afile", "Where to write it it")
pflag.StringVar(&chunkString, "size", "10MB", "Size of chunks to download (whole-numbers with suffixes of B,KB,MB,GB,PB)")
pflag.IntVar(&max, "max", 10, "Maximum number of simultaneous downloaders")
pflag.BoolVar(&debug, "debug", false, "Enable debugging output (disables progress bar)")
pflag.BoolVar(&trash, "trash", false, "Delete the file after downloading (e.g. if benchmarking, etc.)")
pflag.DurationVar(&timeout, "timeout", 1*time.Minute, "Set a general timeout for the download")
pflag.Parse()
if urlString == "" {
fmt.Println("Please at least set --url")
pflag.Usage()
os.Exit(1)
}
var cerr error
chunkSize, cerr = humanity.StringAsBytes(chunkString)
if cerr != nil {
fmt.Printf("Please use wholenumbers with suffixes of B,KB,MB,GB,PB")
pflag.Usage()
os.Exit(1)
}
if debug {
timingOut = log.New(os.Stdout, "[TIMING] ", 0)
debugOut = log.New(os.Stdout, "[DEBUG] ", 0)
}
client := new(http.Client) // make a new Client
rtclient := rangetripper.NewRetryClientWithExponentialBackoff(10, 1*time.Second, timeout) // make a new Client
client.Timeout = timeout
rt, nerr := rangetripper.NewWithLoggers(10, outFile, timingOut, debugOut)
if nerr != nil {
panic(nerr)
}
rt.SetClient(rtclient)
rt.SetChunkSize(chunkSize)
rt.SetMax(max)
if trash {
defer os.Remove(outFile) // clean up after ourselves
}
if !debug {
progressChan = rt.WithProgress()
defer close(doneChan)
go func(done chan interface{}, progress <-chan int64) {
contentLength := <-progress // first item is the contentLength
bar = pb.ProgressBarTemplate(barTmpl).Start64(contentLength)
bar.Set(pb.Bytes, true)
defer bar.Finish()
for {
select {
case <-done:
return
case b := <-progress:
bar.Add64(b)
}
}
}(doneChan, progressChan)
}
client.Transport = rt // Use the RangeTripper as the Transport
debugOut.Printf("GETting %s!\n", urlString)
if _, err := client.Get(urlString); err != nil {
panic(err)
}
}