From 59a664cc8101f1cb5ab74b270f104ef196ce420a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Ledesma?= Date: Sun, 15 Oct 2017 01:18:45 +0200 Subject: [PATCH 1/3] Refactor pluto package to make it usable as a library and to simplify main.go --- cli.go | 200 ------------------------------------------------- main.go | 149 ++++++++++++++++++++++++++++++++++++ pluto/pluto.go | 186 +++++++++++++++++++++++++++++++-------------- 3 files changed, 278 insertions(+), 257 deletions(-) delete mode 100644 cli.go create mode 100644 main.go diff --git a/cli.go b/cli.go deleted file mode 100644 index 0512b0d..0000000 --- a/cli.go +++ /dev/null @@ -1,200 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "io" - "log" - "net/url" - "os" - "os/signal" - "path/filepath" - "strings" - "syscall" - "time" - - "github.com/dustin/go-humanize" - flag "github.com/jessevdk/go-flags" - - "github.com/ishanjain28/pluto/pluto" -) - -var Version string -var Build string - -var options struct { - Verbose bool `long:"verbose" description:"Enable Verbose Mode"` - - Connections uint `short:"n" long:"connections" description:"Number of concurrent connections"` - - Name string `long:"name" description:"Path or Name of save file"` - - LoadFromFile string `short:"f" long:"load-from-file" description:"Load URLs from a file"` - - Headers []string `short:"H" long:"Headers" description:"Headers to send with each request. Useful if a server requires some information in headers"` - - Version bool `short:"v" long:"version" description:"Print Pluto Version and exit"` -} - -func main() { - - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) - - go func() { - <-sig - fmt.Printf("Interrupt Detected, Shutting Down.") - os.Exit(1) - }() - args, err := flag.ParseArgs(&options, os.Args) - if err != nil { - fmt.Printf("%s", err.Error()) - return - } - - if options.Version { - fmt.Println("Pluto - A Fast Multipart File Downloader") - fmt.Printf("Version: %s\n", Version) - fmt.Printf("Build: %s\n", Build) - return - } - - defer func() { - fmt.Scanf("\n", nil) - }() - args = args[1:] - - urls := []string{} - - if options.LoadFromFile != "" { - f, err := os.OpenFile(options.LoadFromFile, os.O_RDONLY, 0x444) - if err != nil { - log.Fatalf("error in opening file %s: %v\n", options.LoadFromFile, err) - } - defer f.Close() - reader := bufio.NewReader(f) - - for { - str, err := reader.ReadString('\n') - if err != nil { - if err == io.EOF { - break - } - log.Fatalf("error in reading file: %v\n", err) - } - u := str[:len(str)-1] - if u != "" { - urls = append(urls, u) - } - } - - fmt.Printf("Queued %d urls\n", len(urls)) - } else { - for _, v := range args { - if v != "" && v != "\n" { - urls = append(urls, v) - } - } - - } - - for i, v := range urls { - up, err := url.Parse(v) - if err != nil { - log.Printf("Invalid URL: %v", err) - continue - } - - download(up, i) - } -} - -func download(up *url.URL, num int) { - - dlFinished := make(chan bool, 1) - - fname := strings.Split(filepath.Base(up.String()), "?")[0] - - meta, err := pluto.FetchMeta(up, options.Headers) - if err != nil { - log.Println(err) - return - } - - if options.Name != "" && num == 0 { - meta.Name = options.Name - } - - if meta.Name == "" { - meta.Name = fname - } - - if meta.MultipartSupported || options.Connections == 0 { - if options.Connections == 0 { - options.Connections = 1 - } - fmt.Printf("Downloading %s(%s) with %d connection\n", meta.Name, humanize.Bytes(meta.Size), options.Connections) - } else { - fmt.Printf("Downloading %s(%s) with 1 connection(Multipart downloads not supported)\n", meta.Name, humanize.Bytes(meta.Size)) - } - - saveFile, err := os.Create(strings.Replace(meta.Name, "/", "\\/", -1)) - if err != nil { - log.Printf("error in creating file: %v", err) - return - } - - config := &pluto.Config{ - Meta: meta, - Connections: options.Connections, - Headers: options.Headers, - Verbose: options.Verbose, - Writer: saveFile, - StatsChan: make(chan *pluto.Stats), - } - - startTime := time.Now() - - go func(dled chan bool) { - if config.StatsChan == nil { - return - } - - for { - select { - case <-dled: - break - case v := <-config.StatsChan: - os.Stdout.WriteString(fmt.Sprintf("%.2f%% - %s/%s - %s/s \r", float64(v.Downloaded)/float64(meta.Size)*100, humanize.IBytes(v.Downloaded), humanize.IBytes(meta.Size), humanize.IBytes(v.Speed))) - os.Stdout.Sync() - } - - } - - }(dlFinished) - - err = pluto.Download(config) - dlFinished <- true - if err != nil { - log.Println(err) - return - } - - timeTaken := time.Since(startTime) - p, err := filepath.Abs(meta.Name) - if err != nil { - fmt.Printf("\nFile saved in %s\n", meta.Name) - } - - s := humanize.IBytes(meta.Size) - htime := timeTaken.String() - ts := timeTaken.Seconds() - if ts == 0 { - ts = 1 - } - as := humanize.IBytes(uint64(float64(meta.Size) / float64(ts))) - - fmt.Printf("Downloaded %s in %s. Avg. Speed - %s/s\n", s, htime, as) - fmt.Printf("File saved in %s\n", p) - -} diff --git a/main.go b/main.go new file mode 100644 index 0000000..3445578 --- /dev/null +++ b/main.go @@ -0,0 +1,149 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "log" + "net/url" + "os" + "os/signal" + "syscall" + + humanize "github.com/dustin/go-humanize" + flag "github.com/jessevdk/go-flags" + + "github.com/ishanjain28/pluto/pluto" +) + +var Version string +var Build string + +var options struct { + Verbose bool `long:"verbose" description:"Enable Verbose Mode"` + + Connections uint `short:"n" long:"connections" default:"1" description:"Number of concurrent connections"` + + Name string `long:"name" description:"Path or Name of save file"` + + LoadFromFile string `short:"f" long:"load-from-file" description:"Load URLs from a file"` + + Headers []string `short:"H" long:"Headers" description:"Headers to send with each request. Useful if a server requires some information in headers"` + + Version bool `short:"v" long:"version" description:"Print Pluto Version and exit"` +} + +func main() { + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-sig + fmt.Printf("Interrupt Detected, Shutting Down.") + os.Exit(1) + }() + + args, err := flag.ParseArgs(&options, os.Args) + if err != nil { + fmt.Printf("%s", err.Error()) + return + } + + if options.Version { + fmt.Println("Pluto - A Fast Multipart File Downloader") + fmt.Printf("Version: %s\n", Version) + fmt.Printf("Build: %s\n", Build) + return + } + + args = args[1:] + + urls := []string{} + + if options.LoadFromFile != "" { + f, err := os.OpenFile(options.LoadFromFile, os.O_RDONLY, 0x444) + if err != nil { + log.Fatalf("error in opening file %s: %v\n", options.LoadFromFile, err) + } + defer f.Close() + reader := bufio.NewReader(f) + + for { + str, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + log.Fatalf("error in reading file: %v\n", err) + } + u := str[:len(str)-1] + if u != "" { + urls = append(urls, u) + } + } + + fmt.Printf("Queued %d urls\n", len(urls)) + } else { + for _, v := range args { + if v != "" && v != "\n" { + urls = append(urls, v) + } + } + + } + if len(urls) == 0 { + log.Fatalf("nothing to do. Please pass some url to fetch") + } + + if options.Connections == 0 { + log.Fatalf("Connections should be > 0") + } + if len(urls) > 1 && options.Name != "" { + log.Fatalf("it is not possible to specify 'name' with more than one url") + } + + for _, v := range urls { + up, err := url.Parse(v) + if err != nil { + log.Printf("Invalid URL: %v", err) + continue + } + + p, err := pluto.New(up, options.Headers, options.Name, options.Connections, options.Verbose) + if err != nil { + log.Printf("error creating pluto instance for url %s: %v", v, err) + } + go func() { + if p.StatsChan == nil { + return + } + + for { + select { + case <-p.Finished: + break + case v := <-p.StatsChan: + os.Stdout.WriteString(fmt.Sprintf("%.2f%% - %s/%s - %s/s \r", float64(v.Downloaded)/float64(v.Size)*100, humanize.IBytes(v.Downloaded), humanize.IBytes(v.Size), humanize.IBytes(v.Speed))) + os.Stdout.Sync() + } + + } + + }() + result, err := p.Download() + if err != nil { + log.Printf("error downloading url %s: %v", v, err) + } else { + s := humanize.IBytes(result.Size) + htime := result.TimeTaken.String() + + as := humanize.IBytes(uint64(result.AvgSpeed)) + + fmt.Printf("Downloaded %s in %s. Avg. Speed - %s/s\n", s, htime, as) + fmt.Printf("File saved in %s\n", result.FileName) + + } + + } +} diff --git a/pluto/pluto.go b/pluto/pluto.go index 4c44210..03381bf 100644 --- a/pluto/pluto.go +++ b/pluto/pluto.go @@ -7,10 +7,15 @@ import ( "log" "net/http" "net/url" + "os" + "path/filepath" "runtime" "strings" "sync" "sync/atomic" + + humanize "github.com/dustin/go-humanize" + "time" ) @@ -26,74 +31,137 @@ type worker struct { url *url.URL } -// FileMeta contains information about the file like it's Size, Name and if the server supports multipart downloads -type FileMeta struct { - u *url.URL - Size uint64 - Name string - MultipartSupported bool -} - // Stats is returned in a channel by Download function every 250ms and contains details like Current download speed in bytes/sec and amount of data Downloaded type Stats struct { Downloaded uint64 Speed uint64 + Size uint64 +} + +type fileMetaData struct { + url *url.URL + Size uint64 + Name string + MultipartSupported bool } -// Config contains all the details that Download needs. +// Pluto contains all the details that Download needs. // Connections is the number of connections to use to download a file // Verbose is to enable verbose mode. // Writer is the place where downloaded data is written. // Headers is any header that you may need to send to download the file. // StatsChan is a channel to which Stats are sent, It can be nil or a channel that can hold data of type *() -type Config struct { - Connections uint - Verbose bool - Headers []string - Writer io.WriterAt - Meta *FileMeta +type Pluto struct { StatsChan chan *Stats + Finished chan struct{} + connections uint + verbose bool + headers []string downloaded uint64 + metaData fileMetaData + startTime time.Time + writer io.WriterAt +} + +//Result is the download results +type Result struct { + FileName string + Size uint64 + AvgSpeed float64 + TimeTaken time.Duration +} + +//New returns a pluto instance +func New(up *url.URL, headers []string, name string, connections uint, verbose bool) (*Pluto, error) { + + p := &Pluto{ + connections: connections, + headers: headers, + verbose: verbose, + StatsChan: make(chan *Stats), + Finished: make(chan struct{}), + } + + err := p.fetchMeta(up, headers) + if err != nil { + return nil, err + } + + if name != "" { + p.metaData.Name = name + } else if p.metaData.Name == "" { + p.metaData.Name = strings.Split(filepath.Base(up.String()), "?")[0] + } + + if p.metaData.MultipartSupported { + fmt.Printf("Downloading %s(%s) with %d connection\n", p.metaData.Name, humanize.Bytes(p.metaData.Size), p.connections) + } else { + fmt.Printf("Downloading %s(%s) with 1 connection(Multipart downloads not supported)\n", p.metaData.Name, humanize.Bytes(p.metaData.Size)) + p.connections = 1 + } + + p.writer, err = os.Create(strings.Replace(p.metaData.Name, "/", "\\/", -1)) + if err != nil { + return nil, fmt.Errorf("error creating file %s: %v", p.metaData.Name, err) + } + + return p, nil + } // Download takes Config struct // then downloads the file by dividing it into given number of parts and downloading all parts concurrently. // If any error occurs in the downloading stage of any part, It'll check if the the program can recover from error by retrying download // And if an error occurs which the program can not recover from, it'll return that error -func Download(c *Config) error { - +func (p *Pluto) Download() (*Result, error) { + p.startTime = time.Now() // Limit number of CPUs it can use runtime.GOMAXPROCS(runtime.NumCPU() / 2) - // If server does not supports multiple connections, Set it to 1 - if !c.Meta.MultipartSupported { - c.Connections = 1 - } - perPartLimit := c.Meta.Size / uint64(c.Connections) - difference := c.Meta.Size % uint64(c.Connections) + perPartLimit := p.metaData.Size / uint64(p.connections) + difference := p.metaData.Size % uint64(p.connections) - workers := make([]*worker, c.Connections) + workers := make([]*worker, p.connections) - var i uint - for i = 0; i < c.Connections; i++ { + for i := uint(0); i < p.connections; i++ { begin := perPartLimit * uint64(i) end := perPartLimit * (uint64(i) + 1) - if i == c.Connections-1 { + if i == p.connections-1 { end += difference } workers[i] = &worker{ begin: begin, end: end, - url: c.Meta.u, + url: p.metaData.url, } } - return startDownload(workers, *c) + err := p.startDownload(workers) + if err != nil { + return nil, err + } + + tt := time.Since(p.startTime) + filename, err := filepath.Abs(p.metaData.Name) + if err != nil { + log.Printf("unable to get absolute path for %s: %v", p.metaData.Name, err) + filename = p.metaData.Name + } + + r := &Result{ + TimeTaken: tt, + FileName: filename, + Size: p.metaData.Size, + AvgSpeed: float64(p.metaData.Size) / float64(tt.Seconds()), + } + + close(p.Finished) + return r, nil } -func startDownload(w []*worker, c Config) error { +func (p *Pluto) startDownload(w []*worker) error { var wg sync.WaitGroup wg.Add(len(w)) @@ -105,14 +173,14 @@ func startDownload(w []*worker, c Config) error { var downloaded uint64 // Stats system, It writes stats to the stats channel - go func(c *Config) { + go func() { var oldSpeed uint64 counter := 0 for { dled := atomic.LoadUint64(&downloaded) - speed := dled - c.downloaded + speed := dled - p.downloaded if speed == 0 && counter < 4 { speed = oldSpeed @@ -121,16 +189,17 @@ func startDownload(w []*worker, c Config) error { counter = 0 } - c.StatsChan <- &Stats{ - Downloaded: c.downloaded, + p.StatsChan <- &Stats{ + Downloaded: p.downloaded, Speed: speed * 2, + Size: p.metaData.Size, } - c.downloaded = dled + p.downloaded = dled oldSpeed = speed time.Sleep(500 * time.Millisecond) } - }(&c) + }() for _, q := range w { // This loop keeps trying to download a file if a recoverable error occurs @@ -146,29 +215,29 @@ func startDownload(w []*worker, c Config) error { }() for { - downloadPart, err := download(begin, end, &c) + downloadPart, err := p.download(begin, end) if err != nil { if err.Error() == "status code: 400" || err.Error() == "status code: 500" || err.Error() == ErrOverflow { cerr <- err return } - if c.Verbose { + if p.verbose { log.Println(err) } continue } - d, err := copyAt(c.Writer, downloadPart, begin, &downloaded) + d, err := p.copyAt(downloadPart, begin, &downloaded) begin += d if err != nil { - if c.Verbose { + if p.verbose { log.Printf("error in copying data at offset %d: %v", v.begin, err) } continue } - if c.Verbose { + if p.verbose { fmt.Printf("Copied %d bytes\n", d) } @@ -193,7 +262,7 @@ func startDownload(w []*worker, c Config) error { } // copyAt reads 64 kilobytes from source and copies them to destination at a given offset -func copyAt(dst io.WriterAt, src io.Reader, offset uint64, dlcounter *uint64) (uint64, error) { +func (p *Pluto) copyAt(src io.Reader, offset uint64, dlcounter *uint64) (uint64, error) { bufBytes := make([]byte, 256*1024) var bytesWritten uint64 @@ -202,7 +271,7 @@ func copyAt(dst io.WriterAt, src io.Reader, offset uint64, dlcounter *uint64) (u for { nsr, serr := src.Read(bufBytes) if nsr > 0 { - ndw, derr := dst.WriteAt(bufBytes[:nsr], int64(offset)) + ndw, derr := p.writer.WriteAt(bufBytes[:nsr], int64(offset)) if ndw > 0 { u64ndw := uint64(ndw) offset += u64ndw @@ -231,13 +300,11 @@ func copyAt(dst io.WriterAt, src io.Reader, offset uint64, dlcounter *uint64) (u return bytesWritten, err } -// FetchMeta fetches information about the file like it's Size, Name and if it supports Multipart Download -// If a link does not supports multipart downloads, Then the provided value of part is ignored and set to 1 -func FetchMeta(u *url.URL, headers []string) (*FileMeta, error) { +func (p *Pluto) fetchMeta(u *url.URL, headers []string) error { req, err := http.NewRequest("HEAD", u.String(), nil) if err != nil { - return nil, fmt.Errorf("error in creating HEAD request: %v", err) + return fmt.Errorf("error in creating HEAD request: %v", err) } for _, v := range headers { @@ -253,17 +320,17 @@ func FetchMeta(u *url.URL, headers []string) (*FileMeta, error) { resp, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("error in sending HEAD request: %v", err) + return fmt.Errorf("error in sending HEAD request: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - return nil, fmt.Errorf("status code is %d", resp.StatusCode) + return fmt.Errorf("status code is %d", resp.StatusCode) } size := resp.ContentLength if size == 0 { - return nil, fmt.Errorf("Incompatible URL, file size is 0") + return fmt.Errorf("Incompatible URL, file size is 0") } msupported := false @@ -274,7 +341,7 @@ func FetchMeta(u *url.URL, headers []string) (*FileMeta, error) { resp, err = http.Get(u.String()) if err != nil { - return nil, fmt.Errorf("error in sending GET request: %v", err) + return fmt.Errorf("error in sending GET request: %v", err) } name := "" @@ -293,21 +360,26 @@ func FetchMeta(u *url.URL, headers []string) (*FileMeta, error) { } resp.Body.Close() - - return &FileMeta{Size: uint64(size), Name: name, u: u, MultipartSupported: msupported}, nil + p.metaData = fileMetaData{ + Size: uint64(size), + Name: name, + url: u, + MultipartSupported: msupported, + } + return nil } -func download(begin, end uint64, config *Config) (io.ReadCloser, error) { +func (p *Pluto) download(begin, end uint64) (io.ReadCloser, error) { client := &http.Client{} - req, err := http.NewRequest("GET", config.Meta.u.String(), nil) + req, err := http.NewRequest("GET", p.metaData.url.String(), nil) if err != nil { return nil, fmt.Errorf("error in creating GET request: %v", err) } req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", begin, end)) - for _, v := range config.Headers { + for _, v := range p.headers { vsp := strings.Index(v, ":") key := v[:vsp] @@ -319,7 +391,7 @@ func download(begin, end uint64, config *Config) (io.ReadCloser, error) { resp, err := client.Do(req) if err != nil { - if config.Verbose { + if p.verbose { fmt.Printf("Requested Bytes %d in range %d-%d. Got %d bytes\n", end-begin, begin, end, resp.ContentLength) } From adc89d902cad37f1dbbc008eb9aded43a11e05d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Ledesma?= Date: Sun, 15 Oct 2017 21:32:19 +0200 Subject: [PATCH 2/3] split worker from pluto. Add context to allow handling interrupt (although we are not there yet). Lacking test update --- main.go | 121 +++++++++++++++++++++-------------- pluto/pluto.go | 167 ++++++++++-------------------------------------- pluto/worker.go | 99 ++++++++++++++++++++++++++++ 3 files changed, 206 insertions(+), 181 deletions(-) create mode 100644 pluto/worker.go diff --git a/main.go b/main.go index 3445578..b3c327e 100644 --- a/main.go +++ b/main.go @@ -2,69 +2,49 @@ package main import ( "bufio" + "context" "fmt" + "hacktober/pluto/pluto" "io" "log" "net/url" "os" "os/signal" + "path/filepath" + "strings" "syscall" humanize "github.com/dustin/go-humanize" flag "github.com/jessevdk/go-flags" - - "github.com/ishanjain28/pluto/pluto" ) var Version string var Build string var options struct { - Verbose bool `long:"verbose" description:"Enable Verbose Mode"` - - Connections uint `short:"n" long:"connections" default:"1" description:"Number of concurrent connections"` - - Name string `long:"name" description:"Path or Name of save file"` - - LoadFromFile string `short:"f" long:"load-from-file" description:"Load URLs from a file"` - - Headers []string `short:"H" long:"Headers" description:"Headers to send with each request. Useful if a server requires some information in headers"` - - Version bool `short:"v" long:"version" description:"Print Pluto Version and exit"` + Verbose bool `long:"verbose" description:"Enable Verbose Mode"` + Connections uint `short:"n" long:"connections" default:"1" description:"Number of concurrent connections"` + Name string `long:"name" description:"Path or Name of save file"` + LoadFromFile string `short:"f" long:"load-from-file" description:"Load URLs from a file"` + Headers []string `short:"H" long:"Headers" description:"Headers to send with each request. Useful if a server requires some information in headers"` + Version bool `short:"v" long:"version" description:"Print Pluto Version and exit"` + urls []string } -func main() { - - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) - - go func() { - <-sig - fmt.Printf("Interrupt Detected, Shutting Down.") - os.Exit(1) - }() - +func parseArgs() error { args, err := flag.ParseArgs(&options, os.Args) if err != nil { - fmt.Printf("%s", err.Error()) - return - } - - if options.Version { - fmt.Println("Pluto - A Fast Multipart File Downloader") - fmt.Printf("Version: %s\n", Version) - fmt.Printf("Build: %s\n", Build) - return + return fmt.Errorf("error parsing args: %v", err) } args = args[1:] - urls := []string{} + options.urls = []string{} if options.LoadFromFile != "" { f, err := os.OpenFile(options.LoadFromFile, os.O_RDONLY, 0x444) if err != nil { - log.Fatalf("error in opening file %s: %v\n", options.LoadFromFile, err) + return fmt.Errorf("error in opening file %s: %v", options.LoadFromFile, err) } defer f.Close() reader := bufio.NewReader(f) @@ -75,45 +55,70 @@ func main() { if err == io.EOF { break } - log.Fatalf("error in reading file: %v\n", err) + return fmt.Errorf("error in reading file: %v", err) } u := str[:len(str)-1] if u != "" { - urls = append(urls, u) + options.urls = append(options.urls, u) } } - fmt.Printf("Queued %d urls\n", len(urls)) + fmt.Printf("queued %d urls\n", len(options.urls)) } else { for _, v := range args { if v != "" && v != "\n" { - urls = append(urls, v) + options.urls = append(options.urls, v) } } } - if len(urls) == 0 { - log.Fatalf("nothing to do. Please pass some url to fetch") + if len(options.urls) == 0 { + return fmt.Errorf("nothing to do. Please pass some url to fetch") } if options.Connections == 0 { - log.Fatalf("Connections should be > 0") + return fmt.Errorf("connections should be > 0") } - if len(urls) > 1 && options.Name != "" { - log.Fatalf("it is not possible to specify 'name' with more than one url") + if len(options.urls) > 1 && options.Name != "" { + return fmt.Errorf("it is not possible to specify 'name' with more than one url") } + return nil +} +func main() { - for _, v := range urls { + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + ctx, cancel := context.WithCancel(context.Background()) + + go func() { + <-sig + fmt.Printf("Interrupt Detected, Shutting Down.") + cancel() + }() + + err := parseArgs() + if err != nil { + log.Fatalf("error parsing args: %v", err) + } + + if options.Version { + fmt.Println("Pluto - A Fast Multipart File Downloader") + fmt.Printf("Version: %s\n", Version) + fmt.Printf("Build: %s\n", Build) + return + } + + for _, v := range options.urls { up, err := url.Parse(v) if err != nil { log.Printf("Invalid URL: %v", err) continue } - - p, err := pluto.New(up, options.Headers, options.Name, options.Connections, options.Verbose) + p, err := pluto.New(up, options.Headers, options.Connections, options.Verbose) if err != nil { - log.Printf("error creating pluto instance for url %s: %v", v, err) + log.Fatalf("error creating pluto instance for url %s: %v\n", v, err) } + go func() { if p.StatsChan == nil { return @@ -131,7 +136,25 @@ func main() { } }() - result, err := p.Download() + + var fileName string + if options.Name != "" { + fileName = options.Name + } else if p.MetaData.Name == "" { + fileName = strings.Split(filepath.Base(up.String()), "?")[0] + } + fileName = strings.Replace(fileName, "/", "\\/", -1) + writer, err := os.Create(fileName) + if err != nil { + log.Fatalf("unable to create file %s: %v\n", fileName, err) + } + defer writer.Close() + + if !p.MetaData.MultipartSupported && options.Connections > 1 { + fmt.Printf("Downloading %s(%s) with 1 connection(Multipart downloads not supported)\n", fileName, humanize.Bytes(p.MetaData.Size)) + } + + result, err := p.Download(ctx, writer) if err != nil { log.Printf("error downloading url %s: %v", v, err) } else { diff --git a/pluto/pluto.go b/pluto/pluto.go index 03381bf..d931c5c 100644 --- a/pluto/pluto.go +++ b/pluto/pluto.go @@ -2,12 +2,12 @@ package pluto import ( + "context" "fmt" "io" "log" "net/http" "net/url" - "os" "path/filepath" "runtime" "strings" @@ -25,12 +25,6 @@ var ( ErrOverflow = "error: Server sent extra bytes" ) -type worker struct { - begin uint64 - end uint64 - url *url.URL -} - // Stats is returned in a channel by Download function every 250ms and contains details like Current download speed in bytes/sec and amount of data Downloaded type Stats struct { Downloaded uint64 @@ -58,9 +52,9 @@ type Pluto struct { verbose bool headers []string downloaded uint64 - metaData fileMetaData + MetaData fileMetaData startTime time.Time - writer io.WriterAt + workers []*worker } //Result is the download results @@ -72,7 +66,7 @@ type Result struct { } //New returns a pluto instance -func New(up *url.URL, headers []string, name string, connections uint, verbose bool) (*Pluto, error) { +func New(up *url.URL, headers []string, connections uint, verbose bool) (*Pluto, error) { p := &Pluto{ connections: connections, @@ -87,24 +81,13 @@ func New(up *url.URL, headers []string, name string, connections uint, verbose b return nil, err } - if name != "" { - p.metaData.Name = name - } else if p.metaData.Name == "" { - p.metaData.Name = strings.Split(filepath.Base(up.String()), "?")[0] - } - - if p.metaData.MultipartSupported { - fmt.Printf("Downloading %s(%s) with %d connection\n", p.metaData.Name, humanize.Bytes(p.metaData.Size), p.connections) + if !p.MetaData.MultipartSupported { + p.connections = 1 + fmt.Printf("Downloading %s(%s) with %d connection\n", p.MetaData.Name, humanize.Bytes(p.MetaData.Size), p.connections) } else { - fmt.Printf("Downloading %s(%s) with 1 connection(Multipart downloads not supported)\n", p.metaData.Name, humanize.Bytes(p.metaData.Size)) p.connections = 1 } - p.writer, err = os.Create(strings.Replace(p.metaData.Name, "/", "\\/", -1)) - if err != nil { - return nil, fmt.Errorf("error creating file %s: %v", p.metaData.Name, err) - } - return p, nil } @@ -113,15 +96,15 @@ func New(up *url.URL, headers []string, name string, connections uint, verbose b // then downloads the file by dividing it into given number of parts and downloading all parts concurrently. // If any error occurs in the downloading stage of any part, It'll check if the the program can recover from error by retrying download // And if an error occurs which the program can not recover from, it'll return that error -func (p *Pluto) Download() (*Result, error) { +func (p *Pluto) Download(ctx context.Context, w io.WriterAt) (*Result, error) { p.startTime = time.Now() // Limit number of CPUs it can use runtime.GOMAXPROCS(runtime.NumCPU() / 2) - perPartLimit := p.metaData.Size / uint64(p.connections) - difference := p.metaData.Size % uint64(p.connections) + perPartLimit := p.MetaData.Size / uint64(p.connections) + difference := p.MetaData.Size % uint64(p.connections) - workers := make([]*worker, p.connections) + p.workers = make([]*worker, p.connections) for i := uint(0); i < p.connections; i++ { begin := perPartLimit * uint64(i) @@ -131,40 +114,44 @@ func (p *Pluto) Download() (*Result, error) { end += difference } - workers[i] = &worker{ - begin: begin, - end: end, - url: p.metaData.url, + p.workers[i] = &worker{ + begin: begin, + end: end, + url: p.MetaData.url, + writer: w, + headers: p.headers, + verbose: p.verbose, + ctx: ctx, } } - err := p.startDownload(workers) + err := p.startDownload() if err != nil { return nil, err } tt := time.Since(p.startTime) - filename, err := filepath.Abs(p.metaData.Name) + filename, err := filepath.Abs(p.MetaData.Name) if err != nil { - log.Printf("unable to get absolute path for %s: %v", p.metaData.Name, err) - filename = p.metaData.Name + log.Printf("unable to get absolute path for %s: %v", p.MetaData.Name, err) + filename = p.MetaData.Name } r := &Result{ TimeTaken: tt, FileName: filename, - Size: p.metaData.Size, - AvgSpeed: float64(p.metaData.Size) / float64(tt.Seconds()), + Size: p.MetaData.Size, + AvgSpeed: float64(p.MetaData.Size) / float64(tt.Seconds()), } close(p.Finished) return r, nil } -func (p *Pluto) startDownload(w []*worker) error { +func (p *Pluto) startDownload() error { var wg sync.WaitGroup - wg.Add(len(w)) + wg.Add(len(p.workers)) var err error errdl := make(chan error, 1) @@ -192,7 +179,7 @@ func (p *Pluto) startDownload(w []*worker) error { p.StatsChan <- &Stats{ Downloaded: p.downloaded, Speed: speed * 2, - Size: p.metaData.Size, + Size: p.MetaData.Size, } p.downloaded = dled @@ -201,11 +188,9 @@ func (p *Pluto) startDownload(w []*worker) error { } }() - for _, q := range w { + for _, w := range p.workers { // This loop keeps trying to download a file if a recoverable error occurs - go func(v *worker, wgroup *sync.WaitGroup, dl *uint64, cerr, dlerr chan error) { - begin := v.begin - end := v.end + go func(w *worker, wgroup *sync.WaitGroup, dl *uint64, cerr, dlerr chan error) { defer func() { @@ -215,7 +200,7 @@ func (p *Pluto) startDownload(w []*worker) error { }() for { - downloadPart, err := p.download(begin, end) + downloadPart, err := w.download() if err != nil { if err.Error() == "status code: 400" || err.Error() == "status code: 500" || err.Error() == ErrOverflow { cerr <- err @@ -228,13 +213,9 @@ func (p *Pluto) startDownload(w []*worker) error { continue } - d, err := p.copyAt(downloadPart, begin, &downloaded) - begin += d + d, err := w.copyAt(downloadPart, &downloaded) if err != nil { - if p.verbose { - log.Printf("error in copying data at offset %d: %v", v.begin, err) - } - continue + cerr <- fmt.Errorf("error copying data at offset %d: %v", w.begin, err) } if p.verbose { @@ -245,7 +226,7 @@ func (p *Pluto) startDownload(w []*worker) error { break } - }(q, &wg, &downloaded, errcopy, errdl) + }(w, &wg, &downloaded, errcopy, errdl) } err = <-errcopy @@ -261,45 +242,6 @@ func (p *Pluto) startDownload(w []*worker) error { return nil } -// copyAt reads 64 kilobytes from source and copies them to destination at a given offset -func (p *Pluto) copyAt(src io.Reader, offset uint64, dlcounter *uint64) (uint64, error) { - bufBytes := make([]byte, 256*1024) - - var bytesWritten uint64 - var err error - - for { - nsr, serr := src.Read(bufBytes) - if nsr > 0 { - ndw, derr := p.writer.WriteAt(bufBytes[:nsr], int64(offset)) - if ndw > 0 { - u64ndw := uint64(ndw) - offset += u64ndw - bytesWritten += u64ndw - atomic.AddUint64(dlcounter, u64ndw) - } - if derr != nil { - err = derr - break - } - if nsr != ndw { - fmt.Printf("Short write error. Read: %d, Wrote: %d", nsr, ndw) - err = io.ErrShortWrite - break - } - } - - if serr != nil { - if serr != io.EOF { - err = serr - } - break - } - } - - return bytesWritten, err -} - func (p *Pluto) fetchMeta(u *url.URL, headers []string) error { req, err := http.NewRequest("HEAD", u.String(), nil) @@ -360,7 +302,7 @@ func (p *Pluto) fetchMeta(u *url.URL, headers []string) error { } resp.Body.Close() - p.metaData = fileMetaData{ + p.MetaData = fileMetaData{ Size: uint64(size), Name: name, url: u, @@ -368,42 +310,3 @@ func (p *Pluto) fetchMeta(u *url.URL, headers []string) error { } return nil } - -func (p *Pluto) download(begin, end uint64) (io.ReadCloser, error) { - - client := &http.Client{} - req, err := http.NewRequest("GET", p.metaData.url.String(), nil) - if err != nil { - return nil, fmt.Errorf("error in creating GET request: %v", err) - } - - req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", begin, end)) - - for _, v := range p.headers { - vsp := strings.Index(v, ":") - - key := v[:vsp] - value := v[vsp:] - - req.Header.Set(key, value) - } - - resp, err := client.Do(req) - if err != nil { - - if p.verbose { - fmt.Printf("Requested Bytes %d in range %d-%d. Got %d bytes\n", end-begin, begin, end, resp.ContentLength) - } - - return nil, fmt.Errorf("error in sending download request: %v", err) - } - - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - return nil, fmt.Errorf("status code: %d", resp.StatusCode) - } - - if uint64(resp.ContentLength) != (end - begin) { - return nil, fmt.Errorf(ErrOverflow) - } - return resp.Body, nil -} diff --git a/pluto/worker.go b/pluto/worker.go new file mode 100644 index 0000000..d1764d0 --- /dev/null +++ b/pluto/worker.go @@ -0,0 +1,99 @@ +package pluto + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync/atomic" +) + +type worker struct { + begin uint64 + end uint64 + url *url.URL + writer io.WriterAt + headers []string + verbose bool + ctx context.Context +} + +// copyAt reads 64 kilobytes from source and copies them to destination at a given offset +func (w *worker) copyAt(src io.Reader, dlcounter *uint64) (uint64, error) { + bufBytes := make([]byte, 256*1024) + + var bytesWritten uint64 + var err error + + for { + nsr, serr := src.Read(bufBytes) + if nsr > 0 { + ndw, derr := w.writer.WriteAt(bufBytes[:nsr], int64(w.begin)) + if ndw > 0 { + u64ndw := uint64(ndw) + w.begin += u64ndw + bytesWritten += u64ndw + atomic.AddUint64(dlcounter, u64ndw) + } + if derr != nil { + err = derr + break + } + if nsr != ndw { + fmt.Printf("Short write error. Read: %d, Wrote: %d", nsr, ndw) + err = io.ErrShortWrite + break + } + } + + if serr != nil { + if serr != io.EOF { + err = serr + } + break + } + } + + return bytesWritten, err +} + +func (w *worker) download() (io.ReadCloser, error) { + + client := &http.Client{} + req, err := http.NewRequest("GET", w.url.String(), nil) + if err != nil { + return nil, fmt.Errorf("error in creating GET request: %v", err) + } + req = req.WithContext(w.ctx) + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", w.begin, w.end)) + + for _, v := range w.headers { + vsp := strings.Index(v, ":") + + key := v[:vsp] + value := v[vsp:] + + req.Header.Set(key, value) + } + + resp, err := client.Do(req) + if err != nil { + + if w.verbose { + fmt.Printf("Requested Bytes %d in range %d-%d. Got %d bytes\n", w.end-w.begin, w.begin, w.end, resp.ContentLength) + } + + return nil, fmt.Errorf("error in sending download request: %v", err) + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + return nil, fmt.Errorf("status code: %d", resp.StatusCode) + } + + if uint64(resp.ContentLength) != (w.end - w.begin) { + return nil, fmt.Errorf(ErrOverflow) + } + return resp.Body, nil +} From f186b32e869aa96248260c0df7a430caf974404d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Ledesma?= Date: Tue, 31 Oct 2017 09:21:11 +0100 Subject: [PATCH 3/3] update tests --- main.go | 3 +- pluto/fixtures/testfile | Bin 0 -> 16384 bytes pluto/pluto_test.go | 76 ++++++++++++++++++++++++++++++++-------- 3 files changed, 63 insertions(+), 16 deletions(-) create mode 100644 pluto/fixtures/testfile diff --git a/main.go b/main.go index b3c327e..48cce82 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ import ( "bufio" "context" "fmt" - "hacktober/pluto/pluto" "io" "log" "net/url" @@ -14,6 +13,8 @@ import ( "strings" "syscall" + "github.com/ishanjain28/pluto/pluto" + humanize "github.com/dustin/go-humanize" flag "github.com/jessevdk/go-flags" ) diff --git a/pluto/fixtures/testfile b/pluto/fixtures/testfile new file mode 100644 index 0000000000000000000000000000000000000000..d9cff3b82325acf000d6c42da511d7bd964cf79c GIT binary patch literal 16384 zcmV+bK>xo`C@}Ug^1L-j8eeTHR3KOhlL8%StHHHr-*P(jtS$s?Sgem!)3kIiGL4{Z}rh1``-Fxz}K|^YmD6#gqXABl`qi`nL1=96tq}gE5tiHq!x~8 zuUX6qK+-TR{}E5l4HRnKjh4~^jj{NIkyy zCSXCE*DMwm;#=|3nFdRx6i_Az$qLkMUEBQ71n}Y^&cccuBDBBh4n{oTz|@zfr{-tV za!=!dCMn|+$+p6y`CqGJ8(;na8fJE@G~@OTqJ(Q1O5LLm+fzvtn6Yk!DKd|ug3|VEMxj*s0i|nE&5l08edSL+wn^-?9h2+MUUz&kJuw8m zK74rcfaPD*Txtd2N@%TKy-){3(8hCjYbZMTkDE57O%b8hqGzdqxa0p%OGA8F8IB;9 z2PZT}4vY3~{(A3I;Vh{ZRC!mb`qrBdc{DaFmo668{~ftp0uLAl5DW?^e=%v04vLsYb8@@q`c;_P3YA%<*XcN0EvFr&w|KA!( z*`-R!Wo~y5IPDb52c#s;>$CPn-8cbbW!;C?Get!YR5L3te0<* zXot#8HW zx+xY!Q>Jgt>!=4N|LW;Ni(3`Qw(Quefrz5j*!w0Q5~WI;1yY&D%E|J6{_k@2g`H|~ zTQpNELeJ_W+Ce0w5U&{b(S~Y%UOTSC{b!W@7i9dZe6)<;K}PH{dfI7Cm^pa;t*^;#jwIb48>s)&9ST6e!Xcw5`AS4G&^W^HA-T(=Ta8tZSe0%%L`ej=q;IK_H=cM@b4r5>BcxHmE&zieGKO z8fy|Lc=kw_yq74_ z+j|T81)6W!V@`#UfT^(EOpFGXay3!xfXM2<%&>Y0N76-uPARIrK6qq?P^xr<(h}G) zgc)*dx3$}pI~6T%a0#%{59}7WySLQA^L|&x5G8_H5IW`N%(yy>DAPpHEE1&wlTiiM zn&?#+(=Zv8T=aqQ1`1=Edn5B$Cd~E)yDtFkS&5;37-twJ+qFq@02GTt)F!YbN&!M- z4WQ=^RW|$?L&0`Qx z6Q2|FxR3thO6D*DY8v(h1wCGC;#fVSUuZ(7K)FF3003Na_@L1~ynn}?{A=ph4r+rY zP}_o(Kn9Eeg?!e457UTng+5wc8(EK+8i*y_*x_BI1y2Q$u>_^ZGQ}9peyF_TbVh;G z+tV&P2{VH4@ZF!lt~9oI#G)P-i6-Zq7(u3VBWcOOEm2b3a`ImbZD%qB+goi zDuK6o-0TJ>u?o$>I?D%X?A!X#fSaJk5fdrK?q;M0EVS*HxlG!?v`)nSNz|k}D$)?) z3!)!ub9g=CzoPUPZt-@MOQ63oeSMWR*;-AaE7@V{#$VL9zQ*EHqHVaZLIB+qQNb1bz}s1fxvY{VpLyX z`-<}6Av?IJefIchGG(Qgs{712&@??gVJmMCo)6f$V_yC=Vz?xh?EGOj%EXpcgY-_^ zTbnZ%5{Eu$8$16v4Ed9;P4@YvH599Ec|2vo#go3>&U%E{-TOtS^o(93+Xv>&N$Kq1 zG`YxsSPI=ACLN<5!TI`z)CygO=n>f;L?_s*3Nj#zw;q|q4$bd$LG;|{rZdTOyHO%Z zGsozZ$y$|H)+J#i+(Fn1ez;GGX|9+e>M3I01l$s(idh|&v(5FRbL6mIaZ^2QDljga zr*=(bn>yC@F&#sRlHMYoUYiVjM9Q$-+DQG_WlB68)!Up=R*_}eV&Th4o-;kw8rZBw zEKRYdur(;PwDyY;Y-Pb+w^hoqPBS^)j|BA4&yxxa1iQ5ijFRjh5PgYmWwTIODR9+1CPE_9>Uo8uz4Rz2>xOrxS zypQQ3KJH47STX;9p1qhSBMRfyJICeFtBYwta*{u;Un3t}TK3YmJ3`ASP@mB!Tm|xq zMmH<2SC`V$crGA{K>}Vh+P#2 zsG*O+AoFUnG=}PfW>P!4UP%NZMfJEk(n;`I0gdISTg*WY3Nj-tLV*honxjbC3vJdH ztA&0M(rdZTZD87J*PuP0!-lJ4UoANWnHSXqp_3B-Lhq}0$ zj)!zK#zgpGlwzF(#S%q*d2m6Pc~su<;sPBN<<(P{6_g{MGXm}<1p@Ml`YuDs+Kh%`oyTMkwY!awC7xIHh!F@ejN}9(q5qN}z)GoLzXc2y)#xy} z34N5$*L8=1OYdN1>)_kS5`Pf#fjMtv?kcTmieY~8b_lM&&DR}e3MIyj;pLv~R7D+- zXNW>MabeD5$G<~^$}MQW(oi5g8Q$W8!?=E;)lrsC-z$v-GF?y-Bqe-bg+mKbz0JJ4 z{&8bLN%D|lZ5rST+YYZ^Ch~J1XLD|LwAL!tmyon8w{460GelQUl+h{A9dm*d1QV^} zMu&8dOp>yRn+D=6v!>j$y#$ zl0)|Z6Mk=m*-L#v>C$ss$*6Kg_;=p4EcMqv`^`};qR%l*0GawP(wb(R6?J!|5Jvse%%WmeQB>B)1>LvsL^1c?N*&-I}9x99f9v4tRb^P(6T3Qwfsv zrOaro*7--Um$F2$MnaYIjU)xG30Enh*ab^AIP3*SI+1$2v?TGB5nK?@;Zx(TXqG$= z0n#2sd*q{^kP}L851*`+vz;D*ESL|yhn?^1JG|((+kb-&_$>A*?R+wdM~j*o-DPIv zlQ=L9W@3h;!>ch>L2pkF|Im>)mS$oFTP+8D>Z~U&D7_MDdl5=#ip`;rX`e3LE_1jN z^!)glejNRg*usc}jtqyje~wqpU9T#g&Yi3P)}G3Mowwx?q2`A3zpze($;!3NTnL+r0-Xp<2C-~#Fm)d{CQF>=qcY%0dHOm8yKyxQ*0>Q!`PwCSnC5iD!2cH>CSj%1_Gi|JC+FH5V}YTb<*s zH~C-5`ob02I3wMTK8*W++bQ@)CV49SpWvEqd@9?t8;G`^3_}l85pxsooPGA{co$2I z$4@%W*Li_30bP`x&UedmnasL9mN|TuUH5%$>0_6Yv6hAe|No1`mK0IlBE)? zJp9yV5iFr$J!?{q{SX6#ejcSl6(eTt_hK|`q`!U*4=vdDGRm#5B%Or|=7%UsXZ9(l z-WvtTfDZr|I)IoXi4-!(-1&VR`WdUeD=E(?XmJTma+F8Fx*Ofyg%-TnD%Sg;Pm z`LmFlX>!_{At!A*CR!qQHT7q(TvKk{BU-$86t4isG(9$cAM z-tpb158&1k3C?^ zO6-~Bqf*=u-yOYPop?!2Jyr?%Cq}-*)J?TxLRY4aSdZ;f-Uk@BD#8UN5Oar>^xi(L z*s=TD6MQ-qLRZ#w4CST5kWXjl6_6>rpEqi@DzLAF`aDxn(=#mqH^$b1oWypW!2rm0 z9>BRuPot(-Q`p*iqm9;kN#=j<3;ZyZac^SJY7R*wv&fz0NYgJkqKd7ak?~%?&RL!w z=Tw)SIhaf_56hHUZ#t30dBPaEZv!& zzTf9^RpT=_ar4t*f?cXgvi#Ddk$+&xvHlCMqD2=eXX{;kC9;!3^Aww^%7HLEBBB{jB$t|-$yD|nPpn+G zMh*|HzlA7^`G;e5Jx5ueE@%+~yl%)Wi!*ubAPV^>FF%nObK~sDCDsK$d95vNLgu*R z3I1b5Ku!6NX(hTN1ke81(nI3ZPGqI%UWMx`T07(inVL&=ng6j*@CF*f>c_m6W+lf0 zkTxQ0hVCF?LI97i{e$E`NqE;Le)1H6lzj%*{)og%I94?E(6YFy?g;Hk+zYOW<{F-x z&L|wGPokazK;OY7;#7ka%YF!PczNJbn6o_7s#(+taAJ z%RUdoT@e#d`=Ka;((R7M#ge`#CA=7#d$ti|m?$Cofg+PC6~7#alnRaG8yPOeWatN zB?7JG?6|C449Ia=fA(Q+)%U_9WhG;cmLVDW_JW@|^%>T9wJU@Cejn$mAS7~If3YXO z6=K-r*cStCYY1dxEp0Y428dcsxTjsl`?e0!KoFmU4Cy(#-`h)XsdQcUF{HZVaF&{{(omCExYYv`S9?2IP_~nYxXe zA8L_O<{;4^#&lP`*L3dw6Io}N_oE{#%7n>+qeoJZQnF{7+K}YkWi6xl?J6W90$Ff4 z#OJ`q+Wbnl{y5K>X5n@-mrylJBM`mjX7^Uf=eSUmGjM-M+pL+Ly2MY_m8)}0QP4!W zV~&2TXyosTeSDi&s7~r58$~2}s*?_>y}&&5^L`WI`>FAd75$BShd}lSpKLGeP!m(Bo2uc!Jp?7_^wiFFun7UV+*Vl95?EVRgRWPmDuw+Ve3(aL zVR(~%KaJ?J((l+Kq_0H?@9P&jLj6<({H*qiheP)}6X;|G2p)bx-sy%a=>Go|v8d8n zKpjlb0H_hxn+@gKNZC1Fi_Bw?r+aS;gYanFTeAxu zrat0X(!L;nFC4{_4|mPD86s+-esI}Rm^DNl-czDs0e?OKlW71-2YLoYW!jeMyRc)0R*|6yj}m#D<$NdAbuO_i=Yj-H6X6^FyHlxqJLP+%{R}}wT%qJI!X4( zq^!9l6GIp$N4ZR7_Ah9a9_o}`tuw)4zY2c2en9QvlQ!aHok-??}N`qbCulBQW zh_<`wi9I%`xRo^W`D7*NsKJA2%JC@tBk7xLD!l3v};+rw#TmYT0rlmHKp; zN+%In`ybwdVAqjK|7@%s;^o^_+SR@qCzSW_E|H2gec5@K8ev)gLMRE8#(pBDhTx_O zI*cw<^9%rs_>6-QCYhSd@foqf4RV=&&x8I3VJ0LXbK=w61XjTFVskNfIk>jCqn9Jm z?*Q?kPfZh>y*uh;pv-}o6M8!#cR~5us<_@sfddQIbCZ~Y@fQSR>Hf7*w5UzP#171V zodJkm;EwE2DxGuaVi952MD>;}CrHTs5VnmSP^x0&njm}3)Hy~Qf!&(EK?BA) z7OS{be`)50V00sOsO8odQxTa)@w-qgpPwQKXR?nfGj>V`s_$8TvN7DQZi#qGq6sMS z5YvjuRq5y6uM(@W!SqKAukbT-QtU3A5wg)o*Wj<8m!oShwUKc*>DQp~20l(=!wGQ0 z%a+JO;RX_=I;HGvZk5WGLGv+pnQf%=?orq!5V(kA5nPdEY$hlyTU(t%zZ+ z2IOP9l3<$()!|KlUFPj|nsIFZW26}lS_42D-7pRS!Ymd6P$vd9@o z%6;I{w=v-0ec9ICCzfejY{1S>37uK)6qM@=;9aSkiLeACnL#Csa6|PFnkSIt<5M~s z?l`?g!Y8D9yfvL}UfD$=GGxN8!2=$6OrL=`J1q)@kzK+nd!GFW6H4cugX16~AA|5g zliivfJA4b+2h3E{ytV~wos}LzJ6)SRc#0q`*9{#(FS7zYJ>%xS%IlmV4qhid&#T4PHGL=w*v+RbPIb!1L z4JkbXqgArM_zT8D$;k>q#KnemQNAew-S?LR4esW2)Y8^#&JCe_Aob)xl92K&s%*xb z+BU>7yur{+*ZnHF^ci66I}=)d6;o!>0AX)$**=<5Cws5JI2oj6V0o7t)LVq_Q{^RK z%m-9dKPn==54>VllXr%etm(9zCME-Pt@6jntREzP{(%hLvI1fUu1`*P?$fSAn>F!1OFzgO1Z=nI=D9qPeGQ_e&hb4#$lJnW(lK(%iS(N%1jYv(t0htT{y+4H zjUE9KG-|-C2%CjV^_Ts>nuMl_U*^!^0)er@TIDYlY?|hn{OJ*Rfq#U50u~Xb%)<0j zQ&-^I?ZvSk2VQ8YGv(Lx2yBwk*M9Fa!P6}VT5k^L}e1} zdXGP2bJ367Q&PL{0OKCyr?GU@SBo5Fi-8Kbw=Sy|dYYef7E-ZbSzs0eJX8t;WiRu6 zcI9IuRrQQ#&m}l--dLgKh7(!E(L3zFSXQz955d!WIzXoZ@kv?tD`yn#<+Vu$wZY%y zb-DOz?H*B7i>2ldH;93MoZtm@_X+qaXy4(1HXiGY+91eLmA@o0N>tbofE!+Xjlo4HDs)6|PV7IMak;VjAHL>W6)#^<-}vU|o7YsX zFLjn~3%}IRmG}`$&M1C_LCM8VU6=T0eyFn_D}l#d@K=K|dp9U%1Pe6JmhHQ|(5^H7 zA}xspMes6O2aRJOQ?W8J(;~ZI+1aD2A3tJ5s5ND)+||26D;S-aM?EuRNoONMscI%K zbr!&l!jFfK0L-hal2kF*5*K2ze^-q~`5_iNJ>8Zy4nfz)!yXtSwlK~rh+cGHhBp$b zU*9_16|HK?!MC}Y!PKGt5v&q0viS2W(Wc-e*l%QC--2(lbzR5kSZ&ai_vQvOQHzdkWXTxN_V7E>gWhbH+Dj+{1WBXP&(uG$YggUB97b~u_PoX>*-0XnAEi6*lAS2-p z2(40L%wt#h0lgg~8%v?ePe7-rSZ;52y_%cPBOB+DSGmD{Nc2$-97ZVRg--?wX1Oj% zeGt`j^NJrQ$Yrpbe6-u@d>uY~djT!n5>=q)S9t#np=u8yk90179AxDR4 zYrf-~k*pXJ`pv33oZ$D`HyW#hvFwK7uswfr6E~Dx!n=@3^a54g-IG;oApPbMpnp3B zD@o(OvbxZZh_T3#i0QdkZrc<|(Y1TLDzpk5#N{0S)u{`%Q+>04Q*u(aelesJC)D0r zW)^!d^`cd6hh+|VA%@r1$#4W}7?m=eI03y)sn>+R46R$tmK)5@Ud*!P5S=somGJ9% zsFlKFfEjVu)Os7){b$W+2S=LZaX8zjm?M)ZrW?YG8}gfy+SBMx8lqbIdlNc*9|YP$ z*wEqju&H$^_0GHLH8o!ZKyTM zvuCY2mlCF*wJ!D!N$c+kg_((pT#!A29SSD?_nXXp@V25@S(Q%}D9rIZ`e_uGMk^IS zbdR%JIx)EF{rU#5)Z)@+NZW5d4Z`k?#4^c-#LQ+A5ZRt~D^-4T-;p83R7%>HJBc8F z4(|fpbr{&pf}UgOWD)J8tun@-3nqaF{OEPDte+F4`Ee?`Vj6UZ@ryylF4R=fSp|rP z2wQ_f?1;`YU~c`oXPRold|mr)zLT2d;hc{zy~FUJ>Qh}KF6yLL$}x!Rvb+JJdM9x8 zYeByM@*~pQH#WmZn!EhN6P$X8;&((HXjip2j3Yf`#)dOHeDJy>l<#F?Xy_-`i1R)eS;e$DefPCh7m7LH1)>v zQgjOocn@8ARMQ*T4wJ6!y@Xn3Y#@w_q-Zfn>3;7fnVIy89nlM^xw>g>Grn$wgboU& zvCililfp9NXZytzqwkMDe)79eh*+JoS|H$qVJN#v&)wn!Rd%S8xR33voUw zl>S<8CE%9se@bT5@;g+q7#uwagF_S=y}8X53xt&MMR=zVn=Z_a@q=%r)}7?GY5hNO z?jh)Y%#?NtQb1t=DYVeIXo9WxL$7*3MA#-ZO&8j)&3Fa}0!op^-~PkCE}n+y##2#k zjb+Ne9dhs>H9z<0XpP@w5TQWOr%^}XCo}UA4l_{r3^OP*VnzjEv#|l3X2M;2$by@# zIMCpoCOjHfp|;L6U08lQ2-`cQXcJ2CO=N=bYKH_J^X;t@yR``ZX(-*sAdf7Rf%n#>DZI0#MS{d~H(gcO3wv4?~?~gqo$<(YOO)8*_%IwO+*7;M`9C z&eq9$n+85|LN!5<71OPX&9Q8SymM$LKQeov?Er}=R_prcb!QWwgYb)qCUzho(EG3g zdM*u+<4w;h<at?@T+kjsa+qwcHPaGoFr;kN<@85RRKxi5u)jeDG)tDTcy z_SogHkT1a?!1AENv>AJ{TP4wKr*%mOu+d`E<9DG4(g+wfeeZJIh=e&4|Hf*XN)Xdm zpG&ds3c%{*gPgI6QoBU>FvZUIJwHA+VtMw*xA9S@07T*IBo1PUyL=_-J4z!11%pKw zOyE4j+F{lzJ_!iK+XBCLbDZ@3azf8Qwb2v~Nd2z50xmXv&2xCjvCoq@ilWTlg1Ee5 zz`yd*xpk_`@9Lc2$e$iDN`r#q6{fPapC($}R)Gu*0VBs~BLHzz)bLxNbg@5eH*@nd z>_z&@gt5i1SB?CTE3!NxN*ZTS5ogQPgjm`^dSLqv3|&5_DRGWRZ~Lxr8ASf2mh+dk zO&+5p!1A58X4M?J&cw5bBre9M=mKxt@vBzz>F02mxI1l#r8s7VspySC&5E*X1_iP9 zYjcr#`T?ZNZEp7cQNJ;KVMac&gGQH0??KMYIVdaXu>cidOGP&SXM+`QAsiOJi0@EZbJAq7s zIgRAYnC}_8{R?H`gHP=N&n{8tz|GKj6de6|PTzcylLh3Anlc2)Mz}I10sK~9geUQy zSh<=IN-E5qE(sAgCXJviZMaTAkDvp7?;DEB0p&}K$J>d(oXZ+7Z3ud)b}9z3a*?!Q zA3ee@8`V4z`ezsBkLjkPfGKItUclM>w0wxmx6=a5ZVCEdBmR9gL9O-JcUhyWW?B++ z`K9Pj(RJ`*IThiUs;ZdTDCDP4)MP0*1kjewtx#CkU3M7W-YWZ6pb_w)#8Cp$qN)97 zSVqeV`gU-;V~-Aj_v-551TuvgzU@ilpaSMt{2+{7(RsO^1$#&WhR%y>b%r&1ppnZB zFpZ1va+s23ViS^v|7j*g?6^Z3vsj=lJtiL7%P^r+e3}1Th6{0Z`haudJ`q=el}o{Y zJ}+x1VIb-c6LB^<8I~|oflP$oO3waR5Cza^a7Cam?*#m>;nNn|)SR`If>UKH^zJzQ9B zj0a`=Uy?$!RdX<%?XehP6!TQG1^e~MaQ-!&3r;xB*aX&2_>|8L+e3Wau_BYSTgW7=LW9LSJc-CG05 zD@%5PN$4v~darebS?7 z$B@uo#8*fxKuy~8y*7FkAH}?SCXX{jb%V;Eav$g&n=5O*ruh+^v&h)TTuifJPjOYS zl&`)=FW=&!)zc1LCOYCg+qBRJqrTS{g_P%SViS(X%uc4jc9`DvSm(k`45FyO=PSUb zGQ+HVQ+>`T#=(>=XluL99R7q#einI-4#BBiZrMt!;bjPURb(&{3s&|4GL%xnU<7a* zX0GZ0pCQ-qteapr{n-}&6MwfewKW&JxM>hlB~kDn`Y15^T8Bl&e}x3c^kIc>J2sSz z7r9OXzUHoA#0R-}cP4pc5{y_I?9r@}QQi_cI5Xk0@Jr00+TXB(4Yb96@ksoavOS(X za!rjnyY6v^4-+mhx>QB?qR%#SHjxz}t%^k_r9!S*;n5C**|9E0hQ?sk7#bpEDhK!p8Agw;JG2Zm5>fw*E&(W6F52?b5xI$>(u)YrL5`^wll?HkY3f~66 zYQ7i7ExG8HXk{Wx9b9_KezCgCy2?yie(xONnA2uUFTl%D8abIYD#rOxkZ^o=kVCOqDEph() z_&ZS!Xd7#?qtyY_&Y5{gt#`>3wi912OFX5(O=xsSkZr5A)QT0fQ(l z!%G?EWjZ|CdL(P6FB&_Q&Z2p8K{sS(7h$5_U-8vKCy)n|QUKark5F`PMAM}B6)xxq zr{`WAeX~%STs^>Ue$ZBf;oTV6vP{_sL&f#s#jRuOt{+C1bxdhiciJWU^=ysO_I@-4 zmx-aDvwYQTU^bX_Ml$wx@Ejpm1&f6o-vgKZIH`SRLnV{=JFU|izpXyfp!@Lb5o&Hq z>DMq{b>0wOqky?xq|e}qx7!g5 ze1nqckYpjfhutmGATr4iPa~3S>uh&={9sPNn%RdUV}CmTHbb2KRWnyxpbz@4PgQC< z)h~~Z*PE5v+bxAmEFnbL%Ta{kH4@`b3+mXytqbQEmxSd!}u%?NKY3k^JH9!70 z6oJiPr4slH|3M0ltQH{*$EEYVA!Dx)L*W&pfrlRyqVf0cQlYoX+;dx5U)?Vmk>5Nu^Ho7aCwI(q&}< zcwE84<3tWVIGZ`F(8b}tz|Jw4{aDnb_AXiL@U(;8p4EN*ZN3l zyCX-YAvB8FK{2%{>=T)LqH=+N9I!?HS%W+a3o@4W=s|X&+!xYUyd{mgrJW&2O|7X!Aj8TwN4XTRF}hKd;UdK1=U@m8EHWqi0HJog z-KczDg{70$N%mS0Vf1%=ngreB7!2*UMK^ati+@s=B9gahZZvRE2hN|KJg0tRyzAD} z8xuyaj|D-NG2iCJFT!B6G&1J@pClb^RB%r$Uf22Fg)io8?ZeqzLwJ`vIde{&Cw`KJ zH%NEN7R@pz7_Iff^ILxw^QS;Z5}*9866(2bTQlvW&H#JD5g|o>kX*rP9Fn!d0-8nH zhn%~nLxOv66hmeeyp!T^d9h~`#Wnx*fd>>T2#5i@!{^ZDjC^NLhN}0&0uFMIlfPt| zjtNmOFlUCfZ zyh8wL7OSERB&8zKa-KKeluQv-RJ(>VgOxd>!=}%>q*q_0Kr?-BQl6V2<>aRx*EYj6 z19%zCod61eWKP!C1rH?b?T!g_wI4Yf=zi$Im65L=vI57QPWV&H69$G(RKbf9#hHkMknE4v*{b=`EPbXkeT>u2DRTp|v^26=7nE|qXY6057ux}MHWpOX7)@1rIb zgA7{F`f|7fomFX~fuVIQRB`glHIk2%fQRYX{gMNW=%>Mvg%|w*f>~%EW+ABRX=;^8?&XJhMXN+N06`2cajE4&-+VZ~PH%Nc^Y1K9aV_Z50$hj%;hAGp}MMv~$ zRkqpfMNLRxh!++!4bjjL;mK{ddlBgcrk8V&y;r%(f1k&+Uy^im&WW>wNXZF0#i>)5 z_eevb=Urt;e-P^vb*Fp$RP$Dw60EBMCE!kDk`{BM+-lhZi`m(z57_;{!}7KhGua&W ztkWD(v9XH0;qb}6x=sUW_2ic@2FS7;wr5=~pB%I84^rd>;>5rjsYYXoUeB4-ItspW zoIV14@$1joB|zQES&t;G$f4K69%_10TK0y}(tf0jJcFOA6Lwx{78y28#a7CFOv~oD z*z+=Oh0f}WQ9nnDO%-kQ|;V|_wbWZhHPPNetq=MF;o4AMx#h@O(O~#o?Y?`tEdr-L zz##uJgw$GbVwK0Yd+!(rN2oELkMV2$r zwZ~snss_xIr)f2>?SBcD1G4VAUHkmPQr!Pqss=cSR0VE3TA^k*aM&UM&RN4R+~gR^ zMFbo>01qm|EmwBbhQ&=P1AEzguIZ*tgxUCYv7}#`dz+<`Ul9>p-eP}=G))VgOd)&x zoTr_;#tOTMy4Nt0*gPKych+$7Z;<>WrMUIcBR;iGJJa)_4I|{M^#&j5%(7ap@Pr_= zaad+cyvLCYl$u^nTtN;)UT2M((-G;URkJ&>hi8_3ZyeMMn4`aZ$Pk*0)8-1~iYVeq82<^4kGC5yKx}(yRjO8{Y_hEh*2HgeJyRr53@l%rndqaw~I;J1ug?uW)2H%on*^J7>#+A{^M&YjM$!cJ5Fc74Jf?>`Sr5 z2-}p5+Slb%CGClAc+~cq?pl_2(_CV-+%KMW>{FY&>@uj83W89flO;@z0+fCuBJZWl zI*mycnowFhsPM^13nIE&(1ddTKo2P)zDURhP%8$WOUX1f6AJ|OCdM3VyB9qeWvR<0 zk3^RtPrwUi&3a?j7(USrt?AHJvz)}f@`rLG#U|f&T-(@0A0mSY0#pKRm+siDwl8oo z7u&kS>GW3Eq47ij^I`<1dPj+H)zrMtRQ(dtTD_lCKy+fHl(>xjkg7INjJltGwI8=v z$TLDB+s(Xe$Y8}I-gV37vIk~&tDOL)CMo&k$k@FZYLz^bF<~HB?P&H z@$AanI9qLajcg!YYp8X0%8s3x{wHutbAqruwczxLSU}n)-OSm!SiUp{G=4WU^5X-l zkKj#O-(CS>MmSl~D`lMHggB~FJcP^mJndRkdJqk(A!jAzdQcHL2Zx0|2QDhQ$2Ziw zXz{26$>zd98nxD?AcfCk0zrUGh*owEOMc|+(mE~rPdpZGu9Wy=!@v5AGYn2Uswy<$ zRO4#*J_96=pv8Yk9mEDHRH_O@%)PFr%ZPUb$S5TAK*zvHeCvxTGOgaro z=Zb`u!}4c^vBx1CFy@%u)dFCxUkrJWej4v+9|(GZvC}8?bHkD@m-yVC7efXHZcVlE z&Zmb<%p!sK!WI)!QZQ$$b;hyRGf`={yc5aBnBSuO8XX8&9N$_v>I|D6pJF(+C85(V zay)NfG6vJV5M9Z6B5ra*Txv+lw8k(}z7mhHksy3W&yds32XNpn_^68sx0rRKbGl$u zub5MN{Yad?pYSe4HA*2H<2sV6A@$=j0AXPwS9fbe-=c0$k z^IhOd#@*Pb$mWa`sc*>s)dC-p~N>9>%DM)o6xemu~a z9Q`FL;m9)NQ&6#%g9T zy}5Xl75nVb+D=wFr{|6iJs=noV_}+GMz`#_;25d3- z3|Sg6G5U9|6dhf5KJMf-GqTZ{7aYa6m~~}I3qFE?lSPWJWhEeAjAe>%@RJ$lT0X&3ZHl*HeiN*mZ+yA+he zdZ(gWR<)7G*6n;XvK*V4_W-Q>+{X7YVYC)xfV|59Qm<5R=(^%d^vu}-cO|xEorL<} z_@k$8!%f_0@xDGc4bL8P=Tcu6VsfPp&fF!IgN60^uYSNSLlc7Q)a|+6me(a z=~=IlZBOj7j{SF!w6Ed`WE8suTijY1KrVvENt}@aNJ4`?$q=GCSb4Fh4o$?R7PtR7 zSxs0-GtD^`2SdNQ4f%3&WX9=z{QZ60I|a&Ewf%q65Ng4wzbIrI4xitxa#xdY-P$=M z7}N^5+gjl6_7P`<#giN*_k-TsaES~vdJ2B~n<`SP!Q0z7@BkXQMZI2KEnIJkt?&uj z!(g;xe>3$F;XEH?z{hL7@xQznvgJ==-c_s1#>srfhkcx)7)gZNnL1nj&8obIeKM~w zX(n5s5Cn=OTp_orxMY7%90pK{c{DO>cD$5?68R?DxNjVjlt~Jo$%a5L@MYf zD(*#^aeq|@T8jVuKCFzL16Vnxti@YasNSjnGG+e7H4x>p7bxs&_FjFSmvYglqV~Xb0GbYT;Gkx;^S3+z zW7lj2fx;H)dTDtARw7VTxM)pE4Sw(vh7KrE#T8$dA%*vX=G9bvFCn>I5hADAkHWaH zByD0>OixU7S1`)XaTRNsRUPRQdPe?}31sj=6t}M!>wLRaNM;EJZk%I_K--4q8;8wj z1KEhNYNkD;LzL{&aqX|EgRmylbX)(cu9&7cv!HD(rJFd z;Q{?!OORM+x4Jlng6(08QW!GvipJ)iH=Blv`_zaUbLY-b6NUE}@tVRK4PNKYA__Vu zq44Qz$`SCVb%78vuS`;z8b7_d&3WuQyizjL-EBaB5>|_^1_)li-6n44-jxc#j`-Krjq^G}`cJ9N@LY>O3qNaI=+E72=Ie@n{