Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds get file size method #10

Merged
merged 30 commits into from
Nov 17, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
584b7cb
Adds named return statements and adds Content Size Header
vmesel Nov 14, 2022
f0448a8
Fixes styling and named returns, fixes testing too
vmesel Nov 14, 2022
00bbf46
Changes ds to req.ContentLength
vmesel Nov 14, 2022
bd0e41a
Better error handling, as suggested by @cuducos
vmesel Nov 14, 2022
9177ffe
Update main.go
vmesel Nov 14, 2022
048ce5f
Fixes content length assign
vmesel Nov 14, 2022
303e775
Merges branch
vmesel Nov 14, 2022
f1ea076
Removes integration of the function to simplify PR
vmesel Nov 14, 2022
88a348e
Adds reviews as requested on PR.
vmesel Nov 15, 2022
670f150
Update main.go
vmesel Nov 15, 2022
9c25f72
Update main.go
vmesel Nov 15, 2022
2d34809
Update main.go
vmesel Nov 15, 2022
3701e8b
Change from named return to explicit
vmesel Nov 15, 2022
b6fbe8a
Merge branch 'feature/implements-head-request' of github.com:vmesel/c…
vmesel Nov 15, 2022
8197ee9
Fixes variable for whole size of the files
vmesel Nov 15, 2022
30f75f7
Update main.go
vmesel Nov 16, 2022
e3fa3ac
Changes Status Code checking.
vmesel Nov 16, 2022
5f38886
Changes Status Code checking.
vmesel Nov 16, 2022
9093ac8
Update main.go
vmesel Nov 16, 2022
a29711a
Merge branch 'feature/implements-head-request' of https://github.com/…
cuducos Nov 16, 2022
758b4de
Closes HEAD body even if the status is unexpected
cuducos Nov 16, 2022
a755472
Cleans up & fixes tests
cuducos Nov 16, 2022
9108ff9
Auto-formats
cuducos Nov 16, 2022
d24201e
Update main.go
vmesel Nov 16, 2022
3f10133
Update main.go
vmesel Nov 16, 2022
4e33e29
Update main.go
vmesel Nov 16, 2022
505b712
Update main.go
vmesel Nov 16, 2022
1d41f33
Adds a TODO comment on retries
vmesel Nov 16, 2022
2f55151
Merge branch 'feature/implements-head-request' of github.com:vmesel/c…
vmesel Nov 16, 2022
24603e3
Merges with Cuducos' Branch
vmesel Nov 17, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 32 additions & 3 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -133,6 +134,28 @@ func (d *Downloader) downloadFileWithTimeout(userCtx context.Context, u string)
}
}

func (d *Downloader) getDownloadSize(ctx context.Context, u string) (uint64, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, u, nil)
vmesel marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return 0, fmt.Errorf("creating the request for %s: %w", u, err)
}
resp, err := d.client.Do(req)
if err != nil {
return 0, fmt.Errorf("sending get http request to %s: %w", u, err)
}
vmesel marked this conversation as resolved.
Show resolved Hide resolved
defer resp.Body.Close()
if resp.StatusCode != 200 {
return 0, fmt.Errorf("got unexpected http response status for %s: %s", u, resp.Status)
}
if resp.ContentLength <= 0 && resp.Header.Get("Content-Range") != "" {
var s uint64
p := strings.Split(resp.Header.Get("Content-Range"), "/")
fmt.Sscan(p[len(p)-1], &s)
return s, nil
}
return uint64(resp.ContentLength), nil
}

func (d *Downloader) downloadFile(ctx context.Context, u string) ([]byte, error) {
ch := make(chan []byte, 1)
defer close(ch)
Expand Down Expand Up @@ -193,9 +216,16 @@ func (d *Downloader) DownloadWithContext(ctx context.Context, urls ...string) <-
wg.Add(1)
go func(u string) {
defer wg.Done()
s := DownloadStatus{URL: u}
path := filepath.Join(os.TempDir(), filepath.Base(u))
s := DownloadStatus{URL: u, DownloadedFilePath: path}
defer func() { ch <- s }()
s.DownloadedFilePath = filepath.Join(os.TempDir(), filepath.Base(u))
t, err := d.getDownloadSize(ctx, u) // TODO: retry
if err != nil {
s.Error = fmt.Errorf("error getting file size: %w", err)
return
}
s.FileSizeBytes = t
ch <- s // send total file size to the user
b, err := d.downloadFile(ctx, u)
if err != nil {
s.Error = err
Expand All @@ -206,7 +236,6 @@ func (d *Downloader) DownloadWithContext(ctx context.Context, urls ...string) <-
return
}
s.DownloadedFileBytes = uint64(len(b))
s.FileSizeBytes = uint64(len(b))
}(u)
}
go func() {
Expand Down
28 changes: 22 additions & 6 deletions main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ func TestDownload_Error(t *testing.T) {
t.Run(tc.desc, func(t *testing.T) {
s := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodHead {
return
}
tc.proc(w)
},
))
Expand All @@ -36,11 +39,12 @@ func TestDownload_Error(t *testing.T) {
WaitBetweenRetries: 0 * time.Second,
}
ch := d.Download(s.URL)
status := <-ch
if status.Error == nil {
<-ch // discard the first got (just the file size)
got := <-ch
if got.Error == nil {
t.Error("expected an error, but got nil")
}
if !strings.Contains(status.Error.Error(), "#4") {
if !strings.Contains(got.Error.Error(), "#4") {
t.Error("expected #4 (configured number of retries), but did not get it")
}
if _, ok := <-ch; ok {
Expand All @@ -59,6 +63,7 @@ func TestDownload_OkWithDefaultDownloader(t *testing.T) {
defer s.Close()

ch := DefaultDownloader().Download(s.URL)
<-ch // discard the first status (just the file size)
got := <-ch
defer os.Remove(got.DownloadedFilePath)

Expand Down Expand Up @@ -99,6 +104,10 @@ func TestDownload_Retry(t *testing.T) {
attempts := int32(0)
s := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodHead {
w.Header().Add("Content-Length", "2")
return
}
if atomic.CompareAndSwapInt32(&attempts, 0, 1) {
tc.proc(w)
}
Expand All @@ -115,6 +124,7 @@ func TestDownload_Retry(t *testing.T) {
WaitBetweenRetries: 0 * time.Second,
}
ch := d.Download(s.URL)
<-ch // discard the first status (just the file size)
got := <-ch
if got.Error != nil {
t.Errorf("invalid error. want:nil got:%q", got.Error)
Expand Down Expand Up @@ -150,6 +160,9 @@ func TestDownloadWithContext_ErrorUserTimeout(t *testing.T) {
timeout := 10 * userTimeout
s := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodHead {
return
}
time.Sleep(2 * userTimeout) // this time is greater than the user timeout, but shorter than the timeout per chunk.
},
))
Expand All @@ -165,11 +178,12 @@ func TestDownloadWithContext_ErrorUserTimeout(t *testing.T) {
defer cancFunc()

ch := d.DownloadWithContext(userCtx, s.URL)
status := <-ch
if status.Error == nil {
<-ch // discard the first got (just the file size)
got := <-ch
if got.Error == nil {
t.Error("expected an error, but got nil")
}
if !strings.Contains(status.Error.Error(), "#4") {
if !strings.Contains(got.Error.Error(), "#4") {
t.Error("expected #4 (configured number of retries), but did not get it")
}
if _, ok := <-ch; ok {
Expand Down Expand Up @@ -202,3 +216,5 @@ func TestDownload_Chunks(t *testing.T) {
}
}
}

// TODO: add tests for getDownloadSize (success with Content-Length, success with Content-Range, failure)