From 00068474b7e8557475d85fddbe3221dcb825a67c Mon Sep 17 00:00:00 2001 From: Jeff Allen Date: Fri, 22 Sep 2023 09:32:12 +0200 Subject: [PATCH 1/4] More tests Signed-off-by: Jeff Allen --- .gitignore | 3 +- seekinghttp.go | 15 ++++++- seekinghttp_test.go | 100 ++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 108 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 035064d..3c80781 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ -cmd/remote-archive-ls/remote-archive-ls \ No newline at end of file +cmd/remote-archive-ls/remote-archive-ls +.vscode \ No newline at end of file diff --git a/seekinghttp.go b/seekinghttp.go index 3c44a72..979db7f 100644 --- a/seekinghttp.go +++ b/seekinghttp.go @@ -86,9 +86,13 @@ func (s *SeekingHTTP) ReadAt(buf []byte, off int64) (n int, err error) { s.Logger.Debugf("ReadAt len %v off %v", len(buf), off) } + if off < 0 { + return 0, io.EOF + } + if s.last != nil && off > s.lastOffset { end := off + int64(len(buf)) - if end < s.lastOffset+int64(s.last.Len()) { + if end <= s.lastOffset+int64(s.last.Len()) { start := off - s.lastOffset if s.Logger != nil { s.Logger.Debugf("cache hit: range (%v-%v) is within cache (%v-%v)", off, off+int64(len(buf)), s.lastOffset, s.lastOffset+int64(s.last.Len())) @@ -111,8 +115,12 @@ func (s *SeekingHTTP) ReadAt(buf []byte, off int64) (n int, err error) { return 0, err } - // Fetch more than what they asked for to reduce round-trips + // Fetch more than what they asked for to reduce round-trips, up to a max of 10 meg wanted := 10 * len(buf) + if wanted > 10e6 { + wanted = 10e6 + } + rng := fmtRange(off, int64(wanted)) req.Header.Add("Range", rng) @@ -154,6 +162,9 @@ func (s *SeekingHTTP) ReadAt(buf []byte, off int64) (n int, err error) { if err != nil { return 0, err } + if s.Logger != nil { + s.Logger.Debugf("loaded %d bytes into last", s.last.Len()) + } s.lastOffset = off var n int diff --git a/seekinghttp_test.go b/seekinghttp_test.go index 6a3a252..8526d38 100644 --- a/seekinghttp_test.go +++ b/seekinghttp_test.go @@ -2,6 +2,7 @@ package seekinghttp import ( "bytes" + "fmt" "io" "net/http" "testing" @@ -9,22 +10,66 @@ import ( "github.com/stretchr/testify/assert" ) +type logger struct { + t *testing.T +} + +func (l logger) Infof(format string, args ...interface{}) { + l.t.Logf(fmt.Sprintf("[INFO] %s", format), args...) +} +func (l logger) Debugf(format string, args ...interface{}) { + l.t.Logf(fmt.Sprintf("[DEBUG] %s", format), args...) +} + // MockHTTPClient is a mock implementation of the http.Client interface for testing purposes. -type MockHTTPClient struct{} +type MockHTTPClient struct { + str string + numReq int +} + +func (c *MockHTTPClient) Do(req *http.Request) (*http.Response, error) { + start := 0 + end := 0 + r := req.Header["Range"][0] + switch r { + case "bytes=0-99": + start = 0 + end = 99 + case "bytes=30-329": + start = 30 + end = 329 + case "bytes=10-109": + start = 10 + end = 109 + case "bytes=20-119": + start = 20 + end = 119 + default: + panic(fmt.Sprintf("unknown range: %s", r)) + } + + if end > len(c.str) { + end = len(c.str) + } + if start > end { + start = end + } -func (c *MockHTTPClient) Do(_ *http.Request) (*http.Response, error) { // Create a mock response for testing purposes. resp := &http.Response{ StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader([]byte("Mock HTTP response body"))), + Body: io.NopCloser(bytes.NewReader([]byte(c.str[start:end]))), } + c.numReq++ return resp, nil } func TestReadAt(t *testing.T) { // Create a new SeekingHTTP instance with a mock HTTP client. s := New("https://example.com") - s.Client = &MockHTTPClient{} + m := &MockHTTPClient{str: "Mock HTTP response body"} + s.Client = m + s.Logger = &logger{t: t} // Define test cases. testCases := []struct { @@ -35,14 +80,55 @@ func TestReadAt(t *testing.T) { }{ {0, 10, 10, nil}, {10, 1, 1, nil}, - {30, 30, 23, nil}, + {30, 30, 0, nil}, + {-1, 0, 0, io.EOF}, } for _, tc := range testCases { buf := make([]byte, tc.bufSize) n, err := s.ReadAt(buf, tc.offset) - assert.ErrorIs(t, err, tc.expectErr, "ReadAt(offset=%d, bufSize=%d) error = %v, expected error = %v", tc.offset, tc.bufSize, err, tc.expectErr) - assert.Equal(t, n, tc.expectLen, "ReadAt(offset=%d, bufSize=%d) len = %d, expected len = %d", tc.offset, tc.bufSize, n, tc.expectLen) + assert.ErrorIs(t, tc.expectErr, err, "ReadAt(offset=%d, bufSize=%d) error = %v, expected error = %v", tc.offset, tc.bufSize, err, tc.expectErr) + assert.Equal(t, tc.expectLen, n, "ReadAt(offset=%d, bufSize=%d) len = %d, expected len = %d", tc.offset, tc.bufSize, n, tc.expectLen) } + // expect 2 reads: one to load the cache, and one to look for bytes past the end for the seek to 30. + assert.Equal(t, 2, m.numReq) +} + +func TestReadNothing(t *testing.T) { + // Create a new SeekingHTTP instance with a mock HTTP client. + s := New("https://example.com") + s.Client = &MockHTTPClient{str: ""} + s.Logger = &logger{t: t} + + buf := make([]byte, 10) + n, err := s.Read(buf) + assert.ErrorIs(t, err, nil) + assert.Equal(t, 0, n) +} + +func TestReadOffEnd(t *testing.T) { + // Create a new SeekingHTTP instance with a mock HTTP client. + s := New("https://example.com") + s.Client = &MockHTTPClient{str: "0123456789abcdefghij"} + s.Logger = &logger{t: t} + + buf := make([]byte, 10) + n, err := s.Read(buf) + assert.ErrorIs(t, err, nil) + assert.Equal(t, n, len(buf)) + assert.Equal(t, "0123456789", string(buf)) + assert.Equal(t, int64(10), s.offset) + + n, err = s.Read(buf) + assert.ErrorIs(t, err, nil) + assert.Equal(t, n, len(buf)) + assert.Equal(t, "abcdefghij", string(buf)) + assert.Equal(t, int64(20), s.offset) + + n, err = s.Read(buf) + assert.ErrorIs(t, err, nil) + assert.Equal(t, 0, n) + assert.Equal(t, int64(20), s.offset) + } From 6c97f893c7304771cfd4084b20e03ca0a33feac6 Mon Sep 17 00:00:00 2001 From: Jeff Allen Date: Fri, 22 Sep 2023 09:50:45 +0200 Subject: [PATCH 2/4] Add link. Signed-off-by: Jeff Allen --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ae4b56f..3047aca 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,5 @@ seekinghttp An implementation of io.ReaderAt that works via GET and the Range header. +This was discussed in an article for [Gopher Academy](https://blog.gopheracademy.com/advent-2017/seekable-http/) in 2017. + From abd2548f1cfe777d7ec4e4b226abadb8ceaaab51 Mon Sep 17 00:00:00 2001 From: Jeff Allen Date: Fri, 22 Sep 2023 11:32:33 +0200 Subject: [PATCH 3/4] Cache 1 meg by default Signed-off-by: Jeff Allen --- .gitignore | 2 +- cmd/remote-archive-ls/main.go | 2 +- seekinghttp.go | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 3c80781..fd9fe31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ *~ -cmd/remote-archive-ls/remote-archive-ls +cmd/remote-archive-ls/remote-archive-ls* .vscode \ No newline at end of file diff --git a/cmd/remote-archive-ls/main.go b/cmd/remote-archive-ls/main.go index b89ec56..1025186 100644 --- a/cmd/remote-archive-ls/main.go +++ b/cmd/remote-archive-ls/main.go @@ -86,7 +86,7 @@ func main() { } for _, f := range z.File { - logger.Infof("File: %s", f.FileHeader.Name) + fmt.Println(f.FileHeader.Name) } return } diff --git a/seekinghttp.go b/seekinghttp.go index 979db7f..7543ca7 100644 --- a/seekinghttp.go +++ b/seekinghttp.go @@ -115,10 +115,10 @@ func (s *SeekingHTTP) ReadAt(buf []byte, off int64) (n int, err error) { return 0, err } - // Fetch more than what they asked for to reduce round-trips, up to a max of 10 meg - wanted := 10 * len(buf) - if wanted > 10e6 { - wanted = 10e6 + // Minimum fetch size is 1 meg + wanted := 1024 * 1024 + if wanted < len(buf) { + wanted = len(buf) } rng := fmtRange(off, int64(wanted)) From 17710ea360f1190a4d869c949fcd97d91bb3526e Mon Sep 17 00:00:00 2001 From: Jeff Allen Date: Mon, 25 Sep 2023 07:04:40 +0200 Subject: [PATCH 4/4] Handle Range properly. Signed-off-by: Jeff Allen --- seekinghttp_test.go | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/seekinghttp_test.go b/seekinghttp_test.go index 8526d38..66a5e9f 100644 --- a/seekinghttp_test.go +++ b/seekinghttp_test.go @@ -5,6 +5,8 @@ import ( "fmt" "io" "net/http" + "strconv" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -28,25 +30,10 @@ type MockHTTPClient struct { } func (c *MockHTTPClient) Do(req *http.Request) (*http.Response, error) { - start := 0 - end := 0 - r := req.Header["Range"][0] - switch r { - case "bytes=0-99": - start = 0 - end = 99 - case "bytes=30-329": - start = 30 - end = 329 - case "bytes=10-109": - start = 10 - end = 109 - case "bytes=20-119": - start = 20 - end = 119 - default: - panic(fmt.Sprintf("unknown range: %s", r)) - } + x := strings.Split(req.Header["Range"][0], "=") + y := strings.Split(x[1], "-") + start, _ := strconv.Atoi(y[0]) + end, _ := strconv.Atoi(y[1]) if end > len(c.str) { end = len(c.str)