diff --git a/.gitignore b/.gitignore index 035064d..fd9fe31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ -cmd/remote-archive-ls/remote-archive-ls \ No newline at end of file +cmd/remote-archive-ls/remote-archive-ls* +.vscode \ No newline at end of file diff --git a/README.md b/README.md index ae4b56f..3047aca 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,5 @@ seekinghttp An implementation of io.ReaderAt that works via GET and the Range header. +This was discussed in an article for [Gopher Academy](https://blog.gopheracademy.com/advent-2017/seekable-http/) in 2017. + diff --git a/cmd/remote-archive-ls/main.go b/cmd/remote-archive-ls/main.go index b89ec56..1025186 100644 --- a/cmd/remote-archive-ls/main.go +++ b/cmd/remote-archive-ls/main.go @@ -86,7 +86,7 @@ func main() { } for _, f := range z.File { - logger.Infof("File: %s", f.FileHeader.Name) + fmt.Println(f.FileHeader.Name) } return } diff --git a/seekinghttp.go b/seekinghttp.go index 3c44a72..7543ca7 100644 --- a/seekinghttp.go +++ b/seekinghttp.go @@ -86,9 +86,13 @@ func (s *SeekingHTTP) ReadAt(buf []byte, off int64) (n int, err error) { s.Logger.Debugf("ReadAt len %v off %v", len(buf), off) } + if off < 0 { + return 0, io.EOF + } + if s.last != nil && off > s.lastOffset { end := off + int64(len(buf)) - if end < s.lastOffset+int64(s.last.Len()) { + if end <= s.lastOffset+int64(s.last.Len()) { start := off - s.lastOffset if s.Logger != nil { s.Logger.Debugf("cache hit: range (%v-%v) is within cache (%v-%v)", off, off+int64(len(buf)), s.lastOffset, s.lastOffset+int64(s.last.Len())) @@ -111,8 +115,12 @@ func (s *SeekingHTTP) ReadAt(buf []byte, off int64) (n int, err error) { return 0, err } - // Fetch more than what they asked for to reduce round-trips - wanted := 10 * len(buf) + // Minimum fetch size is 1 meg + wanted := 1024 * 1024 + if wanted < len(buf) { + wanted = len(buf) + } + rng := fmtRange(off, int64(wanted)) req.Header.Add("Range", rng) @@ -154,6 +162,9 @@ func (s *SeekingHTTP) ReadAt(buf []byte, off int64) (n int, err error) { if err != nil { return 0, err } + if s.Logger != nil { + s.Logger.Debugf("loaded %d bytes into last", s.last.Len()) + } s.lastOffset = off var n int diff --git a/seekinghttp_test.go b/seekinghttp_test.go index 6a3a252..66a5e9f 100644 --- a/seekinghttp_test.go +++ b/seekinghttp_test.go @@ -2,29 +2,61 @@ package seekinghttp import ( "bytes" + "fmt" "io" "net/http" + "strconv" + "strings" "testing" "github.com/stretchr/testify/assert" ) +type logger struct { + t *testing.T +} + +func (l logger) Infof(format string, args ...interface{}) { + l.t.Logf(fmt.Sprintf("[INFO] %s", format), args...) +} +func (l logger) Debugf(format string, args ...interface{}) { + l.t.Logf(fmt.Sprintf("[DEBUG] %s", format), args...) +} + // MockHTTPClient is a mock implementation of the http.Client interface for testing purposes. -type MockHTTPClient struct{} +type MockHTTPClient struct { + str string + numReq int +} + +func (c *MockHTTPClient) Do(req *http.Request) (*http.Response, error) { + x := strings.Split(req.Header["Range"][0], "=") + y := strings.Split(x[1], "-") + start, _ := strconv.Atoi(y[0]) + end, _ := strconv.Atoi(y[1]) + + if end > len(c.str) { + end = len(c.str) + } + if start > end { + start = end + } -func (c *MockHTTPClient) Do(_ *http.Request) (*http.Response, error) { // Create a mock response for testing purposes. resp := &http.Response{ StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader([]byte("Mock HTTP response body"))), + Body: io.NopCloser(bytes.NewReader([]byte(c.str[start:end]))), } + c.numReq++ return resp, nil } func TestReadAt(t *testing.T) { // Create a new SeekingHTTP instance with a mock HTTP client. s := New("https://example.com") - s.Client = &MockHTTPClient{} + m := &MockHTTPClient{str: "Mock HTTP response body"} + s.Client = m + s.Logger = &logger{t: t} // Define test cases. testCases := []struct { @@ -35,14 +67,55 @@ func TestReadAt(t *testing.T) { }{ {0, 10, 10, nil}, {10, 1, 1, nil}, - {30, 30, 23, nil}, + {30, 30, 0, nil}, + {-1, 0, 0, io.EOF}, } for _, tc := range testCases { buf := make([]byte, tc.bufSize) n, err := s.ReadAt(buf, tc.offset) - assert.ErrorIs(t, err, tc.expectErr, "ReadAt(offset=%d, bufSize=%d) error = %v, expected error = %v", tc.offset, tc.bufSize, err, tc.expectErr) - assert.Equal(t, n, tc.expectLen, "ReadAt(offset=%d, bufSize=%d) len = %d, expected len = %d", tc.offset, tc.bufSize, n, tc.expectLen) + assert.ErrorIs(t, tc.expectErr, err, "ReadAt(offset=%d, bufSize=%d) error = %v, expected error = %v", tc.offset, tc.bufSize, err, tc.expectErr) + assert.Equal(t, tc.expectLen, n, "ReadAt(offset=%d, bufSize=%d) len = %d, expected len = %d", tc.offset, tc.bufSize, n, tc.expectLen) } + // expect 2 reads: one to load the cache, and one to look for bytes past the end for the seek to 30. + assert.Equal(t, 2, m.numReq) +} + +func TestReadNothing(t *testing.T) { + // Create a new SeekingHTTP instance with a mock HTTP client. + s := New("https://example.com") + s.Client = &MockHTTPClient{str: ""} + s.Logger = &logger{t: t} + + buf := make([]byte, 10) + n, err := s.Read(buf) + assert.ErrorIs(t, err, nil) + assert.Equal(t, 0, n) +} + +func TestReadOffEnd(t *testing.T) { + // Create a new SeekingHTTP instance with a mock HTTP client. + s := New("https://example.com") + s.Client = &MockHTTPClient{str: "0123456789abcdefghij"} + s.Logger = &logger{t: t} + + buf := make([]byte, 10) + n, err := s.Read(buf) + assert.ErrorIs(t, err, nil) + assert.Equal(t, n, len(buf)) + assert.Equal(t, "0123456789", string(buf)) + assert.Equal(t, int64(10), s.offset) + + n, err = s.Read(buf) + assert.ErrorIs(t, err, nil) + assert.Equal(t, n, len(buf)) + assert.Equal(t, "abcdefghij", string(buf)) + assert.Equal(t, int64(20), s.offset) + + n, err = s.Read(buf) + assert.ErrorIs(t, err, nil) + assert.Equal(t, 0, n) + assert.Equal(t, int64(20), s.offset) + }