Skip to content

Commit

Permalink
readerWrapper: readByte needs to use io.ReadFull (#802)
Browse files Browse the repository at this point in the history
This fixes an "unexpected EOF" error that can occur when streaming
from a network source into the zstd decompressor.

Even when asking for one byte of data, it's necessary to use
io.ReadFull. It's valid to return 0 bytes without EOF on a read,
and this behavior is seen on network sources rather frequently.
  • Loading branch information
jnoxon committed Apr 8, 2023
1 parent b0f7a94 commit 62dad81
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 1 deletion.
2 changes: 1 addition & 1 deletion zstd/bytebuf.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func (r *readerWrapper) readBig(n int, dst []byte) ([]byte, error) {
}

func (r *readerWrapper) readByte() (byte, error) {
n2, err := r.r.Read(r.tmp[:1])
n2, err := io.ReadFull(r.r, r.tmp[:1])
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
Expand Down
42 changes: 42 additions & 0 deletions zstd/decoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,48 @@ func TestNewDecoderSmallFile(t *testing.T) {
t.Logf("Decoded %d bytes with %f.2 MB/s", n, mbpersec)
}

// cursedReader wraps a reader and returns zero bytes every other read.
// This is used to test the ability of the consumer to handle empty reads without EOF,
// which can happen when reading from a network connection.
type cursedReader struct {
io.Reader
numReads int
}

func (r *cursedReader) Read(p []byte) (n int, err error) {
r.numReads++
if r.numReads%2 == 0 {
return 0, nil
}

return r.Reader.Read(p)
}

func TestNewDecoderZeroLengthReads(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
file := "testdata/z000028.zst"
const wantSize = 39807
f, err := os.Open(file)
if err != nil {
t.Fatal(err)
}
defer f.Close()
dec, err := NewReader(&cursedReader{Reader: f})
if err != nil {
t.Fatal(err)
}
defer dec.Close()
n, err := io.Copy(io.Discard, dec)
if err != nil {
t.Fatal(err)
}
if n != wantSize {
t.Errorf("want size %d, got size %d", wantSize, n)
}
}

type readAndBlock struct {
buf []byte
unblock chan struct{}
Expand Down

0 comments on commit 62dad81

Please sign in to comment.