Skip to content

Commit

Permalink
Fix multi-frame Zstandard response decoding
Browse files Browse the repository at this point in the history
  • Loading branch information
Rogdham committed May 14, 2023
1 parent be5e03b commit aca0f01
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 2 deletions.
1 change: 1 addition & 0 deletions changelog/3008.bugfix.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fixed response decoding with Zstandard when compressed data is made of several frames.
9 changes: 7 additions & 2 deletions src/urllib3/response.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,10 +169,15 @@ def __init__(self) -> None:
def decompress(self, data: bytes) -> bytes:
if not data:
return b""
return self._obj.decompress(data) # type: ignore[no-any-return]
data_parts = [self._obj.decompress(data)]
while self._obj.eof and self._obj.unused_data:
unused_data = self._obj.unused_data
self._obj = zstd.ZstdDecompressor().decompressobj()
data_parts.append(self._obj.decompress(unused_data))
return b"".join(data_parts)

def flush(self) -> bytes:
ret = self._obj.flush()
ret = self._obj.flush() # note: this is a no-op
if not self._obj.eof:
raise DecodeError("Zstandard data is incomplete")
return ret # type: ignore[no-any-return]
Expand Down
19 changes: 19 additions & 0 deletions test/test_response.py
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,25 @@ def test_decode_zstd(self) -> None:
r = HTTPResponse(fp, headers={"content-encoding": "zstd"})
assert r.data == b"foo"

@onlyZstd()
def test_decode_multiframe_zstd(self) -> None:
data = (
# Zstandard frame
zstd.compress(b"foo")
# skippable frame (must be ignored)
+ bytes.fromhex(
"50 2A 4D 18" # Magic_Number (little-endian)
"07 00 00 00" # Frame_Size (little-endian)
"00 00 00 00 00 00 00" # User_Data
)
# Zstandard frame
+ zstd.compress(b"bar")
)

fp = BytesIO(data)
r = HTTPResponse(fp, headers={"content-encoding": "zstd"})
assert r.data == b"foobar"

@onlyZstd()
def test_chunked_decoding_zstd(self) -> None:
data = zstd.compress(b"foobarbaz")
Expand Down

0 comments on commit aca0f01

Please sign in to comment.