Permalink
Browse files

bufferedreader: support brotli decompression

rewrite: handle Content-Encoding: br using brotli decompressor
setup: add brotlipy as dependency
  • Loading branch information...
1 parent 3b68ef6 commit 457a1a564c1ef0b6fa8f624fc7efef5208777cff @ikreymer committed Jun 15, 2016
Showing with 23 additions and 4 deletions.
  1. +9 −3 pywb/rewrite/rewrite_content.py
  2. +13 −1 pywb/utils/bufferedreaders.py
  3. +1 −0 setup.py
@@ -77,6 +77,7 @@ def _rewrite_headers(self, urlrewriter, rule, status_headers, stream,
def _check_encoding(self, rewritten_headers, stream, enc):
+ matched = False
if (rewritten_headers.
contains_removed_header('content-encoding', enc)):
@@ -87,8 +88,9 @@ def _check_encoding(self, rewritten_headers, stream, enc):
stream = DecompressingBufferedReader(stream, decomp_type=enc)
rewritten_headers.status_headers.remove_header('content-length')
+ matched = True
- return stream
+ return matched, stream
@@ -139,8 +141,12 @@ def rewrite_content(self, urlrewriter, status_headers, stream,
encoding = None
first_buff = b''
- stream = self._check_encoding(rewritten_headers, stream, 'gzip')
- stream = self._check_encoding(rewritten_headers, stream, 'deflate')
+ for decomp_type in BufferedReader.get_supported_decompressors():
+ matched, stream = self._check_encoding(rewritten_headers,
+ stream,
+ decomp_type)
+ if matched:
+ break
if mod == 'js_':
text_type, stream = self._resolve_text_type('js',
@@ -1,5 +1,6 @@
from io import BytesIO
import zlib
+import brotli
#=================================================================
@@ -17,6 +18,11 @@ def deflate_decompressor():
def deflate_decompressor_alt():
return zlib.decompressobj(-zlib.MAX_WBITS)
+def brotli_decompressor():
+ decomp = brotli.Decompressor()
+ decomp.unused_data = None
+ return decomp
+
#=================================================================
class BufferedReader(object):
@@ -40,7 +46,9 @@ class BufferedReader(object):
DECOMPRESSORS = {'gzip': gzip_decompressor,
'deflate': deflate_decompressor,
- 'deflate_alt': deflate_decompressor_alt}
+ 'deflate_alt': deflate_decompressor_alt,
+ 'br': brotli_decompressor
+ }
def __init__(self, stream, block_size=1024,
decomp_type=None,
@@ -181,6 +189,10 @@ def close(self):
self.stream.close()
self.stream = None
+ @classmethod
+ def get_supported_decompressors(cls):
+ return cls.DECOMPRESSORS.keys()
+
#=================================================================
class DecompressingBufferedReader(BufferedReader):
View
@@ -79,6 +79,7 @@ def run_tests(self):
'redis',
'jinja2',
'surt>=0.3.0',
+ 'brotlipy',
'pyyaml',
'watchdog',
'webencodings',

0 comments on commit 457a1a5

Please sign in to comment.