From 5187f7aef64939cfbce9330007747a417f6ddff3 Mon Sep 17 00:00:00 2001 From: pjanse Date: Fri, 5 Mar 2021 20:07:27 +0100 Subject: [PATCH] feat: Warn about br handling if brotlipy is not installed #4697 --- scrapy/downloadermiddlewares/httpcompression.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f504302e207..2c8480d0f64 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,5 +1,7 @@ import io import zlib +import logging +logger = logging.getLogger(__name__) from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse @@ -13,6 +15,8 @@ import brotli ACCEPTED_ENCODINGS.append(b'br') except ImportError: + msg = ("brotli is not installed") + logger.warning(msg) pass try: @@ -72,8 +76,11 @@ def _decode(self, body, encoding): # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx # http://www.gzip.org/zlib/zlib_faq.html#faq38 body = zlib.decompress(body, -15) - if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: - body = brotli.decompress(body) + if encoding == b'br': + if b'br' in ACCEPTED_ENCODINGS: + body = brotli.decompress(body) + else: + raise ImportError('brotlipy is not installed') if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS: # Using its streaming API since its simple API could handle only cases # where there is content size data embedded in the frame