cleanup

Signed-off-by: Kenneth Reitz <me@kennethreitz.org>
kennethreitz · Mar 21, 2018 · 122b42a · 122b42a
1 parent 30b59cc
commit 122b42a
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 8 deletions.
diff --git a/Pipfile b/Pipfile
@@ -1,23 +1,19 @@
 [[source]]
-
 url = "https://pypi.python.org/simple"
 verify_ssl = true
 name = "pypi"
 
-
 [packages]
-
 requests = "*"
 pyquery = "*"
 fake-useragent = "*"
 parse = "*"
 "bs4" = "*"
 "w3lib" = "*"
 pyppeteer = "*"
-
+"rfc3986" = "*"
 
 [dev-packages]
-
 twine = "*"
 requests-file = "*"
 pytest = "*"
@@ -27,7 +23,5 @@ mypy = "*"
 pytest-asyncio = "*"
 psutil = "*"
 
-
 [scripts]
-
 tests = "pytest -v -m ok"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/requests_html.py b/requests_html.py
@@ -126,6 +126,12 @@ def encoding(self) -> _Encoding:
         # Scan meta tags for charset.
         if self._html:
             self._encoding = html_to_unicode(self.default_encoding, self._html)[0]
+            # Fall back to requests' detected encoding if decode fails.
+            try:
+                self.raw_html.decode(self.encoding)
+            except UnicodeDecodeError:
+                self._encoding = self.default_encoding
+
 
         return self._encoding if self._encoding else self.default_encoding