Encoding autodetection

.gitignore

@@ -14,3 +14,4 @@ Pip*
 .env
 venv
 .idea
+*.ignore.py

diffengine/__init__.py

@@ -24,6 +24,7 @@
 import unicodedata
 
 from diffengine.sendgrid import SendgridHandler
+from diffengine.text import to_utf8
 from diffengine.twitter import TwitterHandler
 
 from exceptions.webdriver import UnknownWebdriverError
@@ -173,8 +174,7 @@ def get_latest(self):
         if resp.status_code != 200:
             logging.warn("Got %s when fetching %s", resp.status_code, self.url)
             return None
-
-        doc = readability.Document(resp.text)
+        doc = readability.Document(to_utf8(resp.text))
         title = doc.title()
         summary = doc.summary(html_partial=True)
         summary = bleach.clean(summary, tags=["p"], strip=True)

diffengine/text_builder.py → diffengine/text.py

@@ -58,3 +58,17 @@ def build_with_default_content(diff):
         text = text[0:225] + "…"
     text += " " + diff.url
     return text
+
+
+def to_utf8(text):
+    for encoding in ["latin1", "ascii"]:
+        try:
+            result = text.encode(encoding).decode("utf8", "strict")
+            break
+        except (UnicodeEncodeError, UnicodeDecodeError):
+            result = None
+
+    if result is None:
+        return text
+
+    return result

diffengine/twitter.py

@@ -3,7 +3,7 @@
 
 from datetime import datetime
 
-from diffengine.text_builder import build_text
+from diffengine.text import build_text
 from exceptions.twitter import (
     AlreadyTweetedError,
     TwitterConfigNotFoundError,

test_diffengine.py

@@ -29,7 +29,7 @@
     SendgridHandler,
     _fingerprint,
 )
-from diffengine.text_builder import build_text
+from diffengine.text import build_text, to_utf8
 from diffengine.utils import generate_config
 from exceptions.sendgrid import (
     SendgridConfigNotFoundError,
@@ -663,8 +663,8 @@ def get_mocked_diff(with_archive_urls=True):
 
 class TextBuilderTest(TestCase):
     @patch("logging.warning")
-    @patch("diffengine.text_builder.build_with_lang")
-    @patch("diffengine.text_builder.build_with_default_content")
+    @patch("diffengine.text.build_with_lang")
+    @patch("diffengine.text.build_with_default_content")
     def test_build_with_default_content_when_no_lang_given(
         self, mocked_build_with_default_content, mocked_build_from_lang, mocked_warning
     ):
@@ -679,8 +679,8 @@ def test_build_with_default_content_when_no_lang_given(
         mocked_build_from_lang.assert_not_called()
 
     @patch("logging.warning")
-    @patch("diffengine.text_builder.build_with_lang")
-    @patch("diffengine.text_builder.build_with_default_content")
+    @patch("diffengine.text.build_with_lang")
+    @patch("diffengine.text.build_with_default_content")
     def test_build_with_default_content_when_lang_is_incomplete(
         self, mocked_build_with_default_content, mocked_build_from_lang, mocked_warning
     ):
@@ -700,8 +700,8 @@ def test_build_with_default_content_when_lang_is_incomplete(
         mocked_build_from_lang.assert_not_called()
 
     @patch("logging.warning")
-    @patch("diffengine.text_builder.build_with_lang")
-    @patch("diffengine.text_builder.build_with_default_content")
+    @patch("diffengine.text.build_with_lang")
+    @patch("diffengine.text.build_with_default_content")
     def test_build_with_lang_when_lang_given(
         self, mocked_build_with_default_content, mocked_build_from_lang, mocked_warning
     ):
@@ -722,7 +722,7 @@ def test_build_with_lang_when_lang_given(
         mocked_build_with_default_content.assert_not_called()
         mocked_build_from_lang.assert_called_once()
 
-    @patch("diffengine.text_builder.build_with_lang")
+    @patch("diffengine.text.build_with_lang")
     def test_default_content_text(self, mocked_build_from_lang):
         diff = get_mocked_diff()
         type(diff.new).title = "Test"
@@ -804,3 +804,16 @@ def test_lang_content_text(self):
         self.assertEqual(
             text, "change in the URL, the title and the summary\n%s" % diff.url
         )
+
+
+class EncodingTest(TestCase):
+    def test_utf8_do_nothingg(self):
+        text_utf8 = "Me preocupa más la parte futbolística"
+        result = to_utf8(text_utf8)
+        self.assertEquals(result, text_utf8)
+
+    def test_latin1_to_utf8(self):
+        text_latin = "Me preocupa más la parte futbolística"
+        text_utf8 = "Me preocupa más la parte futbolística"
+        result = to_utf8(text_latin)
+        self.assertEquals(result, text_utf8)