Don't fail on decoding errors.

fix bug 775264

lib/l10n_utils/dotlang.py

@@ -1,3 +1,5 @@
+# coding=utf-8
+
 """This library parses dotlang files migrated over from the old PHP
 system.
 
@@ -21,10 +23,13 @@ def parse(path):
     if not os.path.exists(path):
         return trans
 
-    with codecs.open(path, 'r', 'utf-8') as lines:
+    with codecs.open(path, 'r', 'utf-8', errors='replace') as lines:
         source = None
 
         for line in lines:
+            if u'�' in line:
+                mail_error(line, path)
+
             line = line.strip()
             if line == '' or line[0] == '#':
                 continue
@@ -40,6 +45,14 @@ def parse(path):
     return trans
 
 
+def mail_error(line, path):
+    """Email admins when a decoding error happened"""
+    from django.core import mail
+    subject = '%s is corrupted' % path
+    message = line
+    mail.mail_admins(subject, message)
+
+
 def fix_case(locale):
     """Convert lowercase locales to uppercase: en-us -> en-US"""
     parts = locale.split('-')

lib/l10n_utils/tests/test_dotlang.py

@@ -1,14 +1,20 @@
+# coding=utf-8
+
 import os
 import unittest
 
 from nose.tools import eq_
 
+from django.core import mail
 from l10n_utils.dotlang import parse
 
 
 class TestDotlang(unittest.TestCase):
+    def setUp(self):
+        self.path = os.path.dirname(os.path.abspath(__file__))
+
     def test_parse(self):
-        path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'test.lang')
+        path = os.path.join(self.path, 'test.lang')
         parsed = parse(path)
         expected = {
             u'Hooray! Your Firefox is up to date.':
@@ -18,4 +24,16 @@ def test_parse(self):
                 u'Votre Firefox ne semble pas \xe0 jour.'
         }
 
-        eq_(parsed,expected)
+        eq_(parsed, expected)
+
+    def test_parse_utf8_error(self):
+        path = os.path.join(self.path, 'test_utf8_error.lang')
+        parsed = parse(path)
+        eq_(len(mail.outbox), 1)
+        eq_(mail.outbox[0].subject, '[Django] %s is corrupted' % path)
+        expected = {
+            u'Update now': u'Niha rojane bike',
+            u'Supported Devices': u'C�haz�n pi�tgiriy'
+        }
+        eq_(parsed, expected)
+        mail.outbox = []

lib/l10n_utils/tests/test_utf8_error.lang

@@ -0,0 +1,6 @@
+;Update now
+Niha rojane bike
+
+
+;Supported Devices
+C�haz�n pi�tgiriy�