Apply the same tweaks during translation as extraction.

Fix bug 763913.

lib/l10n_utils/dotlang.py

@@ -16,6 +16,9 @@
 from django.utils import translation
 from django.utils.functional import lazy
 
+from tower.management.commands.extract import tweak_message
+
+
 FORMAT_IDENTIFIER_RE = re.compile(r"""(%
                                       (?:\((\w+)\))? # Mapping key
                                       s)""", re.VERBOSE)
@@ -70,6 +73,8 @@ def translate(text, files):
     """Search a list of .lang files for a translation"""
     lang = fix_case(translation.get_language())
 
+    tweaked_text = tweak_message(text)
+
     for file_ in files:
         key = "dotlang-%s-%s" % (lang, file_)
 
@@ -80,14 +85,14 @@ def translate(text, files):
             trans = parse(path)
             cache.set(key, trans, settings.DOTLANG_CACHE)
 
-        if text in trans:
+        if tweaked_text in trans:
             original = FORMAT_IDENTIFIER_RE.findall(text)
-            translated = FORMAT_IDENTIFIER_RE.findall(trans[text])
+            translated = FORMAT_IDENTIFIER_RE.findall(trans[tweaked_text])
             if original != translated:
-                message = '%s\n%s' % (text, trans[text])
+                message = '%s\n%s' % (text, trans[tweaked_text])
                 mail_error(file_, message)
                 return text
-            return trans[text]
+            return trans[tweaked_text]
     return text
 
 
@@ -98,8 +103,10 @@ def _(text, *args):
         text = text % args
     return text
 
+
 _lazy = lazy(_, unicode)
 
+
 def get_lang_path(path):
     """Generate the path to a lang file from a django path.
     /apps/foo/templates/foo/bar.html -> /foo/bar.lang

lib/l10n_utils/tests/locale/en-US/tweaked_message_translation.lang

@@ -0,0 +1,2 @@
+;Stuff about many things.
+This is the translation.

lib/l10n_utils/tests/test_dotlang.py

@@ -3,11 +3,13 @@
 import os
 import unittest
 
+from django.conf import settings
+from django.core import mail
+
 from mock import patch
 from nose.tools import eq_
+from tower.management.commands.extract import extract_tower_python
 
-from django.conf import settings
-from django.core import mail
 from l10n_utils.dotlang import FORMAT_IDENTIFIER_RE, parse, translate
 
 ROOT = os.path.dirname(os.path.abspath(__file__))
@@ -54,3 +56,26 @@ def test_format_identifier_mismatch(self):
         eq_(len(mail.outbox), 1)
         eq_(mail.outbox[0].subject, '[Django] %s is corrupted' % path)
         mail.outbox = []
+
+    @patch.object(settings, 'ROOT', ROOT)
+    def test_extract_message_tweaks_do_not_break(self):
+        """
+        Extraction and translation matching should tweak msgids the same.
+        """
+        clean_string = u'Stuff about many things.'
+        dirty_string = u'Stuff\xa0about\r\nmany\t   things.'
+        trans_string = u'This is the translation.'
+
+        # extraction
+        with open(os.path.join(ROOT, 'test_py_extract.py.txt')) as pyfile:
+            vals = extract_tower_python(pyfile, {'_', None}, [], {}).next()
+        eq_(vals[2], clean_string)
+
+        # translation
+        # path won't exist for en-US as there isn't a dir for that
+        # in locale.
+        result = translate(dirty_string, ['does_not_exist'])
+        eq_(result, dirty_string)
+
+        result = translate(dirty_string, ['tweaked_message_translation'])
+        eq_(result, trans_string)

lib/l10n_utils/tests/test_py_extract.py.txt

@@ -0,0 +1,5 @@
+# coding: utf-8
+
+from l10n_utils.dotlang import _
+
+mystring = _(u'Stuff\xa0about\r\nmany\t   things.')

settings/base.py

@@ -44,16 +44,18 @@
     path('locale')
 )
 
-JINJA_CONFIG = {
-    'extensions': [
-        'tower.template.i18n', 'jinja2.ext.do', 'jinja2.ext.with_',
-        'jinja2.ext.loopcontrols', 'l10n_utils.template.l10n_blocks',
-        'l10n_utils.template.lang_blocks'
-    ],
-    # Make None in templates render as ''
-    'finalize': lambda x: x if x is not None else '',
-    'auto_reload': True,
-}
+# has to stay a callable because tower expects that.
+def JINJA_CONFIG():
+    return {
+        'extensions': [
+            'tower.template.i18n', 'jinja2.ext.do', 'jinja2.ext.with_',
+            'jinja2.ext.loopcontrols', 'l10n_utils.template.l10n_blocks',
+            'l10n_utils.template.lang_blocks'
+        ],
+        # Make None in templates render as ''
+        'finalize': lambda x: x if x is not None else '',
+        'auto_reload': True,
+    }
 
 # Bundles is a dictionary of two dictionaries, css and js, which list css files
 # and js files that can be bundled together by the minify app.
@@ -407,7 +409,7 @@
 
     # libs
     'l10n_utils',
-    'captcha'
+    'captcha',
 )
 
 TEMPLATE_CONTEXT_PROCESSORS += (

settings/local.py-dist

@@ -1,7 +1,10 @@
 # This is an example settings_local.py file.
 # Copy it and add your local settings here.
 
-from settings import *
+from settings.base import *
+
+ADMINS = ('foo@bar.com',)
+MANAGERS = ADMINS
 
 # No database yet
 # DATABASES = {
@@ -30,4 +33,4 @@ EMAIL_USE_TLS = False
 STATSD_HOST = ''
 STATSD_PORT = 8125
 STATSD_PREFIX = 'bedrock'
-STATSD_CLIENT = 'django_statsd.clients.normal'
+STATSD_CLIENT = 'django_statsd.clients.normal'