Fixed #5494, #10765, #14924 -- Modified the order in which translatio…

…ns are read when composing the final translation to offer at runtime.

This is slightly backward-incompatible (could result in changed final translations for literals appearing multiple times in different .po files but with different translations).

Translations are now read in the following order (from lower to higher priority):

For the 'django' gettext domain:

 * Django translations
 * INSTALLED_APPS apps translations (with the ones listed first having higher priority)
 * settings/project path translations (deprecated, see below)
 * LOCALE_PATHS translations (with the ones listed first having higher priority)

For the 'djangojs' gettext domain:

 * Python modules whose names are passed to the javascript_catalog view
 * LOCALE_PATHS translations (with the ones listed first having higher priority, previously they weren't included)

Also, automatic loading of translations from the 'locale' subdir of the settings/project path is now deprecated.

Thanks to vanschelven, vbmendes and an anonymous user for reporting issues, to vanschelven, Claude Paroz and an anonymous contributor for their initial work on fixes and to Jannis  Leidel and Claude for review and discussion.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@15441 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/utils/translation/__init__.py

@@ -1,8 +1,11 @@
 """
 Internationalization support.
 """
+from os import path
+
 from django.utils.encoding import force_unicode
-from django.utils.functional import lazy, curry
+from django.utils.functional import lazy
+from django.utils.importlib import import_module
 
 
 __all__ = ['gettext', 'gettext_noop', 'gettext_lazy', 'ngettext',
@@ -33,10 +36,22 @@ class Trans(object):
     performance effect, as access to the function goes the normal path,
     instead of using __getattr__.
     """
+
     def __getattr__(self, real_name):
         from django.conf import settings
         if settings.USE_I18N:
             from django.utils.translation import trans_real as trans
+
+            if settings.SETTINGS_MODULE is not None:
+                import warnings
+                parts = settings.SETTINGS_MODULE.split('.')
+                project = import_module(parts[0])
+                if path.isdir(path.join(path.dirname(project.__file__), 'locale')):
+                    warnings.warn(
+                        "Translations in the project directory aren't supported anymore. Use the LOCALE_PATHS setting instead.",
+                        PendingDeprecationWarning
+                    )
+
         else:
             from django.utils.translation import trans_null as trans
         setattr(self, real_name, getattr(trans, real_name))

django/utils/translation/trans_real.py

@@ -125,12 +125,12 @@ def _fetch(lang, fallback=None):
 
         global _translations
 
-        loc = to_locale(lang)
-
         res = _translations.get(lang, None)
         if res is not None:
             return res
 
+        loc = to_locale(lang)
+
         def _translation(path):
             try:
                 t = gettext_module.translation('django', path, [loc], DjangoTranslation)
@@ -159,11 +159,7 @@ def _merge(path):
                     res.merge(t)
             return res
 
-        for localepath in settings.LOCALE_PATHS:
-            if os.path.isdir(localepath):
-                res = _merge(localepath)
-
-        for appname in settings.INSTALLED_APPS:
+        for appname in reversed(settings.INSTALLED_APPS):
             app = import_module(appname)
             apppath = os.path.join(os.path.dirname(app.__file__), 'locale')
 
@@ -173,6 +169,10 @@ def _merge(path):
         if projectpath and os.path.isdir(projectpath):
             res = _merge(projectpath)
 
+        for localepath in reversed(settings.LOCALE_PATHS):
+            if os.path.isdir(localepath):
+                res = _merge(localepath)
+
         if res is None:
             if fallback is not None:
                 res = fallback

django/views/i18n.py

@@ -193,11 +193,15 @@ def javascript_catalog(request, domain='djangojs', packages=None):
     paths = []
     en_selected = locale.startswith('en')
     en_catalog_missing = True
-    # first load all english languages files for defaults
+    # paths of requested packages
     for package in packages:
         p = importlib.import_module(package)
         path = os.path.join(os.path.dirname(p.__file__), 'locale')
         paths.append(path)
+    # add the filesystem paths listed in the LOCALE_PATHS setting
+    paths.extend(list(reversed(settings.LOCALE_PATHS)))
+    # first load all english languages files for defaults
+    for path in paths:
         try:
             catalog = gettext_module.translation(domain, path, ['en'])
             t.update(catalog._catalog)
@@ -275,4 +279,3 @@ def javascript_catalog(request, domain='djangojs', packages=None):
     src.append(LibFormatFoot)
     src = ''.join(src)
     return http.HttpResponse(src, 'text/javascript')
-

docs/howto/i18n.txt

@@ -4,24 +4,45 @@
 Using internationalization in your own projects
 ===============================================
 
-At runtime, Django looks for translations by following this algorithm:
-
-    * First, it looks for a ``locale`` directory in the directory containing
-      your settings file.
-    * Second, it looks for a ``locale`` directory in the project directory.
-    * Third, it looks for a ``locale`` directory in each of the installed apps.
-      It does this in the reverse order of INSTALLED_APPS
-    * Finally, it checks the Django-provided base translation in
-      ``django/conf/locale``.
+At runtime, Django builds an in-memory unified catalog of literals-translations.
+To achieve this it looks for translations by following this algorithm regarding
+the order in which it examines the different file paths to load the compiled
+:term:`message files <message file>` (``.mo``) and the precedence of multiple
+translations for the same literal:
+
+    1. The directories listed in :setting:`LOCALE_PATHS` have the highest
+       precedence, with the ones appearing first having higher precedence than
+       the ones appearing later.
+    2. Then, it looks for and uses if it exists a ``locale`` directory in each
+       of the installed apps listed in :setting:`INSTALLED_APPS`.  The ones
+       appearing first have higher precedence than the ones appearing later.
+    3. Then, it looks for a ``locale`` directory in the project directory, or
+       more accurately, in the directory containing your settings file.
+    4. Finally, the Django-provided base translation in ``django/conf/locale``
+       is used as a fallback.
+
+.. deprecated:: 1.3
+    Lookup in the ``locale`` subdirectory of the directory containing your
+    settings file (item 3 above) is deprecated since the 1.3 release and will be
+    removed in Django 1.5. You can use the :setting:`LOCALE_PATHS` setting
+    instead, by listing the absolute filesystem path of such ``locale``
+    directory in the setting value.
+
+.. seealso::
+
+    The translations for literals included in JavaScript assets are looked up
+    following a similar but not identical algorithm. See the
+    :ref:`javascript_catalog view documentation <javascript_catalog-view>` for
+    more details.
 
 In all cases the name of the directory containing the translation is expected to
 be named using :term:`locale name` notation. E.g. ``de``, ``pt_BR``, ``es_AR``,
 etc.
 
 This way, you can write applications that include their own translations, and
 you can override base translations in your project path. Or, you can just build
-a big project out of several apps and put all translations into one big project
-message file. The choice is yours.
+a big project out of several apps and put all translations into one big common
+message file specific to the project you are composing. The choice is yours.
 
 .. note::
 
@@ -34,10 +55,11 @@ message file. The choice is yours.
 
 All message file repositories are structured the same way. They are:
 
-    * ``$APPPATH/locale/<language>/LC_MESSAGES/django.(po|mo)``
-    * ``$PROJECTPATH/locale/<language>/LC_MESSAGES/django.(po|mo)``
     * All paths listed in ``LOCALE_PATHS`` in your settings file are
-      searched in that order for ``<language>/LC_MESSAGES/django.(po|mo)``
+      searched for ``<language>/LC_MESSAGES/django.(po|mo)``
+    * ``$PROJECTPATH/locale/<language>/LC_MESSAGES/django.(po|mo)`` --
+      deprecated, see above.
+    * ``$APPPATH/locale/<language>/LC_MESSAGES/django.(po|mo)``
     * ``$PYTHONPATH/django/conf/locale/<language>/LC_MESSAGES/django.(po|mo)``
 
 To create message files, you use the :djadmin:`django-admin.py makemessages <makemessages>`
@@ -50,22 +72,18 @@ You can also run ``django-admin.py compilemessages --settings=path.to.settings``
 to make the compiler process all the directories in your :setting:`LOCALE_PATHS`
 setting.
 
-Application message files are a bit complicated to discover -- they need the
-:class:`~django.middleware.locale.LocaleMiddleware`. If you don't use the
-middleware, only the Django message files and project message files will be
-installed and available at runtime.
-
 Finally, you should give some thought to the structure of your translation
 files. If your applications need to be delivered to other users and will
 be used in other projects, you might want to use app-specific translations.
-But using app-specific translations and project translations could produce
-weird problems with ``makemessages``: It will traverse all directories below
-the current path and so might put message IDs into the project message file
-that are already in application message files.
+But using app-specific translations and project-specific translations could
+produce weird problems with ``makemessages``: It will traverse all directories
+below the current path and so might put message IDs into a unified, common
+message file for the current project that are already in application message
+files.
 
 The easiest way out is to store applications that are not part of the project
 (and so carry their own translations) outside the project tree. That way,
-``django-admin.py makemessages`` on the project level will only translate
+``django-admin.py makemessages``, when ran on a project level will only extract
 strings that are connected to your explicit project and not strings that are
 distributed independently.
 

docs/ref/settings.txt

@@ -1149,6 +1149,17 @@ Default: ``()`` (Empty tuple)
 A tuple of directories where Django looks for translation files.
 See :ref:`using-translations-in-your-own-projects`.
 
+Example::
+
+    LOCALE_PATHS = (
+        '/home/www/project/common_files/locale',
+        '/var/local/translations/locale'
+    )
+
+Note that in the paths you add to the value of this setting, if you have the
+typical ``/path/to/locale/xx/LC_MESSAGES`` hierarchy, you should use the path to
+the ``locale`` directory (i.e. ``'/path/to/locale'``).
+
 .. setting:: LOGGING
 
 LOGGING

docs/releases/1.3.txt

@@ -454,6 +454,52 @@ should either insert it using :ref:`test fixtures
 <topics-testing-fixtures>`, or using the ``setUp()`` method of your
 test case.
 
+Changed priority of translation loading
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Work has been done to homogeneize, simplify, rationalize and properly document
+the algorithm used by Django at runtime to build translations from the
+differents translations found on disk, namely:
+
+For translatable literals found in Python code and templates (``'django'``
+gettext domain):
+
+ * Priorities of translations included with applications listed in the
+   :setting:`INSTALLED_APPS` setting were changed. To provide a behavior
+   consistent with other parts of Django that also use such setting (templates,
+   etc.) now, when building the translation that will be made available, the
+   apps listed first have higher precedence than the ones listed later.
+
+ * Now it is possible to override the translations shipped with applications by
+   using the :setting:`LOCALE_PATHS` setting whose translations have now higher
+   precedence than the translations of ``INSTALLED_APPS`` applications.
+   The relative priority among the values listed in this setting has also been
+   modified so the paths listed first have higher precedence than the
+   ones listed later.
+
+ * The ``locale`` subdirectory of the directory containing the settings, that
+   usually coincides with and is know as the *project directory* is being
+   deprecated in this release as a source of translations. (the precedence of
+   these translations is intermediate between applications and ``LOCALE_PATHS``
+   translations). See the `corresponding deprecated features section`_
+   of this document.
+
+For translatable literals found in Javascript code (``'djangojs'`` gettext
+domain):
+
+ * Similarly to the ``'django'`` domain translations: Overriding of
+   translations shipped with applications by using the :setting:`LOCALE_PATHS`
+   setting is now possible for this domain too. These translations have higher
+   precedence than the translations of Python packages passed to the
+   :ref:`javascript_catalog view <javascript_catalog-view>`.  Paths listed first
+   have higher precedence than the ones listed later.
+
+ * Translations under the ``locale`` sbdirectory of the *project directory* have
+   never been taken in account for JavaScript translations and remain in the
+   same situation considering the deprecation of such location.
+
+.. _corresponding deprecated features section: loading_of_translations_from_the_project_directory_
+
 .. _deprecated-features-1.3:
 
 Features deprecated in 1.3
@@ -631,3 +677,38 @@ Previously, ``django.http`` exposed an undocumented ``CompatCookie`` class,
 which was a bug-fix wrapper around the standard library ``SimpleCookie``. As the
 fixes are moving upstream, this is now deprecated - you should use ``from
 django.http import SimpleCookie`` instead.
+
+.. _loading_of_translations_from_the_project_directory:
+
+Loading of translations from the project directory
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This release of Django starts the deprecation process for inclusion of
+translations located under the *project path* in the translation building
+process performed at runtime. The :setting:`LOCALE_PATHS` setting can be used
+for the same task by including in it the filesystem path to the ``locale``
+directory containing project-level translations.
+
+Rationale for this decision:
+
+ * The *project path* has always been a loosely defined concept (actually, the
+   directory used for locating project-level translations is the directory
+   containing the settings module) and there has been a shift in other parts
+   of the framework to stop using it as a reference for location of assets at
+   runtime.
+
+ * Detection of the ``locale`` subdirectory tends to fail when the deployment
+   scenario is more complex than the basic one. e.g. it fails when the settings
+   module is a directory (ticket #10765).
+
+ * Potential for strange development- and deployment-time problems like the
+   fact that the ``project_dir/locale/`` subdir can generate spurious error
+   messages when the project directory is included in the Python path (default
+   behavior of ``manage.py runserver``) and then it clashes with the equally
+   named standard library module, this is a typical warming message::
+
+     /usr/lib/python2.6/gettext.py:49: ImportWarning: Not importing directory '/path/to/project/dir/locale': missing __init__.py.
+     import locale, copy, os, re, struct, sys
+
+ * This location wasn't included in the translation building process for
+   JavaScript literals.

docs/topics/i18n/deployment.txt

@@ -171,16 +171,36 @@ in ``request.LANGUAGE_CODE``.
 How Django discovers translations
 ---------------------------------
 
-As described in :ref:`using-translations-in-your-own-projects`,
-at runtime, Django looks for translations by following this algorithm:
-
-    * First, it looks for a ``locale`` directory in the directory containing
-      your settings file.
-    * Second, it looks for a ``locale`` directory in the project directory.
-    * Third, it looks for a ``locale`` directory in each of the installed apps.
-      It does this in the reverse order of INSTALLED_APPS
-    * Finally, it checks the Django-provided base translation in
-      ``django/conf/locale``.
+As described in :ref:`using-translations-in-your-own-projects`, Django looks for
+translations by following this algorithm regarding the order in which it
+examines the different file paths to load the compiled :term:`message files
+<message file>` (``.mo``) and the precedence of multiple translations for the
+same literal:
+
+    1. The directories listed in :setting:`LOCALE_PATHS` have the highest
+       precedence, with the ones appearing first having higher precedence than
+       the ones appearing later.
+    2. Then, it looks for and uses if it exists a ``locale`` directory in each
+       of the installed apps listed in :setting:`INSTALLED_APPS`.  The ones
+       appearing first have higher precedence than the ones appearing later.
+    3. Then, it looks for a ``locale`` directory in the project directory, or
+       more accurately, in the directory containing your settings file.
+    4. Finally, the Django-provided base translation in ``django/conf/locale``
+       is used as a fallback.
+
+.. deprecated:: 1.3
+    Lookup in the ``locale`` subdirectory of the directory containing your
+    settings file (item 3 above) is deprecated since the 1.3 release and will be
+    removed in Django 1.5. You can use the :setting:`LOCALE_PATHS` setting
+    instead, by listing the absolute filesystem path of such ``locale``
+    directory in the setting value.
+
+.. seealso::
+
+    The translations for literals included in JavaScript assets are looked up
+    following a similar but not identical algorithm. See the
+    :ref:`javascript_catalog view documentation <javascript_catalog-view>` for
+    more details.
 
 In all cases the name of the directory containing the translation is expected to
 be named using :term:`locale name` notation. E.g. ``de``, ``pt_BR``, ``es_AR``,