Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Enabled makemessages to support several translation directories

Thanks Rémy Hubscher, Ramiro Morales, Unai Zalakain and
Tim Graham for the reviews.
Also fixes #16084.
  • Loading branch information...
commit 50a8ab7cd1e611e6422a148becaec02218577d67 1 parent 9af7e18
Claude Paroz authored
158  django/core/management/commands/makemessages.py
@@ -29,25 +29,28 @@ def check_programs(*programs):
29 29
 
30 30
 @total_ordering
31 31
 class TranslatableFile(object):
32  
-    def __init__(self, dirpath, file_name):
  32
+    def __init__(self, dirpath, file_name, locale_dir):
33 33
         self.file = file_name
34 34
         self.dirpath = dirpath
  35
+        self.locale_dir = locale_dir
35 36
 
36 37
     def __repr__(self):
37 38
         return "<TranslatableFile: %s>" % os.sep.join([self.dirpath, self.file])
38 39
 
39 40
     def __eq__(self, other):
40  
-        return self.dirpath == other.dirpath and self.file == other.file
  41
+        return self.path == other.path
41 42
 
42 43
     def __lt__(self, other):
43  
-        if self.dirpath == other.dirpath:
44  
-            return self.file < other.file
45  
-        return self.dirpath < other.dirpath
  44
+        return self.path < other.path
46 45
 
47  
-    def process(self, command, potfile, domain, keep_pot=False):
  46
+    @property
  47
+    def path(self):
  48
+        return os.path.join(self.dirpath, self.file)
  49
+
  50
+    def process(self, command, domain):
48 51
         """
49  
-        Extract translatable literals from self.file for :param domain:
50  
-        creating or updating the :param potfile: POT file.
  52
+        Extract translatable literals from self.file for :param domain:,
  53
+        creating or updating the POT file.
51 54
 
52 55
         Uses the xgettext GNU gettext utility.
53 56
         """
@@ -127,8 +130,6 @@ def process(self, command, potfile, domain, keep_pot=False):
127 130
             if status != STATUS_OK:
128 131
                 if is_templatized:
129 132
                     os.unlink(work_file)
130  
-                if not keep_pot and os.path.exists(potfile):
131  
-                    os.unlink(potfile)
132 133
                 raise CommandError(
133 134
                     "errors happened while running xgettext on %s\n%s" %
134 135
                     (self.file, errors))
@@ -136,6 +137,8 @@ def process(self, command, potfile, domain, keep_pot=False):
136 137
                 # Print warnings
137 138
                 command.stdout.write(errors)
138 139
         if msgs:
  140
+            # Write/append messages to pot file
  141
+            potfile = os.path.join(self.locale_dir, '%s.pot' % str(domain))
139 142
             if is_templatized:
140 143
                 # Remove '.py' suffix
141 144
                 if os.name == 'nt':
@@ -147,6 +150,7 @@ def process(self, command, potfile, domain, keep_pot=False):
147 150
                     new = '#: ' + orig_file[2:]
148 151
                 msgs = msgs.replace(old, new)
149 152
             write_pot_file(potfile, msgs)
  153
+
150 154
         if is_templatized:
151 155
             os.unlink(work_file)
152 156
 
@@ -242,64 +246,94 @@ def handle_noargs(self, *args, **options):
242 246
                              % get_text_list(list(self.extensions), 'and'))
243 247
 
244 248
         self.invoked_for_django = False
  249
+        self.locale_paths = []
  250
+        self.default_locale_path = None
245 251
         if os.path.isdir(os.path.join('conf', 'locale')):
246  
-            localedir = os.path.abspath(os.path.join('conf', 'locale'))
  252
+            self.locale_paths = [os.path.abspath(os.path.join('conf', 'locale'))]
  253
+            self.default_locale_path = self.locale_paths[0]
247 254
             self.invoked_for_django = True
248 255
             # Ignoring all contrib apps
249 256
             self.ignore_patterns += ['contrib/*']
250  
-        elif os.path.isdir('locale'):
251  
-            localedir = os.path.abspath('locale')
252 257
         else:
253  
-            raise CommandError("This script should be run from the Django Git "
254  
-                    "tree or your project or app tree. If you did indeed run it "
255  
-                    "from the Git checkout or your project or application, "
256  
-                    "maybe you are just missing the conf/locale (in the django "
257  
-                    "tree) or locale (for project and application) directory? It "
258  
-                    "is not created automatically, you have to create it by hand "
259  
-                    "if you want to enable i18n for your project or application.")
260  
-
261  
-        check_programs('xgettext')
262  
-
263  
-        potfile = self.build_pot_file(localedir)
264  
-
265  
-        # Build po files for each selected locale
  258
+            self.locale_paths.extend(list(settings.LOCALE_PATHS))
  259
+            # Allow to run makemessages inside an app dir
  260
+            if os.path.isdir('locale'):
  261
+                self.locale_paths.append(os.path.abspath('locale'))
  262
+            if self.locale_paths:
  263
+                self.default_locale_path = self.locale_paths[0]
  264
+                if not os.path.exists(self.default_locale_path):
  265
+                    os.makedirs(self.default_locale_path)
  266
+
  267
+        # Build locale list
266 268
         locales = []
267 269
         if locale is not None:
268 270
             locales = locale
269 271
         elif process_all:
270  
-            locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir))
  272
+            locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % self.default_locale_path))
271 273
             locales = [os.path.basename(l) for l in locale_dirs]
272  
-
273 274
         if locales:
274 275
             check_programs('msguniq', 'msgmerge', 'msgattrib')
275 276
 
  277
+        check_programs('xgettext')
  278
+
276 279
         try:
  280
+            potfiles = self.build_potfiles()
  281
+
  282
+            # Build po files for each selected locale
277 283
             for locale in locales:
278 284
                 if self.verbosity > 0:
279 285
                     self.stdout.write("processing locale %s\n" % locale)
280  
-                self.write_po_file(potfile, locale)
  286
+                for potfile in potfiles:
  287
+                    self.write_po_file(potfile, locale)
281 288
         finally:
282  
-            if not self.keep_pot and os.path.exists(potfile):
283  
-                os.unlink(potfile)
  289
+            if not self.keep_pot:
  290
+                self.remove_potfiles()
284 291
 
285  
-    def build_pot_file(self, localedir):
  292
+    def build_potfiles(self):
  293
+        """
  294
+        Build pot files and apply msguniq to them.
  295
+        """
286 296
         file_list = self.find_files(".")
287  
-
288  
-        potfile = os.path.join(localedir, '%s.pot' % str(self.domain))
289  
-        if os.path.exists(potfile):
290  
-            # Remove a previous undeleted potfile, if any
291  
-            os.unlink(potfile)
292  
-
  297
+        self.remove_potfiles()
293 298
         for f in file_list:
294 299
             try:
295  
-                f.process(self, potfile, self.domain, self.keep_pot)
  300
+                f.process(self, self.domain)
296 301
             except UnicodeDecodeError:
297 302
                 self.stdout.write("UnicodeDecodeError: skipped file %s in %s" % (f.file, f.dirpath))
298  
-        return potfile
  303
+
  304
+        potfiles = []
  305
+        for path in self.locale_paths:
  306
+            potfile = os.path.join(path, '%s.pot' % str(self.domain))
  307
+            if not os.path.exists(potfile):
  308
+                continue
  309
+            args = ['msguniq', '--to-code=utf-8']
  310
+            if self.wrap:
  311
+                args.append(self.wrap)
  312
+            if self.location:
  313
+                args.append(self.location)
  314
+            args.append(potfile)
  315
+            msgs, errors, status = popen_wrapper(args)
  316
+            if errors:
  317
+                if status != STATUS_OK:
  318
+                    raise CommandError(
  319
+                        "errors happened while running msguniq\n%s" % errors)
  320
+                elif self.verbosity > 0:
  321
+                    self.stdout.write(errors)
  322
+            with open(potfile, 'w') as fp:
  323
+                fp.write(msgs)
  324
+            potfiles.append(potfile)
  325
+        return potfiles
  326
+
  327
+    def remove_potfiles(self):
  328
+        for path in self.locale_paths:
  329
+            pot_path = os.path.join(path, '%s.pot' % str(self.domain))
  330
+            if os.path.exists(pot_path):
  331
+                os.unlink(pot_path)
299 332
 
300 333
     def find_files(self, root):
301 334
         """
302  
-        Helper method to get all files in the given root.
  335
+        Helper method to get all files in the given root. Also check that there
  336
+        is a matching locale dir for each file.
303 337
         """
304 338
 
305 339
         def is_ignored(path, ignore_patterns):
@@ -319,12 +353,26 @@ def is_ignored(path, ignore_patterns):
319 353
                     dirnames.remove(dirname)
320 354
                     if self.verbosity > 1:
321 355
                         self.stdout.write('ignoring directory %s\n' % dirname)
  356
+                elif dirname == 'locale':
  357
+                    dirnames.remove(dirname)
  358
+                    self.locale_paths.insert(0, os.path.join(os.path.abspath(dirpath), dirname))
322 359
             for filename in filenames:
323  
-                if is_ignored(os.path.normpath(os.path.join(dirpath, filename)), self.ignore_patterns):
  360
+                file_path = os.path.normpath(os.path.join(dirpath, filename))
  361
+                if is_ignored(file_path, self.ignore_patterns):
324 362
                     if self.verbosity > 1:
325 363
                         self.stdout.write('ignoring file %s in %s\n' % (filename, dirpath))
326 364
                 else:
327  
-                    all_files.append(TranslatableFile(dirpath, filename))
  365
+                    locale_dir = None
  366
+                    for path in self.locale_paths:
  367
+                        if os.path.abspath(dirpath).startswith(os.path.dirname(path)):
  368
+                            locale_dir = path
  369
+                            break
  370
+                    if not locale_dir:
  371
+                        locale_dir = self.default_locale_path
  372
+                    if not locale_dir:
  373
+                        raise CommandError(
  374
+                            "Unable to find a locale path to store translations for file %s" % file_path)
  375
+                    all_files.append(TranslatableFile(dirpath, filename, locale_dir))
328 376
         return sorted(all_files)
329 377
 
330 378
     def write_po_file(self, potfile, locale):
@@ -332,30 +380,14 @@ def write_po_file(self, potfile, locale):
332 380
         Creates or updates the PO file for self.domain and :param locale:.
333 381
         Uses contents of the existing :param potfile:.
334 382
 
335  
-        Uses mguniq, msgmerge, and msgattrib GNU gettext utilities.
  383
+        Uses msgmerge, and msgattrib GNU gettext utilities.
336 384
         """
337  
-        args = ['msguniq', '--to-code=utf-8']
338  
-        if self.wrap:
339  
-            args.append(self.wrap)
340  
-        if self.location:
341  
-            args.append(self.location)
342  
-        args.append(potfile)
343  
-        msgs, errors, status = popen_wrapper(args)
344  
-        if errors:
345  
-            if status != STATUS_OK:
346  
-                raise CommandError(
347  
-                    "errors happened while running msguniq\n%s" % errors)
348  
-            elif self.verbosity > 0:
349  
-                self.stdout.write(errors)
350  
-
351 385
         basedir = os.path.join(os.path.dirname(potfile), locale, 'LC_MESSAGES')
352 386
         if not os.path.isdir(basedir):
353 387
             os.makedirs(basedir)
354 388
         pofile = os.path.join(basedir, '%s.po' % str(self.domain))
355 389
 
356 390
         if os.path.exists(pofile):
357  
-            with open(potfile, 'w') as fp:
358  
-                fp.write(msgs)
359 391
             args = ['msgmerge', '-q']
360 392
             if self.wrap:
361 393
                 args.append(self.wrap)
@@ -369,8 +401,10 @@ def write_po_file(self, potfile, locale):
369 401
                         "errors happened while running msgmerge\n%s" % errors)
370 402
                 elif self.verbosity > 0:
371 403
                     self.stdout.write(errors)
372  
-        elif not self.invoked_for_django:
373  
-            msgs = self.copy_plural_forms(msgs, locale)
  404
+        else:
  405
+            msgs = open(potfile, 'r').read()
  406
+            if not self.invoked_for_django:
  407
+                msgs = self.copy_plural_forms(msgs, locale)
374 408
         msgs = msgs.replace(
375 409
             "#. #-#-#-#-#  %s.pot (PACKAGE VERSION)  #-#-#-#-#\n" % self.domain, "")
376 410
         with open(pofile, 'w') as fp:
3  docs/man/django-admin.1
@@ -192,7 +192,8 @@ Ignore files or directories matching this glob-style pattern. Use multiple
192 192
 times to ignore more (makemessages command).
193 193
 .TP
194 194
 .I \-\-no\-default\-ignore
195  
-Don't ignore the common private glob-style patterns 'CVS', '.*' and '*~' (makemessages command).
  195
+Don't ignore the common private glob-style patterns 'CVS', '.*', '*~' and '*.pyc'
  196
+(makemessages command).
196 197
 .TP
197 198
 .I \-\-no\-wrap
198 199
 Don't break long message lines into several lines (makemessages command).
4  docs/ref/django-admin.txt
@@ -557,7 +557,7 @@ Example usage::
557 557
 Use the ``--ignore`` or ``-i`` option to ignore files or directories matching
558 558
 the given :mod:`glob`-style pattern. Use multiple times to ignore more.
559 559
 
560  
-These patterns are used by default: ``'CVS'``, ``'.*'``, ``'*~'``
  560
+These patterns are used by default: ``'CVS'``, ``'.*'``, ``'*~'``, ``'*.pyc'``
561 561
 
562 562
 Example usage::
563 563
 
@@ -584,7 +584,7 @@ for technically skilled translators to understand each message's context.
584 584
 .. versionadded:: 1.6
585 585
 
586 586
 Use the ``--keep-pot`` option to prevent Django from deleting the temporary
587  
-.pot file it generates before creating the .po file. This is useful for
  587
+.pot files it generates before creating the .po file. This is useful for
588 588
 debugging errors which may prevent the final language files from being created.
589 589
 
590 590
 makemigrations [<appname>]
5  docs/releases/1.7.txt
@@ -375,6 +375,11 @@ Internationalization
375 375
   in the corresponding entry in the PO file, which makes the translation
376 376
   process easier.
377 377
 
  378
+* When you run :djadmin:`makemessages` from the root directory of your project,
  379
+  any extracted strings will now be automatically distributed to the proper
  380
+  app or project message file. See :ref:`how-to-create-language-files` for
  381
+  details.
  382
+
378 383
 Management Commands
379 384
 ^^^^^^^^^^^^^^^^^^^
380 385
 
28  docs/topics/i18n/translation.txt
@@ -1256,6 +1256,17 @@ is configured correctly). It creates (or updates) a message file in the
1256 1256
 directory ``locale/LANG/LC_MESSAGES``. In the ``de`` example, the file will be
1257 1257
 ``locale/de/LC_MESSAGES/django.po``.
1258 1258
 
  1259
+.. versionchanged:: 1.7
  1260
+
  1261
+    When you run ``makemessages`` from the root directory of your project, the
  1262
+    extracted strings will be automatically distributed to the proper message
  1263
+    files. That is, a string extracted from a file of an app containing a
  1264
+    ``locale`` directory will go in a message file under that directory.
  1265
+    A string extracted from a file of an app without any ``locale`` directory
  1266
+    will either go in a message file under the directory listed first in
  1267
+    :setting:`LOCALE_PATHS` or will generate an error if :setting:`LOCALE_PATHS`
  1268
+    is empty.
  1269
+
1259 1270
 By default :djadmin:`django-admin.py makemessages <makemessages>` examines every
1260 1271
 file that has the ``.html`` or ``.txt`` file extension. In case you want to
1261 1272
 override that default, use the ``--extension`` or ``-e`` option to specify the
@@ -1730,24 +1741,9 @@ All message file repositories are structured the same way. They are:
1730 1741
 * ``$PYTHONPATH/django/conf/locale/<language>/LC_MESSAGES/django.(po|mo)``
1731 1742
 
1732 1743
 To create message files, you use the :djadmin:`django-admin.py makemessages <makemessages>`
1733  
-tool. You only need to be in the same directory where the ``locale/`` directory
1734  
-is located. And you use :djadmin:`django-admin.py compilemessages <compilemessages>`
  1744
+tool. And you use :djadmin:`django-admin.py compilemessages <compilemessages>`
1735 1745
 to produce the binary ``.mo`` files that are used by ``gettext``.
1736 1746
 
1737 1747
 You can also run :djadmin:`django-admin.py compilemessages
1738 1748
 --settings=path.to.settings <compilemessages>` to make the compiler process all
1739 1749
 the directories in your :setting:`LOCALE_PATHS` setting.
1740  
-
1741  
-Finally, you should give some thought to the structure of your translation
1742  
-files. If your applications need to be delivered to other users and will be used
1743  
-in other projects, you might want to use app-specific translations. But using
1744  
-app-specific translations and project-specific translations could produce weird
1745  
-problems with :djadmin:`makemessages`: it will traverse all directories below
1746  
-the current path and so might put message IDs into a unified, common message
1747  
-file for the current project that are already in application message files.
1748  
-
1749  
-The easiest way out is to store applications that are not part of the project
1750  
-(and so carry their own translations) outside the project tree. That way,
1751  
-:djadmin:`django-admin.py makemessages <makemessages>`, when ran on a project
1752  
-level will only extract strings that are connected to your explicit project and
1753  
-not strings that are distributed independently.
4  tests/i18n/project_dir/__init__.py
... ...
@@ -0,0 +1,4 @@
  1
+# Sample project used by test_extraction.CustomLayoutExtractionTests
  2
+from django.utils.translation import ugettext as _
  3
+
  4
+string = _("This is a project-level string")
0  tests/i18n/project_dir/app_no_locale/__init__.py
No changes.
3  tests/i18n/project_dir/app_no_locale/models.py
... ...
@@ -0,0 +1,3 @@
  1
+from django.utils.translation import ugettext as _
  2
+
  3
+string = _("This app has no locale directory")
0  tests/i18n/project_dir/app_with_locale/__init__.py
No changes.
0  tests/i18n/project_dir/app_with_locale/locale/.gitkeep
No changes.
3  tests/i18n/project_dir/app_with_locale/models.py
... ...
@@ -0,0 +1,3 @@
  1
+from django.utils.translation import ugettext as _
  2
+
  3
+string = _("This app has a locale directory")
0  tests/i18n/project_dir/project_locale/.gitkeep
No changes.
46  tests/i18n/test_extraction.py
@@ -8,9 +8,11 @@
8 8
 from unittest import SkipTest, skipUnless
9 9
 import warnings
10 10
 
  11
+from django.conf import settings
11 12
 from django.core import management
12 13
 from django.core.management.utils import find_command
13 14
 from django.test import SimpleTestCase
  15
+from django.test.utils import override_settings
14 16
 from django.utils.encoding import force_text
15 17
 from django.utils._os import upath
16 18
 from django.utils import six
@@ -497,3 +499,47 @@ def test_multiple_locales(self):
497 499
         management.call_command('makemessages', locale=['pt', 'de'], verbosity=0)
498 500
         self.assertTrue(os.path.exists(self.PO_FILE_PT))
499 501
         self.assertTrue(os.path.exists(self.PO_FILE_DE))
  502
+
  503
+
  504
+class CustomLayoutExtractionTests(ExtractorTests):
  505
+    def setUp(self):
  506
+        self._cwd = os.getcwd()
  507
+        self.test_dir = os.path.join(os.path.dirname(upath(__file__)), 'project_dir')
  508
+
  509
+    def test_no_locale_raises(self):
  510
+        os.chdir(self.test_dir)
  511
+        with six.assertRaisesRegex(self, management.CommandError,
  512
+                "Unable to find a locale path to store translations for file"):
  513
+            management.call_command('makemessages', locale=LOCALE, verbosity=0)
  514
+
  515
+    @override_settings(
  516
+        LOCALE_PATHS=(os.path.join(
  517
+            os.path.dirname(upath(__file__)), 'project_dir', 'project_locale'),)
  518
+    )
  519
+    def test_project_locale_paths(self):
  520
+        """
  521
+        Test that:
  522
+          * translations for an app containing a locale folder are stored in that folder
  523
+          * translations outside of that app are in LOCALE_PATHS[0]
  524
+        """
  525
+        os.chdir(self.test_dir)
  526
+        self.addCleanup(shutil.rmtree,
  527
+            os.path.join(settings.LOCALE_PATHS[0], LOCALE), True)
  528
+        self.addCleanup(shutil.rmtree,
  529
+            os.path.join(self.test_dir, 'app_with_locale', 'locale', LOCALE), True)
  530
+
  531
+        management.call_command('makemessages', locale=[LOCALE], verbosity=0)
  532
+        project_de_locale = os.path.join(
  533
+            self.test_dir, 'project_locale', 'de', 'LC_MESSAGES', 'django.po')
  534
+        app_de_locale = os.path.join(
  535
+            self.test_dir, 'app_with_locale', 'locale', 'de', 'LC_MESSAGES', 'django.po')
  536
+        self.assertTrue(os.path.exists(project_de_locale))
  537
+        self.assertTrue(os.path.exists(app_de_locale))
  538
+
  539
+        with open(project_de_locale, 'r') as fp:
  540
+            po_contents = force_text(fp.read())
  541
+            self.assertMsgId('This app has no locale directory', po_contents)
  542
+            self.assertMsgId('This is a project-level string', po_contents)
  543
+        with open(app_de_locale, 'r') as fp:
  544
+            po_contents = force_text(fp.read())
  545
+            self.assertMsgId('This app has a locale directory', po_contents)

0 notes on commit 50a8ab7

Please sign in to comment.
Something went wrong with that request. Please try again.