Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Enabled makemessages to support several translation directories #1706

Closed
wants to merge 1 commit into from

5 participants

@claudep
Collaborator

No description provided.

@ramiro
Collaborator

I've rebased the changes (see https://github.com/ramiro/django/tree/pull_1706) to apply cleanly to current master and to fix a couple of errors in the tests.

One thing I'm seeing is that the tests leave de, ch, pt and es subdirs under tests/i18n/commands/project_dir/app_with_locale/locale. It's like, even when it's invoking makemessages -l de, it's creating the dirs for other, urelated, locales.

Also, there are a couple of empty .gitignore files.

@claudep
Collaborator

Thanks ramiro for the review. I've forced-rebased the commit including your changes.

The subdirs you see appearing are produced by other extraction tests. I have to find a way to better isolate them.

@claudep
Collaborator

Ah... now I remember the reason for empty .gitignore's. It's the 'git-does-not-like-empty-dirs' issue. Anyway, I'm preparing a new version of the patch.

@Natim

Please named them .gitkeep since .gitignore may be confusing.

@claudep
Collaborator

Thanks for the .gitkeep tip, done.

@Natim

:)

@unaizalakain

LGTM, needs rebasing on top of master and fixing a little conflicting doc explanation.

@claudep
Collaborator

I've rebased the patch and added a release note paragraph.

@timgraham timgraham commented on the diff
docs/releases/1.7.txt
@@ -355,6 +355,11 @@ Internationalization
in the corresponding entry in the PO file, which makes the translation
process easier.
+* When you run :djadmin:`makemessages` from the root directory of your project,
+ any extracted string should be automatically distributed to the proper
@timgraham Owner
  1. string->strings
  2. should -> "will now be" (to help clarify this is a new behavior)
  3. does "distributed to the proper app or project message file." convey the same meaning in less words?
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
@timgraham timgraham commented on the diff
docs/topics/i18n/translation.txt
@@ -1256,6 +1256,17 @@ is configured correctly). It creates (or updates) a message file in the
directory ``locale/LANG/LC_MESSAGES``. In the ``de`` example, the file will be
``locale/de/LC_MESSAGES/django.po``.
+.. versionchanged:: 1.7
+
+ When you run ``makemessages`` from the root directory of your project, the
+ extracted strings should be automatically distributed to the proper
@timgraham Owner

"should be" -> "will be" (or there a chance it won't work properly for some reason?)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
@timgraham timgraham commented on the diff
tests/i18n/test_extraction.py
((9 lines not shown))
+ self.test_dir = os.path.join(os.path.dirname(upath(__file__)), 'project_dir')
+
+ def test_no_locale_raises(self):
+ os.chdir(self.test_dir)
+ with six.assertRaisesRegex(self, management.CommandError,
+ "Unable to find a locale path to store translations for file"):
+ management.call_command('makemessages', locale=LOCALE, verbosity=0)
+
+ @override_settings(
+ LOCALE_PATHS=(os.path.join(
+ os.path.dirname(upath(__file__)), 'project_dir', 'project_locale'),)
+ )
+ def test_project_locale_paths(self):
+ """
+ Test that:
+ * translations for app containing locale folder are stored in that folder
@timgraham Owner

an app containing a locale folder

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
@claudep claudep closed this
@claudep claudep deleted the branch
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
View
158 django/core/management/commands/makemessages.py
@@ -29,25 +29,28 @@ def check_programs(*programs):
@total_ordering
class TranslatableFile(object):
- def __init__(self, dirpath, file_name):
+ def __init__(self, dirpath, file_name, locale_dir):
self.file = file_name
self.dirpath = dirpath
+ self.locale_dir = locale_dir
def __repr__(self):
return "<TranslatableFile: %s>" % os.sep.join([self.dirpath, self.file])
def __eq__(self, other):
- return self.dirpath == other.dirpath and self.file == other.file
+ return self.path == other.path
def __lt__(self, other):
- if self.dirpath == other.dirpath:
- return self.file < other.file
- return self.dirpath < other.dirpath
+ return self.path < other.path
- def process(self, command, potfile, domain, keep_pot=False):
+ @property
+ def path(self):
+ return os.path.join(self.dirpath, self.file)
+
+ def process(self, command, domain):
"""
- Extract translatable literals from self.file for :param domain:
- creating or updating the :param potfile: POT file.
+ Extract translatable literals from self.file for :param domain:,
+ creating or updating the POT file.
Uses the xgettext GNU gettext utility.
"""
@@ -127,8 +130,6 @@ def process(self, command, potfile, domain, keep_pot=False):
if status != STATUS_OK:
if is_templatized:
os.unlink(work_file)
- if not keep_pot and os.path.exists(potfile):
- os.unlink(potfile)
raise CommandError(
"errors happened while running xgettext on %s\n%s" %
(self.file, errors))
@@ -136,6 +137,8 @@ def process(self, command, potfile, domain, keep_pot=False):
# Print warnings
command.stdout.write(errors)
if msgs:
+ # Write/append messages to pot file
+ potfile = os.path.join(self.locale_dir, '%s.pot' % str(domain))
if is_templatized:
# Remove '.py' suffix
if os.name == 'nt':
@@ -147,6 +150,7 @@ def process(self, command, potfile, domain, keep_pot=False):
new = '#: ' + orig_file[2:]
msgs = msgs.replace(old, new)
write_pot_file(potfile, msgs)
+
if is_templatized:
os.unlink(work_file)
@@ -242,64 +246,94 @@ def handle_noargs(self, *args, **options):
settings.configure(USE_I18N=True)
self.invoked_for_django = False
+ self.locale_paths = []
+ self.default_locale_path = None
if os.path.isdir(os.path.join('conf', 'locale')):
- localedir = os.path.abspath(os.path.join('conf', 'locale'))
+ self.locale_paths = [os.path.abspath(os.path.join('conf', 'locale'))]
+ self.default_locale_path = self.locale_paths[0]
self.invoked_for_django = True
# Ignoring all contrib apps
self.ignore_patterns += ['contrib/*']
- elif os.path.isdir('locale'):
- localedir = os.path.abspath('locale')
else:
- raise CommandError("This script should be run from the Django Git "
- "tree or your project or app tree. If you did indeed run it "
- "from the Git checkout or your project or application, "
- "maybe you are just missing the conf/locale (in the django "
- "tree) or locale (for project and application) directory? It "
- "is not created automatically, you have to create it by hand "
- "if you want to enable i18n for your project or application.")
-
- check_programs('xgettext')
-
- potfile = self.build_pot_file(localedir)
-
- # Build po files for each selected locale
+ self.locale_paths.extend(list(settings.LOCALE_PATHS))
+ # Allow to run makemessages inside an app dir
+ if os.path.isdir('locale'):
+ self.locale_paths.append(os.path.abspath('locale'))
+ if self.locale_paths:
+ self.default_locale_path = self.locale_paths[0]
+ if not os.path.exists(self.default_locale_path):
+ os.makedirs(self.default_locale_path)
+
+ # Build locale list
locales = []
if locale is not None:
locales += locale.split(',') if not isinstance(locale, list) else locale
elif process_all:
- locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir))
+ locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % self.default_locale_path))
locales = [os.path.basename(l) for l in locale_dirs]
-
if locales:
check_programs('msguniq', 'msgmerge', 'msgattrib')
+ check_programs('xgettext')
+
try:
+ potfiles = self.build_potfiles()
+
+ # Build po files for each selected locale
for locale in locales:
if self.verbosity > 0:
self.stdout.write("processing locale %s\n" % locale)
- self.write_po_file(potfile, locale)
+ for potfile in potfiles:
+ self.write_po_file(potfile, locale)
finally:
- if not self.keep_pot and os.path.exists(potfile):
- os.unlink(potfile)
+ if not self.keep_pot:
+ self.remove_potfiles()
- def build_pot_file(self, localedir):
+ def build_potfiles(self):
+ """
+ Build pot files and apply msguniq to them.
+ """
file_list = self.find_files(".")
-
- potfile = os.path.join(localedir, '%s.pot' % str(self.domain))
- if os.path.exists(potfile):
- # Remove a previous undeleted potfile, if any
- os.unlink(potfile)
-
+ self.remove_potfiles()
for f in file_list:
try:
- f.process(self, potfile, self.domain, self.keep_pot)
+ f.process(self, self.domain)
except UnicodeDecodeError:
self.stdout.write("UnicodeDecodeError: skipped file %s in %s" % (f.file, f.dirpath))
- return potfile
+
+ potfiles = []
+ for path in self.locale_paths:
+ potfile = os.path.join(path, '%s.pot' % str(self.domain))
+ if not os.path.exists(potfile):
+ continue
+ args = ['msguniq', '--to-code=utf-8']
+ if self.wrap:
+ args.append(self.wrap)
+ if self.location:
+ args.append(self.location)
+ args.append(potfile)
+ msgs, errors, status = popen_wrapper(args)
+ if errors:
+ if status != STATUS_OK:
+ raise CommandError(
+ "errors happened while running msguniq\n%s" % errors)
+ elif self.verbosity > 0:
+ self.stdout.write(errors)
+ with open(potfile, 'w') as fp:
+ fp.write(msgs)
+ potfiles.append(potfile)
+ return potfiles
+
+ def remove_potfiles(self):
+ for path in self.locale_paths:
+ pot_path = os.path.join(path, '%s.pot' % str(self.domain))
+ if os.path.exists(pot_path):
+ os.unlink(pot_path)
def find_files(self, root):
"""
- Helper method to get all files in the given root.
+ Helper method to get all files in the given root. Also check that there
+ is a matching locale dir for each file.
"""
def is_ignored(path, ignore_patterns):
@@ -319,12 +353,26 @@ def is_ignored(path, ignore_patterns):
dirnames.remove(dirname)
if self.verbosity > 1:
self.stdout.write('ignoring directory %s\n' % dirname)
+ elif dirname == 'locale':
+ dirnames.remove(dirname)
+ self.locale_paths.insert(0, os.path.join(os.path.abspath(dirpath), dirname))
for filename in filenames:
- if is_ignored(os.path.normpath(os.path.join(dirpath, filename)), self.ignore_patterns):
+ file_path = os.path.normpath(os.path.join(dirpath, filename))
+ if is_ignored(file_path, self.ignore_patterns):
if self.verbosity > 1:
self.stdout.write('ignoring file %s in %s\n' % (filename, dirpath))
else:
- all_files.append(TranslatableFile(dirpath, filename))
+ locale_dir = None
+ for path in self.locale_paths:
+ if os.path.abspath(dirpath).startswith(os.path.dirname(path)):
+ locale_dir = path
+ break
+ if not locale_dir:
+ locale_dir = self.default_locale_path
+ if not locale_dir:
+ raise CommandError(
+ "Unable to find a locale path to store translations for file %s" % file_path)
+ all_files.append(TranslatableFile(dirpath, filename, locale_dir))
return sorted(all_files)
def write_po_file(self, potfile, locale):
@@ -332,30 +380,14 @@ def write_po_file(self, potfile, locale):
Creates or updates the PO file for self.domain and :param locale:.
Uses contents of the existing :param potfile:.
- Uses mguniq, msgmerge, and msgattrib GNU gettext utilities.
+ Uses msgmerge, and msgattrib GNU gettext utilities.
"""
- args = ['msguniq', '--to-code=utf-8']
- if self.wrap:
- args.append(self.wrap)
- if self.location:
- args.append(self.location)
- args.append(potfile)
- msgs, errors, status = popen_wrapper(args)
- if errors:
- if status != STATUS_OK:
- raise CommandError(
- "errors happened while running msguniq\n%s" % errors)
- elif self.verbosity > 0:
- self.stdout.write(errors)
-
basedir = os.path.join(os.path.dirname(potfile), locale, 'LC_MESSAGES')
if not os.path.isdir(basedir):
os.makedirs(basedir)
pofile = os.path.join(basedir, '%s.po' % str(self.domain))
if os.path.exists(pofile):
- with open(potfile, 'w') as fp:
- fp.write(msgs)
args = ['msgmerge', '-q']
if self.wrap:
args.append(self.wrap)
@@ -369,8 +401,10 @@ def write_po_file(self, potfile, locale):
"errors happened while running msgmerge\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
- elif not self.invoked_for_django:
- msgs = self.copy_plural_forms(msgs, locale)
+ else:
+ msgs = open(potfile, 'r').read()
+ if not self.invoked_for_django:
+ msgs = self.copy_plural_forms(msgs, locale)
msgs = msgs.replace(
"#. #-#-#-#-# %s.pot (PACKAGE VERSION) #-#-#-#-#\n" % self.domain, "")
with open(pofile, 'w') as fp:
View
3  docs/man/django-admin.1
@@ -192,7 +192,8 @@ Ignore files or directories matching this glob-style pattern. Use multiple
times to ignore more (makemessages command).
.TP
.I \-\-no\-default\-ignore
-Don't ignore the common private glob-style patterns 'CVS', '.*' and '*~' (makemessages command).
+Don't ignore the common private glob-style patterns 'CVS', '.*', '*~' and '*.pyc'
+(makemessages command).
.TP
.I \-\-no\-wrap
Don't break long message lines into several lines (makemessages command).
View
4 docs/ref/django-admin.txt
@@ -564,7 +564,7 @@ Example usage::
Use the ``--ignore`` or ``-i`` option to ignore files or directories matching
the given :mod:`glob`-style pattern. Use multiple times to ignore more.
-These patterns are used by default: ``'CVS'``, ``'.*'``, ``'*~'``
+These patterns are used by default: ``'CVS'``, ``'.*'``, ``'*~'``, ``'*.pyc'``
Example usage::
@@ -591,7 +591,7 @@ for technically skilled translators to understand each message's context.
.. versionadded:: 1.6
Use the ``--keep-pot`` option to prevent Django from deleting the temporary
-.pot file it generates before creating the .po file. This is useful for
+.pot files it generates before creating the .po file. This is useful for
debugging errors which may prevent the final language files from being created.
makemigrations [<appname>]
View
5 docs/releases/1.7.txt
@@ -355,6 +355,11 @@ Internationalization
in the corresponding entry in the PO file, which makes the translation
process easier.
+* When you run :djadmin:`makemessages` from the root directory of your project,
+ any extracted string should be automatically distributed to the proper
@timgraham Owner
  1. string->strings
  2. should -> "will now be" (to help clarify this is a new behavior)
  3. does "distributed to the proper app or project message file." convey the same meaning in less words?
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ message files, either app-specific or at the project level. See
+ :ref:`how-to-create-language-files` for details.
+
Management Commands
^^^^^^^^^^^^^^^^^^^
View
28 docs/topics/i18n/translation.txt
@@ -1256,6 +1256,17 @@ is configured correctly). It creates (or updates) a message file in the
directory ``locale/LANG/LC_MESSAGES``. In the ``de`` example, the file will be
``locale/de/LC_MESSAGES/django.po``.
+.. versionchanged:: 1.7
+
+ When you run ``makemessages`` from the root directory of your project, the
+ extracted strings should be automatically distributed to the proper
@timgraham Owner

"should be" -> "will be" (or there a chance it won't work properly for some reason?)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ message files. That is, a string extracted from a file of an app containing
+ a ``locale`` directory will go in a message file under that directory.
+ A string extracted from a file of an app without any ``locale`` directory
+ will either go in a message file under the directory listed first in
+ :setting:`LOCALE_PATHS` or will generate an error if :setting:`LOCALE_PATHS`
+ is empty.
+
By default :djadmin:`django-admin.py makemessages <makemessages>` examines every
file that has the ``.html`` or ``.txt`` file extension. In case you want to
override that default, use the ``--extension`` or ``-e`` option to specify the
@@ -1730,24 +1741,9 @@ All message file repositories are structured the same way. They are:
* ``$PYTHONPATH/django/conf/locale/<language>/LC_MESSAGES/django.(po|mo)``
To create message files, you use the :djadmin:`django-admin.py makemessages <makemessages>`
-tool. You only need to be in the same directory where the ``locale/`` directory
-is located. And you use :djadmin:`django-admin.py compilemessages <compilemessages>`
+tool. And you use :djadmin:`django-admin.py compilemessages <compilemessages>`
to produce the binary ``.mo`` files that are used by ``gettext``.
You can also run :djadmin:`django-admin.py compilemessages
--settings=path.to.settings <compilemessages>` to make the compiler process all
the directories in your :setting:`LOCALE_PATHS` setting.
-
-Finally, you should give some thought to the structure of your translation
-files. If your applications need to be delivered to other users and will be used
-in other projects, you might want to use app-specific translations. But using
-app-specific translations and project-specific translations could produce weird
-problems with :djadmin:`makemessages`: it will traverse all directories below
-the current path and so might put message IDs into a unified, common message
-file for the current project that are already in application message files.
-
-The easiest way out is to store applications that are not part of the project
-(and so carry their own translations) outside the project tree. That way,
-:djadmin:`django-admin.py makemessages <makemessages>`, when ran on a project
-level will only extract strings that are connected to your explicit project and
-not strings that are distributed independently.
View
4 tests/i18n/project_dir/__init__.py
@@ -0,0 +1,4 @@
+# Sample project used by test_extraction.CustomLayoutExtractionTests
+from django.utils.translation import ugettext as _
+
+string = _("This is a project-level string")
View
0  tests/i18n/project_dir/app_no_locale/__init__.py
No changes.
View
3  tests/i18n/project_dir/app_no_locale/models.py
@@ -0,0 +1,3 @@
+from django.utils.translation import ugettext as _
+
+string = _("This app has no locale directory")
View
0  tests/i18n/project_dir/app_with_locale/__init__.py
No changes.
View
0  tests/i18n/project_dir/app_with_locale/locale/.gitkeep
No changes.
View
3  tests/i18n/project_dir/app_with_locale/models.py
@@ -0,0 +1,3 @@
+from django.utils.translation import ugettext as _
+
+string = _("This app has a locale directory")
View
0  tests/i18n/project_dir/project_locale/.gitkeep
No changes.
View
45 tests/i18n/test_extraction.py
@@ -8,9 +8,11 @@
from unittest import SkipTest, skipUnless
import warnings
+from django.conf import settings
from django.core import management
from django.core.management.utils import find_command
from django.test import SimpleTestCase
+from django.test.utils import override_settings
from django.utils.encoding import force_text
from django.utils._os import upath
from django.utils import six
@@ -503,3 +505,46 @@ def test_comma_separated_locales(self):
management.call_command('makemessages', locale='pt,de,ch', verbosity=0)
self.assertTrue(os.path.exists(self.PO_FILE_PT))
self.assertTrue(os.path.exists(self.PO_FILE_DE))
+
+
+class CustomLayoutExtractionTests(ExtractorTests):
+ def setUp(self):
+ self._cwd = os.getcwd()
+ self.test_dir = os.path.join(os.path.dirname(upath(__file__)), 'project_dir')
+
+ def test_no_locale_raises(self):
+ os.chdir(self.test_dir)
+ with six.assertRaisesRegex(self, management.CommandError,
+ "Unable to find a locale path to store translations for file"):
+ management.call_command('makemessages', locale=LOCALE, verbosity=0)
+
+ @override_settings(
+ LOCALE_PATHS=(os.path.join(
+ os.path.dirname(upath(__file__)), 'project_dir', 'project_locale'),)
+ )
+ def test_project_locale_paths(self):
+ """
+ Test that:
+ * translations for app containing locale folder are stored in that folder
@timgraham Owner

an app containing a locale folder

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ * translations outside of that app are in LOCALE_PATHS[0]
+ """
+ os.chdir(self.test_dir)
+ self.addCleanup(shutil.rmtree, os.path.join(settings.LOCALE_PATHS[0], LOCALE))
+ self.addCleanup(shutil.rmtree, os.path.join(
+ self.test_dir, 'app_with_locale', 'locale', LOCALE))
+
+ management.call_command('makemessages', locale=LOCALE, verbosity=0)
+ project_de_locale = os.path.join(
+ self.test_dir, 'project_locale', 'de', 'LC_MESSAGES', 'django.po')
+ app_de_locale = os.path.join(
+ self.test_dir, 'app_with_locale', 'locale', 'de', 'LC_MESSAGES', 'django.po')
+ self.assertTrue(os.path.exists(project_de_locale))
+ self.assertTrue(os.path.exists(app_de_locale))
+
+ with open(project_de_locale, 'r') as fp:
+ po_contents = force_text(fp.read())
+ self.assertMsgId('This app has no locale directory', po_contents)
+ self.assertMsgId('This is a project-level string', po_contents)
+ with open(app_de_locale, 'r') as fp:
+ po_contents = force_text(fp.read())
+ self.assertMsgId('This app has a locale directory', po_contents)
Something went wrong with that request. Please try again.