Bug 1226382: Store Firefox OS strings in separate pofiles.

bin/jenkins/makemessages.sh

@@ -17,14 +17,15 @@ virtualenv $TDIR
 . $TDIR/bin/activate
 pip install fig
 
-rm -rf locale db-strings.txt run-output
+rm -rf locale db-changes.diff run-output
 git clone $LOCALE_REPOSITORY locale
 
 deis login $DEIS_CONTROLLER --username $DEIS_USERNAME --password $DEIS_PASSWORD
-deis run -a $DEIS_APP -- "./manage.py runscript db_strings && echo CUTHERE && cat db-strings.txt | gzip -9 | uuencode -" > run-output
-awk '{if (nowprint) {print;}}/CUTHERE/ {nowprint = 1}' run-output | uudecode | gunzip > db-strings.txt
-fig --project-name jenkins${JOB_NAME}${BUILD_NUMBER} run -T web ./manage.py runscript makemessages_all_locales
-fig --project-name jenkins${JOB_NAME}${BUILD_NUMBER} run -T web ./manage.py runscript cleanup_po
+deis run -a $DEIS_APP -- "./manage.py runscript db_strings && echo CUTHERE && git diff locale | gzip -9 | uuencode -" > run-output
+awk '{if (nowprint) {print;}}/CUTHERE/ {nowprint = 1}' run-output | uudecode | gunzip > db-changes.diff
+fig --project-name jenkins${JOB_NAME}${BUILD_NUMBER} run -T web git apply db-changes.diff
+fig --project-name jenkins${JOB_NAME}${BUILD_NUMBER} run -T web ./manage.py runscript extract
+fig --project-name jenkins${JOB_NAME}${BUILD_NUMBER} run -T web ./manage.py runscript merge
 fig --project-name jenkins${JOB_NAME}${BUILD_NUMBER} run -T web chmod a+wx -R locale
 
 cd locale

bin/run-common.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
 ./manage.py syncdb --noinput
+./manage.py collectstrings
 ./manage.py compilemessages
 ./manage.py runscript create_db_locales > /dev/null 2>&1 &

docs/index.rst

@@ -14,6 +14,7 @@ Contents:
    authoring_content
    versions_and_locales
    translation_using_verbatim
+   porting_translations
    gulp
    importing_legacy_data
 

docs/porting_translations.rst

@@ -0,0 +1,29 @@
+Porting Translations
+====================
+
+Prior to the addition of the Puente and the `collectstrings` script,
+translations were stored in a single `django.po` file per-locale, and the
+`cleanup_po` script was used to remove strings that weren't necessary for
+certain locales.
+
+This following describes the steps for porting the old translation file layout
+to the new one while preserving existing translations.
+
+1. Delete the ``django.pot`` file, as well as the
+   ``templates/LC_MESSAGES/django.pot`` symlink.
+
+2. Run the extraction commands for database and template strings::
+
+    ./manage.py extract
+    ./manage.py merge
+    ./manage.py runscript db_strings
+
+3. Run the one-time import script to populate the new files using translations
+   from the existing ``django.po`` files::
+
+    ./manage.py runscript import_collected_translations
+
+4. Delete all the remaining ``django.po`` files and add ``django.po``
+   and ``django.pot`` to the ``.gitignore`` file within the locale directory.
+
+5. Commit your changes. You're done!

docs/versions_and_locales.rst

@@ -6,7 +6,7 @@ Adding new Versions of the Documentation
 
 To add more Versions of MasterFirefoxOS Documentation you need to create
 the corresponding CMS pages and edit settings. For example to add
-version 2.0 based on version 1.3T::
+version 2.0 based on version 1.3T:
 
 1. Go to /admin/page/page
 2. Select the root page of version 1.3T
@@ -52,18 +52,18 @@ change `LANGUAGES` to::
 
 Then you need to generate messages for the first time for the added locale::
 
-  ./manage.py makemessages --locale de
+  ./manage.py extract
+  ./manage.py merge -c
 
 .. note::
 
-   It's important to explicitly define the new locale using the
-   `--locale` flag when executing the `makemessages` command. Using
-   the `-a` flag to make messages for all locales will ignore the new
-   locales.
+   It's important to include the `-c` flag when adding a new locale. Without it,
+   `merge` will ignore new locales that don't have directories yet and will not
+   create new directories for them.
 
 
 Managing the Versions of Documentation per locale
-------------------------------------------------
+-------------------------------------------------
 
 A locale gets activated once it gets linked with a Version of the
 Documentation. For example to link locale `foo` with version `1.1` edit
@@ -111,70 +111,52 @@ MasterFirefoxOS stores strings for localization in three different places:
   * HTML files (.html)
   * In the database
 
-Python and HTML files are automatically handled by Django's
-`makemessages` command. The database strings must first get extracted
-into a text file be parsable by `makemessages` command. To extract the
-database strings run::
+Python and HTML files are automatically handled by Puente_'s `extract` and
+`merge` commands and stored in `messages.po` files. To generate or update these
+files, run::
 
-  ./manage.py runscript db_strings
-
-.. note::
-
-  Extracted database strings are stored in `db-strings.txt` file. This
-  file should *not* be edited manually.
-
-Now all strings are in Python, HTML and Text files that `makemessages`
-command can parse. To generate `.po` files for all supported languages
-run::
-
-  ./manage.py makemessages -a
-
-.. warning::
-
-   Always extract database strings before running
-   `makemessages`. Failing to do so may remove all database strings
-   from `.po` files.
+  ./manage.py extract
+  ./manage.py merge
 
+The database strings are extracted and placed into translation files via the
+`db_strings` script::
 
-Now you can distributed your `.po` files to the translators.
-
-The generated po files contain strings for all versions of the CMS
-content. The final step is to keep only the strings from the versions
-of the CMS content activated per locale::
+  ./manage.py runscript db_strings
 
-  ./manage.py runscript cleanup_po
+.. _Puente: https://github.com/mozilla/puente
 
 
 Compile strings
 ---------------
 
-Given that you have translated `.po` files you need to compile them
-into `.mo` files. To do this run the `compilemessages` command::
+Strings are stored in several `.po` files. To load them into the site, you must
+first collect them together into a single `django.po` file for each locale using
+the `collectstrings` script::
 
-  ./manage.py compilemessages
+  ./manage.py runscript collectstrings
 
+Once the strings have been collected, you must compile them into `.mo` files
+using the `compilemessages` command::
+
+  ./manage.py compilemessages
 
 This is required step for translations to work.
 
 
 How does database localization work?
 ------------------------------------
 
-The following command will iterate through all FeinCMS Pages and
-through all Content Types defined in each Page, and extract strings
-from fields named in each Content Type model's `_l10n_fields`
-attribute, and output to a template text file:
-
-  ./manage.py db_strings
-
-By default, the command outputs to `db-strings.txt` but accepts an
-optional `filename` argument.
+The `db_strings` script iterates through all FeinCMS Pages and through all
+Content Types defined in each Page, and extracts strings from fields named in
+each Content Type model's `_l10n_fields` attribute. It groups these strings by
+the Firefox OS version that they belong to (strings shared among multiple
+versions are in shared groups) and creates `.pot` files with these strings.
 
-This text file can be parsed with `./manage.py makemessages` command
-to generate a `.po` file.
+Once the `.pot` files are created, the script creates or updates the `.po` files
+that translators edit with their translations.
 
-We use a custom `render` method that calls `ugettext` on
-each localizable field::
+We use a custom `render` method that calls `ugettext` on each localizable
+field::
 
   from django.utils.translation import ugettext as _
 

masterfirefoxos/base/tests/test_utils.py

@@ -1,3 +1,4 @@
+import os
 from unittest.mock import Mock, patch
 
 from django.test import override_settings, RequestFactory
@@ -31,27 +32,6 @@ def test_entry_strings():
         'test title', 'test text', 'sub 2', 'sub 3'])
 
 
-def test_pages_l10n_template():
-    parent = Page(title='Parent Title', slug='parent-slug')
-    page = Page(title='Page Title', slug='page-slug')
-    page.parent = parent
-    parent_entry = models.RichTextEntry(text='<p>Parent Text</p>')
-    entry = models.RichTextEntry(text='<p>Page Text</p>')
-    parent.content.all_of_type = lambda content_type: [parent_entry]
-    page.content.all_of_type = lambda content_type: [entry]
-    l10n_template = utils.pages_l10n_template([parent, page])
-    assert 'Page path: /parent-slug/\n' in l10n_template
-    assert 'Page path: /parent-slug/page-slug/\n' in l10n_template
-    assert ('{% blocktrans trimmed %}\nParent Title\n{% endblocktrans %}'
-            in l10n_template)
-    assert ('{% blocktrans trimmed %}\nPage Title\n{% endblocktrans %}'
-            in l10n_template)
-    assert ('{% blocktrans trimmed %}\n<p>Parent Text</p>\n{% endblocktrans %}'
-            in l10n_template)
-    assert ('{% blocktrans trimmed %}\n<p>Page Text</p>\n{% endblocktrans %}'
-            in l10n_template)
-
-
 @patch('masterfirefoxos.base.utils.copy_content_and_children')
 @patch('masterfirefoxos.base.utils.Page.objects')
 @patch('masterfirefoxos.base.utils.datetime')
@@ -111,3 +91,101 @@ def test_youtube_embed_url_en():
     request = RequestFactory().get('/en/introduction/')
     expected = 'https://www.youtube.com/embed/en-youtube-id'
     assert utils.youtube_embed_url(request, 'en-youtube-id') == expected
+
+
+def test_page_strings():
+    page = Page(title='page title')
+    rich_text_entry = models.RichTextEntry(
+        title='title', subheader_2='sub 2', subheader_3='sub 3', text='test text')
+    faq_entry = models.FAQEntry(
+        question='test question', answer='test answer')
+
+    page._feincms_content_types = [Mock()]
+    page.content.all_of_type = lambda type: [rich_text_entry, faq_entry]
+
+    assert set(utils.page_strings(page)) == set([
+        'page title', 'title', 'sub 2', 'sub 3', 'test text', 'test question',
+        'test answer'])
+
+
+TEST_VERSIONS_LOCALE_MAP = {
+    'v1': {
+        'locales': ['fr', 'pt-BR'],
+        'pending_locales': ['es'],
+        'slug': 'v1',
+    },
+    'v2': {
+        'locales': ['fr', 'sl'],
+        'slug': 'v2',
+    },
+}
+
+
+@override_settings(VERSIONS_LOCALE_MAP=TEST_VERSIONS_LOCALE_MAP)
+def test_versions_for_locale():
+    assert set(utils.versions_for_locale('fr')) == set(['v1', 'v2'])
+    assert set(utils.versions_for_locale('sl')) == set(['v2'])
+    assert set(utils.versions_for_locale('es')) == set(['v1'])
+
+
+@override_settings(VERSIONS_LOCALE_MAP=TEST_VERSIONS_LOCALE_MAP)
+def test_all_versions():
+    assert set(utils.all_versions()) == set(['v1', 'v2'])
+
+
+def base_path(*parts):
+    return os.path.join(os.sep, 'app', *parts)
+
+
+@override_settings(BASE_DIR=base_path())
+def test_versions_po_filename():
+    assert (utils.versions_po_filename('es', ['v1'])
+            == base_path('locale', 'es', 'LC_MESSAGES', 'firefox_os_v1.po'))
+    assert (utils.versions_po_filename('pt-br', ['v2'], template=True)
+            == base_path('locale', 'pt_BR', 'LC_MESSAGES', 'firefox_os_v2.pot'))
+    assert (utils.versions_po_filename('es', ['z2', 'v1'])
+            == base_path('locale', 'es', 'LC_MESSAGES', 'firefox_os_v1_z2.po'))
+
+
+@override_settings(BASE_DIR=base_path())
+def test_po_filename():
+    assert (utils.po_filename('es', 'messages')
+            == base_path('locale', 'es', 'LC_MESSAGES', 'messages.po'))
+    assert (utils.po_filename('pt-br', 'template', template=True)
+            == base_path('locale', 'pt_BR', 'LC_MESSAGES', 'template.pot'))
+
+
+@override_settings(BASE_DIR=base_path())
+def test_locale_dir():
+    assert utils.locale_dir('es') == base_path('locale', 'es')
+    assert utils.locale_dir('pt-br') == base_path('locale', 'pt_BR')
+    assert utils.locale_dir('en-US') == base_path('locale', 'en_US')
+
+
+@override_settings(BASE_DIR=base_path())
+def test_po_filenames_for_locale():
+    with patch('masterfirefoxos.base.utils.glob') as mock_glob:
+        mock_glob.return_value = ['/app/test.po', '/app/test2.po', '/app/django.po']
+        assert (set(utils.po_filenames_for_locale('es'))
+                == set(['/app/test.po', '/app/test2.po']))
+        mock_glob.assert_called_with(base_path('locale', 'es', '**', '*.po'))
+
+
+def test_execute():
+    with patch('masterfirefoxos.base.utils.subprocess') as mock_subprocess:
+        proc = mock_subprocess.Popen.return_value
+        proc.returncode = 0
+        proc.communicate.return_value = 'output', 'error'
+
+        assert utils.execute(['foo', 'bar']) == (0, 'output', 'error')
+        pipe = mock_subprocess.PIPE
+        mock_subprocess.Popen.assert_called_with(
+            args=['foo', 'bar'], stdout=pipe, stderr=pipe, stdin=pipe)
+
+
+def test_execute_oserror():
+    with patch('masterfirefoxos.base.utils.subprocess') as mock_subprocess:
+        error = OSError('Could not find file')
+        mock_subprocess.Popen.side_effect = error
+
+        assert utils.execute(['foo', 'bar']) == (-1, '', error)