diff --git a/CHANGES b/CHANGES index da6ba170357..8e0b7616e88 100644 --- a/CHANGES +++ b/CHANGES @@ -53,6 +53,11 @@ Features added via :confval:`linkcheck_anchors_ignore_for_url` while still checking the validity of the page itself. Patch by Bénédikt Tran +* #1246: Add translation progress statistics and inspection support, + via a new substitution (``|translation progress|``) and a new + configuration variable (:confval:`translation_progress_classes`). + These enable determining the percentage of translated elements within + a document, and the remaining translated and untranslated elements. Bugs fixed ---------- diff --git a/doc/usage/advanced/intl.rst b/doc/usage/advanced/intl.rst index 1a98ebb1f31..ae6e7dc9d6b 100644 --- a/doc/usage/advanced/intl.rst +++ b/doc/usage/advanced/intl.rst @@ -340,6 +340,23 @@ There is a `sphinx translation page`_ for Sphinx (master) documentation. Detail is here: https://docs.transifex.com/getting-started-1/translators + +Translation progress and statistics +----------------------------------- + +.. versionadded:: 7.1.0 + +During the rendering process, +Sphinx marks each translatable node with a ``translated`` attribute, +indicating if a translation was found for the text in that node. + +The :confval:`translation_progress_classes` configuration value +can be used to add a class to each element, +depending on the value of the ``translated`` attribute. + +The ``|translation progress|`` substitution can be used to display the +percentage of nodes that have been translated on a per-document basis. + .. rubric:: Footnotes .. [1] See the `GNU gettext utilities diff --git a/doc/usage/configuration.rst b/doc/usage/configuration.rst index 30c552a6c0f..aec911a292f 100644 --- a/doc/usage/configuration.rst +++ b/doc/usage/configuration.rst @@ -1002,6 +1002,21 @@ documentation on :ref:`intl` for details. .. versionchanged:: 3.2 Added ``{docpath}`` token. +.. confval:: translation_progress_classes + + Control which, if any, classes are added to indicate translation progress. + This setting would likely only be used by translators of documentation, + in order to quickly indicate translated and untranslated content. + + * ``True``: add ``translated`` and ``untranslated`` classes + to all nodes with translatable content. + * ``translated``: only add the ``translated`` class. + * ``untranslated``: only add the ``untranslated`` class. + * ``False``: do not add any classes to indicate translation progress. + + Defaults to ``False``. + + .. versionadded:: 7.1 .. _math-options: diff --git a/doc/usage/restructuredtext/roles.rst b/doc/usage/restructuredtext/roles.rst index 62fcaa4c4dd..e468de9c244 100644 --- a/doc/usage/restructuredtext/roles.rst +++ b/doc/usage/restructuredtext/roles.rst @@ -528,3 +528,9 @@ default. They are set in the build configuration file. Replaced by either today's date (the date on which the document is read), or the date set in the build configuration file. Normally has the format ``April 14, 2007``. Set by :confval:`today_fmt` and :confval:`today`. + +.. describe:: |translation progress| + + Replaced by the translation progress of the document. + This substitution is intented for use by document translators + as a marker for the translation progress of the document. diff --git a/sphinx/config.py b/sphinx/config.py index b8cf1eda2ca..8b8a136e185 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -58,7 +58,7 @@ class ENUM: Example: app.add_config_value('latex_show_urls', 'no', None, ENUM('no', 'footnote', 'inline')) """ - def __init__(self, *candidates: str) -> None: + def __init__(self, *candidates: str | bool) -> None: self.candidates = candidates def match(self, value: str | list | tuple) -> bool: @@ -101,6 +101,8 @@ class Config: 'locale_dirs': (['locales'], 'env', []), 'figure_language_filename': ('{root}.{language}{ext}', 'env', [str]), 'gettext_allow_fuzzy_translations': (False, 'gettext', []), + 'translation_progress_classes': (False, 'env', + ENUM(True, False, 'translated', 'untranslated')), 'master_doc': ('index', 'env', []), 'root_doc': (lambda config: config.master_doc, 'env', []), diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index d2beffdd9db..1d34e867a1e 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -629,7 +629,7 @@ def get_and_resolve_doctree( prune=prune_toctrees, includehidden=includehidden) if result is None: - toctreenode.replace_self([]) + toctreenode.parent.replace(toctreenode, []) else: toctreenode.replace_self(result) diff --git a/sphinx/themes/basic/static/basic.css_t b/sphinx/themes/basic/static/basic.css_t index 9ae18026720..d816abc3f1d 100644 --- a/sphinx/themes/basic/static/basic.css_t +++ b/sphinx/themes/basic/static/basic.css_t @@ -748,6 +748,14 @@ abbr, acronym { cursor: help; } +.translated { + background-color: rgba(207, 255, 207, 0.2) +} + +.untranslated { + background-color: rgba(255, 207, 207, 0.2) +} + /* -- code displays --------------------------------------------------------- */ pre { diff --git a/sphinx/transforms/__init__.py b/sphinx/transforms/__init__.py index 4cfc2b5bcaa..be38c6f244f 100644 --- a/sphinx/transforms/__init__.py +++ b/sphinx/transforms/__init__.py @@ -34,6 +34,7 @@ 'version', 'release', 'today', + 'translation progress', } @@ -103,7 +104,11 @@ def apply(self, **kwargs: Any) -> None: for ref in self.document.findall(nodes.substitution_reference): refname = ref['refname'] if refname in to_handle: - text = self.config[refname] + if refname == 'translation progress': + # special handling: calculate translation progress + text = _calculate_translation_progress(self.document) + else: + text = self.config[refname] if refname == 'today' and not text: # special handling: can also specify a strftime format text = format_date(self.config.today_fmt or _('%b %d, %Y'), @@ -111,6 +116,19 @@ def apply(self, **kwargs: Any) -> None: ref.replace_self(nodes.Text(text)) +def _calculate_translation_progress(document: nodes.document) -> str: + try: + translation_progress = document['translation_progress'] + except KeyError: + return _('could not calculate translation progress!') + + total = translation_progress['total'] + translated = translation_progress['translated'] + if total <= 0: + return _('no translated elements!') + return f'{translated / total:.2%}' + + class MoveModuleTargets(SphinxTransform): """ Move module targets that are the first thing in a section to the section diff --git a/sphinx/transforms/i18n.py b/sphinx/transforms/i18n.py index c412a0d12fb..d16b8f75488 100644 --- a/sphinx/transforms/i18n.py +++ b/sphinx/transforms/i18n.py @@ -14,6 +14,7 @@ from sphinx import addnodes from sphinx.config import Config from sphinx.domains.std import make_glossary_term, split_term_classifiers +from sphinx.errors import ConfigError from sphinx.locale import __ from sphinx.locale import init as init_locale from sphinx.transforms import SphinxTransform @@ -360,9 +361,9 @@ def apply(self, **kwargs: Any) -> None: if not isinstance(node, LITERAL_TYPE_NODES): msgstr, _ = parse_noqa(msgstr) - # XXX add marker to untranslated parts if not msgstr or msgstr == msg or not msgstr.strip(): # as-of-yet untranslated + node['translated'] = False continue # Avoid "Literal block expected; none found." warnings. @@ -404,10 +405,12 @@ def apply(self, **kwargs: Any) -> None: if processed: updater.update_leaves() node['translated'] = True # to avoid double translation + else: + node['translated'] = False # phase2: translation for node, msg in extract_messages(self.document): - if node.get('translated', False): # to avoid double translation + if node.setdefault('translated', False): # to avoid double translation continue # skip if the node is already translated by phase1 msgstr = catalog.gettext(msg) @@ -417,8 +420,8 @@ def apply(self, **kwargs: Any) -> None: if not isinstance(node, LITERAL_TYPE_NODES): msgstr, noqa = parse_noqa(msgstr) - # XXX add marker to untranslated parts if not msgstr or msgstr == msg: # as-of-yet untranslated + node['translated'] = False continue # update translatable nodes @@ -429,6 +432,7 @@ def apply(self, **kwargs: Any) -> None: # update meta nodes if isinstance(node, nodes.meta): # type: ignore[attr-defined] node['content'] = msgstr + node['translated'] = True continue if isinstance(node, nodes.image) and node.get('alt') == msg: @@ -490,6 +494,7 @@ def apply(self, **kwargs: Any) -> None: if isinstance(node, nodes.image) and node.get('alt') != msg: node['uri'] = patch['uri'] + node['translated'] = False continue # do not mark translated node['translated'] = True # to avoid double translation @@ -514,6 +519,64 @@ def apply(self, **kwargs: Any) -> None: node['entries'] = new_entries +class TranslationProgressTotaliser(SphinxTransform): + """ + Calculate the number of translated and untranslated nodes. + """ + default_priority = 25 # MUST happen after Locale + + def apply(self, **kwargs: Any) -> None: + from sphinx.builders.gettext import MessageCatalogBuilder + if isinstance(self.app.builder, MessageCatalogBuilder): + return + + total = translated = 0 + for node in self.document.findall(NodeMatcher(translated=Any)): # type: nodes.Element + total += 1 + if node['translated']: + translated += 1 + + self.document['translation_progress'] = { + 'total': total, + 'translated': translated, + } + + +class AddTranslationClasses(SphinxTransform): + """ + Add ``translated`` or ``untranslated`` classes to indicate translation status. + """ + default_priority = 950 + + def apply(self, **kwargs: Any) -> None: + from sphinx.builders.gettext import MessageCatalogBuilder + if isinstance(self.app.builder, MessageCatalogBuilder): + return + + if not self.config.translation_progress_classes: + return + + if self.config.translation_progress_classes is True: + add_translated = add_untranslated = True + elif self.config.translation_progress_classes == 'translated': + add_translated = True + add_untranslated = False + elif self.config.translation_progress_classes == 'untranslated': + add_translated = False + add_untranslated = True + else: + raise ConfigError('translation_progress_classes must be' + ' True, False, "translated" or "untranslated"') + + for node in self.document.findall(NodeMatcher(translated=Any)): # type: nodes.Element + if node['translated']: + if add_translated: + node.setdefault('classes', []).append('translated') + else: + if add_untranslated: + node.setdefault('classes', []).append('untranslated') + + class RemoveTranslatableInline(SphinxTransform): """ Remove inline nodes used for translation as placeholders. @@ -534,6 +597,8 @@ def apply(self, **kwargs: Any) -> None: def setup(app: Sphinx) -> dict[str, Any]: app.add_transform(PreserveTranslatableMessages) app.add_transform(Locale) + app.add_transform(TranslationProgressTotaliser) + app.add_transform(AddTranslationClasses) app.add_transform(RemoveTranslatableInline) return { diff --git a/tests/roots/test-intl/index.txt b/tests/roots/test-intl/index.txt index 1e09294f984..9de15d5463b 100644 --- a/tests/roots/test-intl/index.txt +++ b/tests/roots/test-intl/index.txt @@ -29,6 +29,7 @@ CONTENTS raw refs section + translation_progress topic .. toctree:: diff --git a/tests/roots/test-intl/translation_progress.txt b/tests/roots/test-intl/translation_progress.txt new file mode 100644 index 00000000000..61f893e2245 --- /dev/null +++ b/tests/roots/test-intl/translation_progress.txt @@ -0,0 +1,36 @@ +Translation Progress +==================== + +When, in disgrace with fortune and men’s eyes, + +I all alone beweep my outcast state, + +And trouble deaf heaven with my bootless cries, + +And look upon myself, and curse my fate, + +Wishing me like to one more rich in hope, + +Featur’d like him, like him with friends possess’d, + +Desiring this man’s art and that man’s scope, + +With what I most enjoy contented least; + +Yet in these thoughts myself almost despising, + +Haply I think on thee, and then my state, + +Like to the lark at break of day arising + +.. untranslated (3 out of 14 lines): + +From sullen earth, sings hymns at heaven’s gate; + +For thy sweet love remember’d such wealth brings + +That then I scorn to change my state with kings. + +.. translation progress substitution + +|translation progress| diff --git a/tests/roots/test-intl/xx/LC_MESSAGES/translation_progress.po b/tests/roots/test-intl/xx/LC_MESSAGES/translation_progress.po new file mode 100644 index 00000000000..a83112f48f6 --- /dev/null +++ b/tests/roots/test-intl/xx/LC_MESSAGES/translation_progress.po @@ -0,0 +1,48 @@ +msgid "" +msgstr "" +"Project-Id-Version: \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2000-01-01 00:00\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: \n" +"Language: xx\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "Translation Progress" +msgstr "TRANSLATION PROGRESS" + +msgid "When, in disgrace with fortune and men’s eyes," +msgstr "WHEN, IN DISGRACE WITH FORTUNE AND MEN’S EYES," + +msgid "I all alone beweep my outcast state," +msgstr "I ALL ALONE BEWEEP MY OUTCAST STATE," + +msgid "And trouble deaf heaven with my bootless cries," +msgstr "AND TROUBLE DEAF HEAVEN WITH MY BOOTLESS CRIES," + +msgid "And look upon myself, and curse my fate," +msgstr "AND LOOK UPON MYSELF, AND CURSE MY FATE," + +msgid "Wishing me like to one more rich in hope," +msgstr "WISHING ME LIKE TO ONE MORE RICH IN HOPE," + +msgid "Featur’d like him, like him with friends possess’d," +msgstr "FEATUR’D LIKE HIM, LIKE HIM WITH FRIENDS POSSESS’D," + +msgid "Desiring this man’s art and that man’s scope," +msgstr "DESIRING THIS MAN’S ART AND THAT MAN’S SCOPE," + +msgid "With what I most enjoy contented least;" +msgstr "WITH WHAT I MOST ENJOY CONTENTED LEAST;" + +msgid "Yet in these thoughts myself almost despising," +msgstr "YET IN THESE THOUGHTS MYSELF ALMOST DESPISING," + +msgid "Haply I think on thee, and then my state," +msgstr "HAPLY I THINK ON THEE, AND THEN MY STATE," + +msgid "Like to the lark at break of day arising" +msgstr "LIKE TO THE LARK AT BREAK OF DAY ARISING" diff --git a/tests/test_intl.py b/tests/test_intl.py index 45cf8f5436d..e451a664e27 100644 --- a/tests/test_intl.py +++ b/tests/test_intl.py @@ -23,6 +23,7 @@ path, strip_escseq, ) +from sphinx.util.nodes import NodeMatcher sphinx_intl = pytest.mark.sphinx( testroot='intl', @@ -521,7 +522,7 @@ def test_text_toctree(app): # --- toctree (toctree.rst) result = (app.outdir / 'toctree.txt').read_text(encoding='utf8') expect = read_po(app.srcdir / 'xx' / 'LC_MESSAGES' / 'toctree.po') - for expect_msg in [m for m in expect if m.id]: + for expect_msg in (m for m in expect if m.id): assert expect_msg.string in result @@ -619,14 +620,58 @@ def test_gettext_buildr_ignores_only_directive(app): def test_node_translated_attribute(app): app.build() - expected = 23 - translated_nodes = 0 + doctree = app.env.get_doctree('translation_progress') + + translated_nodes = sum(1 for _ in doctree.findall(NodeMatcher(translated=True))) + assert translated_nodes == 11 + 1 # 11 lines + title + + untranslated_nodes = sum(1 for _ in doctree.findall(NodeMatcher(translated=False))) + assert untranslated_nodes == 3 + 1 # 3 lines + substitution reference + + +@sphinx_intl +def test_translation_progress_substitution(app): + app.build() + + doctree = app.env.get_doctree('translation_progress') + + assert doctree[0][17][0] == '75.00%' # 12 out of 16 lines are translated + + +@pytest.mark.sphinx(testroot='intl', freshenv=True, confoverrides={ + 'language': 'xx', 'locale_dirs': ['.'], + 'gettext_compact': False, + 'translation_progress_classes': True, +}) +def test_translation_progress_classes_true(app): + app.build() + + doctree = app.env.get_doctree('translation_progress') + + assert 'translated' in doctree[0][0]['classes'] + assert 'translated' in doctree[0][1]['classes'] + assert 'translated' in doctree[0][2]['classes'] + assert 'translated' in doctree[0][3]['classes'] + assert 'translated' in doctree[0][4]['classes'] + assert 'translated' in doctree[0][5]['classes'] + assert 'translated' in doctree[0][6]['classes'] + assert 'translated' in doctree[0][7]['classes'] + assert 'translated' in doctree[0][8]['classes'] + assert 'translated' in doctree[0][9]['classes'] + assert 'translated' in doctree[0][10]['classes'] + assert 'translated' in doctree[0][11]['classes'] + + assert doctree[0][12]['classes'] == [] # comment node + + assert 'untranslated' in doctree[0][13]['classes'] + assert 'untranslated' in doctree[0][14]['classes'] + assert 'untranslated' in doctree[0][15]['classes'] + + assert doctree[0][16]['classes'] == [] # comment node + + assert 'untranslated' in doctree[0][17]['classes'] - doctree = app.env.get_doctree('admonitions') - for node in doctree.findall(): - if hasattr(node, 'get') and node.get('translated', False): - translated_nodes += 1 - assert translated_nodes == expected + assert len(doctree[0]) == 18 @sphinx_intl @@ -682,9 +727,9 @@ def test_html_meta(app): app.build() # --- test for meta result = (app.outdir / 'index.html').read_text(encoding='utf8') - expected_expr = '' + expected_expr = '' assert expected_expr in result - expected_expr = '' + expected_expr = '' assert expected_expr in result expected_expr = '

HIDDEN TOC

' assert expected_expr in result