Add translation progress information (#11509)

Co-authored-by: Manuel Kaufmann <humitos@gmail.com>

CHANGES

@@ -53,6 +53,11 @@ Features added
   via :confval:`linkcheck_anchors_ignore_for_url` while
   still checking the validity of the page itself.
   Patch by Bénédikt Tran
+* #1246: Add translation progress statistics and inspection support,
+  via a new substitution (``|translation progress|``) and a new
+  configuration variable (:confval:`translation_progress_classes`).
+  These enable determining the percentage of translated elements within
+  a document, and the remaining translated and untranslated elements.
 
 Bugs fixed
 ----------

doc/usage/advanced/intl.rst

@@ -340,6 +340,23 @@ There is a `sphinx translation page`_ for Sphinx (master) documentation.
 
 Detail is here: https://docs.transifex.com/getting-started-1/translators
 
+
+Translation progress and statistics
+-----------------------------------
+
+.. versionadded:: 7.1.0
+
+During the rendering process,
+Sphinx marks each translatable node with a ``translated`` attribute,
+indicating if a translation was found for the text in that node.
+
+The :confval:`translation_progress_classes` configuration value
+can be used to add a class to each element,
+depending on the value of the ``translated`` attribute.
+
+The ``|translation progress|`` substitution can be used to display the
+percentage of nodes that have been translated on a per-document basis.
+
 .. rubric:: Footnotes
 
 .. [1] See the `GNU gettext utilities

doc/usage/configuration.rst

@@ -1002,6 +1002,21 @@ documentation on :ref:`intl` for details.
    .. versionchanged:: 3.2
       Added ``{docpath}`` token.
 
+.. confval:: translation_progress_classes
+
+   Control which, if any, classes are added to indicate translation progress.
+   This setting would likely only be used by translators of documentation,
+   in order to quickly indicate translated and untranslated content.
+
+   * ``True``: add ``translated`` and ``untranslated`` classes
+     to all nodes with translatable content.
+   * ``translated``: only add the ``translated`` class.
+   * ``untranslated``: only add the ``untranslated`` class.
+   * ``False``: do not add any classes to indicate translation progress.
+
+   Defaults to ``False``.
+
+   .. versionadded:: 7.1
 
 .. _math-options:
 

doc/usage/restructuredtext/roles.rst

@@ -528,3 +528,9 @@ default. They are set in the build configuration file.
    Replaced by either today's date (the date on which the document is read), or
    the date set in the build configuration file.  Normally has the format
    ``April 14, 2007``.  Set by :confval:`today_fmt` and :confval:`today`.
+
+.. describe:: |translation progress|
+
+   Replaced by the translation progress of the document.
+   This substitution is intented for use by document translators
+   as a marker for the translation progress of the document.

sphinx/config.py

@@ -58,7 +58,7 @@ class ENUM:
     Example:
         app.add_config_value('latex_show_urls', 'no', None, ENUM('no', 'footnote', 'inline'))
     """
-    def __init__(self, *candidates: str) -> None:
+    def __init__(self, *candidates: str | bool) -> None:
         self.candidates = candidates
 
     def match(self, value: str | list | tuple) -> bool:
@@ -101,6 +101,8 @@ class Config:
         'locale_dirs': (['locales'], 'env', []),
         'figure_language_filename': ('{root}.{language}{ext}', 'env', [str]),
         'gettext_allow_fuzzy_translations': (False, 'gettext', []),
+        'translation_progress_classes': (False, 'env',
+                                         ENUM(True, False, 'translated', 'untranslated')),
 
         'master_doc': ('index', 'env', []),
         'root_doc': (lambda config: config.master_doc, 'env', []),

sphinx/environment/__init__.py

@@ -629,7 +629,7 @@ def get_and_resolve_doctree(
                                            prune=prune_toctrees,
                                            includehidden=includehidden)
             if result is None:
-                toctreenode.replace_self([])
+                toctreenode.parent.replace(toctreenode, [])
             else:
                 toctreenode.replace_self(result)
 

sphinx/themes/basic/static/basic.css_t

@@ -748,6 +748,14 @@ abbr, acronym {
     cursor: help;
 }
 
+.translated {
+    background-color: rgba(207, 255, 207, 0.2)
+}
+
+.untranslated {
+    background-color: rgba(255, 207, 207, 0.2)
+}
+
 /* -- code displays --------------------------------------------------------- */
 
 pre {

sphinx/transforms/__init__.py

@@ -34,6 +34,7 @@
     'version',
     'release',
     'today',
+    'translation progress',
 }
 
 
@@ -103,14 +104,31 @@ def apply(self, **kwargs: Any) -> None:
         for ref in self.document.findall(nodes.substitution_reference):
             refname = ref['refname']
             if refname in to_handle:
-                text = self.config[refname]
+                if refname == 'translation progress':
+                    # special handling: calculate translation progress
+                    text = _calculate_translation_progress(self.document)
+                else:
+                    text = self.config[refname]
                 if refname == 'today' and not text:
                     # special handling: can also specify a strftime format
                     text = format_date(self.config.today_fmt or _('%b %d, %Y'),
                                        language=self.config.language)
                 ref.replace_self(nodes.Text(text))
 
 
+def _calculate_translation_progress(document: nodes.document) -> str:
+    try:
+        translation_progress = document['translation_progress']
+    except KeyError:
+        return _('could not calculate translation progress!')
+
+    total = translation_progress['total']
+    translated = translation_progress['translated']
+    if total <= 0:
+        return _('no translated elements!')
+    return f'{translated / total:.2%}'
+
+
 class MoveModuleTargets(SphinxTransform):
     """
     Move module targets that are the first thing in a section to the section

sphinx/transforms/i18n.py

@@ -14,6 +14,7 @@
 from sphinx import addnodes
 from sphinx.config import Config
 from sphinx.domains.std import make_glossary_term, split_term_classifiers
+from sphinx.errors import ConfigError
 from sphinx.locale import __
 from sphinx.locale import init as init_locale
 from sphinx.transforms import SphinxTransform
@@ -360,9 +361,9 @@ def apply(self, **kwargs: Any) -> None:
             if not isinstance(node, LITERAL_TYPE_NODES):
                 msgstr, _ = parse_noqa(msgstr)
 
-            # XXX add marker to untranslated parts
             if not msgstr or msgstr == msg or not msgstr.strip():
                 # as-of-yet untranslated
+                node['translated'] = False
                 continue
 
             # Avoid "Literal block expected; none found." warnings.
@@ -404,10 +405,12 @@ def apply(self, **kwargs: Any) -> None:
             if processed:
                 updater.update_leaves()
                 node['translated'] = True  # to avoid double translation
+            else:
+                node['translated'] = False
 
         # phase2: translation
         for node, msg in extract_messages(self.document):
-            if node.get('translated', False):  # to avoid double translation
+            if node.setdefault('translated', False):  # to avoid double translation
                 continue  # skip if the node is already translated by phase1
 
             msgstr = catalog.gettext(msg)
@@ -417,8 +420,8 @@ def apply(self, **kwargs: Any) -> None:
             if not isinstance(node, LITERAL_TYPE_NODES):
                 msgstr, noqa = parse_noqa(msgstr)
 
-            # XXX add marker to untranslated parts
             if not msgstr or msgstr == msg:  # as-of-yet untranslated
+                node['translated'] = False
                 continue
 
             # update translatable nodes
@@ -429,6 +432,7 @@ def apply(self, **kwargs: Any) -> None:
             # update meta nodes
             if isinstance(node, nodes.meta):  # type: ignore[attr-defined]
                 node['content'] = msgstr
+                node['translated'] = True
                 continue
 
             if isinstance(node, nodes.image) and node.get('alt') == msg:
@@ -490,6 +494,7 @@ def apply(self, **kwargs: Any) -> None:
 
             if isinstance(node, nodes.image) and node.get('alt') != msg:
                 node['uri'] = patch['uri']
+                node['translated'] = False
                 continue  # do not mark translated
 
             node['translated'] = True  # to avoid double translation
@@ -514,6 +519,64 @@ def apply(self, **kwargs: Any) -> None:
                 node['entries'] = new_entries
 
 
+class TranslationProgressTotaliser(SphinxTransform):
+    """
+    Calculate the number of translated and untranslated nodes.
+    """
+    default_priority = 25  # MUST happen after Locale
+
+    def apply(self, **kwargs: Any) -> None:
+        from sphinx.builders.gettext import MessageCatalogBuilder
+        if isinstance(self.app.builder, MessageCatalogBuilder):
+            return
+
+        total = translated = 0
+        for node in self.document.findall(NodeMatcher(translated=Any)):  # type: nodes.Element
+            total += 1
+            if node['translated']:
+                translated += 1
+
+        self.document['translation_progress'] = {
+            'total': total,
+            'translated': translated,
+        }
+
+
+class AddTranslationClasses(SphinxTransform):
+    """
+    Add ``translated`` or ``untranslated`` classes to indicate translation status.
+    """
+    default_priority = 950
+
+    def apply(self, **kwargs: Any) -> None:
+        from sphinx.builders.gettext import MessageCatalogBuilder
+        if isinstance(self.app.builder, MessageCatalogBuilder):
+            return
+
+        if not self.config.translation_progress_classes:
+            return
+
+        if self.config.translation_progress_classes is True:
+            add_translated = add_untranslated = True
+        elif self.config.translation_progress_classes == 'translated':
+            add_translated = True
+            add_untranslated = False
+        elif self.config.translation_progress_classes == 'untranslated':
+            add_translated = False
+            add_untranslated = True
+        else:
+            raise ConfigError('translation_progress_classes must be'
+                              ' True, False, "translated" or "untranslated"')
+
+        for node in self.document.findall(NodeMatcher(translated=Any)):  # type: nodes.Element
+            if node['translated']:
+                if add_translated:
+                    node.setdefault('classes', []).append('translated')
+            else:
+                if add_untranslated:
+                    node.setdefault('classes', []).append('untranslated')
+
+
 class RemoveTranslatableInline(SphinxTransform):
     """
     Remove inline nodes used for translation as placeholders.
@@ -534,6 +597,8 @@ def apply(self, **kwargs: Any) -> None:
 def setup(app: Sphinx) -> dict[str, Any]:
     app.add_transform(PreserveTranslatableMessages)
     app.add_transform(Locale)
+    app.add_transform(TranslationProgressTotaliser)
+    app.add_transform(AddTranslationClasses)
     app.add_transform(RemoveTranslatableInline)
 
     return {

tests/roots/test-intl/index.txt

@@ -29,6 +29,7 @@ CONTENTS
    raw
    refs
    section
+   translation_progress
    topic
 
 .. toctree::

tests/roots/test-intl/translation_progress.txt

@@ -0,0 +1,36 @@
+Translation Progress
+====================
+
+When, in disgrace with fortune and men’s eyes,
+
+I all alone beweep my outcast state,
+
+And trouble deaf heaven with my bootless cries,
+
+And look upon myself, and curse my fate,
+
+Wishing me like to one more rich in hope,
+
+Featur’d like him, like him with friends possess’d,
+
+Desiring this man’s art and that man’s scope,
+
+With what I most enjoy contented least;
+
+Yet in these thoughts myself almost despising,
+
+Haply I think on thee, and then my state,
+
+Like to the lark at break of day arising
+
+.. untranslated (3 out of 14 lines):
+
+From sullen earth, sings hymns at heaven’s gate;
+
+For thy sweet love remember’d such wealth brings
+
+That then I scorn to change my state with kings.
+
+.. translation progress substitution
+
+|translation progress|

tests/roots/test-intl/xx/LC_MESSAGES/translation_progress.po

@@ -0,0 +1,48 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2000-01-01 00:00\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: \n"
+"Language: xx\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+msgid "Translation Progress"
+msgstr "TRANSLATION PROGRESS"
+
+msgid "When, in disgrace with fortune and men’s eyes,"
+msgstr "WHEN, IN DISGRACE WITH FORTUNE AND MEN’S EYES,"
+
+msgid "I all alone beweep my outcast state,"
+msgstr "I ALL ALONE BEWEEP MY OUTCAST STATE,"
+
+msgid "And trouble deaf heaven with my bootless cries,"
+msgstr "AND TROUBLE DEAF HEAVEN WITH MY BOOTLESS CRIES,"
+
+msgid "And look upon myself, and curse my fate,"
+msgstr "AND LOOK UPON MYSELF, AND CURSE MY FATE,"
+
+msgid "Wishing me like to one more rich in hope,"
+msgstr "WISHING ME LIKE TO ONE MORE RICH IN HOPE,"
+
+msgid "Featur’d like him, like him with friends possess’d,"
+msgstr "FEATUR’D LIKE HIM, LIKE HIM WITH FRIENDS POSSESS’D,"
+
+msgid "Desiring this man’s art and that man’s scope,"
+msgstr "DESIRING THIS MAN’S ART AND THAT MAN’S SCOPE,"
+
+msgid "With what I most enjoy contented least;"
+msgstr "WITH WHAT I MOST ENJOY CONTENTED LEAST;"
+
+msgid "Yet in these thoughts myself almost despising,"
+msgstr "YET IN THESE THOUGHTS MYSELF ALMOST DESPISING,"
+
+msgid "Haply I think on thee, and then my state,"
+msgstr "HAPLY I THINK ON THEE, AND THEN MY STATE,"
+
+msgid "Like to the lark at break of day arising"
+msgstr "LIKE TO THE LARK AT BREAK OF DAY ARISING"