gh-130057: Pygettext: Support translator comments

Lib/test/test_tools/i18n_data/comments.pot

@@ -0,0 +1,110 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: comments.py:4
+msgid "foo"
+msgstr ""
+
+#. i18n: This is a translator comment
+#: comments.py:7
+msgid "bar"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. i18n: This is another translator comment
+#: comments.py:11
+msgid "baz"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. with multiple
+#. lines
+#: comments.py:16
+msgid "qux"
+msgstr ""
+
+#. i18n: This is a translator comment
+#: comments.py:21
+msgid "quux"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. with multiple lines
+#. i18n: This is another translator comment
+#. with multiple lines
+#: comments.py:27
+msgid "corge"
+msgstr ""
+
+#: comments.py:31
+msgid "grault"
+msgstr ""
+
+#. i18n: This is another translator comment
+#: comments.py:36
+msgid "garply"
+msgstr ""
+
+#: comments.py:40
+msgid "george"
+msgstr ""
+
+#. i18n: This is another translator comment
+#: comments.py:45
+msgid "waldo"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. i18n: This is also a translator comment
+#. i18n: This is another translator comment
+#: comments.py:50
+msgid "waldo2"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. i18n: This is another translator comment
+#. i18n: This is yet another translator comment
+#. i18n: This is a translator comment
+#. with multiple lines
+#: comments.py:53 comments.py:56 comments.py:59 comments.py:63
+msgid "fred"
+msgstr ""
+
+#: comments.py:65
+msgid "plugh"
+msgstr ""
+
+#: comments.py:67
+msgid "foobar"
+msgstr ""
+
+#. i18n: This is a translator comment
+#: comments.py:71
+msgid "xyzzy"
+msgstr ""
+
+#: comments.py:72
+msgid "thud"
+msgstr ""
+
+#. i18n: This is a translator comment
+#. i18n: This is another translator comment
+#. i18n: This is yet another translator comment
+#: comments.py:78
+msgid "foos"
+msgstr ""
+

Lib/test/test_tools/i18n_data/comments.py

@@ -0,0 +1,78 @@
+from gettext import gettext as _
+
+# Not a translator comment
+_('foo')
+
+# i18n: This is a translator comment
+_('bar')
+
+# i18n: This is a translator comment
+# i18n: This is another translator comment
+_('baz')
+
+# i18n: This is a translator comment
+# with multiple
+# lines
+_('qux')
+
+# This comment should not be included because
+# it does not start with the prefix
+# i18n: This is a translator comment
+_('quux')
+
+# i18n: This is a translator comment
+# with multiple lines
+# i18n: This is another translator comment
+# with multiple lines
+_('corge')
+
+# i18n: This comment should be ignored
+
+_('grault')
+
+# i18n: This comment should be ignored
+
+# i18n: This is another translator comment
+_('garply')
+
+# i18n: comment should be ignored
+x = 1
+_('george')
+
+# i18n: This comment should be ignored
+x = 1
+# i18n: This is another translator comment
+_('waldo')
+
+# i18n: This is a translator comment
+x = 1  # i18n: This is also a translator comment
+# i18n: This is another translator comment
+_('waldo2')
+
+# i18n: This is a translator comment
+_('fred')
+
+# i18n: This is another translator comment
+_('fred')
+
+# i18n: This is yet another translator comment
+_('fred')
+
+# i18n: This is a translator comment
+# with multiple lines
+_('fred')
+
+_('plugh')  # i18n: This comment should be ignored
+
+_('foo'  # i18n: This comment should be ignored
+  'bar')  # i18n: This comment should be ignored
+
+# i18n: This is a translator comment
+_('xyzzy')
+_('thud')
+
+
+## i18n: This is a translator comment
+# # i18n: This is another translator comment
+### ###    i18n: This is yet another translator comment
+_('foos')

Lib/test/test_tools/test_i18n.py

@@ -87,7 +87,8 @@ def assert_POT_equal(self, expected, actual):
         self.maxDiff = None
         self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual))
 
-    def extract_from_str(self, module_content, *, args=(), strict=True, with_stderr=False):
+    def extract_from_str(self, module_content, *, args=(), strict=True,
+                         with_stderr=False, raw=False):
         """Return all msgids extracted from module_content."""
         filename = 'test.py'
         with temp_cwd(None):
@@ -98,10 +99,11 @@ def extract_from_str(self, module_content, *, args=(), strict=True, with_stderr=
                 self.assertEqual(res.err, b'')
             with open('messages.pot', encoding='utf-8') as fp:
                 data = fp.read()
-        msgids = self.get_msgids(data)
+        if not raw:
+            data = self.get_msgids(data)
         if not with_stderr:
-            return msgids
-        return msgids, res.err
+            return data
+        return data, res.err
 
     def extract_docstrings_from_str(self, module_content):
         """Return all docstrings extracted from module_content."""
@@ -381,7 +383,8 @@ def test_pygettext_output(self):
                 contents = input_file.read_text(encoding='utf-8')
                 with temp_cwd(None):
                     Path(input_file.name).write_text(contents)
-                    assert_python_ok('-Xutf8', self.script, '--docstrings', input_file.name)
+                    assert_python_ok('-Xutf8', self.script, '--docstrings',
+                                     '--add-comments=i18n:', input_file.name)
                     output = Path('messages.pot').read_text(encoding='utf-8')
 
                 expected = output_file.read_text(encoding='utf-8')
@@ -431,14 +434,60 @@ def test_error_messages(self):
             "*** test.py:3: Variable positional arguments are not allowed in gettext calls\n"
         )
 
+    def test_extract_all_comments(self):
+        """
+        Test that the --add-comments option without an
+        explicit tag extracts all translator comments.
+        """
+        for arg in ('--add-comments', '-c'):
+            with self.subTest(arg=arg):
+                data = self.extract_from_str(dedent('''\
+                # Translator comment
+                _("foo")
+                '''), args=(arg,), raw=True)
+                self.assertIn('#. Translator comment', data)
+
+    def test_comments_with_multiple_tags(self):
+        """
+        Test that multiple --add-comments tags can be specified.
+        """
+        for arg in ('--add-comments={}', '-c{}'):
+            with self.subTest(arg=arg):
+                args = (arg.format('foo:'), arg.format('bar:'))
+                data = self.extract_from_str(dedent('''\
+                # foo: comment
+                _("foo")
+
+                # bar: comment
+                _("bar")
+
+                # baz: comment
+                _("baz")
+                '''), args=args, raw=True)
+                self.assertIn('#. foo: comment', data)
+                self.assertIn('#. bar: comment', data)
+                self.assertNotIn('#. baz: comment', data)
+
+    def test_comments_not_extracted_without_tags(self):
+        """
+        Test that translator comments are not extracted without
+        specifying --add-comments.
+        """
+        data = self.extract_from_str(dedent('''\
+        # Translator comment
+        _("foo")
+        '''), raw=True)
+        self.assertNotIn('#.', data)
+
 
 def update_POT_snapshots():
     for input_file in DATA_DIR.glob('*.py'):
         output_file = input_file.with_suffix('.pot')
         contents = input_file.read_bytes()
         with temp_cwd(None):
             Path(input_file.name).write_bytes(contents)
-            assert_python_ok('-Xutf8', Test_pygettext.script, '--docstrings', input_file.name)
+            assert_python_ok('-Xutf8', Test_pygettext.script, '--docstrings',
+                             '--add-comments=i18n:', input_file.name)
             output = Path('messages.pot').read_text(encoding='utf-8')
 
         output = normalize_POT_file(output)

Misc/NEWS.d/next/Tools-Demos/2025-02-12-23-24-37.gh-issue-130057.TKUKI6.rst

@@ -0,0 +1 @@
+Add support for translator comments in :program:`pygettext.py`.