gh-154668: Add escaping mode to PyUnicode_FromFormat()

Doc/c-api/unicode.rst

@@ -477,6 +477,30 @@ APIs:
    | ``-`` | The converted value is left adjusted (overrides the ``0``   |
    |       | flag if both are given).                                    |
    +-------+-------------------------------------------------------------+
+   | ``#`` | Escape special and non-printable characters for the ``c``,  |
+   |       | ``s``, ``S``, ``U`` and ``V`` conversions.                  |
+   |       | Use a colon as a separator for the ``T`` and ``N``          |
+   |       | conversions.                                                |
+   +-------+-------------------------------------------------------------+
+   | ``+`` | Escape also all non-ASCII characters (only in combination   |
+   |       | with the ``#`` flag for the ``c``, ``s``, ``S``, ``U`` and  |
+   |       | ``V`` conversions).                                         |
+   +-------+-------------------------------------------------------------+
+
+   With the ``#`` flag, the ``c``, ``s``, ``S``, ``U`` and ``V``
+   conversions escape special and non-printable characters
+   like :func:`repr` does,
+   but without adding surrounding quotes:
+   backslash, tab, carriage return and line feed are escaped as
+   ``\\``, ``\t``, ``\r`` and ``\n`` respectively,
+   other non-printable characters are escaped as ``\xNN`` if they are
+   ASCII, and as ``\uNNNN`` or ``\UNNNNNNNN`` otherwise.
+   With the ``+`` flag, all non-ASCII characters are escaped as well.
+   For the ``s`` and ``V`` conversions,
+   bytes which cannot be decoded from UTF-8 are escaped as ``\xNN``,
+   so they can always be distinguished from valid non-ASCII characters.
+   The precision is applied to the string before escaping,
+   the width -- after escaping.
 
    The length modifiers for following integer conversions (``d``, ``i``,
    ``o``, ``u``, ``x``, or ``X``) specify the type of the argument
@@ -630,6 +654,10 @@ APIs:
    .. versionchanged:: 3.13
       Support for ``%T``, ``%#T``, ``%N`` and ``%#N`` formats added.
 
+   .. versionchanged:: next
+      Support for flags ``#`` and ``+`` for the ``c``, ``s``, ``S``,
+      ``U`` and ``V`` conversions added.
+
 
 .. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs)
 

Doc/whatsnew/3.16.rst

@@ -814,7 +814,12 @@ C API changes
 New features
 ------------
 
-* TODO
+* :c:func:`PyUnicode_FromFormat` now supports the escaping mode for
+  the ``c``, ``s``, ``S``, ``U`` and ``V`` conversions:
+  with the ``#`` flag, special and non-printable characters in the output
+  are escaped,
+  and with the additional ``+`` flag -- also all non-ASCII characters.
+  (Contributed by Serhiy Storchaka in :gh:`154668`.)
 
 Porting to Python 3.16
 ----------------------

Lib/test/test_capi/test_unicode.py

@@ -535,6 +535,114 @@ def check_format(expected, format, *args):
         check_format("repr=   12",
                      b'repr=%5.2V', None, b'123')
 
+        # test the escaping mode ("#" flag) with %c
+        check_format('A',
+                     b'%#c', c_int(ord('A')))
+        check_format('\\n',
+                     b'%#c', c_int(ord('\n')))
+        check_format('\\\\',
+                     b'%#c', c_int(ord('\\')))
+        check_format('\\x1b',
+                     b'%#c', c_int(0x1b))
+        check_format('\\x7f',
+                     b'%#c', c_int(0x7f))
+        check_format('\\u009f',
+                     b'%#c', c_int(0x9f))
+        check_format('\xe9',
+                     b'%#c', c_int(0xe9))
+        check_format('\\u2028',
+                     b'%#c', c_int(0x2028))
+        check_format('\U0001f600',
+                     b'%#c', c_int(0x1f600))
+        check_format('\\ud800',
+                     b'%#c', c_int(0xd800))
+        check_format('\\udc9f',
+                     b'%#c', c_int(0xdc9f))
+        check_format('\\u00e9',
+                     b'%+#c', c_int(0xe9))
+        check_format('\\u20ac',
+                     b'%+#c', c_int(0x20ac))
+        check_format('\\U0001f600',
+                     b'%+#c', c_int(0x1f600))
+
+        # test the escaping mode with %s
+        check_format('a\\tb\\r\\nc\\\\d',
+                     b'%#s', b'a\tb\r\nc\\d')
+        check_format('quote\'"',
+                     b'%#s', b'quote\'"')
+        check_format('caf\xe9',
+                     b'%#s', b'caf\xc3\xa9')
+        check_format('caf\\u00e9',
+                     b'%+#s', b'caf\xc3\xa9')
+        # valid UTF-8 is escaped as a character, invalid -- as a raw byte
+        check_format('\\u009f',
+                     b'%#s', b'\xc2\x9f')
+        check_format('\\x9f',
+                     b'%#s', b'\x9f')
+        check_format('\\x9f',
+                     b'%+#s', b'\x9f')
+        check_format('a\\xffb',
+                     b'%#s', b'a\xffb')
+        check_format('abc\\xc3',
+                     b'%#s', b'abc\xc3')
+        # the precision is applied before escaping, the width -- after
+        check_format('ab\\n',
+                     b'%#.3s', b'ab\ncd')
+        check_format('  ab\\n',
+                     b'%#6.3s', b'ab\ncd')
+        check_format('ab\\n  ',
+                     b'%-#6.3s', b'ab\ncd')
+        check_format('a\\xff',
+                     b'%#.2s', b'a\xffbc')
+        check_format('abc',
+                     b'%#.4s', b'abc\xc3\xa9')
+
+        # test the escaping mode with %U, %V and %S
+        check_format('a\\tb',
+                     b'%#U', 'a\tb')
+        check_format('caf\xe9',
+                     b'%#U', 'caf\xe9')
+        check_format('caf\\u00e9',
+                     b'%+#U', 'caf\xe9')
+        check_format('a\\tb',
+                     b'%#S', 'a\tb')
+        check_format('caf\xe9',
+                     b'%#S', 'caf\xe9')
+        check_format('caf\\u00e9',
+                     b'%+#S', 'caf\xe9')
+        check_format('   ab',
+                     b'%#5.2S', 'ab\ncd')
+        # a lone surrogate in a string argument stays a surrogate
+        check_format('\\udc9f',
+                     b'%#U', '\udc9f')
+        check_format('\\udc9f',
+                     b'%+#U', '\udc9f')
+        check_format('   ab',
+                     b'%#5.2U', 'ab\ncd')
+        check_format('a\\tb',
+                     b'%#V', 'a\tb', b'ignored')
+        check_format('\\u009f',
+                     b'%#V', None, b'\xc2\x9f')
+        check_format('\\x9f',
+                     b'%#V', None, b'\x9f')
+
+        # test the escaping mode with %ls and %lV
+        check_format('a\\tb',
+                     b'%#ls', c_wchar_p('a\tb'))
+        check_format('caf\xe9',
+                     b'%#ls', c_wchar_p('caf\xe9'))
+        check_format('caf\\u00e9',
+                     b'%+#ls', c_wchar_p('caf\xe9'))
+        check_format('a\\tb',
+                     b'%#lV', None, c_wchar_p('a\tb'))
+
+        # "+" requires "#" and is only supported for %c, %s, %S, %U and %V
+        for format in (b'%+c', b'%+s', b'%+S', b'%+U', b'%+d', b'%+#d',
+                       b'%+#R', b'%+#A', b'%+#T', b'%+#p'):
+            with self.subTest(format=format):
+                with self.assertRaises(SystemError):
+                    PyUnicode_FromFormat(format, py_object('abc'))
+
         # test integer formats (%i, %d, %u, %o, %x, %X)
         check_format('010',
                      b'%03i', c_int(10))
@@ -2003,6 +2111,11 @@ def test_format(self):
         writer.write_char(ord('.'))
         self.assertEqual(writer.finish(), 'abc 123.')
 
+    def test_format_escape(self):
+        writer = self.create_writer(0)
+        self.writer_format(writer, b'%#s %+#s', b'a\tb\xff', b'caf\xc3\xa9')
+        self.assertEqual(writer.finish(), 'a\\tb\\xff caf\\u00e9')
+
     def test_recover_error(self):
         # test recovering from PyUnicodeWriter_Format() error
         writer = self.create_writer(0)

Misc/NEWS.d/next/C_API/2026-07-25-10-40-00.gh-issue-154668.fEsc4p.rst

@@ -0,0 +1,5 @@
+Add the escaping mode to :c:func:`PyUnicode_FromFormat`:
+with the ``#`` flag,
+the ``c``, ``s``, ``S``, ``U`` and ``V`` conversions now escape special
+and non-printable characters,
+and with the additional ``+`` flag -- also all non-ASCII characters.

Modules/_testlimitedcapi/unicode.c

@@ -1824,6 +1824,29 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored))
     CHECK_FORMAT_2("%1.5V",  "None",    NULL,    "None");
     CHECK_FORMAT_2("%1.5lV", "None",    NULL,   L"None");
 
+    // Strings: escaping mode ('#' and '+' flags)
+    CHECK_FORMAT_1("%#c",      "c",       'c');
+    CHECK_FORMAT_1("%#c",    "\\n",      '\n');
+    CHECK_FORMAT_1("%+#c",   "\\n",      '\n');
+    CHECK_FORMAT_1("%#s",  "a\\tb",   "a\tb");
+    CHECK_FORMAT_1("%+#s", "a\\tb",   "a\tb");
+    CHECK_FORMAT_1("%#ls", "a\\tb",  L"a\tb");
+    CHECK_FORMAT_1("%#S",   "None",   Py_None);
+    CHECK_FORMAT_1("%+#S",  "None",   Py_None);
+    CHECK_FORMAT_1("%#U",   "None",   unicode);
+    CHECK_FORMAT_2("%#V",   "None",   unicode, "ignored");
+    CHECK_FORMAT_2("%#V",  "a\\tb",      NULL,   "a\tb");
+    CHECK_FORMAT_2("%#lV", "a\\tb",      NULL,  L"a\tb");
+    CHECK_FORMAT_1("%#6.3s", "  ab\\n", "ab\ncd");
+
+    // The '+' flag requires the '#' flag and is only supported for
+    // the 'c', 's', 'S', 'U' and 'V' conversions
+    CHECK_FORMAT_1("%+c",  NULL, 'c');
+    CHECK_FORMAT_1("%+s",  NULL, "abc");
+    CHECK_FORMAT_1("%+d",  NULL, 1);
+    CHECK_FORMAT_1("%+#d", NULL, 1);
+    CHECK_FORMAT_1("%+#R", NULL, Py_None);
+
     Py_XDECREF(unicode);
     Py_RETURN_NONE;