From 1f65ec6e53eb88c022368460b4c8d9579cfdd9ff Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 17 Apr 2024 22:59:37 +0100 Subject: [PATCH 01/19] Redirect stdout and stderr to system log when embedded in an Android app --- Lib/_android_support.py | 99 +++++++ Lib/test/test_android.py | 269 ++++++++++++++++++ ...-04-17-22-49-15.gh-issue-116622.tthNUF.rst | 1 + Python/pylifecycle.c | 27 ++ 4 files changed, 396 insertions(+) create mode 100644 Lib/_android_support.py create mode 100644 Lib/test/test_android.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst diff --git a/Lib/_android_support.py b/Lib/_android_support.py new file mode 100644 index 00000000000000..41fecf68f6dcba --- /dev/null +++ b/Lib/_android_support.py @@ -0,0 +1,99 @@ +import io +import sys +from ctypes import CDLL, c_char_p, c_int + + +# The maximum length of a log message in bytes, including the level marker and +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in +# platform/system/logging/liblog/include/log/log.h. As of API level 30, messages +# longer than this will be be truncated by logcat. This limit has already been +# reduced at least once in the history of Android (from 4076 to 4068 between API +# level 23 and 26), so leave some headroom. +MAX_BYTES_PER_WRITE = 4000 + +# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this +# size ensures that TextIOWrapper can always avoid exceeding MAX_BYTES_PER_WRITE. +# However, if the actual number of bytes per character is smaller than that, +# then TextIOWrapper may still join multiple consecutive text writes into binary +# writes containing a larger number of characters. +MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 + + +# When embedded in an app on current versions of Android, there's no easy way to +# monitor the C-level stdout and stderr. The testbed comes with a .c file to +# redirect them to the system log using a pipe, but that wouldn't be convenient +# or appropriate for all apps. So we redirect at the Python level instead. +def init_streams(): + if sys.executable: + return # Not embedded in an app. + + # Despite its name, this function is part of the public API + # (https://developer.android.com/ndk/reference/group/logging). + # Use `getattr` to avoid private name mangling. + global android_log_write + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + + # These log levels match those used by Java's System.out and System.err. + ANDROID_LOG_INFO = 4 + ANDROID_LOG_WARN = 5 + + sys.stdout = TextLogStream( + ANDROID_LOG_INFO, "python.stdout", errors=sys.stdout.errors) + sys.stderr = TextLogStream( + ANDROID_LOG_WARN, "python.stderr", errors=sys.stderr.errors) + + +class TextLogStream(io.TextIOWrapper): + def __init__(self, level, tag, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(BinaryLogStream(level, tag), **kwargs) + self._CHUNK_SIZE = MAX_BYTES_PER_WRITE + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # We want to emit one log message per line wherever possible, so split + # the string before sending it to the superclass. Note that + # "".splitlines() == [], so nothing will be logged for an empty string. + for line, line_keepends in zip( + s.splitlines(), s.splitlines(keepends=True) + ): + # Simplify the later stages by translating all newlines into "\n". + if line != line_keepends: + line += "\n" + while line: + super().write(line[:MAX_CHARS_PER_WRITE]) + line = line[MAX_CHARS_PER_WRITE:] + + return len(s) + + +class BinaryLogStream(io.RawIOBase): + def __init__(self, level, tag): + self.level = level + self.tag = tag + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if hasattr(b, "__buffer__"): + b = bytes(b) + else: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}") + + # Writing an empty string to the stream should have no effect. + if b: + android_log_write(self.level, self.tag.encode("UTF-8"), b) + return len(b) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py new file mode 100644 index 00000000000000..d8885184fee12d --- /dev/null +++ b/Lib/test/test_android.py @@ -0,0 +1,269 @@ +import platform +import queue +import re +import subprocess +import sys +import unittest +from contextlib import contextmanager +from ctypes import CDLL, c_char_p, c_int +from threading import Thread +from time import time + + +if sys.platform != "android": + raise unittest.SkipTest("Android-specific") + +api_level = platform.android_ver().api_level + + +# Test redirection of stdout and stderr to the Android log. +class TestAndroidOutput(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.logcat_process = subprocess.Popen( + ["logcat", "-v", "tag"], stdout=subprocess.PIPE, + errors="backslashreplace" + ) + self.logcat_queue = queue.Queue() + + def logcat_thread(): + for line in self.logcat_process.stdout: + self.logcat_queue.put(line.rstrip("\n")) + self.logcat_process.stdout.close() + Thread(target=logcat_thread).start() + + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + ANDROID_LOG_INFO = 4 + + # Separate tests using a marker line with a different tag. + tag, message = "python.test", f"{self.id()} {time()}" + android_log_write( + ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8")) + self.assert_log("I", tag, message, skip=True, timeout=5) + + def assert_logs(self, level, tag, expected, **kwargs): + for line in expected: + self.assert_log(level, tag, line, **kwargs) + + def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): + deadline = time() + timeout + while True: + try: + line = self.logcat_queue.get(timeout=(deadline - time())) + except queue.Empty: + self.fail(f"line not found: {expected!r}") + if match := re.fullmatch(fr"(.)/{tag}: (.*)", line): + try: + self.assertEqual(level, match[1]) + self.assertEqual(expected, match[2]) + break + except AssertionError: + if not skip: + raise + + def tearDown(self): + self.logcat_process.terminate() + self.logcat_process.wait(0.1) + + @contextmanager + def unbuffered(self, stream): + stream.reconfigure(write_through=True) + try: + yield + finally: + stream.reconfigure(write_through=False) + + def test_str(self): + for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + with self.subTest(stream=stream_name): + stream = getattr(sys, stream_name) + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + + self.assertTrue(stream.writable()) + self.assertFalse(stream.readable()) + self.assertEqual("UTF-8", stream.encoding) + self.assertTrue(stream.line_buffering) + self.assertFalse(stream.write_through) + + # stderr is backslashreplace by default; stdout is configured + # that way by libregrtest.main. + self.assertEqual("backslashreplace", stream.errors) + + def write(s, lines=None): + self.assertEqual(len(s), stream.write(s)) + if lines is None: + lines = [s] + self.assert_logs(level, tag, lines) + + # Single-line messages, + with self.unbuffered(stream): + write("", []) + + write("a") + write("Hello") + write("Hello world") + write(" ") + write(" ") + + # Non-ASCII text + write("ol\u00e9") # Spanish + write("\u4e2d\u6587") # Chinese + + # Non-BMP emoji + write("\U0001f600", + [r"\xed\xa0\xbd\xed\xb8\x80" if api_level < 23 + else "\U0001f600"]) + + # Null characters will truncate a message. + write("\u0000", [""]) + write("a\u0000", ["a"]) + write("\u0000b", [""]) + write("a\u0000b", ["a"]) + + # Multi-line messages. Avoid identical consecutive lines, as + # they may activate "chatty" filtering and break the tests. + write("\nx", [""]) + write("\na\n", ["x", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d"]) + write("xx", []) + write("f\n\ng", ["exxf", ""]) + write("\n", ["g"]) + + with self.unbuffered(stream): + write("\nx", ["", "x"]) + write("\na\n", ["", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d", "e"]) + write("xx", ["xx"]) + write("f\n\ng", ["f", "", "g"]) + write("\n", [""]) + + # "\r\n" should be translated into "\n". + write("hello\r\n", ["hello"]) + write("hello\r\nworld\r\n", ["hello", "world"]) + write("\r\n", [""]) + + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + # Manual flushing is supported. + write("hello", []) + stream.flush() + self.assert_log(level, tag, "hello") + write("hello", []) + write("world", []) + stream.flush() + self.assert_log(level, tag, "helloworld") + + # Long lines are split into blocks of 1000 *characters*, but + # TextIOWrapper should then join them back together as much as + # possible without exceeding 4000 UTF-8 *bytes*. + # + # ASCII (1 byte per character) + write(("foobar" * 700) + "\n", + [("foobar" * 666) + "foob", # 4000 bytes + "ar" + ("foobar" * 33)]) # 200 bytes + + # "Full-width" digits 0-9 (3 bytes per character) + s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" + write((s * 150) + "\n", + [s * 100, # 3000 bytes + s * 50]) # 1500 bytes + + s = "0123456789" + write(s * 200, []) + write(s * 150, []) + write(s * 51, [s * 350]) # 3500 bytes + write("\n", [s * 51]) # 510 bytes + + def test_bytes(self): + for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + with self.subTest(stream=stream_name): + stream = getattr(sys, stream_name).buffer + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + self.assertTrue(stream.writable()) + self.assertFalse(stream.readable()) + + def write(b, lines=None): + self.assertEqual(len(b), stream.write(b)) + if lines is None: + lines = [b.decode()] + self.assert_logs(level, tag, lines) + + # Single-line messages, + write(b"", []) + + write(b"a") + write(b"Hello") + write(b"Hello world") + write(b" ") + write(b" ") + + # Non-ASCII text + write(b"ol\xc3\xa9") # Spanish + write(b"\xe4\xb8\xad\xe6\x96\x87") # Chinese + + # Non-BMP emoji + write(b"\xf0\x9f\x98\x80", + [r"\xed\xa0\xbd\xed\xb8\x80" if api_level < 23 + else "\U0001f600"]) + + # Null characters will truncate a message. + write(b"\x00", [""]) + write(b"a\x00", ["a"]) + write(b"\x00b", [""]) + write(b"a\x00b", ["a"]) + + # Invalid UTF-8 + write(b"\xff", [r"\xff"]) + write(b"a\xff", [r"a\xff"]) + write(b"\xffb", [r"\xffb"]) + write(b"a\xffb", [r"a\xffb"]) + + # Log entries containing newlines are shown differently by + # `logcat -v tag`, `logcat -v long`, and Android Studio. We + # currently use `logcat -v tag`, which shows each line as if it + # was a separate log entry, but strips a single trailing + # newline. + # + # On newer versions of Android, all three of the above tools (or + # maybe Logcat itself) will also strip any number of leading + # newlines. + write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) + write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) + write(b"\n", [""]) + write(b"b\n", ["b"]) + write(b"c\n\n", ["c", ""]) + write(b"d\ne", ["d", "e"]) + write(b"xx", ["xx"]) + write(b"f\n\ng", ["f", "", "g"]) + write(b"\n", [""]) + + # "\r\n" should be translated into "\n". + write(b"hello\r\n", ["hello"]) + write(b"hello\r\nworld\r\n", ["hello", "world"]) + write(b"\r\n", [""]) + + for obj in ["", "hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst new file mode 100644 index 00000000000000..04f84792f667a0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst @@ -0,0 +1 @@ +Redirect stdout and stderr to system log when embedded in an Android app. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index efb25878312d85..ddd5f30e8af476 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -71,6 +71,7 @@ static PyStatus add_main_module(PyInterpreterState *interp); static PyStatus init_import_site(void); static PyStatus init_set_builtins_open(void); static PyStatus init_sys_streams(PyThreadState *tstate); +static void init_android_streams(PyThreadState *tstate); static void wait_for_thread_shutdown(PyThreadState *tstate); static void call_ll_exitfuncs(_PyRuntimeState *runtime); @@ -1223,6 +1224,10 @@ init_interp_main(PyThreadState *tstate) return status; } +#ifdef __ANDROID__ + init_android_streams(tstate); +#endif + #ifdef Py_DEBUG run_presite(tstate); #endif @@ -2719,6 +2724,28 @@ init_sys_streams(PyThreadState *tstate) } +/* Redirecting streams to the log is a convenience, but won't be critical for + every app, so failures of this function are non-fatal. */ +static void +init_android_streams(PyThreadState *tstate) +{ + PyObject *_android_support = PyImport_ImportModule("_android_support"); + if (_android_support == NULL) { + fprintf(stderr, "_android_support import failed:\n"); + _PyErr_Print(tstate); + } else { + PyObject *result = PyObject_CallMethod(_android_support, + "init_streams", NULL); + if (result == NULL) { + fprintf(stderr, "_android_support.init_streams failed:\n"); + _PyErr_Print(tstate); + } + Py_XDECREF(result); + } + Py_XDECREF(_android_support); +} + + static void _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, PyThreadState *tstate) From d5cadd57acd941f41b0e5884540f7e67884a6e8a Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 18 Apr 2024 19:22:58 +0100 Subject: [PATCH 02/19] Deal with variations in older Android versions --- Lib/test/test_android.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index d8885184fee12d..0a00c33fab6960 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -17,6 +17,10 @@ # Test redirection of stdout and stderr to the Android log. +@unittest.skipIf( + api_level < 23 and platform.machine() == "aarch64", + "SELinux blocks reading logs on older ARM64 emulators" +) class TestAndroidOutput(unittest.TestCase): maxDiff = None @@ -118,9 +122,9 @@ def write(s, lines=None): else "\U0001f600"]) # Null characters will truncate a message. - write("\u0000", [""]) + write("\u0000", [] if api_level < 24 else [""]) write("a\u0000", ["a"]) - write("\u0000b", [""]) + write("\u0000b", [] if api_level < 24 else [""]) write("a\u0000b", ["a"]) # Multi-line messages. Avoid identical consecutive lines, as @@ -224,9 +228,9 @@ def write(b, lines=None): else "\U0001f600"]) # Null characters will truncate a message. - write(b"\x00", [""]) + write(b"\x00", [] if api_level < 24 else [""]) write(b"a\x00", ["a"]) - write(b"\x00b", [""]) + write(b"\x00b", [] if api_level < 24 else [""]) write(b"a\x00b", ["a"]) # Invalid UTF-8 From 8e4d56adf64f9b1a57a28fc70a266f196a8d1946 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 18 Apr 2024 19:34:48 +0100 Subject: [PATCH 03/19] Deal with even older Android versions --- Lib/test/test_android.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 0a00c33fab6960..ca6f1eb96edb26 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -117,9 +117,7 @@ def write(s, lines=None): write("\u4e2d\u6587") # Chinese # Non-BMP emoji - write("\U0001f600", - [r"\xed\xa0\xbd\xed\xb8\x80" if api_level < 23 - else "\U0001f600"]) + write("\U0001f600") # Null characters will truncate a message. write("\u0000", [] if api_level < 24 else [""]) @@ -223,9 +221,7 @@ def write(b, lines=None): write(b"\xe4\xb8\xad\xe6\x96\x87") # Chinese # Non-BMP emoji - write(b"\xf0\x9f\x98\x80", - [r"\xed\xa0\xbd\xed\xb8\x80" if api_level < 23 - else "\U0001f600"]) + write(b"\xf0\x9f\x98\x80") # Null characters will truncate a message. write(b"\x00", [] if api_level < 24 else [""]) From 6da6ef0b5a8d7a07121b2e442d398e989fdcda47 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 18 Apr 2024 20:00:03 +0100 Subject: [PATCH 04/19] Update stdlib_module_names.h --- Python/stdlib_module_names.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index ac9d91b5e12885..a1e4e94c47b322 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -5,6 +5,7 @@ static const char* _Py_stdlib_module_names[] = { "__future__", "_abc", "_aix_support", +"_android_support", "_ast", "_asyncio", "_bisect", From c6d9809398fdf78c0578f7c8818a9192f284851e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 18 Apr 2024 20:08:56 +0100 Subject: [PATCH 05/19] Move ctypes import from top level into a function, since it's not available on WASI --- Lib/test/test_android.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index ca6f1eb96edb26..8d5bebd6ca92dc 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -5,7 +5,6 @@ import sys import unittest from contextlib import contextmanager -from ctypes import CDLL, c_char_p, c_int from threading import Thread from time import time @@ -37,6 +36,7 @@ def logcat_thread(): self.logcat_process.stdout.close() Thread(target=logcat_thread).start() + from ctypes import CDLL, c_char_p, c_int android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") android_log_write.argtypes = (c_int, c_char_p, c_char_p) ANDROID_LOG_INFO = 4 From 1f2114121851d3b58d2d0caf8dd8099fb3cea049 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 18 Apr 2024 20:20:24 +0100 Subject: [PATCH 06/19] Add #ifdef to avoid unused-function warning on other platforms --- Python/pylifecycle.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ddd5f30e8af476..8eb91a57dba10c 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -71,7 +71,9 @@ static PyStatus add_main_module(PyInterpreterState *interp); static PyStatus init_import_site(void); static PyStatus init_set_builtins_open(void); static PyStatus init_sys_streams(PyThreadState *tstate); +#ifdef __ANDROID__ static void init_android_streams(PyThreadState *tstate); +#endif static void wait_for_thread_shutdown(PyThreadState *tstate); static void call_ll_exitfuncs(_PyRuntimeState *runtime); @@ -2724,6 +2726,7 @@ init_sys_streams(PyThreadState *tstate) } +#ifdef __ANDROID__ /* Redirecting streams to the log is a convenience, but won't be critical for every app, so failures of this function are non-fatal. */ static void @@ -2744,6 +2747,7 @@ init_android_streams(PyThreadState *tstate) } Py_XDECREF(_android_support); } +#endif static void From a8b67607c312e8c65cd8ad474d9a2adb830029cf Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 24 Apr 2024 18:19:37 +0100 Subject: [PATCH 07/19] Avoid importing ctypes on startup --- Lib/_android_support.py | 29 +++++---------- Python/pylifecycle.c | 81 +++++++++++++++++++++++++++++++---------- configure | 3 ++ configure.ac | 3 ++ 4 files changed, 77 insertions(+), 39 deletions(-) diff --git a/Lib/_android_support.py b/Lib/_android_support.py index 41fecf68f6dcba..f272a5b06b5385 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -1,6 +1,5 @@ import io import sys -from ctypes import CDLL, c_char_p, c_int # The maximum length of a log message in bytes, including the level marker and @@ -23,32 +22,21 @@ # monitor the C-level stdout and stderr. The testbed comes with a .c file to # redirect them to the system log using a pipe, but that wouldn't be convenient # or appropriate for all apps. So we redirect at the Python level instead. -def init_streams(): +def init_streams(android_log_write, stdout_prio, stderr_prio): if sys.executable: return # Not embedded in an app. - # Despite its name, this function is part of the public API - # (https://developer.android.com/ndk/reference/group/logging). - # Use `getattr` to avoid private name mangling. - global android_log_write - android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") - android_log_write.argtypes = (c_int, c_char_p, c_char_p) - - # These log levels match those used by Java's System.out and System.err. - ANDROID_LOG_INFO = 4 - ANDROID_LOG_WARN = 5 - sys.stdout = TextLogStream( - ANDROID_LOG_INFO, "python.stdout", errors=sys.stdout.errors) + android_log_write, stdout_prio, "python.stdout", errors=sys.stdout.errors) sys.stderr = TextLogStream( - ANDROID_LOG_WARN, "python.stderr", errors=sys.stderr.errors) + android_log_write, stderr_prio, "python.stderr", errors=sys.stderr.errors) class TextLogStream(io.TextIOWrapper): - def __init__(self, level, tag, **kwargs): + def __init__(self, android_log_write, prio, tag, **kwargs): kwargs.setdefault("encoding", "UTF-8") kwargs.setdefault("line_buffering", True) - super().__init__(BinaryLogStream(level, tag), **kwargs) + super().__init__(BinaryLogStream(android_log_write, prio, tag), **kwargs) self._CHUNK_SIZE = MAX_BYTES_PER_WRITE def __repr__(self): @@ -76,8 +64,9 @@ def write(self, s): class BinaryLogStream(io.RawIOBase): - def __init__(self, level, tag): - self.level = level + def __init__(self, android_log_write, prio, tag): + self.android_log_write = android_log_write + self.prio = prio self.tag = tag def __repr__(self): @@ -95,5 +84,5 @@ def write(self, b): # Writing an empty string to the stream should have no effect. if b: - android_log_write(self.level, self.tag.encode("UTF-8"), b) + self.android_log_write(self.prio, self.tag, b) return len(b) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8eb91a57dba10c..dd07c0836d711f 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -72,7 +72,7 @@ static PyStatus init_import_site(void); static PyStatus init_set_builtins_open(void); static PyStatus init_sys_streams(PyThreadState *tstate); #ifdef __ANDROID__ -static void init_android_streams(PyThreadState *tstate); +static PyStatus init_android_streams(PyThreadState *tstate); #endif static void wait_for_thread_shutdown(PyThreadState *tstate); static void call_ll_exitfuncs(_PyRuntimeState *runtime); @@ -1227,7 +1227,10 @@ init_interp_main(PyThreadState *tstate) } #ifdef __ANDROID__ - init_android_streams(tstate); + status = init_android_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } #endif #ifdef Py_DEBUG @@ -2727,27 +2730,67 @@ init_sys_streams(PyThreadState *tstate) #ifdef __ANDROID__ -/* Redirecting streams to the log is a convenience, but won't be critical for - every app, so failures of this function are non-fatal. */ -static void +#include + +static PyObject * +android_log_write_impl(PyObject *self, PyObject *args) +{ + int prio = 0; + char *tag = NULL; + char *text = NULL; + Py_ssize_t *text_len = 0; + if (!PyArg_ParseTuple(args, "iss#", &prio, &tag, &text, &text_len)) + return NULL; + + // Despite its name, this function is part of the public API + // (https://developer.android.com/ndk/reference/group/logging). + __android_log_write(prio, tag, text); + Py_RETURN_NONE; +} + + +static PyMethodDef android_log_write_method = { + "android_log_write", android_log_write_impl, METH_VARARGS +}; + + +static PyStatus init_android_streams(PyThreadState *tstate) { - PyObject *_android_support = PyImport_ImportModule("_android_support"); - if (_android_support == NULL) { - fprintf(stderr, "_android_support import failed:\n"); - _PyErr_Print(tstate); - } else { - PyObject *result = PyObject_CallMethod(_android_support, - "init_streams", NULL); - if (result == NULL) { - fprintf(stderr, "_android_support.init_streams failed:\n"); - _PyErr_Print(tstate); - } - Py_XDECREF(result); - } + PyStatus status = _PyStatus_OK(); + PyObject *_android_support = NULL; + PyObject *android_log_write = NULL; + PyObject *result = NULL; + + _android_support = PyImport_ImportModule("_android_support"); + if (_android_support == NULL) + goto error; + + android_log_write = PyCFunction_New(&android_log_write_method, NULL); + if (android_log_write == NULL) + goto error; + + // These log priorities match those used by Java's System.out and System.err. + result = PyObject_CallMethod( + _android_support, "init_streams", "Oii", + android_log_write, ANDROID_LOG_INFO, ANDROID_LOG_WARN); + if (result == NULL) + goto error; + + goto done; + +error: + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Android streams"); + +done: + Py_XDECREF(result); + Py_XDECREF(android_log_write); Py_XDECREF(_android_support); + return status; } -#endif + +#endif // __ANDROID__ static void diff --git a/configure b/configure index 80403255a814af..41cca712f77676 100755 --- a/configure +++ b/configure @@ -7096,6 +7096,9 @@ printf "%s\n" "$ANDROID_API_LEVEL" >&6; } printf "%s\n" "#define ANDROID_API_LEVEL $ANDROID_API_LEVEL" >>confdefs.h + # For __android_log_write in pylifecycle.c. + LIBS="$LIBS -llog" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the Android arm ABI" >&5 printf %s "checking for the Android arm ABI... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $_arm_arch" >&5 diff --git a/configure.ac b/configure.ac index ec925d4d4a0a5a..9d05a8351dee85 100644 --- a/configure.ac +++ b/configure.ac @@ -1192,6 +1192,9 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then AC_DEFINE_UNQUOTED([ANDROID_API_LEVEL], [$ANDROID_API_LEVEL], [The Android API level.]) + # For __android_log_write in pylifecycle.c. + LIBS="$LIBS -llog" + AC_MSG_CHECKING([for the Android arm ABI]) AC_MSG_RESULT([$_arm_arch]) if test "$_arm_arch" = 7; then From bbe816ff1507edf9a1a0325f7fec3b7544574477 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 24 Apr 2024 18:23:45 +0100 Subject: [PATCH 08/19] Clarify comment --- Lib/_android_support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_android_support.py b/Lib/_android_support.py index f272a5b06b5385..2d4dd88530b98c 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -53,7 +53,7 @@ def write(self, s): for line, line_keepends in zip( s.splitlines(), s.splitlines(keepends=True) ): - # Simplify the later stages by translating all newlines into "\n". + # Normalize all newlines to "\n". if line != line_keepends: line += "\n" while line: From aea0b8b5ee9115dce29391b7ad924d6f7b795b1b Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 24 Apr 2024 21:57:46 +0100 Subject: [PATCH 09/19] Fix issues with nulls and newlines --- Lib/_android_support.py | 17 ++++++++--------- Lib/test/test_android.py | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/Lib/_android_support.py b/Lib/_android_support.py index 2d4dd88530b98c..ce112dfc948e15 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -50,12 +50,7 @@ def write(self, s): # We want to emit one log message per line wherever possible, so split # the string before sending it to the superclass. Note that # "".splitlines() == [], so nothing will be logged for an empty string. - for line, line_keepends in zip( - s.splitlines(), s.splitlines(keepends=True) - ): - # Normalize all newlines to "\n". - if line != line_keepends: - line += "\n" + for line in s.splitlines(keepends=True): while line: super().write(line[:MAX_CHARS_PER_WRITE]) line = line[MAX_CHARS_PER_WRITE:] @@ -77,12 +72,16 @@ def writable(self): def write(self, b): if hasattr(b, "__buffer__"): - b = bytes(b) + b_out = bytes(b) else: raise TypeError( f"write() argument must be bytes-like, not {type(b).__name__}") + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. + b_out = b_out.replace(b"\x00", b"\xc0\x80") + # Writing an empty string to the stream should have no effect. - if b: - self.android_log_write(self.prio, self.tag, b) + if b_out: + self.android_log_write(self.prio, self.tag, b_out) return len(b) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 8d5bebd6ca92dc..4f020a0efbc1ae 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -119,11 +119,11 @@ def write(s, lines=None): # Non-BMP emoji write("\U0001f600") - # Null characters will truncate a message. - write("\u0000", [] if api_level < 24 else [""]) - write("a\u0000", ["a"]) - write("\u0000b", [] if api_level < 24 else [""]) - write("a\u0000b", ["a"]) + # Null characters are logged using "modified UTF-8". + write("\u0000", [r"\xc0\x80"]) + write("a\u0000", [r"a\xc0\x80"]) + write("\u0000b", [r"\xc0\x80b"]) + write("a\u0000b", [r"a\xc0\x80b"]) # Multi-line messages. Avoid identical consecutive lines, as # they may activate "chatty" filtering and break the tests. @@ -223,11 +223,11 @@ def write(b, lines=None): # Non-BMP emoji write(b"\xf0\x9f\x98\x80") - # Null characters will truncate a message. - write(b"\x00", [] if api_level < 24 else [""]) - write(b"a\x00", ["a"]) - write(b"\x00b", [] if api_level < 24 else [""]) - write(b"a\x00b", ["a"]) + # Null characters are logged using "modified UTF-8". + write(b"\x00", [r"\xc0\x80"]) + write(b"a\x00", [r"a\xc0\x80"]) + write(b"\x00b", [r"\xc0\x80b"]) + write(b"a\x00b", [r"a\xc0\x80b"]) # Invalid UTF-8 write(b"\xff", [r"\xff"]) From 41736d6a9487dfe94c057178068e37e2da8fef6d Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 24 Apr 2024 22:35:53 +0100 Subject: [PATCH 10/19] Improve identification of bytes-like objects --- Lib/_android_support.py | 11 +++++++---- Python/pylifecycle.c | 3 +-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/_android_support.py b/Lib/_android_support.py index ce112dfc948e15..1cec6a8f1b95c8 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -71,11 +71,14 @@ def writable(self): return True def write(self, b): - if hasattr(b, "__buffer__"): - b_out = bytes(b) - else: + try: + memoryview(b) + except TypeError: raise TypeError( - f"write() argument must be bytes-like, not {type(b).__name__}") + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + else: + b_out = bytes(b) # Encode null bytes using "modified UTF-8" to avoid truncating the # message. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index dd07c0836d711f..fafd1842d4a9b9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2738,8 +2738,7 @@ android_log_write_impl(PyObject *self, PyObject *args) int prio = 0; char *tag = NULL; char *text = NULL; - Py_ssize_t *text_len = 0; - if (!PyArg_ParseTuple(args, "iss#", &prio, &tag, &text, &text_len)) + if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) return NULL; // Despite its name, this function is part of the public API From 02cf414c7309777563eeb74a1b61c3ed26dc630b Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 24 Apr 2024 22:44:23 +0100 Subject: [PATCH 11/19] Fix C coding style --- Python/pylifecycle.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index fafd1842d4a9b9..97cfe920d83d1a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2736,10 +2736,11 @@ static PyObject * android_log_write_impl(PyObject *self, PyObject *args) { int prio = 0; - char *tag = NULL; - char *text = NULL; - if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) + const char *tag = NULL; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) { return NULL; + } // Despite its name, this function is part of the public API // (https://developer.android.com/ndk/reference/group/logging). @@ -2762,19 +2763,22 @@ init_android_streams(PyThreadState *tstate) PyObject *result = NULL; _android_support = PyImport_ImportModule("_android_support"); - if (_android_support == NULL) + if (_android_support == NULL) { goto error; + } android_log_write = PyCFunction_New(&android_log_write_method, NULL); - if (android_log_write == NULL) + if (android_log_write == NULL) { goto error; + } // These log priorities match those used by Java's System.out and System.err. result = PyObject_CallMethod( _android_support, "init_streams", "Oii", android_log_write, ANDROID_LOG_INFO, ANDROID_LOG_WARN); - if (result == NULL) + if (result == NULL) { goto error; + } goto done; From 61d7e98135b2d0831fe000426ea4de6b11acb6fd Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 27 Apr 2024 12:19:13 +0100 Subject: [PATCH 12/19] Use LOOPBACK_TIMEOUT to wait for subprocesses, and add breadcrumb to comment --- Lib/test/test_android.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 4f020a0efbc1ae..55b1a51cd63d00 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -6,6 +6,7 @@ import unittest from contextlib import contextmanager from threading import Thread +from test.support import LOOPBACK_TIMEOUT from time import time @@ -69,7 +70,7 @@ def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): def tearDown(self): self.logcat_process.terminate() - self.logcat_process.wait(0.1) + self.logcat_process.wait(LOOPBACK_TIMEOUT) @contextmanager def unbuffered(self, stream): @@ -171,9 +172,10 @@ def write(s, lines=None): stream.flush() self.assert_log(level, tag, "helloworld") - # Long lines are split into blocks of 1000 *characters*, but - # TextIOWrapper should then join them back together as much as - # possible without exceeding 4000 UTF-8 *bytes*. + # Long lines are split into blocks of 1000 characters + # (MAX_CHARS_PER_WRITE), but TextIOWrapper should then join them + # back together as much as possible without exceeding 4000 UTF-8 + # bytes (MAX_BYTES_PER_WRITE). # # ASCII (1 byte per character) write(("foobar" * 700) + "\n", From 3014e6c375a81dd4fc809d1e5efe64c31f8ea446 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 27 Apr 2024 12:58:00 +0100 Subject: [PATCH 13/19] Add tests for str subclasses --- Lib/_android_support.py | 4 ++++ Lib/test/test_android.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/Lib/_android_support.py b/Lib/_android_support.py index 1cec6a8f1b95c8..6227f85ef921cf 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -47,6 +47,10 @@ def write(self, s): raise TypeError( f"write() argument must be str, not {type(s).__name__}") + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + # We want to emit one log message per line wherever possible, so split # the string before sending it to the superclass. Note that # "".splitlines() == [], so nothing will be logged for an empty string. diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 55b1a51cd63d00..c1885d8374991d 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -154,6 +154,16 @@ def write(s, lines=None): write("hello\r\nworld\r\n", ["hello", "world"]) write("\r\n", [""]) + # String subclasses are accepted, and if their methods write + # themselves, this doesn't cause infinite recursion. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + sys.stdout.write(self) + return super().splitlines(*args, **kwargs) + + write(CustomStr("custom\n"), ["custom"]) + + # Non-string classes are not accepted. for obj in [b"", b"hello", None, 42]: with self.subTest(obj=obj): with self.assertRaisesRegex( From 839de0f30b3042a93b3452d69f16904c2895c349 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 27 Apr 2024 14:44:48 +0100 Subject: [PATCH 14/19] Add tests for other bytes-like objects --- Lib/_android_support.py | 15 +++++++++------ Lib/test/test_android.py | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/Lib/_android_support.py b/Lib/_android_support.py index 6227f85ef921cf..36f20a4dcda43d 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -81,14 +81,17 @@ def write(self, b): raise TypeError( f"write() argument must be bytes-like, not {type(b).__name__}" ) from None - else: - b_out = bytes(b) - # Encode null bytes using "modified UTF-8" to avoid truncating the - # message. - b_out = b_out.replace(b"\x00", b"\xc0\x80") + b_out = bytes(b) + b_len = len(b_out) # May be different from len(b) if b is an array. # Writing an empty string to the stream should have no effect. if b_out: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + b_out = b_out.replace(b"\x00", b"\xc0\x80") + self.android_log_write(self.prio, self.tag, b_out) - return len(b) + + return b_len diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index c1885d8374991d..3ef99ff22ac65e 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -4,6 +4,7 @@ import subprocess import sys import unittest +from array import array from contextlib import contextmanager from threading import Thread from test.support import LOOPBACK_TIMEOUT @@ -213,8 +214,10 @@ def test_bytes(self): self.assertTrue(stream.writable()) self.assertFalse(stream.readable()) - def write(b, lines=None): - self.assertEqual(len(b), stream.write(b)) + def write(b, lines=None, *, write_len=None): + if write_len is None: + write_len = len(b) + self.assertEqual(write_len, stream.write(b)) if lines is None: lines = [b.decode()] self.assert_logs(level, tag, lines) @@ -271,6 +274,34 @@ def write(b, lines=None): write(b"hello\r\nworld\r\n", ["hello", "world"]) write(b"\r\n", [""]) + # Other bytes-like objects are accepted. + write(bytearray(b"bytearray")) + + mv = memoryview(b"memoryview") + write(mv, ["memoryview"]) # Continuous + write(mv[::2], ["mmrve"]) # Discontinuous + + write( + # Android only supports little-endian architectures, so the + # bytes representation is as follows: + array("H", [ + 0, # 00 00 + 1, # 01 00 + 65534, # FE FF + 65535, # FF FF + ]), + + # After encoding null bytes with modified UTF-8, the only + # valid UTF-8 sequence is \x01. All other bytes are handled + # by backslashreplace. + ["\\xc0\\x80\\xc0\\x80" + "\x01\\xc0\\x80" + "\\xfe\\xff" + "\\xff\\xff"], + write_len=8, + ) + + # Non-bytes-like classes are not accepted. for obj in ["", "hello", None, 42]: with self.subTest(obj=obj): with self.assertRaisesRegex( From 0ffb19f921afd4855aeb821221c4246ec67f3e2b Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 27 Apr 2024 15:18:16 +0100 Subject: [PATCH 15/19] Add tests for Unicode surrogates and non-standard line separators --- Lib/test/test_android.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 3ef99ff22ac65e..70930fae1eb9a3 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -121,11 +121,17 @@ def write(s, lines=None): # Non-BMP emoji write("\U0001f600") + # Non-encodable surrogates + write("\ud800\udc00", ["\\ud800\\udc00"]) + + # Code used by surrogateescape (which isn't enabled here) + write("\udc80", ["\\udc80"]) + # Null characters are logged using "modified UTF-8". - write("\u0000", [r"\xc0\x80"]) - write("a\u0000", [r"a\xc0\x80"]) - write("\u0000b", [r"\xc0\x80b"]) - write("a\u0000b", [r"a\xc0\x80b"]) + write("\u0000", ["\\xc0\\x80"]) + write("a\u0000", ["a\\xc0\\x80"]) + write("\u0000b", ["\\xc0\\x80b"]) + write("a\u0000b", ["a\\xc0\\x80b"]) # Multi-line messages. Avoid identical consecutive lines, as # they may activate "chatty" filtering and break the tests. @@ -155,6 +161,12 @@ def write(s, lines=None): write("hello\r\nworld\r\n", ["hello", "world"]) write("\r\n", [""]) + # Non-standard line separators should be preserved. + write("before form feed\x0cafter form feed\n", + ["before form feed\x0cafter form feed"]) + write("before line separator\u2028after line separator\n", + ["before line separator\u2028after line separator"]) + # String subclasses are accepted, and if their methods write # themselves, this doesn't cause infinite recursion. class CustomStr(str): @@ -238,17 +250,17 @@ def write(b, lines=None, *, write_len=None): # Non-BMP emoji write(b"\xf0\x9f\x98\x80") - # Null characters are logged using "modified UTF-8". - write(b"\x00", [r"\xc0\x80"]) - write(b"a\x00", [r"a\xc0\x80"]) - write(b"\x00b", [r"\xc0\x80b"]) - write(b"a\x00b", [r"a\xc0\x80b"]) + # Null bytes are logged using "modified UTF-8". + write(b"\x00", ["\\xc0\\x80"]) + write(b"a\x00", ["a\\xc0\\x80"]) + write(b"\x00b", ["\\xc0\\x80b"]) + write(b"a\x00b", ["a\\xc0\\x80b"]) # Invalid UTF-8 - write(b"\xff", [r"\xff"]) - write(b"a\xff", [r"a\xff"]) - write(b"\xffb", [r"\xffb"]) - write(b"a\xffb", [r"a\xffb"]) + write(b"\xff", ["\\xff"]) + write(b"a\xff", ["a\\xff"]) + write(b"\xffb", ["\\xffb"]) + write(b"a\xffb", ["a\\xffb"]) # Log entries containing newlines are shown differently by # `logcat -v tag`, `logcat -v long`, and Android Studio. We From b2e2ef6f6e317eae0c11749d70723867cf03a86e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 30 Apr 2024 01:06:25 +0100 Subject: [PATCH 16/19] Bytes cleanups --- Lib/_android_support.py | 25 +++++++++++-------------- Lib/test/test_android.py | 16 ++++++++-------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/Lib/_android_support.py b/Lib/_android_support.py index 36f20a4dcda43d..590e85ea8c2db1 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -75,23 +75,20 @@ def writable(self): return True def write(self, b): - try: - memoryview(b) - except TypeError: - raise TypeError( - f"write() argument must be bytes-like, not {type(b).__name__}" - ) from None - - b_out = bytes(b) - b_len = len(b_out) # May be different from len(b) if b is an array. + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None # Writing an empty string to the stream should have no effect. - if b_out: + if b: # Encode null bytes using "modified UTF-8" to avoid truncating the # message. This should not affect the return value, as the caller # may be expecting it to match the length of the input. - b_out = b_out.replace(b"\x00", b"\xc0\x80") - - self.android_log_write(self.prio, self.tag, b_out) + self.android_log_write(self.prio, self.tag, + b.replace(b"\x00", b"\xc0\x80")) - return b_len + return len(b) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 70930fae1eb9a3..ad4487a3e564ca 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -251,16 +251,16 @@ def write(b, lines=None, *, write_len=None): write(b"\xf0\x9f\x98\x80") # Null bytes are logged using "modified UTF-8". - write(b"\x00", ["\\xc0\\x80"]) - write(b"a\x00", ["a\\xc0\\x80"]) - write(b"\x00b", ["\\xc0\\x80b"]) - write(b"a\x00b", ["a\\xc0\\x80b"]) + write(b"\x00", [r"\xc0\x80"]) + write(b"a\x00", [r"a\xc0\x80"]) + write(b"\x00b", [r"\xc0\x80b"]) + write(b"a\x00b", [r"a\xc0\x80b"]) # Invalid UTF-8 - write(b"\xff", ["\\xff"]) - write(b"a\xff", ["a\\xff"]) - write(b"\xffb", ["\\xffb"]) - write(b"a\xffb", ["a\\xffb"]) + write(b"\xff", [r"\xff"]) + write(b"a\xff", [r"a\xff"]) + write(b"\xffb", [r"\xffb"]) + write(b"a\xffb", [r"a\xffb"]) # Log entries containing newlines are shown differently by # `logcat -v tag`, `logcat -v long`, and Android Studio. We From 56e6f85b83a3a105a528eb6d6c3f605dae9ff508 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 30 Apr 2024 01:07:22 +0100 Subject: [PATCH 17/19] Str cleanups --- Lib/test/test_android.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index ad4487a3e564ca..115882a4c281f6 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -98,8 +98,10 @@ def test_str(self): # that way by libregrtest.main. self.assertEqual("backslashreplace", stream.errors) - def write(s, lines=None): - self.assertEqual(len(s), stream.write(s)) + def write(s, lines=None, *, write_len=None): + if write_len is None: + write_len = len(s) + self.assertEqual(write_len, stream.write(s)) if lines is None: lines = [s] self.assert_logs(level, tag, lines) @@ -122,16 +124,16 @@ def write(s, lines=None): write("\U0001f600") # Non-encodable surrogates - write("\ud800\udc00", ["\\ud800\\udc00"]) + write("\ud800\udc00", [r"\ud800\udc00"]) # Code used by surrogateescape (which isn't enabled here) - write("\udc80", ["\\udc80"]) + write("\udc80", [r"\udc80"]) # Null characters are logged using "modified UTF-8". - write("\u0000", ["\\xc0\\x80"]) - write("a\u0000", ["a\\xc0\\x80"]) - write("\u0000b", ["\\xc0\\x80b"]) - write("a\u0000b", ["a\\xc0\\x80b"]) + write("\u0000", [r"\xc0\x80"]) + write("a\u0000", [r"a\xc0\x80"]) + write("\u0000b", [r"\xc0\x80b"]) + write("a\u0000b", [r"a\xc0\x80b"]) # Multi-line messages. Avoid identical consecutive lines, as # they may activate "chatty" filtering and break the tests. @@ -167,14 +169,19 @@ def write(s, lines=None): write("before line separator\u2028after line separator\n", ["before line separator\u2028after line separator"]) - # String subclasses are accepted, and if their methods write - # themselves, this doesn't cause infinite recursion. + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. class CustomStr(str): def splitlines(self, *args, **kwargs): - sys.stdout.write(self) - return super().splitlines(*args, **kwargs) + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() - write(CustomStr("custom\n"), ["custom"]) + write(CustomStr("custom\n"), ["custom"], write_len=7) # Non-string classes are not accepted. for obj in [b"", b"hello", None, 42]: @@ -196,9 +203,10 @@ def splitlines(self, *args, **kwargs): self.assert_log(level, tag, "helloworld") # Long lines are split into blocks of 1000 characters - # (MAX_CHARS_PER_WRITE), but TextIOWrapper should then join them - # back together as much as possible without exceeding 4000 UTF-8 - # bytes (MAX_BYTES_PER_WRITE). + # (MAX_CHARS_PER_WRITE in _android_support.py), but + # TextIOWrapper should then join them back together as much as + # possible without exceeding 4000 UTF-8 bytes + # (MAX_BYTES_PER_WRITE). # # ASCII (1 byte per character) write(("foobar" * 700) + "\n", From 53ba406cd95b5908722d9fa6bcc0cbf2251fe731 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 30 Apr 2024 12:54:08 +0100 Subject: [PATCH 18/19] Update configure.ac Co-authored-by: Victor Stinner --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 9d05a8351dee85..96ae9b527257bb 100644 --- a/configure.ac +++ b/configure.ac @@ -1192,7 +1192,7 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then AC_DEFINE_UNQUOTED([ANDROID_API_LEVEL], [$ANDROID_API_LEVEL], [The Android API level.]) - # For __android_log_write in pylifecycle.c. + # For __android_log_write() in Python/pylifecycle.c. LIBS="$LIBS -llog" AC_MSG_CHECKING([for the Android arm ABI]) From 58746877568db74ad3cd111ffec0cb60dfcd7ed5 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 30 Apr 2024 13:46:48 +0100 Subject: [PATCH 19/19] Autoreconf --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index 41cca712f77676..ad54a679f3ac9c 100755 --- a/configure +++ b/configure @@ -7096,7 +7096,7 @@ printf "%s\n" "$ANDROID_API_LEVEL" >&6; } printf "%s\n" "#define ANDROID_API_LEVEL $ANDROID_API_LEVEL" >>confdefs.h - # For __android_log_write in pylifecycle.c. + # For __android_log_write() in Python/pylifecycle.c. LIBS="$LIBS -llog" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the Android arm ABI" >&5