diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9303190ea6dbba..33a5950c1483a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -354,7 +354,7 @@ jobs: with: persist-credentials: false - name: Build and test - run: ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android + run: python3 Platforms/Android ci --fast-ci ${{ matrix.arch }}-linux-android build-ios: name: iOS diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 8ba11b7d12d552..0091bf95f0cef0 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1970,7 +1970,7 @@ FileType objects run and then use the :keyword:`with`-statement to manage the files. .. versionchanged:: 3.4 - Added the *encodings* and *errors* parameters. + Added the *encoding* and *errors* parameters. .. deprecated:: 3.14 diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 5bbea187394db6..68d353b213c097 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -292,7 +292,7 @@ _PyDict_NotifyEvent(PyDict_WatchEvent event, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = FT_ATOMIC_LOAD_UINT64_RELAXED(mp->_ma_watcher_tag) & DICT_WATCHER_MASK; + int watcher_bits = FT_ATOMIC_LOAD_UINT64_ACQUIRE(mp->_ma_watcher_tag) & DICT_WATCHER_MASK; if (watcher_bits) { RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 3155481bb5c36b..fafdd728a8229a 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -49,6 +49,8 @@ extern "C" { _Py_atomic_load_uint16_relaxed(&value) #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ _Py_atomic_load_uint32_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) \ + _Py_atomic_load_uint64_acquire(&value) #define FT_ATOMIC_LOAD_UINT64_RELAXED(value) \ _Py_atomic_load_uint64_relaxed(&value) #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \ @@ -154,6 +156,7 @@ extern "C" { #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) value #define FT_ATOMIC_LOAD_UINT64_RELAXED(value) value #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 0bf3bdded40200..8ee09b38469d4c 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -162,17 +162,29 @@ def interrupt(self) -> None: "ps", help="Display a table of all pending tasks in a process" ) ps.add_argument("pid", type=int, help="Process ID to inspect") + ps.add_argument( + "--retries", + type=int, + default=3, + help="Number of retries on transient attach errors", + ) pstree = subparsers.add_parser( "pstree", help="Display a tree of all pending tasks in a process" ) pstree.add_argument("pid", type=int, help="Process ID to inspect") + pstree.add_argument( + "--retries", + type=int, + default=3, + help="Number of retries on transient attach errors", + ) args = parser.parse_args() match args.command: case "ps": - asyncio.tools.display_awaited_by_tasks_table(args.pid) + asyncio.tools.display_awaited_by_tasks_table(args.pid, retries=args.retries) sys.exit(0) case "pstree": - asyncio.tools.display_awaited_by_tasks_tree(args.pid) + asyncio.tools.display_awaited_by_tasks_tree(args.pid, retries=args.retries) sys.exit(0) case None: pass # continue to the interactive shell diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 62d6a71557fa37..2ac1738d15c6c7 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -231,27 +231,38 @@ def exit_with_permission_help_text(): print( "Error: The specified process cannot be attached to due to insufficient permissions.\n" "See the Python documentation for details on required privileges and troubleshooting:\n" - "https://docs.python.org/3.14/howto/remote_debugging.html#permission-requirements\n" + "https://docs.python.org/3/howto/remote_debugging.html#permission-requirements\n", + file=sys.stderr, ) sys.exit(1) -def _get_awaited_by_tasks(pid: int) -> list: - try: - return get_all_awaited_by(pid) - except RuntimeError as e: - while e.__context__ is not None: - e = e.__context__ - print(f"Error retrieving tasks: {e}") - sys.exit(1) - except PermissionError: - exit_with_permission_help_text() +_TRANSIENT_ERRORS = (RuntimeError, OSError, UnicodeDecodeError, MemoryError) + + +def _get_awaited_by_tasks(pid: int, retries: int = 3) -> list: + for attempt in range(retries + 1): + try: + return get_all_awaited_by(pid) + except PermissionError: + exit_with_permission_help_text() + except ProcessLookupError: + print(f"Error: process {pid} not found.", file=sys.stderr) + sys.exit(1) + except _TRANSIENT_ERRORS as e: + if attempt < retries: + continue + if isinstance(e, RuntimeError): + while e.__context__ is not None: + e = e.__context__ + print(f"Error retrieving tasks: {e}", file=sys.stderr) + sys.exit(1) -def display_awaited_by_tasks_table(pid: int) -> None: +def display_awaited_by_tasks_table(pid: int, retries: int = 3) -> None: """Build and print a table of all pending tasks under `pid`.""" - tasks = _get_awaited_by_tasks(pid) + tasks = _get_awaited_by_tasks(pid, retries=retries) table = build_task_table(tasks) # Print the table in a simple tabular format print( @@ -262,10 +273,10 @@ def display_awaited_by_tasks_table(pid: int) -> None: print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") -def display_awaited_by_tasks_tree(pid: int) -> None: +def display_awaited_by_tasks_tree(pid: int, retries: int = 3) -> None: """Build and print a tree of all pending tasks under `pid`.""" - tasks = _get_awaited_by_tasks(pid) + tasks = _get_awaited_by_tasks(pid, retries=retries) try: result = build_async_tree(tasks) except CycleFoundException as e: diff --git a/Lib/configparser.py b/Lib/configparser.py index e76647d339e913..a53ac87276445a 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -315,12 +315,15 @@ def __init__(self, source, *args): def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + self.message += f'\n\t[line {lineno:2d}]: {line!r}' def combine(self, others): + messages = [self.message] for other in others: - for error in other.errors: - self.append(*error) + for lineno, line in other.errors: + self.errors.append((lineno, line)) + messages.append(f'\n\t[line {lineno:2d}]: {line!r}') + self.message = "".join(messages) return self @staticmethod diff --git a/Lib/socket.py b/Lib/socket.py index 3073c012b19877..34d897a82edd4f 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -649,18 +649,22 @@ def _fallback_socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): # Authenticating avoids using a connection from something else # able to connect to {host}:{port} instead of us. # We expect only AF_INET and AF_INET6 families. - try: - if ( - ssock.getsockname() != csock.getpeername() - or csock.getsockname() != ssock.getpeername() - ): - raise ConnectionError("Unexpected peer connection") - except: - # getsockname() and getpeername() can fail - # if either socket isn't connected. - ssock.close() - csock.close() - raise + # + # Note that we skip this on WASI because on that platorm the client socket + # may not have finished connecting by the time we've reached this point (gh-146139). + if sys.platform != "wasi": + try: + if ( + ssock.getsockname() != csock.getpeername() + or csock.getsockname() != ssock.getpeername() + ): + raise ConnectionError("Unexpected peer connection") + except: + # getsockname() and getpeername() can fail + # if either socket isn't connected. + ssock.close() + csock.close() + raise return (ssock, csock) diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index d7c4f19c1a5ef0..8d8dd2a2bf27fb 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -1729,6 +1729,19 @@ def test_error(self): self.assertEqual(e1.message, e2.message) self.assertEqual(repr(e1), repr(e2)) + def test_combine_error_linear_complexity(self): + # Ensure that ParsingError.combine() has linear complexity. + # See https://github.com/python/cpython/issues/148370. + n = 50000 + s = '[*]\n' + (err_line := '=\n') * n + p = configparser.ConfigParser(strict=False) + with self.assertRaises(configparser.ParsingError) as cm: + p.read_string(s) + errlines = cm.exception.message.splitlines() + self.assertEqual(len(errlines), n + 1) + self.assertStartsWith(errlines[0], "Source contains parsing errors: ") + self.assertEqual(errlines[42], f"\t[line {43:2d}]: {err_line!r}") + def test_nosectionerror(self): import pickle e1 = configparser.NoSectionError('section') diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 299dc185fcf211..2ba3af8d5bf22f 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -119,6 +119,15 @@ def test_open_bad_new_parameter(self): arguments=[URL], kw=dict(new=999)) + def test_reject_action_dash_prefixes(self): + browser = self.browser_class(name=CMD_NAME) + with self.assertRaises(ValueError): + browser.open('%action--incognito') + # new=1: action is "--new-window", so "%action" itself expands to + # a dash-prefixed flag even with no dash in the original URL. + with self.assertRaises(ValueError): + browser.open('%action', new=1) + class EdgeCommandTest(CommandTestMixin, unittest.TestCase): diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 0e0b5034e5f53d..97aad6eea509eb 100644 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -274,7 +274,6 @@ def _invoke(self, args, remote, autoraise, url=None): def open(self, url, new=0, autoraise=True): sys.audit("webbrowser.open", url) - self._check_url(url) if new == 0: action = self.remote_action elif new == 1: @@ -288,7 +287,9 @@ def open(self, url, new=0, autoraise=True): raise Error("Bad 'new' parameter to open(); " f"expected 0, 1, or 2, got {new}") - args = [arg.replace("%s", url).replace("%action", action) + self._check_url(url.replace("%action", action)) + + args = [arg.replace("%action", action).replace("%s", url) for arg in self.remote_args] args = [arg for arg in args if arg] success = self._invoke(args, True, autoraise, url) diff --git a/Misc/NEWS.d/next/Build/2026-04-09-11-42-32.gh-issue-146445.Z1vccC.rst b/Misc/NEWS.d/next/Build/2026-04-09-11-42-32.gh-issue-146445.Z1vccC.rst new file mode 100644 index 00000000000000..e51454b9701e12 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-04-09-11-42-32.gh-issue-146445.Z1vccC.rst @@ -0,0 +1 @@ +The Android build tools have been moved to the Platforms folder. diff --git a/Misc/NEWS.d/next/Library/2026-04-12-16-40-11.gh-issue-148370.0Li2EK.rst b/Misc/NEWS.d/next/Library/2026-04-12-16-40-11.gh-issue-148370.0Li2EK.rst new file mode 100644 index 00000000000000..3bb662350796f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-12-16-40-11.gh-issue-148370.0Li2EK.rst @@ -0,0 +1,2 @@ +:mod:`configparser`: prevent quadratic behavior when a :exc:`~configparser.ParsingError` +is raised after a parser fails to parse multiple lines. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-04-13-21-38-50.gh-issue-144881.3kPqXw.rst b/Misc/NEWS.d/next/Library/2026-04-13-21-38-50.gh-issue-144881.3kPqXw.rst new file mode 100644 index 00000000000000..0812dc9efb6d8b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-13-21-38-50.gh-issue-144881.3kPqXw.rst @@ -0,0 +1,4 @@ +:mod:`asyncio` debugging tools (``python -m asyncio ps`` and ``pstree``) +now retry automatically on transient errors that can occur when attaching +to a process under active thread delegation. The number of retries can be +controlled with the ``--retries`` flag. Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst b/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst new file mode 100644 index 00000000000000..45cdeebe1b6d64 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst @@ -0,0 +1,2 @@ +A bypass in :mod:`webbrowser` allowed URLs prefixed with ``%action`` to pass +the dash-prefix safety check. diff --git a/Misc/NEWS.d/next/Security/2026-04-06-13-55-00.gh-issue-148178.Rs7kLm.rst b/Misc/NEWS.d/next/Security/2026-04-06-13-55-00.gh-issue-148178.Rs7kLm.rst new file mode 100644 index 00000000000000..ed138a54a859de --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-06-13-55-00.gh-issue-148178.Rs7kLm.rst @@ -0,0 +1,2 @@ +Hardened :mod:`!_remote_debugging` by validating remote debug offset tables +before using them to size memory reads or interpret remote layouts. diff --git a/Misc/NEWS.d/next/Security/2026-04-08-14-25-47.gh-issue-148252.IEp9Rt.rst b/Misc/NEWS.d/next/Security/2026-04-08-14-25-47.gh-issue-148252.IEp9Rt.rst new file mode 100644 index 00000000000000..adc2c9287149db --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-08-14-25-47.gh-issue-148252.IEp9Rt.rst @@ -0,0 +1,3 @@ +Fixed stack depth calculation in :mod:`!_remote_debugging` when decoding +certain ``.pyb`` inputs on 32-bit builds. Issue originally identified and +diagnosed by Tristan Madani (@TristanInSec on GitHub). diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 3722273dfd2998..07738d45e42d24 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -120,9 +120,10 @@ typedef enum _WIN32_THREADSTATE { * MACROS AND CONSTANTS * ============================================================================ */ -#define GET_MEMBER(type, obj, offset) (*(type*)((char*)(obj) + (offset))) +#define GET_MEMBER(type, obj, offset) \ + (*(const type *)memcpy(&(type){0}, (const char *)(obj) + (offset), sizeof(type))) #define CLEAR_PTR_TAG(ptr) (((uintptr_t)(ptr) & ~Py_TAG_BITS)) -#define GET_MEMBER_NO_TAG(type, obj, offset) (type)(CLEAR_PTR_TAG(*(type*)((char*)(obj) + (offset)))) +#define GET_MEMBER_NO_TAG(type, obj, offset) (type)(CLEAR_PTR_TAG(GET_MEMBER(type, obj, offset))) /* Size macros for opaque buffers */ #define SIZEOF_BYTES_OBJ sizeof(PyBytesObject) @@ -417,6 +418,7 @@ extern void cached_code_metadata_destroy(void *ptr); /* Validation */ extern int is_prerelease_version(uint64_t version); extern int validate_debug_offsets(struct _Py_DebugOffsets *debug_offsets); +#define PY_REMOTE_DEBUG_INVALID_ASYNC_DEBUG_OFFSETS (-2) /* ============================================================================ * MEMORY READING FUNCTION DECLARATIONS diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index 263c502a857004..fc7487d4044bfb 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -6,6 +6,7 @@ ******************************************************************************/ #include "_remote_debugging.h" +#include "debug_offsets_validation.h" /* ============================================================================ * ASYNCIO DEBUG ADDRESS FUNCTIONS @@ -71,8 +72,13 @@ read_async_debug(RemoteUnwinderObject *unwinder) int result = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, async_debug_addr, size, &unwinder->async_debug_offsets); if (result < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read AsyncioDebug offsets"); + return result; } - return result; + if (_PyRemoteDebug_ValidateAsyncDebugOffsets(&unwinder->async_debug_offsets) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Invalid AsyncioDebug offsets"); + return PY_REMOTE_DEBUG_INVALID_ASYNC_DEBUG_OFFSETS; + } + return 0; } int @@ -85,7 +91,11 @@ ensure_async_debug_offsets(RemoteUnwinderObject *unwinder) // Try to load async debug offsets (the target process may have // loaded asyncio since we last checked) - if (read_async_debug(unwinder) < 0) { + int result = read_async_debug(unwinder); + if (result == PY_REMOTE_DEBUG_INVALID_ASYNC_DEBUG_OFFSETS) { + return -1; + } + if (result < 0) { PyErr_Clear(); PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); set_exception_cause(unwinder, PyExc_RuntimeError, diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 616213541e12e1..aca93e9cb1a30e 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -601,6 +601,20 @@ reader_get_or_create_thread_state(BinaryReader *reader, uint64_t thread_id, * STACK DECODING HELPERS * ============================================================================ */ +/* Validate that final_depth fits in the stack buffer. + * Uses uint64_t to prevent overflow on 32-bit platforms. */ +static inline int +validate_stack_depth(ReaderThreadState *ts, uint64_t final_depth) +{ + if (final_depth > ts->current_stack_capacity) { + PyErr_Format(PyExc_ValueError, + "Final stack depth %llu exceeds capacity %zu", + (unsigned long long)final_depth, ts->current_stack_capacity); + return -1; + } + return 0; +} + /* Decode a full stack from sample data. * Updates ts->current_stack and ts->current_stack_depth. * Returns 0 on success, -1 on error (bounds violation). */ @@ -658,12 +672,9 @@ decode_stack_suffix(ReaderThreadState *ts, const uint8_t *data, return -1; } - /* Validate final depth doesn't exceed capacity */ - size_t final_depth = (size_t)shared + new_count; - if (final_depth > ts->current_stack_capacity) { - PyErr_Format(PyExc_ValueError, - "Final stack depth %zu exceeds capacity %zu", - final_depth, ts->current_stack_capacity); + /* Use uint64_t to prevent overflow on 32-bit platforms */ + uint64_t final_depth = (uint64_t)shared + new_count; + if (validate_stack_depth(ts, final_depth) < 0) { return -1; } @@ -713,12 +724,9 @@ decode_stack_pop_push(ReaderThreadState *ts, const uint8_t *data, } size_t keep = (ts->current_stack_depth > pop) ? ts->current_stack_depth - pop : 0; - /* Validate final depth doesn't exceed capacity */ - size_t final_depth = keep + push; - if (final_depth > ts->current_stack_capacity) { - PyErr_Format(PyExc_ValueError, - "Final stack depth %zu exceeds capacity %zu", - final_depth, ts->current_stack_capacity); + /* Use uint64_t to prevent overflow on 32-bit platforms */ + uint64_t final_depth = (uint64_t)keep + push; + if (validate_stack_depth(ts, final_depth) < 0) { return -1; } diff --git a/Modules/_remote_debugging/debug_offsets_validation.h b/Modules/_remote_debugging/debug_offsets_validation.h new file mode 100644 index 00000000000000..32800e767b3169 --- /dev/null +++ b/Modules/_remote_debugging/debug_offsets_validation.h @@ -0,0 +1,483 @@ +#ifndef Py_REMOTE_DEBUG_OFFSETS_VALIDATION_H +#define Py_REMOTE_DEBUG_OFFSETS_VALIDATION_H + +/* + * The remote debugging tables are read from the target process and must be + * treated as untrusted input. This header centralizes the one-time validation + * that runs immediately after those tables are read, before the unwinder uses + * any reported sizes or offsets to copy remote structs into fixed local + * buffers or to interpret those local copies. + * + * The key rule is simple: every offset that is later dereferenced against a + * local buffer or local object view must appear in one of the field lists + * below. Validation then checks two bounds for each field: + * + * 1. The field must fit within the section size reported by the target. + * 2. The same field must also fit within the local buffer or local layout the + * debugger will actually use. + * + * Sections that are copied into fixed local buffers also have their reported + * size checked against the corresponding local buffer size up front. + * + * This is intentionally front-loaded. Once validation succeeds, the hot path + * can keep using the raw offsets without adding per-sample bounds checks. + * + * Maintenance rule: if either exported table grows, the static_asserts below + * should yell at you. When that happens, update the matching field lists in + * this file in the same change. And if you add a new field that the unwinder + * is going to poke at later, put it in the right list here too, so nobody has + * to rediscover this the annoying way. + */ +#define FIELD_SIZE(type, member) sizeof(((type *)0)->member) + +enum { + PY_REMOTE_DEBUG_OFFSETS_TOTAL_SIZE = 840, + PY_REMOTE_ASYNC_DEBUG_OFFSETS_TOTAL_SIZE = 104, +}; + +/* + * These asserts are the coordination tripwire for table growth. If either + * exported table changes size, update the validation lists below in the same + * change. + */ +static_assert( + sizeof(_Py_DebugOffsets) == PY_REMOTE_DEBUG_OFFSETS_TOTAL_SIZE, + "Update _remote_debugging validation for _Py_DebugOffsets"); +static_assert( + sizeof(struct _Py_AsyncioModuleDebugOffsets) == + PY_REMOTE_ASYNC_DEBUG_OFFSETS_TOTAL_SIZE, + "Update _remote_debugging validation for _Py_AsyncioModuleDebugOffsets"); + +/* + * This logic lives in a private header because it is shared by module.c and + * asyncio.c. Keep the helpers static inline so they stay local to those users + * without adding another compilation unit or exported symbols. + */ +static inline int +validate_section_size(const char *section_name, uint64_t size) +{ + if (size == 0) { + PyErr_Format( + PyExc_RuntimeError, + "Invalid debug offsets: %s.size must be greater than zero", + section_name); + return -1; + } + return 0; +} + +static inline int +validate_read_size(const char *section_name, uint64_t size, size_t buffer_size) +{ + if (validate_section_size(section_name, size) < 0) { + return -1; + } + if (size > buffer_size) { + PyErr_Format( + PyExc_RuntimeError, + "Invalid debug offsets: %s.size=%llu exceeds local buffer size %zu", + section_name, + (unsigned long long)size, + buffer_size); + return -1; + } + return 0; +} + +static inline int +validate_span( + const char *field_name, + uint64_t offset, + size_t width, + uint64_t limit, + const char *limit_name) +{ + uint64_t span = (uint64_t)width; + if (span > limit || offset > limit - span) { + PyErr_Format( + PyExc_RuntimeError, + "Invalid debug offsets: %s=%llu with width %zu exceeds %s %llu", + field_name, + (unsigned long long)offset, + width, + limit_name, + (unsigned long long)limit); + return -1; + } + return 0; +} + +static inline int +validate_alignment( + const char *field_name, + uint64_t offset, + size_t alignment) +{ + if (alignment > 1 && offset % alignment != 0) { + PyErr_Format( + PyExc_RuntimeError, + "Invalid debug offsets: %s=%llu is not aligned to %zu bytes", + field_name, + (unsigned long long)offset, + alignment); + return -1; + } + return 0; +} + +static inline int +validate_field( + const char *field_name, + uint64_t reported_size, + uint64_t offset, + size_t width, + size_t alignment, + size_t buffer_size) +{ + if (validate_alignment(field_name, offset, alignment) < 0) { + return -1; + } + if (validate_span(field_name, offset, width, reported_size, "reported size") < 0) { + return -1; + } + return validate_span(field_name, offset, width, buffer_size, "local buffer size"); +} + +static inline int +validate_nested_field( + const char *field_name, + uint64_t reported_size, + uint64_t base_offset, + uint64_t nested_offset, + size_t width, + size_t alignment, + size_t buffer_size) +{ + if (base_offset > UINT64_MAX - nested_offset) { + PyErr_Format( + PyExc_RuntimeError, + "Invalid debug offsets: %s overflows the offset calculation", + field_name); + return -1; + } + return validate_field( + field_name, + reported_size, + base_offset + nested_offset, + width, + alignment, + buffer_size); +} + +static inline int +validate_fixed_field( + const char *field_name, + uint64_t offset, + size_t width, + size_t alignment, + size_t buffer_size) +{ + if (validate_alignment(field_name, offset, alignment) < 0) { + return -1; + } + return validate_span(field_name, offset, width, buffer_size, "local buffer size"); +} + +#define PY_REMOTE_DEBUG_VALIDATE_SECTION(section) \ + do { \ + if (validate_section_size(#section, debug_offsets->section.size) < 0) { \ + return -1; \ + } \ + } while (0) + +#define PY_REMOTE_DEBUG_VALIDATE_READ_SECTION(section, buffer_size) \ + do { \ + if (validate_read_size(#section, debug_offsets->section.size, buffer_size) < 0) { \ + return -1; \ + } \ + } while (0) + +#define PY_REMOTE_DEBUG_VALIDATE_FIELD(section, field, field_size, field_alignment, buffer_size) \ + do { \ + if (validate_field( \ + #section "." #field, \ + debug_offsets->section.size, \ + debug_offsets->section.field, \ + field_size, \ + field_alignment, \ + buffer_size) < 0) { \ + return -1; \ + } \ + } while (0) + +#define PY_REMOTE_DEBUG_VALIDATE_NESTED_FIELD(section, base, nested_section, field, field_size, field_alignment, buffer_size) \ + do { \ + if (validate_nested_field( \ + #section "." #base "." #field, \ + debug_offsets->section.size, \ + debug_offsets->section.base, \ + debug_offsets->nested_section.field, \ + field_size, \ + field_alignment, \ + buffer_size) < 0) { \ + return -1; \ + } \ + } while (0) + +#define PY_REMOTE_DEBUG_VALIDATE_FIXED_FIELD(section, field, field_size, field_alignment, buffer_size) \ + do { \ + if (validate_fixed_field( \ + #section "." #field, \ + debug_offsets->section.field, \ + field_size, \ + field_alignment, \ + buffer_size) < 0) { \ + return -1; \ + } \ + } while (0) + +/* + * Each list below must include every offset that is later dereferenced against + * a local buffer or local object view. The validator checks that each field + * stays within both the remote table's reported section size and the local + * buffer size we use when reading that section. If a new dereferenced field is + * added to the offset tables, add it to the matching list here. + * + * Sections not listed here are present in the offset tables but not used by + * the unwinder, so no validation is needed for them. + */ +#define PY_REMOTE_DEBUG_RUNTIME_STATE_FIELDS(APPLY, buffer_size) \ + APPLY(runtime_state, interpreters_head, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size) + +#define PY_REMOTE_DEBUG_THREAD_STATE_FIELDS(APPLY, buffer_size) \ + APPLY(thread_state, native_thread_id, sizeof(unsigned long), _Alignof(long), buffer_size); \ + APPLY(thread_state, interp, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(thread_state, datastack_chunk, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(thread_state, status, FIELD_SIZE(PyThreadState, _status), _Alignof(unsigned int), buffer_size); \ + APPLY(thread_state, holds_gil, sizeof(int), _Alignof(int), buffer_size); \ + APPLY(thread_state, gil_requested, sizeof(int), _Alignof(int), buffer_size); \ + APPLY(thread_state, current_exception, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(thread_state, thread_id, sizeof(unsigned long), _Alignof(long), buffer_size); \ + APPLY(thread_state, next, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(thread_state, current_frame, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(thread_state, base_frame, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(thread_state, last_profiled_frame, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size) + +#define PY_REMOTE_DEBUG_INTERPRETER_STATE_FIELDS(APPLY, buffer_size) \ + APPLY(interpreter_state, id, sizeof(int64_t), _Alignof(int64_t), buffer_size); \ + APPLY(interpreter_state, next, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(interpreter_state, threads_head, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(interpreter_state, threads_main, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(interpreter_state, gil_runtime_state_locked, sizeof(int), _Alignof(int), buffer_size); \ + APPLY(interpreter_state, gil_runtime_state_holder, sizeof(PyThreadState *), _Alignof(PyThreadState *), buffer_size); \ + APPLY(interpreter_state, code_object_generation, sizeof(uint64_t), _Alignof(uint64_t), buffer_size); \ + APPLY(interpreter_state, tlbc_generation, sizeof(uint32_t), _Alignof(uint32_t), buffer_size) + +#define PY_REMOTE_DEBUG_INTERPRETER_FRAME_FIELDS(APPLY, buffer_size) \ + APPLY(interpreter_frame, previous, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(interpreter_frame, executable, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(interpreter_frame, instr_ptr, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(interpreter_frame, owner, sizeof(char), _Alignof(char), buffer_size); \ + APPLY(interpreter_frame, stackpointer, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(interpreter_frame, tlbc_index, sizeof(int32_t), _Alignof(int32_t), buffer_size) + +#define PY_REMOTE_DEBUG_CODE_OBJECT_FIELDS(APPLY, buffer_size) \ + APPLY(code_object, qualname, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(code_object, filename, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(code_object, linetable, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(code_object, firstlineno, sizeof(int), _Alignof(int), buffer_size); \ + APPLY(code_object, co_code_adaptive, sizeof(char), _Alignof(char), buffer_size); \ + APPLY(code_object, co_tlbc, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size) + +#define PY_REMOTE_DEBUG_SET_OBJECT_FIELDS(APPLY, buffer_size) \ + APPLY(set_object, used, sizeof(Py_ssize_t), _Alignof(Py_ssize_t), buffer_size); \ + APPLY(set_object, mask, sizeof(Py_ssize_t), _Alignof(Py_ssize_t), buffer_size); \ + APPLY(set_object, table, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size) + +#define PY_REMOTE_DEBUG_LONG_OBJECT_FIELDS(APPLY, buffer_size) \ + APPLY(long_object, lv_tag, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(long_object, ob_digit, sizeof(digit), _Alignof(digit), buffer_size) + +#define PY_REMOTE_DEBUG_BYTES_OBJECT_FIELDS(APPLY, buffer_size) \ + APPLY(bytes_object, ob_size, sizeof(Py_ssize_t), _Alignof(Py_ssize_t), buffer_size); \ + APPLY(bytes_object, ob_sval, sizeof(char), _Alignof(char), buffer_size) + +#define PY_REMOTE_DEBUG_UNICODE_OBJECT_FIELDS(APPLY, buffer_size) \ + APPLY(unicode_object, length, sizeof(Py_ssize_t), _Alignof(Py_ssize_t), buffer_size); \ + APPLY(unicode_object, asciiobject_size, sizeof(char), _Alignof(char), buffer_size) + +#define PY_REMOTE_DEBUG_GEN_OBJECT_FIELDS(APPLY, buffer_size) \ + APPLY(gen_object, gi_frame_state, sizeof(int8_t), _Alignof(int8_t), buffer_size); \ + APPLY(gen_object, gi_iframe, FIELD_SIZE(PyGenObject, gi_iframe), _Alignof(_PyInterpreterFrame), buffer_size) + +#define PY_REMOTE_DEBUG_TASK_OBJECT_FIELDS(APPLY, buffer_size) \ + APPLY(asyncio_task_object, task_name, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(asyncio_task_object, task_awaited_by, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(asyncio_task_object, task_is_task, sizeof(char), _Alignof(char), buffer_size); \ + APPLY(asyncio_task_object, task_awaited_by_is_set, sizeof(char), _Alignof(char), buffer_size); \ + APPLY(asyncio_task_object, task_coro, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(asyncio_task_object, task_node, SIZEOF_LLIST_NODE, _Alignof(struct llist_node), buffer_size) + +#define PY_REMOTE_DEBUG_ASYNC_INTERPRETER_STATE_FIELDS(APPLY, buffer_size) \ + APPLY(asyncio_interpreter_state, asyncio_tasks_head, SIZEOF_LLIST_NODE, _Alignof(struct llist_node), buffer_size) + +#define PY_REMOTE_DEBUG_ASYNC_THREAD_STATE_FIELDS(APPLY, buffer_size) \ + APPLY(asyncio_thread_state, asyncio_running_loop, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(asyncio_thread_state, asyncio_running_task, sizeof(uintptr_t), _Alignof(uintptr_t), buffer_size); \ + APPLY(asyncio_thread_state, asyncio_tasks_head, SIZEOF_LLIST_NODE, _Alignof(struct llist_node), buffer_size) + +/* Called once after reading _Py_DebugOffsets, before any hot-path use. */ +static inline int +_PyRemoteDebug_ValidateDebugOffsetsLayout(struct _Py_DebugOffsets *debug_offsets) +{ + /* Validate every field the unwinder dereferences against a local buffer + * or local object view. Fields used only for remote address arithmetic + * (e.g. runtime_state.interpreters_head) are also checked as a sanity + * bound on the offset value. */ + PY_REMOTE_DEBUG_VALIDATE_SECTION(runtime_state); + PY_REMOTE_DEBUG_RUNTIME_STATE_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + sizeof(_PyRuntimeState)); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(interpreter_state); + PY_REMOTE_DEBUG_INTERPRETER_STATE_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + INTERP_STATE_BUFFER_SIZE); + + PY_REMOTE_DEBUG_VALIDATE_READ_SECTION(thread_state, SIZEOF_THREAD_STATE); + PY_REMOTE_DEBUG_THREAD_STATE_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + SIZEOF_THREAD_STATE); + PY_REMOTE_DEBUG_VALIDATE_FIXED_FIELD( + err_stackitem, + exc_value, + sizeof(uintptr_t), + _Alignof(uintptr_t), + sizeof(_PyErr_StackItem)); + PY_REMOTE_DEBUG_VALIDATE_NESTED_FIELD( + thread_state, + exc_state, + err_stackitem, + exc_value, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_THREAD_STATE); + + PY_REMOTE_DEBUG_VALIDATE_READ_SECTION(gc, SIZEOF_GC_RUNTIME_STATE); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + gc, + frame, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_GC_RUNTIME_STATE); + PY_REMOTE_DEBUG_VALIDATE_NESTED_FIELD( + interpreter_state, + gc, + gc, + frame, + sizeof(uintptr_t), + _Alignof(uintptr_t), + INTERP_STATE_BUFFER_SIZE); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(interpreter_frame); + PY_REMOTE_DEBUG_INTERPRETER_FRAME_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + SIZEOF_INTERP_FRAME); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(code_object); + PY_REMOTE_DEBUG_CODE_OBJECT_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + SIZEOF_CODE_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(pyobject); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + pyobject, + ob_type, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_PYOBJECT); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(type_object); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + type_object, + tp_flags, + sizeof(unsigned long), + _Alignof(unsigned long), + SIZEOF_TYPE_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(set_object); + PY_REMOTE_DEBUG_SET_OBJECT_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + SIZEOF_SET_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_READ_SECTION(long_object, SIZEOF_LONG_OBJ); + PY_REMOTE_DEBUG_LONG_OBJECT_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + SIZEOF_LONG_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(bytes_object); + PY_REMOTE_DEBUG_BYTES_OBJECT_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + SIZEOF_BYTES_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(unicode_object); + PY_REMOTE_DEBUG_UNICODE_OBJECT_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + SIZEOF_UNICODE_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(gen_object); + PY_REMOTE_DEBUG_GEN_OBJECT_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + SIZEOF_GEN_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_FIXED_FIELD( + llist_node, + next, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_LLIST_NODE); + + return 0; +} + +/* Called once when AsyncioDebug is loaded, before any task inspection uses it. */ +static inline int +_PyRemoteDebug_ValidateAsyncDebugOffsets( + struct _Py_AsyncioModuleDebugOffsets *debug_offsets) +{ + PY_REMOTE_DEBUG_VALIDATE_READ_SECTION(asyncio_task_object, SIZEOF_TASK_OBJ); + PY_REMOTE_DEBUG_TASK_OBJECT_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + SIZEOF_TASK_OBJ); + PY_REMOTE_DEBUG_VALIDATE_SECTION(asyncio_interpreter_state); + PY_REMOTE_DEBUG_ASYNC_INTERPRETER_STATE_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + sizeof(PyInterpreterState)); + PY_REMOTE_DEBUG_VALIDATE_SECTION(asyncio_thread_state); + PY_REMOTE_DEBUG_ASYNC_THREAD_STATE_FIELDS( + PY_REMOTE_DEBUG_VALIDATE_FIELD, + sizeof(_PyThreadStateImpl)); + return 0; +} + +#undef PY_REMOTE_DEBUG_VALIDATE_SECTION +#undef PY_REMOTE_DEBUG_VALIDATE_READ_SECTION +#undef PY_REMOTE_DEBUG_VALIDATE_FIELD +#undef PY_REMOTE_DEBUG_VALIDATE_NESTED_FIELD +#undef PY_REMOTE_DEBUG_VALIDATE_FIXED_FIELD +#undef PY_REMOTE_DEBUG_RUNTIME_STATE_FIELDS +#undef PY_REMOTE_DEBUG_THREAD_STATE_FIELDS +#undef PY_REMOTE_DEBUG_INTERPRETER_STATE_FIELDS +#undef PY_REMOTE_DEBUG_INTERPRETER_FRAME_FIELDS +#undef PY_REMOTE_DEBUG_CODE_OBJECT_FIELDS +#undef PY_REMOTE_DEBUG_SET_OBJECT_FIELDS +#undef PY_REMOTE_DEBUG_LONG_OBJECT_FIELDS +#undef PY_REMOTE_DEBUG_BYTES_OBJECT_FIELDS +#undef PY_REMOTE_DEBUG_UNICODE_OBJECT_FIELDS +#undef PY_REMOTE_DEBUG_GEN_OBJECT_FIELDS +#undef PY_REMOTE_DEBUG_TASK_OBJECT_FIELDS +#undef PY_REMOTE_DEBUG_ASYNC_INTERPRETER_STATE_FIELDS +#undef PY_REMOTE_DEBUG_ASYNC_THREAD_STATE_FIELDS +#undef FIELD_SIZE + +#endif diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index a0b4a1e8a1e542..bbdfce3f7201d9 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -148,7 +148,9 @@ find_frame_in_chunks(StackChunkList *chunks, uintptr_t remote_ptr) uintptr_t base = chunks->chunks[i].remote_addr + offsetof(_PyStackChunk, data); size_t payload = chunks->chunks[i].size - offsetof(_PyStackChunk, data); - if (remote_ptr >= base && remote_ptr < base + payload) { + if (payload >= SIZEOF_INTERP_FRAME && + remote_ptr >= base && + remote_ptr <= base + payload - SIZEOF_INTERP_FRAME) { return (char *)chunks->chunks[i].local_copy + (remote_ptr - chunks->chunks[i].remote_addr); } } diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index f86bbf8ce5526e..32f2cbacf2143b 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -7,6 +7,7 @@ #include "_remote_debugging.h" #include "binary_io.h" +#include "debug_offsets_validation.h" /* Forward declarations for clinic-generated code */ typedef struct { @@ -240,7 +241,7 @@ validate_debug_offsets(struct _Py_DebugOffsets *debug_offsets) return -1; } - return 0; + return _PyRemoteDebug_ValidateDebugOffsetsLayout(debug_offsets); } /* ============================================================================ @@ -374,7 +375,11 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, // Try to read async debug offsets, but don't fail if they're not available self->async_debug_offsets_available = 1; - if (read_async_debug(self) < 0) { + int async_debug_result = read_async_debug(self); + if (async_debug_result == PY_REMOTE_DEBUG_INVALID_ASYNC_DEBUG_OFFSETS) { + return -1; + } + if (async_debug_result < 0) { PyErr_Clear(); memset(&self->async_debug_offsets, 0, sizeof(self->async_debug_offsets)); self->async_debug_offsets_available = 0; diff --git a/Android/README.md b/Platforms/Android/README.md similarity index 63% rename from Android/README.md rename to Platforms/Android/README.md index 0004f26e72b21c..d6f95c365c63a0 100644 --- a/Android/README.md +++ b/Platforms/Android/README.md @@ -11,7 +11,6 @@ Instead, use one of the tools listed [here](https://docs.python.org/3/using/android.html), which will provide a much easier experience. - ## Prerequisites If you already have an Android SDK installed, export the `ANDROID_HOME` @@ -25,7 +24,7 @@ it: `android-sdk/cmdline-tools/latest`. * `export ANDROID_HOME=/path/to/android-sdk` -The `android.py` script will automatically use the SDK's `sdkmanager` to install +The `Platforms/Android` script will automatically use the SDK's `sdkmanager` to install any packages it needs. The script also requires the following commands to be on the `PATH`: @@ -33,7 +32,6 @@ The script also requires the following commands to be on the `PATH`: * `curl` * `java` (or set the `JAVA_HOME` environment variable) - ## Building Python can be built for Android on any POSIX platform supported by the Android @@ -43,29 +41,28 @@ First we'll make a "build" Python (for your development machine), then use it to help produce a "host" Python for Android. So make sure you have all the usual tools and libraries needed to build Python for your development machine. -The easiest way to do a build is to use the `android.py` script. You can either +The easiest way to do a build is to use the `Platforms/Android` script. You can either have it perform the entire build process from start to finish in one step, or you can do it in discrete steps that mirror running `configure` and `make` for each of the two builds of Python you end up producing. -The discrete steps for building via `android.py` are: +The discrete steps for building via `Platforms/Android` are: ```sh -./android.py configure-build -./android.py make-build -./android.py configure-host HOST -./android.py make-host HOST +python3 Platforms/Android configure-build +python3 Platforms/Android make-build +python3 Platforms/Android configure-host HOST +python3 Platforms/Android make-host HOST ``` `HOST` identifies which architecture to build. To see the possible values, run -`./android.py configure-host --help`. +`python3 Platforms/Android configure-host --help`. To do all steps in a single command, run: ```sh -./android.py build HOST +python3 Platforms/Android build HOST ``` - In the end you should have a build Python in `cross-build/build`, and a host Python in `cross-build/HOST`. @@ -75,17 +72,16 @@ call. For example, if you want a pydebug build that also caches the results from `configure`, you can do: ```sh -./android.py build HOST -- -C --with-pydebug +python3 Platforms/Android build HOST -- -C --with-pydebug ``` - ## Packaging After building an architecture as described in the section above, you can package it for release with this command: ```sh -./android.py package HOST +python3 Platforms/Android package HOST ``` `HOST` is defined in the section above. @@ -93,25 +89,16 @@ package it for release with this command: This will generate a tarball in `cross-build/HOST/dist`, whose structure is similar to the `Android` directory of the CPython source tree. - ## Testing -The Python test suite can be run on Linux, macOS, or Windows. +Tests can be run on Linux, macOS, or Windows, using either an Android emulator +or a physical device. On Linux, the emulator needs access to the KVM virtualization interface. This may require adding your user to a group, or changing your udev rules. On GitHub Actions, the test script will do this automatically using the commands shown [here](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). -You can run the test suite either: - -* Within the CPython repository, after doing a build as described above. On - Windows, you won't be able to do the build on the same machine, so you'll have - to copy the `cross-build/HOST/prefix` directory from somewhere else. - -* Or by taking a release package built using the `package` command, extracting - it wherever you want, and using its own copy of `android.py`. - The test script supports the following modes: * In `--connected` mode, it runs on a device or emulator you have already @@ -120,7 +107,7 @@ The test script supports the following modes: script like this: ```sh - ./android.py test --connected emulator-5554 + python3 Platforms/Android test --connected emulator-5554 ``` * In `--managed` mode, it uses a temporary headless emulator defined in the @@ -131,29 +118,55 @@ The test script supports the following modes: to our minimum and maximum supported Android versions. For example: ```sh - ./android.py test --managed maxVersion + python3 Platforms/Android test --managed maxVersion ``` By default, the only messages the script will show are Python's own stdout and stderr. Add the `-v` option to also show Gradle output, and non-Python logcat messages. -Any other arguments on the `android.py test` command line will be passed through -to `python -m test` – use `--` to separate them from android.py's own options. +### Testing Python + +You can run the test suite by doing a build as described above, and then running +`python3 Platforms/Android test`. On Windows, you won't be able to do the build +on the same machine, so you'll have to copy the `cross-build/HOST/prefix` directory +from somewhere else. + +Extra arguments on the `Platforms/Android test` command line will be passed through +to `python -m test` – use `--` to separate them from `Platforms/Android`'s own options. See the [Python Developer's Guide](https://devguide.python.org/testing/run-write-tests/) for common options – most of them will work on Android, except for those that involve subprocesses, such as `-j`. -Every time you run `android.py test`, changes in pure-Python files in the +Every time you run `python3 Platforms/Android test`, changes in pure-Python files in the repository's `Lib` directory will be picked up immediately. Changes in C files, and architecture-specific files such as sysconfigdata, will not take effect -until you re-run `android.py make-host` or `build`. +until you re-run `python3 Platforms/Android make-host` or `build`. + +### Testing a third-party package + +The `Platforms/Android` script is also included as `android.py` in the root of a +release package (i.e., the one built using `Platforms/Android package`). + +You can use this script to test third-party packages by taking a release +package, extracting it wherever you want, and using the `android.py` script to +run the test suite for your third-party package. + +Any argument that can be passed to `python3 Platforms/Android test` can also be +passed to `android.py`. The following options will be of particular use when +configuring the execution of a third-party test suite: + +* `--cwd`: the directory of content to copy into the testbed app as the working + directory. +* `--site-packages`: the directory to copy into the testbed app to use as site + packages. -The testbed app can also be used to test third-party packages. For more details, -run `android.py test --help`, paying attention to the options `--site-packages`, -`--cwd`, `-c` and `-m`. +Extra arguments on the `android.py test` command line will be passed through to +Python – use `--` to separate them from `android.py`'s own options. You must include +either a `-c` or `-m` argument to specify how the test suite should be started. +For more details, run `android.py test --help`. ## Using in your own app diff --git a/Android/android.py b/Platforms/Android/__main__.py similarity index 98% rename from Android/android.py rename to Platforms/Android/__main__.py index 9d452ea87fced9..315632ea12c07d 100755 --- a/Android/android.py +++ b/Platforms/Android/__main__.py @@ -24,8 +24,11 @@ SCRIPT_NAME = Path(__file__).name +if SCRIPT_NAME.startswith("__"): + SCRIPT_NAME = "Platforms/Android" + ANDROID_DIR = Path(__file__).resolve().parent -PYTHON_DIR = ANDROID_DIR.parent +PYTHON_DIR = ANDROID_DIR.parent.parent in_source_tree = ( ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() ) @@ -756,7 +759,7 @@ def package(context): prefix_dir = subdir(context.host, "prefix") version = package_version(prefix_dir) - with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + with TemporaryDirectory(prefix=SCRIPT_NAME.replace("/", "-")) as temp_dir: temp_dir = Path(temp_dir) # Include all tracked files from the Android directory. @@ -765,7 +768,10 @@ def package(context): cwd=ANDROID_DIR, capture_output=True, text=True, log=False, ).stdout.splitlines(): src = ANDROID_DIR / line - dst = temp_dir / line + # "__main__.py" is renamed "android.py" for distribution purpose + dst = temp_dir / { + "__main__.py": "android.py" + }.get(line, line) dst.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src, dst, follow_symlinks=False) @@ -831,7 +837,7 @@ def ci(context): "emulator on this platform." ) else: - with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + with TemporaryDirectory(prefix=SCRIPT_NAME.replace("/", "-")) as temp_dir: print("::group::Tests") # Prove the package is self-contained by using it to run the tests. diff --git a/Android/android-env.sh b/Platforms/Android/android-env.sh similarity index 100% rename from Android/android-env.sh rename to Platforms/Android/android-env.sh diff --git a/Android/testbed/.gitignore b/Platforms/Android/testbed/.gitignore similarity index 100% rename from Android/testbed/.gitignore rename to Platforms/Android/testbed/.gitignore diff --git a/Android/testbed/.idea/inspectionProfiles/Project_Default.xml b/Platforms/Android/testbed/.idea/inspectionProfiles/Project_Default.xml similarity index 100% rename from Android/testbed/.idea/inspectionProfiles/Project_Default.xml rename to Platforms/Android/testbed/.idea/inspectionProfiles/Project_Default.xml diff --git a/Android/testbed/app/.gitignore b/Platforms/Android/testbed/app/.gitignore similarity index 100% rename from Android/testbed/app/.gitignore rename to Platforms/Android/testbed/app/.gitignore diff --git a/Android/testbed/app/build.gradle.kts b/Platforms/Android/testbed/app/build.gradle.kts similarity index 99% rename from Android/testbed/app/build.gradle.kts rename to Platforms/Android/testbed/app/build.gradle.kts index bd8334b64bb0a8..e51398fce81e26 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Platforms/Android/testbed/app/build.gradle.kts @@ -7,7 +7,7 @@ plugins { } val ANDROID_DIR = file("../..") -val PYTHON_DIR = ANDROID_DIR.parentFile!! +val PYTHON_DIR = ANDROID_DIR.parentFile.parentFile!! val PYTHON_CROSS_DIR = file(System.getenv("CROSS_BUILD_DIR") ?: "$PYTHON_DIR/cross-build") val inSourceTree = ( ANDROID_DIR.name == "Android" && file("$PYTHON_DIR/pyconfig.h.in").exists() diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Platforms/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt similarity index 100% rename from Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt rename to Platforms/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt diff --git a/Android/testbed/app/src/main/AndroidManifest.xml b/Platforms/Android/testbed/app/src/main/AndroidManifest.xml similarity index 100% rename from Android/testbed/app/src/main/AndroidManifest.xml rename to Platforms/Android/testbed/app/src/main/AndroidManifest.xml diff --git a/Android/testbed/app/src/main/c/CMakeLists.txt b/Platforms/Android/testbed/app/src/main/c/CMakeLists.txt similarity index 100% rename from Android/testbed/app/src/main/c/CMakeLists.txt rename to Platforms/Android/testbed/app/src/main/c/CMakeLists.txt diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Platforms/Android/testbed/app/src/main/c/main_activity.c similarity index 100% rename from Android/testbed/app/src/main/c/main_activity.c rename to Platforms/Android/testbed/app/src/main/c/main_activity.c diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Platforms/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt similarity index 100% rename from Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt rename to Platforms/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt diff --git a/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png b/Platforms/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png similarity index 100% rename from Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png rename to Platforms/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png diff --git a/Android/testbed/app/src/main/res/layout/activity_main.xml b/Platforms/Android/testbed/app/src/main/res/layout/activity_main.xml similarity index 100% rename from Android/testbed/app/src/main/res/layout/activity_main.xml rename to Platforms/Android/testbed/app/src/main/res/layout/activity_main.xml diff --git a/Android/testbed/app/src/main/res/values/strings.xml b/Platforms/Android/testbed/app/src/main/res/values/strings.xml similarity index 100% rename from Android/testbed/app/src/main/res/values/strings.xml rename to Platforms/Android/testbed/app/src/main/res/values/strings.xml diff --git a/Android/testbed/build.gradle.kts b/Platforms/Android/testbed/build.gradle.kts similarity index 100% rename from Android/testbed/build.gradle.kts rename to Platforms/Android/testbed/build.gradle.kts diff --git a/Android/testbed/gradle.properties b/Platforms/Android/testbed/gradle.properties similarity index 100% rename from Android/testbed/gradle.properties rename to Platforms/Android/testbed/gradle.properties diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Platforms/Android/testbed/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from Android/testbed/gradle/wrapper/gradle-wrapper.properties rename to Platforms/Android/testbed/gradle/wrapper/gradle-wrapper.properties diff --git a/Android/testbed/settings.gradle.kts b/Platforms/Android/testbed/settings.gradle.kts similarity index 100% rename from Android/testbed/settings.gradle.kts rename to Platforms/Android/testbed/settings.gradle.kts