Merge branch 'main' into ac/libmpdec

.gitattributes

@@ -77,6 +77,7 @@ Include/internal/pycore_opcode.h                    generated
 Include/internal/pycore_opcode_metadata.h           generated
 Include/internal/pycore_*_generated.h               generated
 Include/internal/pycore_uop_ids.h                   generated
+Include/internal/pycore_uop_metadata.h              generated
 Include/opcode.h                                    generated
 Include/opcode_ids.h                                generated
 Include/token.h                                     generated
@@ -94,6 +95,7 @@ Programs/test_frozenmain.h                          generated
 Python/Python-ast.c                                 generated
 Python/executor_cases.c.h                           generated
 Python/generated_cases.c.h                          generated
+Python/optimizer_cases.c.h                          generated
 Python/opcode_targets.h                             generated
 Python/stdlib_module_names.h                        generated
 Tools/peg_generator/pegen/grammar_parser.py         generated

.github/CODEOWNERS

@@ -37,6 +37,8 @@ Python/flowgraph.c            @markshannon @iritkatriel
 Python/ast_opt.c              @isidentical
 Python/bytecodes.c            @markshannon @gvanrossum
 Python/optimizer*.c           @markshannon @gvanrossum
+Python/optimizer_analysis.c   @Fidget-Spinner
+Python/optimizer_bytecodes.c  @Fidget-Spinner
 Lib/test/test_patma.py        @brandtbucher
 Lib/test/test_type_*.py       @JelleZijlstra
 Lib/test/test_capi/test_misc.py  @markshannon @gvanrossum
@@ -247,3 +249,7 @@ Lib/test/test_interpreters/   @ericsnowcurrently
 # SBOM
 /Misc/sbom.spdx.json          @sethmlarson
 /Tools/build/generate_sbom.py @sethmlarson
+
+# Config Parser
+Lib/configparser.py           @jaraco
+Lib/test/test_configparser.py @jaraco

.github/workflows/build.yml

@@ -131,11 +131,13 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
+      - name: Runner image version
+        run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
       - name: Restore config.cache
         uses: actions/cache@v4
         with:
           path: config.cache
-          key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
+          key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
       - name: Add ccache to PATH
@@ -160,7 +162,7 @@ jobs:
       - name: Build CPython
         run: |
           make -j4 regen-all
-          make regen-stdlib-module-names
+          make regen-stdlib-module-names regen-sbom
       - name: Check for changes
         run: |
           git add -u
@@ -258,11 +260,13 @@ jobs:
       LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
     steps:
     - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
@@ -341,11 +345,13 @@ jobs:
       run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: |
@@ -420,11 +426,13 @@ jobs:
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
     - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies

.github/workflows/jit.yml

@@ -4,10 +4,14 @@ on:
     paths:
       - '**jit**'
       - 'Python/bytecodes.c'
+      - 'Python/optimizer*.c'
+      - 'Python/optimizer_bytecodes.c'
   push:
     paths:
       - '**jit**'
       - 'Python/bytecodes.c'
+      - 'Python/optimizer*.c'
+      - 'Python/optimizer_bytecodes.c'
   workflow_dispatch:
 
 concurrency:

.github/workflows/reusable-macos.yml

@@ -28,11 +28,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ matrix.os }}-${{ inputs.config_hash }}
+        key: ${{ github.job }}-${{ matrix.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
     - name: Install Homebrew dependencies
       run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk
     - name: Configure CPython

.github/workflows/reusable-ubuntu.yml

@@ -52,11 +52,13 @@ jobs:
       run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: ${{ inputs.options }}

.gitignore

@@ -69,6 +69,7 @@ Lib/test/data/*
 /_bootstrap_python
 /Makefile
 /Makefile.pre
+iOS/Resources/Info.plist
 Mac/Makefile
 Mac/PythonLauncher/Info.plist
 Mac/PythonLauncher/Makefile

Doc/c-api/code.rst

@@ -30,9 +30,13 @@ bound into a function.
    Return true if *co* is a :ref:`code object <code-objects>`.
    This function always succeeds.
 
-.. c:function:: int PyCode_GetNumFree(PyCodeObject *co)
+.. c:function:: Py_ssize_t PyCode_GetNumFree(PyCodeObject *co)
 
-   Return the number of free variables in *co*.
+   Return the number of free variables in a code object.
+
+.. c:function:: int PyCode_GetFirstFree(PyCodeObject *co)
+
+   Return the position of the first free variable in a code object.
 
 .. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
 

Doc/c-api/exceptions.rst

@@ -105,7 +105,7 @@ Printing and clearing
    parameters help format the warning message; they have the same meaning and
    values as in :c:func:`PyUnicode_FromFormat`.
    ``PyErr_WriteUnraisable(obj)`` is roughtly equivalent to
-   ``PyErr_FormatUnraisable("Exception ignored in: %R, obj)``.
+   ``PyErr_FormatUnraisable("Exception ignored in: %R", obj)``.
    If *format* is ``NULL``, only the traceback is printed.
 
    .. versionadded:: 3.13

Doc/c-api/long.rst

@@ -358,46 +358,86 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
 
    Copy the Python integer value to a native *buffer* of size *n_bytes*::
 
-      int value;
-      Py_ssize_t bytes = PyLong_AsNativeBytes(v, &value, sizeof(value), -1);
+      int32_t value;
+      Py_ssize_t bytes = PyLong_AsNativeBits(pylong, &value, sizeof(value), -1);
       if (bytes < 0) {
-          // Error occurred
+          // A Python exception was set with the reason.
           return NULL;
       }
       else if (bytes <= (Py_ssize_t)sizeof(value)) {
           // Success!
       }
       else {
-          // Overflow occurred, but 'value' contains truncated value
+          // Overflow occurred, but 'value' contains the truncated
+          // lowest bits of pylong.
       }
 
+   The above example may look *similar* to
+   :c:func:`PyLong_As* <PyLong_AsSize_t>`
+   but instead fills in a specific caller defined type and never raises an
+   error about of the :class:`int` *pylong*'s value regardless of *n_bytes*
+   or the returned byte count.
+
+   To get at the entire potentially big Python value, this can be used to
+   reserve enough space and copy it::
+
+      // Ask how much space we need.
+      Py_ssize_t expected = PyLong_AsNativeBits(pylong, NULL, 0, -1);
+      if (expected < 0) {
+          // A Python exception was set with the reason.
+          return NULL;
+      }
+      assert(expected != 0);  // Impossible per the API definition.
+      uint8_t *bignum = malloc(expected);
+      if (!bignum) {
+          PyErr_SetString(PyExc_MemoryError, "bignum malloc failed.");
+          return NULL;
+      }
+      // Safely get the entire value.
+      Py_ssize_t bytes = PyLong_AsNativeBits(pylong, bignum, expected, -1);
+      if (bytes < 0) {  // Exception set.
+          free(bignum);
+          return NULL;
+      }
+      else if (bytes > expected) {  // Be safe, should not be possible.
+          PyErr_SetString(PyExc_RuntimeError,
+              "Unexpected bignum truncation after a size check.");
+          free(bignum);
+          return NULL;
+      }
+      // The expected success given the above pre-check.
+      // ... use bignum ...
+      free(bignum);
+
    *endianness* may be passed ``-1`` for the native endian that CPython was
    compiled with, or ``0`` for big endian and ``1`` for little.
 
-   Return ``-1`` with an exception raised if *pylong* cannot be interpreted as
+   Returns ``-1`` with an exception raised if *pylong* cannot be interpreted as
    an integer. Otherwise, return the size of the buffer required to store the
    value. If this is equal to or less than *n_bytes*, the entire value was
-   copied.
+   copied. ``0`` will never be returned.
 
-   Unless an exception is raised, all *n_bytes* of the buffer will be written
-   with as much of the value as can fit. This allows the caller to ignore all
-   non-negative results if the intent is to match the typical behavior of a
-   C-style downcast. No exception is set for this case.
+   Unless an exception is raised, all *n_bytes* of the buffer will always be
+   written. In the case of truncation, as many of the lowest bits of the value
+   as could fit are written. This allows the caller to ignore all non-negative
+   results if the intent is to match the typical behavior of a C-style
+   downcast. No exception is set on truncation.
 
-   Values are always copied as two's-complement, and sufficient buffer will be
+   Values are always copied as two's-complement and sufficient buffer will be
    requested to include a sign bit. For example, this may cause an value that
    fits into 8 bytes when treated as unsigned to request 9 bytes, even though
    all eight bytes were copied into the buffer. What has been omitted is the
-   zero sign bit, which is redundant when the intention is to treat the value as
-   unsigned.
+   zero sign bit -- redundant if the caller's intention is to treat the value
+   as unsigned.
 
-   Passing zero to *n_bytes* will return the requested buffer size.
+   Passing zero to *n_bytes* will return the size of a buffer that would
+   be large enough to hold the value. This may be larger than technically
+   necessary, but not unreasonably so.
 
    .. note::
 
-      When the value does not fit in the provided buffer, the requested size
-      returned from the function may be larger than necessary. Passing 0 to this
-      function is not an accurate way to determine the bit length of a value.
+      Passing *n_bytes=0* to this function is not an accurate way to determine
+      the bit length of a value.
 
    .. versionadded:: 3.13
 

Doc/c-api/structures.rst

@@ -187,26 +187,26 @@ Implementing functions and methods
                                         PyObject *kwargs);
 
 
-.. c:type:: _PyCFunctionFast
+.. c:type:: PyCFunctionFast
 
    Type of the functions used to implement Python callables in C
    with signature :c:macro:`METH_FASTCALL`.
    The function signature is::
 
-      PyObject *_PyCFunctionFast(PyObject *self,
-                                 PyObject *const *args,
-                                 Py_ssize_t nargs);
+      PyObject *PyCFunctionFast(PyObject *self,
+                                PyObject *const *args,
+                                Py_ssize_t nargs);
 
-.. c:type:: _PyCFunctionFastWithKeywords
+.. c:type:: PyCFunctionFastWithKeywords
 
    Type of the functions used to implement Python callables in C
    with signature :ref:`METH_FASTCALL | METH_KEYWORDS <METH_FASTCALL-METH_KEYWORDS>`.
    The function signature is::
 
-      PyObject *_PyCFunctionFastWithKeywords(PyObject *self,
-                                             PyObject *const *args,
-                                             Py_ssize_t nargs,
-                                             PyObject *kwnames);
+      PyObject *PyCFunctionFastWithKeywords(PyObject *self,
+                                            PyObject *const *args,
+                                            Py_ssize_t nargs,
+                                            PyObject *kwnames);
 
 .. c:type:: PyCMethod
 
@@ -290,7 +290,7 @@ There are these calling conventions:
 .. c:macro:: METH_FASTCALL
 
    Fast calling convention supporting only positional arguments.
-   The methods have the type :c:type:`_PyCFunctionFast`.
+   The methods have the type :c:type:`PyCFunctionFast`.
    The first parameter is *self*, the second parameter is a C array
    of :c:expr:`PyObject*` values indicating the arguments and the third
    parameter is the number of arguments (the length of the array).
@@ -306,7 +306,7 @@ There are these calling conventions:
 
 :c:expr:`METH_FASTCALL | METH_KEYWORDS`
    Extension of :c:macro:`METH_FASTCALL` supporting also keyword arguments,
-   with methods of type :c:type:`_PyCFunctionFastWithKeywords`.
+   with methods of type :c:type:`PyCFunctionFastWithKeywords`.
    Keyword arguments are passed the same way as in the
    :ref:`vectorcall protocol <vectorcall>`:
    there is an additional fourth :c:expr:`PyObject*` parameter

Doc/c-api/unicode.rst

@@ -854,7 +854,12 @@ wchar_t Support
    Copy the Unicode object contents into the :c:type:`wchar_t` buffer *wstr*.  At most
    *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing
    null termination character).  Return the number of :c:type:`wchar_t` characters
-   copied or ``-1`` in case of an error.  Note that the resulting :c:expr:`wchar_t*`
+   copied or ``-1`` in case of an error.
+
+   When *wstr* is ``NULL``, instead return the *size* that would be required
+   to store all of *unicode* including a terminating null.
+
+   Note that the resulting :c:expr:`wchar_t*`
    string may or may not be null-terminated.  It is the responsibility of the caller
    to make sure that the :c:expr:`wchar_t*` string is null-terminated in case this is
    required by the application. Also, note that the :c:expr:`wchar_t*` string

Doc/conf.py

@@ -66,6 +66,8 @@
 
 rst_epilog = f"""
 .. |python_version_literal| replace:: ``Python {version}``
+.. |python_x_dot_y_literal| replace:: ``python{version}``
+.. |usr_local_bin_python_x_dot_y_literal| replace:: ``/usr/local/bin/python{version}``
 """
 
 # There are two options for replacing |today|: either, you set today to some
@@ -99,11 +101,13 @@
     ('c:func', 'dlopen'),
     ('c:func', 'exec'),
     ('c:func', 'fcntl'),
+    ('c:func', 'flock'),
     ('c:func', 'fork'),
     ('c:func', 'free'),
     ('c:func', 'gettimeofday'),
     ('c:func', 'gmtime'),
     ('c:func', 'grantpt'),
+    ('c:func', 'ioctl'),
     ('c:func', 'localeconv'),
     ('c:func', 'localtime'),
     ('c:func', 'main'),
@@ -273,6 +277,7 @@
     ('py:attr', '__annotations__'),
     ('py:meth', '__missing__'),
     ('py:attr', '__wrapped__'),
+    ('py:attr', 'decimal.Context.clamp'),
     ('py:meth', 'index'),  # list.index, tuple.index, etc.
 ]