diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3422ef835279bc..d40519e40d3cc2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,6 +7,9 @@ # GitHub .github/** @ezio-melotti @hugovk +# pre-commit +.pre-commit-config.yaml @hugovk @AlexWaygood + # Build system configure* @erlend-aasland @corona10 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000000..4481ea80bfd936 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,22 @@ +name: Lint + +on: [push, pull_request, workflow_dispatch] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + - uses: pre-commit/action@v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000000..808622f19a3dbf --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + - id: trailing-whitespace + types_or: [c, python, rst] diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index f02abb01f0220b..0ca8ad624b2034 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -154,7 +154,7 @@ Dictionary Objects .. c:function:: Py_ssize_t PyDict_Size(PyObject *p) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the number of items in the dictionary. This is equivalent to ``len(p)`` on a dictionary. diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8e5af32b65e02c..79843ba521ab93 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -41,7 +41,7 @@ Importing Modules .. c:function:: PyObject* PyImport_ImportModuleEx(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) - .. index:: builtin: __import__ + .. index:: pair: built-in function; __import__ Import a module. This is best described by referring to the built-in Python function :func:`__import__`. @@ -120,7 +120,7 @@ Importing Modules .. c:function:: PyObject* PyImport_ExecCodeModule(const char *name, PyObject *co) - .. index:: builtin: compile + .. index:: pair: built-in function; compile Given a module name (possibly of the form ``package.module``) and a code object read from a Python bytecode file or obtained from the built-in function diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 317421f0db8453..dbf35611eccd3e 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -45,7 +45,7 @@ List Objects .. c:function:: Py_ssize_t PyList_Size(PyObject *list) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the length of the list object in *list*; this is equivalent to ``len(list)`` on a list object. @@ -138,7 +138,7 @@ List Objects .. c:function:: PyObject* PyList_AsTuple(PyObject *list) - .. index:: builtin: tuple + .. index:: pair: built-in function; tuple Return a new tuple object containing the contents of *list*; equivalent to ``tuple(list)``. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 3c9d282c6d0ab0..cffb0ed50fb77d 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -20,7 +20,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: Py_ssize_t PyMapping_Size(PyObject *o) Py_ssize_t PyMapping_Length(PyObject *o) - .. index:: builtin: len + .. index:: pair: built-in function; len Returns the number of keys in object *o* on success, and ``-1`` on failure. This is equivalent to the Python expression ``len(o)``. diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst index 70b91f8c2d0ca1..13d3c5af956905 100644 --- a/Doc/c-api/number.rst +++ b/Doc/c-api/number.rst @@ -64,7 +64,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Divmod(PyObject *o1, PyObject *o2) - .. index:: builtin: divmod + .. index:: pair: built-in function; divmod See the built-in function :func:`divmod`. Returns ``NULL`` on failure. This is the equivalent of the Python expression ``divmod(o1, o2)``. @@ -72,7 +72,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Power(PyObject *o1, PyObject *o2, PyObject *o3) - .. index:: builtin: pow + .. index:: pair: built-in function; pow See the built-in function :func:`pow`. Returns ``NULL`` on failure. This is the equivalent of the Python expression ``pow(o1, o2, o3)``, where *o3* is optional. @@ -94,7 +94,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Absolute(PyObject *o) - .. index:: builtin: abs + .. index:: pair: built-in function; abs Returns the absolute value of *o*, or ``NULL`` on failure. This is the equivalent of the Python expression ``abs(o)``. @@ -192,7 +192,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_InPlacePower(PyObject *o1, PyObject *o2, PyObject *o3) - .. index:: builtin: pow + .. index:: pair: built-in function; pow See the built-in function :func:`pow`. Returns ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python @@ -238,7 +238,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Long(PyObject *o) - .. index:: builtin: int + .. index:: pair: built-in function; int Returns the *o* converted to an integer object on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``int(o)``. @@ -246,7 +246,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Float(PyObject *o) - .. index:: builtin: float + .. index:: pair: built-in function; float Returns the *o* converted to a float object on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``float(o)``. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a0c3194ab0fb90..a25ff244c9f07c 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -190,7 +190,7 @@ Object Protocol .. c:function:: PyObject* PyObject_Repr(PyObject *o) - .. index:: builtin: repr + .. index:: pair: built-in function; repr Compute a string representation of object *o*. Returns the string representation on success, ``NULL`` on failure. This is the equivalent of the @@ -202,7 +202,7 @@ Object Protocol .. c:function:: PyObject* PyObject_ASCII(PyObject *o) - .. index:: builtin: ascii + .. index:: pair: built-in function; ascii As :c:func:`PyObject_Repr`, compute a string representation of object *o*, but escape the non-ASCII characters in the string returned by @@ -227,7 +227,7 @@ Object Protocol .. c:function:: PyObject* PyObject_Bytes(PyObject *o) - .. index:: builtin: bytes + .. index:: pair: built-in function; bytes Compute a bytes representation of object *o*. ``NULL`` is returned on failure and a bytes object on success. This is equivalent to the Python @@ -278,7 +278,7 @@ Object Protocol .. c:function:: Py_hash_t PyObject_Hash(PyObject *o) - .. index:: builtin: hash + .. index:: pair: built-in function; hash Compute and return the hash value of an object *o*. On failure, return ``-1``. This is the equivalent of the Python expression ``hash(o)``. @@ -312,7 +312,7 @@ Object Protocol .. c:function:: PyObject* PyObject_Type(PyObject *o) - .. index:: builtin: type + .. index:: pair: built-in function; type When *o* is non-``NULL``, returns a type object corresponding to the object type of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This @@ -332,7 +332,7 @@ Object Protocol .. c:function:: Py_ssize_t PyObject_Size(PyObject *o) Py_ssize_t PyObject_Length(PyObject *o) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the length of object *o*. If the object *o* provides either the sequence and mapping protocols, the sequence length is returned. On error, ``-1`` is diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index c78d273f9f149f..402a3e5e09ff56 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -18,7 +18,7 @@ Sequence Protocol .. c:function:: Py_ssize_t PySequence_Size(PyObject *o) Py_ssize_t PySequence_Length(PyObject *o) - .. index:: builtin: len + .. index:: pair: built-in function; len Returns the number of objects in sequence *o* on success, and ``-1`` on failure. This is equivalent to the Python expression ``len(o)``. @@ -120,7 +120,7 @@ Sequence Protocol .. c:function:: PyObject* PySequence_Tuple(PyObject *o) - .. index:: builtin: tuple + .. index:: pair: built-in function; tuple Return a tuple object with the same contents as the sequence or iterable *o*, or ``NULL`` on failure. If *o* is a tuple, a new reference will be returned, diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 8e8af60252250e..d642a5f1902e2e 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -107,7 +107,7 @@ or :class:`frozenset` or instances of their subtypes. .. c:function:: Py_ssize_t PySet_Size(PyObject *anyset) - .. index:: builtin: len + .. index:: pair: built-in function; len Return the length of a :class:`set` or :class:`frozenset` object. Equivalent to ``len(anyset)``. Raises a :exc:`PyExc_SystemError` if *anyset* is not a diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 338db6378d240e..aae1b951804491 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -347,7 +347,7 @@ method. .. data:: METH_CLASS - .. index:: builtin: classmethod + .. index:: pair: built-in function; classmethod The method will be passed the type object as the first parameter rather than an instance of the type. This is used to create *class methods*, @@ -357,7 +357,7 @@ method. .. data:: METH_STATIC - .. index:: builtin: staticmethod + .. index:: pair: built-in function; staticmethod The method will be passed ``NULL`` as the first parameter rather than an instance of the type. This is used to create *static methods*, similar to diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index e13db3fb2211e2..0584989233de3f 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -805,7 +805,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_repr - .. index:: builtin: repr + .. index:: pair: built-in function; repr An optional pointer to a function that implements the built-in function :func:`repr`. @@ -870,7 +870,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: hashfunc PyTypeObject.tp_hash - .. index:: builtin: hash + .. index:: pair: built-in function; hash An optional pointer to a function that implements the built-in function :func:`hash`. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 56b40acdb69fed..6852a385f0c63c 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -149,7 +149,7 @@ done. This can be done using the :c:func:`PyErr_Fetch` and .. index:: single: string; object representation - builtin: repr + pair: built-in function; repr Object Presentation ------------------- diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index f8727b98066990..b2d7362a9de213 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -560,6 +560,13 @@ Eager Task Factory using the provided *custom_task_constructor* when creating a new task instead of the default :class:`Task`. + *custom_task_constructor* must be a *callable* with the signature matching + the signature of :class:`Task.__init__ `. + The callable must return a :class:`asyncio.Task`-compatible object. + + This function returns a *callable* intended to be used as a task factory of an + event loop via :meth:`loop.set_task_factory(factory) `). + .. versionadded:: 3.12 @@ -1014,7 +1021,7 @@ Introspection Task Object =========== -.. class:: Task(coro, *, loop=None, name=None) +.. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False) A :class:`Future-like ` object that runs a Python :ref:`coroutine `. Not thread-safe. @@ -1049,9 +1056,17 @@ Task Object APIs except :meth:`Future.set_result` and :meth:`Future.set_exception`. - Tasks support the :mod:`contextvars` module. When a Task - is created it copies the current context and later runs its - coroutine in the copied context. + An optional keyword-only *context* argument allows specifying a + custom :class:`contextvars.Context` for the *coro* to run in. + If no *context* is provided, the Task copies the current context + and later runs its coroutine in the copied context. + + An optional keyword-only *eager_start* argument allows eagerly starting + the execution of the :class:`asyncio.Task` at task creation time. + If set to ``True`` and the event loop is running, the task will start + executing the coroutine immediately, until the first time the coroutine + blocks. If the coroutine returns or raises without blocking, the task + will be finished eagerly and will skip scheduling to the event loop. .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. @@ -1063,6 +1078,12 @@ Task Object Deprecation warning is emitted if *loop* is not specified and there is no running event loop. + .. versionchanged:: 3.11 + Added the *context* parameter. + + .. versionchanged:: 3.12 + Added the *eager_start* parameter. + .. method:: done() Return ``True`` if the Task is *done*. diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index e3c8c801904b61..8022c596f0af97 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -24,6 +24,8 @@ method to determine whether a value has been found. Instead, the functions only call the :meth:`__lt__` method and will return an insertion point between values in an array. +.. _bisect functions: + The following functions are provided: @@ -55,7 +57,7 @@ The following functions are provided: .. function:: bisect_right(a, x, lo=0, hi=len(a), *, key=None) bisect(a, x, lo=0, hi=len(a), *, key=None) - Similar to :func:`bisect_left`, but returns an insertion point which comes + Similar to :py:func:`~bisect.bisect_left`, but returns an insertion point which comes after (to the right of) any existing entries of *x* in *a*. The returned insertion point *ip* partitions the array *a* into two slices @@ -70,7 +72,7 @@ The following functions are provided: Insert *x* in *a* in sorted order. - This function first runs :func:`bisect_left` to locate an insertion point. + This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. Next, it runs the :meth:`insert` method on *a* to insert *x* at the appropriate position to maintain sort order. @@ -87,10 +89,10 @@ The following functions are provided: .. function:: insort_right(a, x, lo=0, hi=len(a), *, key=None) insort(a, x, lo=0, hi=len(a), *, key=None) - Similar to :func:`insort_left`, but inserting *x* in *a* after any existing + Similar to :py:func:`~bisect.insort_left`, but inserting *x* in *a* after any existing entries of *x*. - This function first runs :func:`bisect_right` to locate an insertion point. + This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. Next, it runs the :meth:`insert` method on *a* to insert *x* at the appropriate position to maintain sort order. @@ -120,7 +122,7 @@ thoughts in mind: they are used. Consequently, if the search functions are used in a loop, the key function may be called again and again on the same array elements. If the key function isn't fast, consider wrapping it with - :func:`functools.cache` to avoid duplicate computations. Alternatively, + :py:func:`functools.cache` to avoid duplicate computations. Alternatively, consider searching an array of precomputed keys to locate the insertion point (as shown in the examples section below). @@ -140,7 +142,7 @@ thoughts in mind: Searching Sorted Lists ---------------------- -The above :func:`bisect` functions are useful for finding insertion points but +The above `bisect functions`_ are useful for finding insertion points but can be tricky or awkward to use for common searching tasks. The following five functions show how to transform them into the standard lookups for sorted lists:: @@ -186,8 +188,8 @@ Examples .. _bisect-example: -The :func:`bisect` function can be useful for numeric table lookups. This -example uses :func:`bisect` to look up a letter grade for an exam score (say) +The :py:func:`~bisect.bisect` function can be useful for numeric table lookups. This +example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam score (say) based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: @@ -198,8 +200,8 @@ a 'B', and so on:: >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] ['F', 'A', 'C', 'C', 'B', 'A', 'A'] -The :func:`bisect` and :func:`insort` functions also work with lists of -tuples. The *key* argument can serve to extract the field used for ordering +The :py:func:`~bisect.bisect` and :py:func:`~bisect.insort` functions also work with +lists of tuples. The *key* argument can serve to extract the field used for ordering records in a table:: >>> from collections import namedtuple diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 7fae0fee7e724b..248743b8fa0a87 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1375,7 +1375,7 @@ iterations of the loop. .. opcode:: BUILD_SLICE (argc) - .. index:: builtin: slice + .. index:: pair: built-in function; slice Pushes a slice object on the stack. *argc* must be 2 or 3. If it is 2, implements:: diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 80b7b8b4f4ed07..48a832db60e919 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -562,7 +562,7 @@ are always available. They are listed here in alphabetical order. Raises an :ref:`auditing event ` ``exec`` with the code object as the argument. Code compilation events may also be raised. -.. index:: builtin: exec +.. index:: pair: built-in function; exec .. function:: exec(object, globals=None, locals=None, /, *, closure=None) diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 3097bcf47b627f..d2cc769e2c8400 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -1,11 +1,11 @@ .. _using: -================================= - Using :mod:`!importlib.metadata` -================================= +======================================================== +:mod:`!importlib.metadata` -- Accessing package metadata +======================================================== .. module:: importlib.metadata - :synopsis: The implementation of the importlib metadata. + :synopsis: Accessing package metadata .. versionadded:: 3.8 .. versionchanged:: 3.10 @@ -13,7 +13,7 @@ **Source code:** :source:`Lib/importlib/metadata/__init__.py` -``importlib_metadata`` is a library that provides access to +``importlib.metadata`` is a library that provides access to the metadata of an installed `Distribution Package `_, such as its entry points or its top-level names (`Import Package `_\s, modules, if any). @@ -24,7 +24,7 @@ API`_ and `metadata API`_ of ``pkg_resources``. Along with this package can eliminate the need to use the older and less efficient ``pkg_resources`` package. -``importlib_metadata`` operates on third-party *distribution packages* +``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as `pip `_. Specifically, it works with distributions with discoverable @@ -368,7 +368,7 @@ system :ref:`finders `. To find a distribution package's m ``importlib.metadata`` queries the list of :term:`meta path finders ` on :data:`sys.meta_path`. -By default ``importlib_metadata`` installs a finder for distribution packages +By default ``importlib.metadata`` installs a finder for distribution packages found on the file system. This finder doesn't actually find any *distributions*, but it can find their metadata. diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 4c6aa59bf9f58f..755693840fecd8 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources` -- Resources ---------------------------------------- +:mod:`importlib.resources` -- Package resource reading, opening and access +-------------------------------------------------------------------------- .. module:: importlib.resources :synopsis: Package resource reading, opening, and access @@ -97,7 +97,7 @@ for example, a package and its resources can be imported from a zip file using Deprecated functions --------------------- +^^^^^^^^^^^^^^^^^^^^ An older, deprecated set of functions is still available, but is scheduled for removal in a future version of Python. diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 4e29192311fc21..d8269ef48cb36a 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -159,7 +159,7 @@ The :mod:`pprint` module defines one class: .. function:: isreadable(object) - .. index:: builtin: eval + .. index:: pair: built-in function; eval Determine if the formatted representation of *object* is "readable", or can be used to reconstruct the value using :func:`eval`. This always returns ``False`` @@ -218,7 +218,7 @@ created. .. method:: PrettyPrinter.isreadable(object) - .. index:: builtin: eval + .. index:: pair: built-in function; eval Determine if the formatted representation of the object is "readable," or can be used to reconstruct the value using :func:`eval`. Note that this returns diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c739485c5564da..9203afbf6a4e8a 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -244,9 +244,9 @@ and imaginary parts. .. index:: single: arithmetic - builtin: int - builtin: float - builtin: complex + pair: built-in function; int + pair: built-in function; float + pair: built-in function; complex single: operator; + (plus) single: + (plus); unary operator single: + (plus); binary operator @@ -945,9 +945,9 @@ operations have the same priority as the corresponding numeric operations. [3]_ .. index:: triple: operations on; sequence; types - builtin: len - builtin: min - builtin: max + pair: built-in function; len + pair: built-in function; min + pair: built-in function; max pair: concatenation; operation pair: repetition; operation pair: subscript; operation @@ -1113,7 +1113,7 @@ Immutable Sequence Types .. index:: triple: immutable; sequence; types pair: object; tuple - builtin: hash + pair: built-in function; hash The only operation that immutable sequence types generally implement that is not also implemented by mutable sequence types is support for the :func:`hash` @@ -4419,7 +4419,7 @@ Mapping Types --- :class:`dict` triple: operations on; mapping; types triple: operations on; dictionary; type pair: statement; del - builtin: len + pair: built-in function; len A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. Mappings are mutable objects. There is currently only one standard mapping @@ -5348,7 +5348,7 @@ Code Objects ------------ .. index:: - builtin: compile + pair: built-in function; compile single: __code__ (function object attribute) Code objects are used by the implementation to represent "pseudo-compiled" @@ -5362,8 +5362,8 @@ Accessing ``__code__`` raises an :ref:`auditing event ` ``object.__getattr__`` with arguments ``obj`` and ``"__code__"``. .. index:: - builtin: exec - builtin: eval + pair: built-in function; exec + pair: built-in function; eval A code object can be executed or evaluated by passing it (instead of a source string) to the :func:`exec` or :func:`eval` built-in functions. @@ -5377,7 +5377,7 @@ Type Objects ------------ .. index:: - builtin: type + pair: built-in function; type pair: module; types Type objects represent the various object types. An object's type is accessed diff --git a/Doc/library/types.rst b/Doc/library/types.rst index a15fb5cfa49473..8cbe17df16f107 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -186,7 +186,7 @@ Standard names are defined for the following types: .. class:: CodeType(**kwargs) - .. index:: builtin: compile + .. index:: pair: built-in function; compile The type for code objects such as returned by :func:`compile`. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 162041fc7a846e..ebab1389f07e58 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2130,15 +2130,10 @@ Corresponding to collections in :mod:`collections.abc` .. class:: ByteString(Sequence[int]) - A generic version of :class:`collections.abc.ByteString`. - This type represents the types :class:`bytes`, :class:`bytearray`, and :class:`memoryview` of byte sequences. - As a shorthand for this type, :class:`bytes` can be used to - annotate arguments of any of the types mentioned above. - - .. deprecated:: 3.9 + .. deprecated-removed:: 3.9 3.14 Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) @@ -2977,6 +2972,8 @@ convenience. This is subject to change, and not all deprecations are listed. | ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | | collections | | | | +----------------------------------+---------------+-------------------+----------------+ +| ``typing.ByteString`` | 3.9 | 3.14 | :gh:`91896` | ++----------------------------------+---------------+-------------------+----------------+ | ``typing.Text`` | 3.11 | Undecided | :gh:`92332` | +----------------------------------+---------------+-------------------+----------------+ | ``typing.Hashable`` and | 3.12 | Undecided | :gh:`94309` | diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index e9c1c493ae4278..9d1e5b6c596d9f 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -188,7 +188,7 @@ those made in the suite of the for-loop:: .. index:: - builtin: range + pair: built-in function; range Names in the target list are not deleted when the loop is finished, but if the sequence is empty, they will not have been assigned to at all by the loop. Hint: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 0a9cabc158b9e4..c0734e49f29192 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -21,8 +21,8 @@ conformance to Von Neumann's model of a "stored program computer", code is also represented by objects.) .. index:: - builtin: id - builtin: type + pair: built-in function; id + pair: built-in function; type single: identity of an object single: value of an object single: type of an object @@ -267,7 +267,7 @@ Ellipsis Sequences .. index:: - builtin: len + pair: built-in function; len pair: object; sequence single: index operation single: item selection @@ -308,8 +308,8 @@ Sequences Strings .. index:: - builtin: chr - builtin: ord + pair: built-in function; chr + pair: built-in function; ord single: character single: integer single: Unicode @@ -384,7 +384,7 @@ Sequences Set types .. index:: - builtin: len + pair: built-in function; len pair: object; set type These represent unordered, finite sets of unique, immutable objects. As such, @@ -418,7 +418,7 @@ Set types Mappings .. index:: - builtin: len + pair: built-in function; len single: subscription pair: object; mapping @@ -908,7 +908,7 @@ Class instances I/O objects (also known as file objects) .. index:: - builtin: open + pair: built-in function; open pair: module; io single: popen() (in module os) single: makefile() (socket method) @@ -1177,7 +1177,7 @@ Internal types and the ``tb_next`` attribute of existing instances can be updated. Slice objects - .. index:: builtin: slice + .. index:: pair: built-in function; slice Slice objects are used to represent slices for :meth:`~object.__getitem__` @@ -1411,7 +1411,7 @@ Basic customization .. method:: object.__bytes__(self) - .. index:: builtin: bytes + .. index:: pair: built-in function; bytes Called by :ref:`bytes ` to compute a byte-string representation of an object. This should return a :class:`bytes` object. @@ -1419,7 +1419,7 @@ Basic customization .. index:: single: string; __format__() (object method) pair: string; conversion - builtin: print + pair: built-in function; print .. method:: object.__format__(self, format_spec) @@ -1499,7 +1499,7 @@ Basic customization .. index:: pair: object; dictionary - builtin: hash + pair: built-in function; hash Called by built-in function :func:`hash` and for operations on members of hashed collections including :class:`set`, :class:`frozenset`, and @@ -2050,7 +2050,7 @@ Metaclasses .. index:: single: metaclass - builtin: type + pair: built-in function; type single: = (equals); class definition By default, classes are constructed using :func:`type`. The class body is @@ -2477,7 +2477,7 @@ through the object's keys; for sequences, it should iterate through the values. .. method:: object.__len__(self) .. index:: - builtin: len + pair: built-in function; len single: __bool__() (object method) Called to implement the built-in function :func:`len`. Should return the length @@ -2635,9 +2635,9 @@ left undefined. object.__or__(self, other) .. index:: - builtin: divmod - builtin: pow - builtin: pow + pair: built-in function; divmod + pair: built-in function; pow + pair: built-in function; pow These methods are called to implement the binary arithmetic operations (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, @@ -2670,8 +2670,8 @@ left undefined. object.__ror__(self, other) .. index:: - builtin: divmod - builtin: pow + pair: built-in function; divmod + pair: built-in function; pow These methods are called to implement the binary arithmetic operations (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, @@ -2683,7 +2683,7 @@ left undefined. ``type(y).__rsub__(y, x)`` is called if ``type(x).__sub__(x, y)`` returns *NotImplemented*. - .. index:: builtin: pow + .. index:: pair: built-in function; pow Note that ternary :func:`pow` will not try calling :meth:`__rpow__` (the coercion rules would become too complicated). @@ -2730,7 +2730,7 @@ left undefined. object.__abs__(self) object.__invert__(self) - .. index:: builtin: abs + .. index:: pair: built-in function; abs Called to implement the unary arithmetic operations (``-``, ``+``, :func:`abs` and ``~``). @@ -2741,9 +2741,9 @@ left undefined. object.__float__(self) .. index:: - builtin: complex - builtin: int - builtin: float + pair: built-in function; complex + pair: built-in function; int + pair: built-in function; float Called to implement the built-in functions :func:`complex`, :func:`int` and :func:`float`. Should return a value @@ -2768,7 +2768,7 @@ left undefined. object.__floor__(self) object.__ceil__(self) - .. index:: builtin: round + .. index:: pair: built-in function; round Called to implement the built-in function :func:`round` and :mod:`math` functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index ca1aa744389bda..f7a8b44d195417 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -53,7 +53,7 @@ An expression statement evaluates the expression list (which may be a single expression). .. index:: - builtin: repr + pair: built-in function; repr pair: object; None pair: string; conversion single: output @@ -970,9 +970,9 @@ annotation. them or silently change the meaning of the program. .. index:: - builtin: exec - builtin: eval - builtin: compile + pair: built-in function; exec + pair: built-in function; eval + pair: built-in function; compile **Programmer's note:** :keyword:`global` is a directive to the parser. It applies only to code parsed at the same time as the :keyword:`!global` statement. diff --git a/Doc/reference/toplevel_components.rst b/Doc/reference/toplevel_components.rst index ee472ace6e2129..dd3d3d6878e289 100644 --- a/Doc/reference/toplevel_components.rst +++ b/Doc/reference/toplevel_components.rst @@ -98,7 +98,7 @@ Expression input ================ .. index:: single: input -.. index:: builtin: eval +.. index:: pair: built-in function; eval :func:`eval` is used for expression input. It ignores leading whitespace. The string argument to :func:`eval` must have the following form: diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8fc44cab44455a..cd8d9febb0d13b 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -674,7 +674,14 @@ def process_audit_events(app, doctree, fromdocname): node.replace_self(table) -def patch_pairindextypes(app) -> None: +def patch_pairindextypes(app, _env) -> None: + """Remove all entries from ``pairindextypes`` before writing POT files. + + We want to run this just before writing output files, as the check to + circumvent is in ``I18nBuilder.write_doc()``. + As such, we link this to ``env-check-consistency``, even though it has + nothing to do with the environment consistency check. + """ if app.builder.name != 'gettext': return @@ -688,14 +695,7 @@ def patch_pairindextypes(app) -> None: # the Sphinx-translated pairindextypes values. As we intend to move # away from this, we need Sphinx to believe that these values don't # exist, by deleting them when using the gettext builder. - - pairindextypes.pop('module', None) - pairindextypes.pop('keyword', None) - pairindextypes.pop('operator', None) - pairindextypes.pop('object', None) - pairindextypes.pop('exception', None) - pairindextypes.pop('statement', None) - # pairindextypes.pop('builtin', None) + pairindextypes.clear() def setup(app): @@ -719,7 +719,7 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) - app.connect('builder-inited', patch_pairindextypes) + app.connect('env-check-consistency', patch_pairindextypes) app.connect('doctree-resolved', process_audit_events) app.connect('env-merge-info', audit_events_merge) app.connect('env-purge-doc', audit_events_purge) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 3fcf4e3f43a300..f5cdd84cbadefe 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -285,7 +285,7 @@ Reading and Writing Files ========================= .. index:: - builtin: open + pair: built-in function; open pair: object; file :func:`open` returns a :term:`file object`, and is most commonly used with diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 4f5ada90eb57bc..6bae279c5e9cde 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -24,7 +24,7 @@ Be sure to use the ``import os`` style instead of ``from os import *``. This will keep :func:`os.open` from shadowing the built-in :func:`open` function which operates much differently. -.. index:: builtin: help +.. index:: pair: built-in function; help The built-in :func:`dir` and :func:`help` functions are useful as interactive aids for working with large modules like :mod:`os`:: diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index d5ccf17491c339..918f1953ba90f3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -366,8 +366,9 @@ inspect (Contributed by Thomas Krennwallner in :issue:`35759`.) * The performance of :func:`inspect.getattr_static` has been considerably - improved. Most calls to the function should be around 2x faster than they - were in Python 3.11. (Contributed by Alex Waygood in :gh:`103193`.) + improved. Most calls to the function should be at least 2x faster than they + were in Python 3.11, and some may be 6x faster or more. (Contributed by Alex + Waygood in :gh:`103193`.) pathlib ------- @@ -621,7 +622,7 @@ typing :func:`runtime-checkable protocols ` has changed significantly. Most ``isinstance()`` checks against protocols with only a few members should be at least 2x faster than in 3.11, and some may be 20x - faster or more. However, ``isinstance()`` checks against protocols with seven + faster or more. However, ``isinstance()`` checks against protocols with fourteen or more members may be slower than in Python 3.11. (Contributed by Alex Waygood in :gh:`74690` and :gh:`103193`.) @@ -868,6 +869,19 @@ Pending Removal in Python 3.14 use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) +* The following :mod:`ast` features have been deprecated in documentation since + Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at runtime + when they are accessed or used, and will be removed in Python 3.14: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. + (Contributed by Serhiy Storchaka in :gh:`90953`.) + Pending Removal in Future Versions ---------------------------------- diff --git a/Include/cpython/memoryobject.h b/Include/cpython/memoryobject.h index deab3cc89f726e..3837fa8c6ab5aa 100644 --- a/Include/cpython/memoryobject.h +++ b/Include/cpython/memoryobject.h @@ -24,6 +24,7 @@ typedef struct { #define _Py_MEMORYVIEW_FORTRAN 0x004 /* Fortran contiguous layout */ #define _Py_MEMORYVIEW_SCALAR 0x008 /* scalar: ndim = 0 */ #define _Py_MEMORYVIEW_PIL 0x010 /* PIL-style layout */ +#define _Py_MEMORYVIEW_RESTRICTED 0x020 /* Disallow new references to the memoryview's buffer */ typedef struct { PyObject_VAR_HEAD diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index b7a9bf40425bc7..3c8b368bd2af4e 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -21,8 +21,7 @@ struct _ceval_runtime_state; extern void _Py_FinishPendingCalls(PyThreadState *tstate); -extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *); -extern void _PyEval_InitState(struct _ceval_state *, PyThread_type_lock); +extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock); extern void _PyEval_FiniState(struct _ceval_state *ceval); PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp); PyAPI_FUNC(int) _PyEval_AddPendingCall( @@ -100,7 +99,9 @@ extern int _PyEval_ThreadsInitialized(void); extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); +extern void _PyEval_AcquireLock(PyThreadState *tstate); extern void _PyEval_ReleaseLock(PyThreadState *tstate); +extern PyThreadState * _PyThreadState_SwapNoGIL(PyThreadState *); extern void _PyEval_DeactivateOpCache(void); diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 4781dd5735dcf6..b352801673c40a 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -49,9 +49,6 @@ struct _ceval_runtime_state { the main thread of the main interpreter can handle signals: see _Py_ThreadCanHandleSignals(). */ _Py_atomic_int signals_pending; - - /* This is (only) used indirectly through PyInterpreterState.ceval.gil. */ - struct _gil_runtime_state gil; }; #ifdef PY_HAVE_PERF_TRAMPOLINE diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 80e2a03bca9faf..499f55f3e276be 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -100,7 +100,8 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( PyObject *ast, PyObject *filename, PyCompilerFlags *flags, - int optimize); + int optimize, + int compile_mode); PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( PyObject *instructions, diff --git a/Include/internal/pycore_format.h b/Include/internal/pycore_format.h index 1899609e77ef20..1b8d57539ca505 100644 --- a/Include/internal/pycore_format.h +++ b/Include/internal/pycore_format.h @@ -14,14 +14,12 @@ extern "C" { * F_BLANK ' ' * F_ALT '#' * F_ZERO '0' - * F_NO_NEG_0 'z' */ #define F_LJUST (1<<0) #define F_SIGN (1<<1) #define F_BLANK (1<<2) #define F_ALT (1<<3) #define F_ZERO (1<<4) -#define F_NO_NEG_0 (1<<5) #ifdef __cplusplus } diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9377fd8526e3a2..7e495817981f06 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -847,6 +847,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(consts)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(context)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cookie)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index ed9b2bb44ddffc..8ebfee85c87c23 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -335,6 +335,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(code) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) + STRUCT_FOR_ID(compile_mode) STRUCT_FOR_ID(consts) STRUCT_FOR_ID(context) STRUCT_FOR_ID(cookie) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 7276ce35ba68f0..527b2121148f4c 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -178,6 +178,9 @@ struct _is { basis. Also see _PyRuntimeState regarding the various mutex fields. */ + /* The per-interpreter GIL, which might not be used. */ + struct _gil_runtime_state _gil; + /* the initial PyInterpreterState.threads.head */ PyThreadState _initial_thread; }; diff --git a/Include/internal/pycore_memoryobject.h b/Include/internal/pycore_memoryobject.h index acc12c9275172c..fe19e3f9611a16 100644 --- a/Include/internal/pycore_memoryobject.h +++ b/Include/internal/pycore_memoryobject.h @@ -9,7 +9,8 @@ extern "C" { #endif PyObject * -PyMemoryView_FromObjectAndFlags(PyObject *v, int flags); +_PyMemoryView_FromBufferProc(PyObject *v, int flags, + getbufferproc bufferproc); #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index d1b165d0ab9c38..6e06e874711bc2 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -32,8 +32,6 @@ struct _getargs_runtime_state { struct _PyArg_Parser *static_parsers; }; -/* ceval state */ - /* GIL state */ struct _gilstate_runtime_state { diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 6ade8fb6eade03..7b9c73dd1edf3b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -841,6 +841,7 @@ extern "C" { INIT_ID(code), \ INIT_ID(command), \ INIT_ID(comment_factory), \ + INIT_ID(compile_mode), \ INIT_ID(consts), \ INIT_ID(context), \ INIT_ID(cookie), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 0b33ea187e60ff..8e086edbdf8193 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -858,6 +858,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(comment_factory); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(compile_mode); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(consts); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/argparse.py b/Lib/argparse.py index a819d2650e85f0..f5f44ff02c0d38 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -345,21 +345,22 @@ def _format_usage(self, usage, actions, groups, prefix): def get_lines(parts, indent, prefix=None): lines = [] line = [] + indent_length = len(indent) if prefix is not None: line_len = len(prefix) - 1 else: - line_len = len(indent) - 1 + line_len = indent_length - 1 for part in parts: if line_len + 1 + len(part) > text_width and line: lines.append(indent + ' '.join(line)) line = [] - line_len = len(indent) - 1 + line_len = indent_length - 1 line.append(part) line_len += len(part) + 1 if line: lines.append(indent + ' '.join(line)) if prefix is not None: - lines[0] = lines[0][len(indent):] + lines[0] = lines[0][indent_length:] return lines # if prog is short, follow it with optionals or positionals @@ -2605,9 +2606,11 @@ def print_help(self, file=None): def _print_message(self, message, file=None): if message: - if file is None: - file = _sys.stderr - file.write(message) + file = file or _sys.stderr + try: + file.write(message) + except (AttributeError, OSError): + pass # =============== # Exiting methods diff --git a/Lib/ast.py b/Lib/ast.py index d9733a79d3a78f..65152047a22370 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -294,9 +294,7 @@ def get_docstring(node, clean=True): if not(node.body and isinstance(node.body[0], Expr)): return None node = node.body[0].value - if isinstance(node, Str): - text = node.s - elif isinstance(node, Constant) and isinstance(node.value, str): + if isinstance(node, Constant) and isinstance(node.value, str): text = node.value else: return None @@ -499,20 +497,52 @@ def generic_visit(self, node): return node +_DEPRECATED_VALUE_ALIAS_MESSAGE = ( + "{name} is deprecated and will be removed in Python {remove}; use value instead" +) +_DEPRECATED_CLASS_MESSAGE = ( + "{name} is deprecated and will be removed in Python {remove}; " + "use ast.Constant instead" +) + + # If the ast module is loaded more than once, only add deprecated methods once if not hasattr(Constant, 'n'): # The following code is for backward compatibility. # It will be removed in future. - def _getter(self): + def _n_getter(self): + """Deprecated. Use value instead.""" + import warnings + warnings._deprecated( + "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) + return self.value + + def _n_setter(self, value): + import warnings + warnings._deprecated( + "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) + self.value = value + + def _s_getter(self): """Deprecated. Use value instead.""" + import warnings + warnings._deprecated( + "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) return self.value - def _setter(self, value): + def _s_setter(self, value): + import warnings + warnings._deprecated( + "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) + ) self.value = value - Constant.n = property(_getter, _setter) - Constant.s = property(_getter, _setter) + Constant.n = property(_n_getter, _n_setter) + Constant.s = property(_s_getter, _s_setter) class _ABC(type): @@ -520,6 +550,13 @@ def __init__(cls, *args): cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead""" def __instancecheck__(cls, inst): + if cls in _const_types: + import warnings + warnings._deprecated( + f"ast.{cls.__qualname__}", + message=_DEPRECATED_CLASS_MESSAGE, + remove=(3, 14) + ) if not isinstance(inst, Constant): return False if cls in _const_types: @@ -543,6 +580,10 @@ def _new(cls, *args, **kwargs): if pos < len(args): raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") if cls in _const_types: + import warnings + warnings._deprecated( + f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) + ) return Constant(*args, **kwargs) return Constant.__new__(cls, *args, **kwargs) @@ -565,10 +606,19 @@ class Ellipsis(Constant, metaclass=_ABC): _fields = () def __new__(cls, *args, **kwargs): - if cls is Ellipsis: + if cls is _ast_Ellipsis: + import warnings + warnings._deprecated( + "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) + ) return Constant(..., *args, **kwargs) return Constant.__new__(cls, *args, **kwargs) +# Keep another reference to Ellipsis in the global namespace +# so it can be referenced in Ellipsis.__new__ +# (The original "Ellipsis" name is removed from the global namespace later on) +_ast_Ellipsis = Ellipsis + _const_types = { Num: (int, float, complex), Str: (str,), @@ -1699,6 +1749,22 @@ def unparse(ast_obj): return unparser.visit(ast_obj) +_deprecated_globals = { + name: globals().pop(name) + for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis') +} + +def __getattr__(name): + if name in _deprecated_globals: + globals()[name] = value = _deprecated_globals[name] + import warnings + warnings._deprecated( + f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) + ) + return value + raise AttributeError(f"module 'ast' has no attribute '{name}'") + + def main(): import argparse diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 0fdea3697ece3d..06b2e0db86a1fe 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -164,8 +164,14 @@ def create_task(self, coro, *, name=None, context=None): else: task = self._loop.create_task(coro, context=context) tasks._set_task_name(task, name) - task.add_done_callback(self._on_task_done) - self._tasks.add(task) + # optimization: Immediately call the done callback if the task is + # already done (e.g. if the coro was able to complete eagerly), + # and skip scheduling a done callback + if task.done(): + self._on_task_done(task) + else: + self._tasks.add(task) + task.add_done_callback(self._on_task_done) return task # Since Python 3.8 Tasks propagate all exceptions correctly, diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index aa5269ade19a7f..8d5bde09ea9b5b 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -813,6 +813,7 @@ def _done_callback(fut): children = [] nfuts = 0 nfinished = 0 + done_futs = [] loop = None outer = None # bpo-46672 for arg in coros_or_futures: @@ -829,7 +830,10 @@ def _done_callback(fut): nfuts += 1 arg_to_fut[arg] = fut - fut.add_done_callback(_done_callback) + if fut.done(): + done_futs.append(fut) + else: + fut.add_done_callback(_done_callback) else: # There's a duplicate Future object in coros_or_futures. @@ -838,6 +842,13 @@ def _done_callback(fut): children.append(fut) outer = _GatheringFuture(children, loop=loop) + # Run done callbacks after GatheringFuture created so any post-processing + # can be performed at this point + # optimization: in the special case that *all* futures finished eagerly, + # this will effectively complete the gather eagerly, with the last + # callback setting the result (or exception) on outer before returning it + for fut in done_futs: + _done_callback(fut) return outer @@ -931,18 +942,31 @@ def callback(): def create_eager_task_factory(custom_task_constructor): + """Create a function suitable for use as a task factory on an event-loop. + + Example usage: + + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) - if "eager_start" not in inspect.signature(custom_task_constructor).parameters: - raise TypeError( - "Provided constructor does not support eager task execution") + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any callable + that returns a Task-compatible object and has a signature compatible + with `Task.__init__`; it must have the `eager_start` keyword argument. + + Most applications will use `Task` for `custom_task_constructor` and in + this case there's no need to call `create_eager_task_factory()` + directly. Instead the global `eager_task_factory` instance can be + used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. + """ def factory(loop, coro, *, name=None, context=None): return custom_task_constructor( coro, loop=loop, name=name, context=context, eager_start=True) - return factory + eager_task_factory = create_eager_task_factory(Task) diff --git a/Lib/gzip.py b/Lib/gzip.py index 75c6ddc3f2cffb..8796c8d9fd9a2d 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -22,6 +22,7 @@ _COMPRESS_LEVEL_BEST = 9 READ_BUFFER_SIZE = 128 * 1024 +_WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, @@ -120,6 +121,21 @@ class BadGzipFile(OSError): """Exception raised in some cases for invalid gzip files.""" +class _WriteBufferStream(io.RawIOBase): + """Minimal object to pass WriteBuffer flushes into GzipFile""" + def __init__(self, gzip_file): + self.gzip_file = gzip_file + + def write(self, data): + return self.gzip_file._write_raw(data) + + def seekable(self): + return False + + def writable(self): + return True + + class GzipFile(_compression.BaseStream): """The GzipFile class simulates most of the methods of a file object with the exception of the truncate() method. @@ -184,6 +200,7 @@ def __init__(self, filename=None, mode=None, if mode is None: mode = getattr(fileobj, 'mode', 'rb') + if mode.startswith('r'): self.mode = READ raw = _GzipReader(fileobj) @@ -206,6 +223,9 @@ def __init__(self, filename=None, mode=None, zlib.DEF_MEM_LEVEL, 0) self._write_mtime = mtime + self._buffer_size = _WRITE_BUFFER_SIZE + self._buffer = io.BufferedWriter(_WriteBufferStream(self), + buffer_size=self._buffer_size) else: raise ValueError("Invalid mode: {!r}".format(mode)) @@ -231,6 +251,11 @@ def _init_write(self, filename): self.bufsize = 0 self.offset = 0 # Current file offset for seek(), tell(), etc + def tell(self): + self._check_not_closed() + self._buffer.flush() + return super().tell() + def _write_gzip_header(self, compresslevel): self.fileobj.write(b'\037\213') # magic header self.fileobj.write(b'\010') # compression method @@ -272,6 +297,10 @@ def write(self,data): if self.fileobj is None: raise ValueError("write() on closed GzipFile object") + return self._buffer.write(data) + + def _write_raw(self, data): + # Called by our self._buffer underlying WriteBufferStream. if isinstance(data, (bytes, bytearray)): length = len(data) else: @@ -322,9 +351,9 @@ def close(self): fileobj = self.fileobj if fileobj is None: return - self.fileobj = None try: if self.mode == WRITE: + self._buffer.flush() fileobj.write(self.compress.flush()) write32u(fileobj, self.crc) # self.size may exceed 2 GiB, or even 4 GiB @@ -332,6 +361,7 @@ def close(self): elif self.mode == READ: self._buffer.close() finally: + self.fileobj = None myfileobj = self.myfileobj if myfileobj: self.myfileobj = None @@ -341,7 +371,7 @@ def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): self._check_not_closed() if self.mode == WRITE: # Ensure the compressor's buffer is flushed - self.fileobj.write(self.compress.flush(zlib_mode)) + self._buffer.flush() self.fileobj.flush() def fileno(self): @@ -378,10 +408,10 @@ def seek(self, offset, whence=io.SEEK_SET): if offset < self.offset: raise OSError('Negative seek in write mode') count = offset - self.offset - chunk = b'\0' * 1024 - for i in range(count // 1024): + chunk = b'\0' * self._buffer_size + for i in range(count // self._buffer_size): self.write(chunk) - self.write(b'\0' * (count % 1024)) + self.write(b'\0' * (count % self._buffer_size)) elif self.mode == READ: self._check_not_closed() return self._buffer.seek(offset, whence) diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 5294578cc26cf3..b1d9271f8e47ca 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -112,6 +112,43 @@ def find_spec(name, package=None): return spec +# Normally we would use contextlib.contextmanager. However, this module +# is imported by runpy, which means we want to avoid any unnecessary +# dependencies. Thus we use a class. + +class allowing_all_extensions: + """A context manager that lets users skip the compatibility check. + + Normally, extensions that do not support multiple interpreters + may not be imported in a subinterpreter. That implies modules + that do not implement multi-phase init. + + Likewise for modules import in a subinterpeter with its own GIL + when the extension does not support a per-interpreter GIL. This + implies the module does not have a Py_mod_multiple_interpreters slot + set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. + + In both cases, this context manager may be used to temporarily + disable the check for compatible extension modules. + """ + + def __init__(self, disable_check=True): + self.disable_check = disable_check + + def __enter__(self): + self.old = _imp._override_multi_interp_extensions_check(self.override) + return self + + def __exit__(self, *args): + old = self.old + del self.old + _imp._override_multi_interp_extensions_check(old) + + @property + def override(self): + return -1 if self.disable_check else 1 + + class _LazyModule(types.ModuleType): """A subclass of the module type which triggers loading upon attribute access.""" diff --git a/Lib/inspect.py b/Lib/inspect.py index 95da7fb71a3997..a64e85e4fd67a4 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1794,8 +1794,9 @@ def _check_class(klass, attr): return entry.__dict__[attr] return _sentinel -def _shadowed_dict(klass): - for entry in _static_getmro(klass): +@functools.lru_cache() +def _shadowed_dict_from_mro_tuple(mro): + for entry in mro: dunder_dict = _get_dunder_dict_of_class(entry) if '__dict__' in dunder_dict: class_dict = dunder_dict['__dict__'] @@ -1805,6 +1806,9 @@ def _shadowed_dict(klass): return class_dict return _sentinel +def _shadowed_dict(klass): + return _shadowed_dict_from_mro_tuple(_static_getmro(klass)) + def getattr_static(obj, attr, default=_sentinel): """Retrieve attributes without triggering dynamic lookup via the descriptor protocol, __getattr__ or __getattribute__. diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 9aa3c1e52447d1..20ec1ce9d80374 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -64,17 +64,25 @@ def _is_case_sensitive(flavour): @functools.lru_cache() def _make_selector(pattern_parts, flavour, case_sensitive): pat = pattern_parts[0] - child_parts = pattern_parts[1:] if not pat: return _TerminatingSelector() if pat == '**': - cls = _RecursiveWildcardSelector - elif pat == '..': - cls = _ParentSelector - elif '**' in pat: - raise ValueError("Invalid pattern: '**' can only be an entire path component") + child_parts_idx = 1 + while child_parts_idx < len(pattern_parts) and pattern_parts[child_parts_idx] == '**': + child_parts_idx += 1 + child_parts = pattern_parts[child_parts_idx:] + if '**' in child_parts: + cls = _DoubleRecursiveWildcardSelector + else: + cls = _RecursiveWildcardSelector else: - cls = _WildcardSelector + child_parts = pattern_parts[1:] + if pat == '..': + cls = _ParentSelector + elif '**' in pat: + raise ValueError("Invalid pattern: '**' can only be an entire path component") + else: + cls = _WildcardSelector return cls(pat, child_parts, flavour, case_sensitive) @@ -183,20 +191,32 @@ def _iterate_directories(self, parent_path, scandir): def _select_from(self, parent_path, scandir): try: - yielded = set() - try: - successor_select = self.successor._select_from - for starting_point in self._iterate_directories(parent_path, scandir): - for p in successor_select(starting_point, scandir): - if p not in yielded: - yield p - yielded.add(p) - finally: - yielded.clear() + successor_select = self.successor._select_from + for starting_point in self._iterate_directories(parent_path, scandir): + for p in successor_select(starting_point, scandir): + yield p except PermissionError: return +class _DoubleRecursiveWildcardSelector(_RecursiveWildcardSelector): + """ + Like _RecursiveWildcardSelector, but also de-duplicates results from + successive selectors. This is necessary if the pattern contains + multiple non-adjacent '**' segments. + """ + + def _select_from(self, parent_path, scandir): + yielded = set() + try: + for p in super()._select_from(parent_path, scandir): + if p not in yielded: + yield p + yielded.add(p) + finally: + yielded.clear() + + # # Public API # @@ -664,7 +684,7 @@ def is_absolute(self): # ntpath.isabs() is defective - see GH-44626 . if self._flavour is ntpath: return bool(self.drive and self.root) - return self._flavour.isabs(self) + return self._flavour.isabs(self._raw_path) def is_reserved(self): """Return True if the path contains one of the special names reserved @@ -740,216 +760,196 @@ class Path(PurePath): """ __slots__ = () - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def _make_child_relpath(self, name): - path_str = str(self) - tail = self._tail - if tail: - path_str = f'{path_str}{self._flavour.sep}{name}' - elif path_str != '.': - path_str = f'{path_str}{name}' - else: - path_str = name - path = self.with_segments(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = tail + [name] - return path - - def __enter__(self): - # In previous versions of pathlib, __exit__() marked this path as - # closed; subsequent attempts to perform I/O would raise an IOError. - # This functionality was never documented, and had the effect of - # making Path objects mutable, contrary to PEP 428. - # In Python 3.9 __exit__() was made a no-op. - # In Python 3.11 __enter__() began emitting DeprecationWarning. - # In Python 3.13 __enter__() and __exit__() should be removed. - warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " - "for removal in Python 3.13; Path objects as a context " - "manager is a no-op", - DeprecationWarning, stacklevel=2) - return self - - def __exit__(self, t, v, tb): - pass - - # Public API - - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory.""" - # We call 'absolute()' rather than using 'os.getcwd()' directly to - # enable users to replace the implementation of 'absolute()' in a - # subclass and benefit from the new behaviour here. This works because - # os.path.abspath('.') == os.getcwd(). - return cls().absolute() - - @classmethod - def home(cls): - """Return a new path pointing to the user's home directory (as - returned by os.path.expanduser('~')). + def stat(self, *, follow_symlinks=True): """ - return cls("~").expanduser() - - def samefile(self, other_path): - """Return whether other_path is the same or not as this file - (as returned by os.path.samefile()). + Return the result of the stat() system call on this path, like + os.stat() does. """ - st = self.stat() - try: - other_st = other_path.stat() - except AttributeError: - other_st = self.with_segments(other_path).stat() - return self._flavour.samestat(st, other_st) - - def iterdir(self): - """Yield path objects of the directory contents. + return os.stat(self, follow_symlinks=follow_symlinks) - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. + def lstat(self): """ - for name in os.listdir(self): - yield self._make_child_relpath(name) - - def _scandir(self): - # bpo-24132: a future version of pathlib will support subclassing of - # pathlib.Path to customize how the filesystem is accessed. This - # includes scandir(), which is used to implement glob(). - return os.scandir(self) - - def glob(self, pattern, *, case_sensitive=None): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. """ - sys.audit("pathlib.Path.glob", self, pattern) - if not pattern: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - if pattern[-1] in (self._flavour.sep, self._flavour.altsep): - pattern_parts.append('') - selector = _make_selector(tuple(pattern_parts), self._flavour, case_sensitive) - for p in selector.select_from(self): - yield p + return self.stat(follow_symlinks=False) - def rglob(self, pattern, *, case_sensitive=None): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - drv, root, pattern_parts = self._parse_path(pattern) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): - pattern_parts.append('') - selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour, case_sensitive) - for p in selector.select_from(self): - yield p - def absolute(self): - """Return an absolute version of this path by prepending the current - working directory. No normalization or symlink resolution is performed. + # Convenience functions for querying the stat results - Use resolve() to get the canonical path to a file. + def exists(self, *, follow_symlinks=True): """ - if self.is_absolute(): - return self - elif self.drive: - # There is a CWD on each drive-letter drive. - cwd = self._flavour.abspath(self.drive) - else: - cwd = os.getcwd() - return self.with_segments(cwd, self) + Whether this path exists. - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. """ - - def check_eloop(e): - winerror = getattr(e, 'winerror', 0) - if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME: - raise RuntimeError("Symlink loop from %r" % e.filename) - try: - s = self._flavour.realpath(self, strict=strict) + self.stat(follow_symlinks=follow_symlinks) except OSError as e: - check_eloop(e) - raise - p = self.with_segments(s) - - # In non-strict mode, realpath() doesn't raise on symlink loops. - # Ensure we get an exception by calling stat() - if not strict: - try: - p.stat() - except OSError as e: - check_eloop(e) - return p + if not _ignore_error(e): + raise + return False + except ValueError: + # Non-encodable path + return False + return True - def stat(self, *, follow_symlinks=True): + def is_dir(self): """ - Return the result of the stat() system call on this path, like - os.stat() does. + Whether this path is a directory. """ - return os.stat(self, follow_symlinks=follow_symlinks) + try: + return S_ISDIR(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False - def owner(self): + def is_file(self): """ - Return the login name of the file owner. + Whether this path is a regular file (also True for symlinks pointing + to regular files). """ try: - import pwd - return pwd.getpwuid(self.stat().st_uid).pw_name - except ImportError: - raise NotImplementedError("Path.owner() is unsupported on this system") + return S_ISREG(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False - def group(self): + def is_mount(self): """ - Return the group name of the file gid. + Check if this path is a mount point """ + return self._flavour.ismount(self) - try: - import grp - return grp.getgrgid(self.stat().st_gid).gr_name - except ImportError: - raise NotImplementedError("Path.group() is unsupported on this system") - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): + def is_symlink(self): """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. + Whether this path is a symbolic link. """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) + try: + return S_ISLNK(self.lstat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist + return False + except ValueError: + # Non-encodable path + return False - def read_bytes(self): + def is_junction(self): """ - Open the file in bytes mode, read it, and close the file. + Whether this path is a junction. """ - with self.open(mode='rb') as f: - return f.read() + return self._flavour.isjunction(self) - def read_text(self, encoding=None, errors=None): + def is_block_device(self): """ - Open the file in text mode, read it, and close the file. + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return self._flavour.samestat(st, other_st) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with self.open(mode='rb') as f: + return f.read() + + def read_text(self, encoding=None, errors=None): + """ + Open the file in text mode, read it, and close the file. """ encoding = io.text_encoding(encoding) with self.open(mode='r', encoding=encoding, errors=errors) as f: @@ -975,6 +975,232 @@ def write_text(self, data, encoding=None, errors=None, newline=None): with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + for name in os.listdir(self): + yield self._make_child_relpath(name) + + def _scandir(self): + # bpo-24132: a future version of pathlib will support subclassing of + # pathlib.Path to customize how the filesystem is accessed. This + # includes scandir(), which is used to implement glob(). + return os.scandir(self) + + def _make_child_relpath(self, name): + path_str = str(self) + tail = self._tail + if tail: + path_str = f'{path_str}{self._flavour.sep}{name}' + elif path_str != '.': + path_str = f'{path_str}{name}' + else: + path_str = name + path = self.with_segments(path_str) + path._str = path_str + path._drv = self.drive + path._root = self.root + path._tail_cached = tail + [name] + return path + + def glob(self, pattern, *, case_sensitive=None): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if not pattern: + raise ValueError("Unacceptable pattern: {!r}".format(pattern)) + drv, root, pattern_parts = self._parse_path(pattern) + if drv or root: + raise NotImplementedError("Non-relative patterns are unsupported") + if pattern[-1] in (self._flavour.sep, self._flavour.altsep): + pattern_parts.append('') + selector = _make_selector(tuple(pattern_parts), self._flavour, case_sensitive) + for p in selector.select_from(self): + yield p + + def rglob(self, pattern, *, case_sensitive=None): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + drv, root, pattern_parts = self._parse_path(pattern) + if drv or root: + raise NotImplementedError("Non-relative patterns are unsupported") + if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): + pattern_parts.append('') + selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour, case_sensitive) + for p in selector.select_from(self): + yield p + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + paths = [self] + + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + + # We may not have read permission for self, in which case we can't + # get a list of the files the directory contains. os.walk() + # always suppressed the exception in that instance, rather than + # blow up for a minor reason when (say) a thousand readable + # directories are still left to visit. That logic is copied here. + try: + scandir_it = path._scandir() + except OSError as error: + if on_error is not None: + on_error(error) + continue + + with scandir_it: + dirnames = [] + filenames = [] + for entry in scandir_it: + try: + is_dir = entry.is_dir(follow_symlinks=follow_symlinks) + except OSError: + # Carried over from os.path.isdir(). + is_dir = False + + if is_dir: + dirnames.append(entry.name) + else: + filenames.append(entry.name) + + if top_down: + yield path, dirnames, filenames + else: + paths.append((path, dirnames, filenames)) + + paths += [path._make_child_relpath(d) for d in reversed(dirnames)] + + def __init__(self, *args, **kwargs): + if kwargs: + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def __enter__(self): + # In previous versions of pathlib, __exit__() marked this path as + # closed; subsequent attempts to perform I/O would raise an IOError. + # This functionality was never documented, and had the effect of + # making Path objects mutable, contrary to PEP 428. + # In Python 3.9 __exit__() was made a no-op. + # In Python 3.11 __enter__() began emitting DeprecationWarning. + # In Python 3.13 __enter__() and __exit__() should be removed. + warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " + "for removal in Python 3.13; Path objects as a context " + "manager is a no-op", + DeprecationWarning, stacklevel=2) + return self + + def __exit__(self, t, v, tb): + pass + + # Public API + + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + # We call 'absolute()' rather than using 'os.getcwd()' directly to + # enable users to replace the implementation of 'absolute()' in a + # subclass and benefit from the new behaviour here. This works because + # os.path.abspath('.') == os.getcwd(). + return cls().absolute() + + @classmethod + def home(cls): + """Return a new path pointing to the user's home directory (as + returned by os.path.expanduser('~')). + """ + return cls("~").expanduser() + + def absolute(self): + """Return an absolute version of this path by prepending the current + working directory. No normalization or symlink resolution is performed. + + Use resolve() to get the canonical path to a file. + """ + if self.is_absolute(): + return self + elif self.drive: + # There is a CWD on each drive-letter drive. + cwd = self._flavour.abspath(self.drive) + else: + cwd = os.getcwd() + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + if not self.root and not self._tail: + result = self.with_segments(cwd) + result._str = cwd + return result + return self.with_segments(cwd, self) + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + def check_eloop(e): + winerror = getattr(e, 'winerror', 0) + if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME: + raise RuntimeError("Symlink loop from %r" % e.filename) + + try: + s = self._flavour.realpath(self, strict=strict) + except OSError as e: + check_eloop(e) + raise + p = self.with_segments(s) + + # In non-strict mode, realpath() doesn't raise on symlink loops. + # Ensure we get an exception by calling stat() + if not strict: + try: + p.stat() + except OSError as e: + check_eloop(e) + return p + + def owner(self): + """ + Return the login name of the file owner. + """ + try: + import pwd + return pwd.getpwuid(self.stat().st_uid).pw_name + except ImportError: + raise NotImplementedError("Path.owner() is unsupported on this system") + + def group(self): + """ + Return the group name of the file gid. + """ + + try: + import grp + return grp.getgrgid(self.stat().st_gid).gr_name + except ImportError: + raise NotImplementedError("Path.group() is unsupported on this system") + def readlink(self): """ Return the path to which the symbolic link points. @@ -1052,13 +1278,6 @@ def rmdir(self): """ os.rmdir(self) - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - def rename(self, target): """ Rename this path to the target path. @@ -1104,151 +1323,6 @@ def hardlink_to(self, target): raise NotImplementedError("os.link() not available on this system") os.link(target, self) - - # Convenience functions for querying the stat results - - def exists(self, *, follow_symlinks=True): - """ - Whether this path exists. - - This method normally follows symlinks; to check whether a symlink exists, - add the argument follow_symlinks=False. - """ - try: - self.stat(follow_symlinks=follow_symlinks) - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path - return False - return True - - def is_dir(self): - """ - Whether this path is a directory. - """ - try: - return S_ISDIR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_file(self): - """ - Whether this path is a regular file (also True for symlinks pointing - to regular files). - """ - try: - return S_ISREG(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_mount(self): - """ - Check if this path is a mount point - """ - return self._flavour.ismount(self) - - def is_symlink(self): - """ - Whether this path is a symbolic link. - """ - try: - return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path - return False - - def is_junction(self): - """ - Whether this path is a junction. - """ - return self._flavour.isjunction(self) - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - def expanduser(self): """ Return a new path with expanded ~ and ~user constructs (as returned by os.path.expanduser) @@ -1263,51 +1337,6 @@ def expanduser(self): return self - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - paths = [self] - - while paths: - path = paths.pop() - if isinstance(path, tuple): - yield path - continue - - # We may not have read permission for self, in which case we can't - # get a list of the files the directory contains. os.walk() - # always suppressed the exception in that instance, rather than - # blow up for a minor reason when (say) a thousand readable - # directories are still left to visit. That logic is copied here. - try: - scandir_it = path._scandir() - except OSError as error: - if on_error is not None: - on_error(error) - continue - - with scandir_it: - dirnames = [] - filenames = [] - for entry in scandir_it: - try: - is_dir = entry.is_dir(follow_symlinks=follow_symlinks) - except OSError: - # Carried over from os.path.isdir(). - is_dir = False - - if is_dir: - dirnames.append(entry.name) - else: - filenames.append(entry.name) - - if top_down: - yield path, dirnames, filenames - else: - paths.append((path, dirnames, filenames)) - - paths += [path._make_child_relpath(d) for d in reversed(dirnames)] - class PosixPath(Path, PurePosixPath): """Path subclass for non-Windows systems. diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 772c0987c2ebef..67f18e530edc4b 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -115,6 +115,8 @@ def multi_interp_extensions_check(enabled=True): It overrides the PyInterpreterConfig.check_multi_interp_extensions setting (see support.run_in_subinterp_with_config() and _xxsubinterpreters.create()). + + Also see importlib.utils.allowing_all_extensions(). """ old = _imp._override_multi_interp_extensions_check(1 if enabled else -1) try: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 861da2326d1214..0659d244d35686 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1,5 +1,7 @@ # Author: Steven J. Bethard . +import contextlib +import functools import inspect import io import operator @@ -35,6 +37,35 @@ def getvalue(self): return self.buffer.raw.getvalue().decode('utf-8') +class StdStreamTest(unittest.TestCase): + + def test_skip_invalid_stderr(self): + parser = argparse.ArgumentParser() + with ( + contextlib.redirect_stderr(None), + mock.patch('argparse._sys.exit') + ): + parser.exit(status=0, message='foo') + + def test_skip_invalid_stdout(self): + parser = argparse.ArgumentParser() + for func in ( + parser.print_usage, + parser.print_help, + functools.partial(parser.parse_args, ['-h']) + ): + with ( + self.subTest(func=func), + contextlib.redirect_stdout(None), + # argparse uses stderr as a fallback + StdIOBuffer() as mocked_stderr, + contextlib.redirect_stderr(mocked_stderr), + mock.patch('argparse._sys.exit'), + ): + func() + self.assertRegex(mocked_stderr.getvalue(), r'usage:') + + class TestCase(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 8eef7baec70118..fdd21aca06ffdd 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -8,9 +8,11 @@ import unittest import warnings import weakref +from functools import partial from textwrap import dedent from test import support +from test.support.import_helper import import_fresh_module from test.support import os_helper, script_helper from test.support.ast_helper import ASTTestMixin @@ -267,6 +269,7 @@ def to_tuple(t): # excepthandler, arguments, keywords, alias class AST_Tests(unittest.TestCase): + maxDiff = None def _is_ast_node(self, name, node): if not isinstance(node, type): @@ -435,16 +438,42 @@ def test_base_classes(self): self.assertTrue(issubclass(ast.comprehension, ast.AST)) self.assertTrue(issubclass(ast.Gt, ast.AST)) + def test_import_deprecated(self): + ast = import_fresh_module('ast') + depr_regex = ( + r'ast\.{} is deprecated and will be removed in Python 3.14; ' + r'use ast\.Constant instead' + ) + for name in 'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis': + with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): + getattr(ast, name) + + def test_field_attr_existence_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'): + item = getattr(ast, name) + if self._is_ast_node(name, item): + with self.subTest(item): + with self.assertWarns(DeprecationWarning): + x = item() + if isinstance(x, ast.AST): + self.assertIs(type(x._fields), tuple) + def test_field_attr_existence(self): for name, item in ast.__dict__.items(): + # These emit DeprecationWarnings + if name in {'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'}: + continue + # constructor has a different signature + if name == 'Index': + continue if self._is_ast_node(name, item): - if name == 'Index': - # Index(value) just returns value now. - # The argument is required. - continue x = item() if isinstance(x, ast.AST): - self.assertEqual(type(x._fields), tuple) + self.assertIs(type(x._fields), tuple) def test_arguments(self): x = ast.arguments() @@ -459,25 +488,108 @@ def test_arguments(self): self.assertEqual(x.args, 2) self.assertEqual(x.vararg, 3) + def test_field_attr_writable_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + x = ast.Num() + # We can assign to _fields + x._fields = 666 + self.assertEqual(x._fields, 666) + def test_field_attr_writable(self): - x = ast.Num() + x = ast.Constant() # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) + def test_classattrs_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + x = ast.Num() + self.assertEqual(x._fields, ('value', 'kind')) + + with self.assertRaises(AttributeError): + x.value + + with self.assertRaises(AttributeError): + x.n + + x = ast.Num(42) + self.assertEqual(x.value, 42) + self.assertEqual(x.n, 42) + + with self.assertRaises(AttributeError): + x.lineno + + with self.assertRaises(AttributeError): + x.foobar + + x = ast.Num(lineno=2) + self.assertEqual(x.lineno, 2) + + x = ast.Num(42, lineno=0) + self.assertEqual(x.lineno, 0) + self.assertEqual(x._fields, ('value', 'kind')) + self.assertEqual(x.value, 42) + self.assertEqual(x.n, 42) + + self.assertRaises(TypeError, ast.Num, 1, None, 2) + self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) + + # Arbitrary keyword arguments are supported + self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') + + with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): + ast.Num(1, n=2) + + self.assertEqual(ast.Num(42).n, 42) + self.assertEqual(ast.Num(4.25).n, 4.25) + self.assertEqual(ast.Num(4.25j).n, 4.25j) + self.assertEqual(ast.Str('42').s, '42') + self.assertEqual(ast.Bytes(b'42').s, b'42') + self.assertIs(ast.NameConstant(True).value, True) + self.assertIs(ast.NameConstant(False).value, False) + self.assertIs(ast.NameConstant(None).value, None) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) + def test_classattrs(self): - x = ast.Num() + x = ast.Constant() self.assertEqual(x._fields, ('value', 'kind')) with self.assertRaises(AttributeError): x.value - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) + x = ast.Constant(42) self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) with self.assertRaises(AttributeError): x.lineno @@ -485,36 +597,23 @@ def test_classattrs(self): with self.assertRaises(AttributeError): x.foobar - x = ast.Num(lineno=2) + x = ast.Constant(lineno=2) self.assertEqual(x.lineno, 2) - x = ast.Num(42, lineno=0) + x = ast.Constant(42, lineno=0) self.assertEqual(x.lineno, 0) self.assertEqual(x._fields, ('value', 'kind')) self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) + self.assertRaises(TypeError, ast.Constant, 1, None, 2) + self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) # Arbitrary keyword arguments are supported self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') - self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') - with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): - ast.Num(1, n=2) with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): ast.Constant(1, value=2) - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str('42').s, '42') - self.assertEqual(ast.Bytes(b'42').s, b'42') - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - self.assertEqual(ast.Constant(42).value, 42) self.assertEqual(ast.Constant(4.25).value, 4.25) self.assertEqual(ast.Constant(4.25j).value, 4.25j) @@ -526,85 +625,211 @@ def test_classattrs(self): self.assertIs(ast.Constant(...).value, ...) def test_realtype(self): - self.assertEqual(type(ast.Num(42)), ast.Constant) - self.assertEqual(type(ast.Num(4.25)), ast.Constant) - self.assertEqual(type(ast.Num(4.25j)), ast.Constant) - self.assertEqual(type(ast.Str('42')), ast.Constant) - self.assertEqual(type(ast.Bytes(b'42')), ast.Constant) - self.assertEqual(type(ast.NameConstant(True)), ast.Constant) - self.assertEqual(type(ast.NameConstant(False)), ast.Constant) - self.assertEqual(type(ast.NameConstant(None)), ast.Constant) - self.assertEqual(type(ast.Ellipsis()), ast.Constant) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + self.assertIs(type(ast.Num(42)), ast.Constant) + self.assertIs(type(ast.Num(4.25)), ast.Constant) + self.assertIs(type(ast.Num(4.25j)), ast.Constant) + self.assertIs(type(ast.Str('42')), ast.Constant) + self.assertIs(type(ast.Bytes(b'42')), ast.Constant) + self.assertIs(type(ast.NameConstant(True)), ast.Constant) + self.assertIs(type(ast.NameConstant(False)), ast.Constant) + self.assertIs(type(ast.NameConstant(None)), ast.Constant) + self.assertIs(type(ast.Ellipsis()), ast.Constant) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) def test_isinstance(self): - self.assertTrue(isinstance(ast.Num(42), ast.Num)) - self.assertTrue(isinstance(ast.Num(4.2), ast.Num)) - self.assertTrue(isinstance(ast.Num(4.2j), ast.Num)) - self.assertTrue(isinstance(ast.Str('42'), ast.Str)) - self.assertTrue(isinstance(ast.Bytes(b'42'), ast.Bytes)) - self.assertTrue(isinstance(ast.NameConstant(True), ast.NameConstant)) - self.assertTrue(isinstance(ast.NameConstant(False), ast.NameConstant)) - self.assertTrue(isinstance(ast.NameConstant(None), ast.NameConstant)) - self.assertTrue(isinstance(ast.Ellipsis(), ast.Ellipsis)) - - self.assertTrue(isinstance(ast.Constant(42), ast.Num)) - self.assertTrue(isinstance(ast.Constant(4.2), ast.Num)) - self.assertTrue(isinstance(ast.Constant(4.2j), ast.Num)) - self.assertTrue(isinstance(ast.Constant('42'), ast.Str)) - self.assertTrue(isinstance(ast.Constant(b'42'), ast.Bytes)) - self.assertTrue(isinstance(ast.Constant(True), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(False), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(None), ast.NameConstant)) - self.assertTrue(isinstance(ast.Constant(...), ast.Ellipsis)) - - self.assertFalse(isinstance(ast.Str('42'), ast.Num)) - self.assertFalse(isinstance(ast.Num(42), ast.Str)) - self.assertFalse(isinstance(ast.Str('42'), ast.Bytes)) - self.assertFalse(isinstance(ast.Num(42), ast.NameConstant)) - self.assertFalse(isinstance(ast.Num(42), ast.Ellipsis)) - self.assertFalse(isinstance(ast.NameConstant(True), ast.Num)) - self.assertFalse(isinstance(ast.NameConstant(False), ast.Num)) - - self.assertFalse(isinstance(ast.Constant('42'), ast.Num)) - self.assertFalse(isinstance(ast.Constant(42), ast.Str)) - self.assertFalse(isinstance(ast.Constant('42'), ast.Bytes)) - self.assertFalse(isinstance(ast.Constant(42), ast.NameConstant)) - self.assertFalse(isinstance(ast.Constant(42), ast.Ellipsis)) - self.assertFalse(isinstance(ast.Constant(True), ast.Num)) - self.assertFalse(isinstance(ast.Constant(False), ast.Num)) - - self.assertFalse(isinstance(ast.Constant(), ast.Num)) - self.assertFalse(isinstance(ast.Constant(), ast.Str)) - self.assertFalse(isinstance(ast.Constant(), ast.Bytes)) - self.assertFalse(isinstance(ast.Constant(), ast.NameConstant)) - self.assertFalse(isinstance(ast.Constant(), ast.Ellipsis)) + from ast import Constant + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + cls_depr_msg = ( + 'ast.{} is deprecated and will be removed in Python 3.14; ' + 'use ast.Constant instead' + ) + + assertNumDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") + ) + assertStrDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") + ) + assertBytesDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") + ) + assertNameConstantDeprecated = partial( + self.assertWarnsRegex, + DeprecationWarning, + cls_depr_msg.format("NameConstant") + ) + assertEllipsisDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") + ) + + for arg in 42, 4.2, 4.2j: + with self.subTest(arg=arg): + with assertNumDeprecated(): + n = Num(arg) + with assertNumDeprecated(): + self.assertIsInstance(n, Num) + + with assertStrDeprecated(): + s = Str('42') + with assertStrDeprecated(): + self.assertIsInstance(s, Str) + + with assertBytesDeprecated(): + b = Bytes(b'42') + with assertBytesDeprecated(): + self.assertIsInstance(b, Bytes) + + for arg in True, False, None: + with self.subTest(arg=arg): + with assertNameConstantDeprecated(): + n = NameConstant(arg) + with assertNameConstantDeprecated(): + self.assertIsInstance(n, NameConstant) + + with assertEllipsisDeprecated(): + e = Ellipsis() + with assertEllipsisDeprecated(): + self.assertIsInstance(e, Ellipsis) + + for arg in 42, 4.2, 4.2j: + with self.subTest(arg=arg): + with assertNumDeprecated(): + self.assertIsInstance(Constant(arg), Num) + + with assertStrDeprecated(): + self.assertIsInstance(Constant('42'), Str) + + with assertBytesDeprecated(): + self.assertIsInstance(Constant(b'42'), Bytes) + + for arg in True, False, None: + with self.subTest(arg=arg): + with assertNameConstantDeprecated(): + self.assertIsInstance(Constant(arg), NameConstant) + + with assertEllipsisDeprecated(): + self.assertIsInstance(Constant(...), Ellipsis) + + with assertStrDeprecated(): + s = Str('42') + assertNumDeprecated(self.assertNotIsInstance, s, Num) + assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) + + with assertNumDeprecated(): + n = Num(42) + assertStrDeprecated(self.assertNotIsInstance, n, Str) + assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) + + with assertNameConstantDeprecated(): + n = NameConstant(True) + with assertNumDeprecated(): + self.assertNotIsInstance(n, Num) + + with assertNameConstantDeprecated(): + n = NameConstant(False) + with assertNumDeprecated(): + self.assertNotIsInstance(n, Num) + + for arg in '42', True, False: + with self.subTest(arg=arg): + with assertNumDeprecated(): + self.assertNotIsInstance(Constant(arg), Num) + + assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) + assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) + assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) + assertNumDeprecated(self.assertNotIsInstance, Constant(), Num) + assertStrDeprecated(self.assertNotIsInstance, Constant(), Str) + assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes) + assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis) class S(str): pass - self.assertTrue(isinstance(ast.Constant(S('42')), ast.Str)) - self.assertFalse(isinstance(ast.Constant(S('42')), ast.Num)) + with assertStrDeprecated(): + self.assertIsInstance(Constant(S('42')), Str) + with assertNumDeprecated(): + self.assertNotIsInstance(Constant(S('42')), Num) + + def test_constant_subclasses_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num - def test_subclasses(self): - class N(ast.Num): + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + class N(ast.Num): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.z = 'spam' + class N2(ast.Num): + pass + + n = N(42) + self.assertEqual(n.n, 42) + self.assertEqual(n.z, 'spam') + self.assertIs(type(n), N) + self.assertIsInstance(n, N) + self.assertIsInstance(n, ast.Num) + self.assertNotIsInstance(n, N2) + self.assertNotIsInstance(ast.Num(42), N) + n = N(n=42) + self.assertEqual(n.n, 42) + self.assertIs(type(n), N) + + self.assertEqual([str(w.message) for w in wlog], [ + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', + ]) + + def test_constant_subclasses(self): + class N(ast.Constant): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.z = 'spam' - class N2(ast.Num): + class N2(ast.Constant): pass n = N(42) - self.assertEqual(n.n, 42) + self.assertEqual(n.value, 42) self.assertEqual(n.z, 'spam') self.assertEqual(type(n), N) self.assertTrue(isinstance(n, N)) - self.assertTrue(isinstance(n, ast.Num)) + self.assertTrue(isinstance(n, ast.Constant)) self.assertFalse(isinstance(n, N2)) - self.assertFalse(isinstance(ast.Num(42), N)) - n = N(n=42) - self.assertEqual(n.n, 42) + self.assertFalse(isinstance(ast.Constant(42), N)) + n = N(value=42) + self.assertEqual(n.value, 42) self.assertEqual(type(n), N) def test_module(self): - body = [ast.Num(42)] + body = [ast.Constant(42)] x = ast.Module(body, []) self.assertEqual(x.body, body) @@ -617,8 +842,8 @@ def test_nodeclasses(self): x.foobarbaz = 5 self.assertEqual(x.foobarbaz, 5) - n1 = ast.Num(1) - n3 = ast.Num(3) + n1 = ast.Constant(1) + n3 = ast.Constant(3) addop = ast.Add() x = ast.BinOp(n1, addop, n3) self.assertEqual(x.left, n1) @@ -987,7 +1212,7 @@ def test_dump_incomplete(self): def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') - src.body.right = ast.copy_location(ast.Num(2), src.body.right) + src.body.right = ast.copy_location(ast.Constant(2), src.body.right) self.assertEqual(ast.dump(src, include_attributes=True), 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, ' 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, ' @@ -1004,7 +1229,7 @@ def test_copy_location(self): def test_fix_missing_locations(self): src = ast.parse('write("spam")') src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), - [ast.Str('eggs')], []))) + [ast.Constant('eggs')], []))) self.assertEqual(src, ast.fix_missing_locations(src)) self.maxDiff = None self.assertEqual(ast.dump(src, include_attributes=True), @@ -1317,9 +1542,9 @@ def arguments(args=None, posonlyargs=None, vararg=None, check(arguments(args=args), "must have Load context") check(arguments(posonlyargs=args), "must have Load context") check(arguments(kwonlyargs=args), "must have Load context") - check(arguments(defaults=[ast.Num(3)]), + check(arguments(defaults=[ast.Constant(3)]), "more positional defaults than args") - check(arguments(kw_defaults=[ast.Num(4)]), + check(arguments(kw_defaults=[ast.Constant(4)]), "length of kwonlyargs is not the same as kw_defaults") args = [ast.arg("x", ast.Name("x", ast.Load()))] check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]), @@ -1372,9 +1597,9 @@ def test_delete(self): "must have Del context") def test_assign(self): - self.stmt(ast.Assign([], ast.Num(3)), "empty targets on Assign") - self.stmt(ast.Assign([None], ast.Num(3)), "None disallowed") - self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Num(3)), + self.stmt(ast.Assign([], ast.Constant(3)), "empty targets on Assign") + self.stmt(ast.Assign([None], ast.Constant(3)), "None disallowed") + self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Constant(3)), "must have Store context") self.stmt(ast.Assign([ast.Name("x", ast.Store())], ast.Name("y", ast.Store())), @@ -1402,39 +1627,39 @@ def test_for(self): self.stmt(ast.For(x, y, [p], [e]), "must have Load context") def test_while(self): - self.stmt(ast.While(ast.Num(3), [], []), "empty body on While") + self.stmt(ast.While(ast.Constant(3), [], []), "empty body on While") self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), "must have Load context") - self.stmt(ast.While(ast.Num(3), [ast.Pass()], + self.stmt(ast.While(ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))]), "must have Load context") def test_if(self): - self.stmt(ast.If(ast.Num(3), [], []), "empty body on If") + self.stmt(ast.If(ast.Constant(3), [], []), "empty body on If") i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) self.stmt(i, "must have Load context") - i = ast.If(ast.Num(3), [ast.Expr(ast.Name("x", ast.Store()))], []) + i = ast.If(ast.Constant(3), [ast.Expr(ast.Name("x", ast.Store()))], []) self.stmt(i, "must have Load context") - i = ast.If(ast.Num(3), [ast.Pass()], + i = ast.If(ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(i, "must have Load context") def test_with(self): p = ast.Pass() self.stmt(ast.With([], [p]), "empty items on With") - i = ast.withitem(ast.Num(3), None) + i = ast.withitem(ast.Constant(3), None) self.stmt(ast.With([i], []), "empty body on With") i = ast.withitem(ast.Name("x", ast.Store()), None) self.stmt(ast.With([i], [p]), "must have Load context") - i = ast.withitem(ast.Num(3), ast.Name("x", ast.Load())) + i = ast.withitem(ast.Constant(3), ast.Name("x", ast.Load())) self.stmt(ast.With([i], [p]), "must have Store context") def test_raise(self): - r = ast.Raise(None, ast.Num(3)) + r = ast.Raise(None, ast.Constant(3)) self.stmt(r, "Raise with cause but no exception") r = ast.Raise(ast.Name("x", ast.Store()), None) self.stmt(r, "must have Load context") - r = ast.Raise(ast.Num(4), ast.Name("x", ast.Store())) + r = ast.Raise(ast.Constant(4), ast.Name("x", ast.Store())) self.stmt(r, "must have Load context") def test_try(self): @@ -1505,11 +1730,11 @@ def test_expr(self): def test_boolop(self): b = ast.BoolOp(ast.And(), []) self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Num(3)]) + b = ast.BoolOp(ast.And(), [ast.Constant(3)]) self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Num(4), None]) + b = ast.BoolOp(ast.And(), [ast.Constant(4), None]) self.expr(b, "None disallowed") - b = ast.BoolOp(ast.And(), [ast.Num(4), ast.Name("x", ast.Store())]) + b = ast.BoolOp(ast.And(), [ast.Constant(4), ast.Name("x", ast.Store())]) self.expr(b, "must have Load context") def test_unaryop(self): @@ -1597,11 +1822,11 @@ def test_compare(self): left = ast.Name("x", ast.Load()) comp = ast.Compare(left, [ast.In()], []) self.expr(comp, "no comparators") - comp = ast.Compare(left, [ast.In()], [ast.Num(4), ast.Num(5)]) + comp = ast.Compare(left, [ast.In()], [ast.Constant(4), ast.Constant(5)]) self.expr(comp, "different number of comparators and operands") - comp = ast.Compare(ast.Num("blah"), [ast.In()], [left]) + comp = ast.Compare(ast.Constant("blah"), [ast.In()], [left]) self.expr(comp) - comp = ast.Compare(left, [ast.In()], [ast.Num("blah")]) + comp = ast.Compare(left, [ast.In()], [ast.Constant("blah")]) self.expr(comp) def test_call(self): @@ -1617,23 +1842,37 @@ def test_call(self): self.expr(call, "must have Load context") def test_num(self): - class subint(int): - pass - class subfloat(float): - pass - class subcomplex(complex): - pass - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import Num + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + class subint(int): + pass + class subfloat(float): + pass + class subcomplex(complex): + pass + for obj in "0", "hello": + self.expr(ast.Num(obj)) + for obj in subint(), subfloat(), subcomplex(): + self.expr(ast.Num(obj), "invalid type", exc=TypeError) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context") def test_subscript(self): - sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Num(3), + sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), ast.Load()) self.expr(sub, "must have Load context") x = ast.Name("x", ast.Load()) @@ -1653,7 +1892,7 @@ def test_subscript(self): def test_starred(self): left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], ast.Store()) - assign = ast.Assign([left], ast.Num(4)) + assign = ast.Assign([left], ast.Constant(4)) self.stmt(assign, "must have Store context") def _sequence(self, fac): @@ -1668,7 +1907,17 @@ def test_tuple(self): self._sequence(ast.Tuple) def test_nameconstant(self): - self.expr(ast.NameConstant(4)) + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('ignore', '', DeprecationWarning) + from ast import NameConstant + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + self.expr(ast.NameConstant(4)) + + self.assertEqual([str(w.message) for w in wlog], [ + 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', + ]) def test_stdlib_validates(self): stdlib = os.path.dirname(ast.__file__) @@ -2357,10 +2606,15 @@ def visit_Ellipsis(self, node): ]) self.assertEqual([str(w.message) for w in wlog], [ 'visit_Num is deprecated; add visit_Constant', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'visit_Num is deprecated; add visit_Constant', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'visit_Num is deprecated; add visit_Constant', + 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'visit_Str is deprecated; add visit_Constant', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', 'visit_Bytes is deprecated; add visit_Constant', + 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', 'visit_NameConstant is deprecated; add visit_Constant', 'visit_NameConstant is deprecated; add visit_Constant', 'visit_Ellipsis is deprecated; add visit_Constant', diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index b6e82ad4db266a..2c65ae8114818f 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4579,6 +4579,176 @@ def test_c_buffer(self): buf.__release_buffer__(mv) self.assertEqual(buf.references, 0) + def test_inheritance(self): + class A(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + a = A(b"hello") + mv = memoryview(a) + self.assertEqual(mv.tobytes(), b"hello") + + def test_inheritance_releasebuffer(self): + rb_call_count = 0 + class B(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + super().__release_buffer__(view) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + + def test_inherit_but_return_something_else(self): + class A(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + + a = A(b"hello") + with memoryview(a) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + rb_call_count = 0 + rb_raised = False + class B(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + try: + super().__release_buffer__(view) + except ValueError: + nonlocal rb_raised + rb_raised = True + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + self.assertIs(rb_raised, True) + + def test_override_only_release(self): + class C(bytearray): + def __release_buffer__(self, buffer): + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + def test_release_saves_reference(self): + smuggled_buffer = None + + class C(bytearray): + def __release_buffer__(s, buffer: memoryview): + with self.assertRaises(ValueError): + memoryview(buffer) + with self.assertRaises(ValueError): + buffer.cast("b") + with self.assertRaises(ValueError): + buffer.toreadonly() + with self.assertRaises(ValueError): + buffer[:1] + with self.assertRaises(ValueError): + buffer.__buffer__(0) + nonlocal smuggled_buffer + smuggled_buffer = buffer + self.assertEqual(buffer.tobytes(), b"hello") + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + with self.assertRaises(ValueError): + smuggled_buffer.tobytes() + + def test_release_saves_reference_no_subclassing(self): + ba = bytearray(b"hello") + + class C: + def __buffer__(self, flags): + return memoryview(ba) + + def __release_buffer__(self, buffer): + self.buffer = buffer + + c = C() + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(c.buffer.tobytes(), b"hello") + + with self.assertRaises(BufferError): + ba.clear() + c.buffer.release() + ba.clear() + + def test_multiple_inheritance_buffer_last(self): + class A: + def __buffer__(self, flags): + return memoryview(b"hello A") + + class B(A, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello A") + + class Releaser: + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(Releaser, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello C") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello C") + c.clear() + with self.assertRaises(ValueError): + c.buffer.tobytes() + + def test_multiple_inheritance_buffer_last(self): + class A: + def __buffer__(self, flags): + raise RuntimeError("should not be called") + + def __release_buffer__(self, buffer): + raise RuntimeError("should not be called") + + class B(bytearray, A): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + class Releaser: + buffer = None + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(bytearray, Releaser): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + self.assertIs(c.buffer, None) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 022753e0c99483..ea57df9cd2400b 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -25,6 +25,8 @@ def test_if_expression(self): ('LOAD_CONST', 2, 1), exit_lbl, ('POP_TOP', None), + ('LOAD_CONST', 3), + ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) @@ -46,5 +48,7 @@ def test_for_loop(self): ('JUMP', loop_lbl), exit_lbl, ('END_FOR', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 8fb97bc0c1a1a7..de7ac97d72cb8e 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -10,7 +10,7 @@ import gc import pickle from test import support -from test.support import warnings_helper +from test.support import warnings_helper, import_helper, check_disallow_instantiation from itertools import permutations from textwrap import dedent from collections import OrderedDict @@ -1430,5 +1430,12 @@ def test_subclassable(self): # issue 44089 class Foo(csv.Error): ... + @support.cpython_only + def test_disallow_instantiation(self): + _csv = import_helper.import_module("_csv") + for tp in _csv.Reader, _csv.Writer: + with self.subTest(tp=tp): + check_disallow_instantiation(self, tp) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index cc782ea1ee5dff..31680b5a92e0f3 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -2141,11 +2141,10 @@ def printsolution(self, x): ... SyntaxError: 'yield' outside function -# Pegen does not produce this error message yet -# >>> def f(): x = yield = y -# Traceback (most recent call last): -# ... -# SyntaxError: assignment to yield expression not possible +>>> def f(): x = yield = y +Traceback (most recent call last): + ... +SyntaxError: assignment to yield expression not possible >>> def f(): (yield bar) = y Traceback (most recent call last): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 773b7094c6b8ce..e2384a08ecaa90 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1861,6 +1861,26 @@ def test_multi_init_extension_non_isolated_compat(self): with self.subTest(f'{modname}: not strict'): self.check_compatible_here(modname, filename, strict=False) + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_per_interpreter_gil_compat(self): + modname = '_test_shared_gil_only' + filename = _testmultiphase.__file__ + loader = ExtensionFileLoader(modname, filename) + spec = importlib.util.spec_from_loader(modname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + sys.modules[modname] = module + + require_extension(module) + with self.subTest(f'{modname}: isolated, strict'): + self.check_incompatible_here(modname, filename, isolated=True) + with self.subTest(f'{modname}: not isolated, strict'): + self.check_compatible_here(modname, filename, + strict=True, isolated=False) + with self.subTest(f'{modname}: not isolated, not strict'): + self.check_compatible_here(modname, filename, + strict=False, isolated=False) + def test_python_compat(self): module = 'threading' require_pure_python(module) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 3bf2bbdcdcc4e6..3a74b821eaee49 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -348,6 +348,8 @@ def test_bad_modules(self): 'exec_err', 'exec_raise', 'exec_unreported_exception', + 'multiple_create_slots', + 'multiple_multiple_interpreters_slots', ]: with self.subTest(name_base): name = self.name + '_' + name_base diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 08a615ecf5288b..0be504925ecc6a 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -8,14 +8,29 @@ import importlib.util import os import pathlib +import re import string import sys from test import support +import textwrap import types import unittest import unittest.mock import warnings +try: + import _testsinglephase +except ImportError: + _testsinglephase = None +try: + import _testmultiphase +except ImportError: + _testmultiphase = None +try: + import _xxsubinterpreters as _interpreters +except ModuleNotFoundError: + _interpreters = None + class DecodeSourceBytesTests: @@ -637,5 +652,111 @@ def test_magic_number(self): self.assertEqual(EXPECTED_MAGIC_NUMBER, actual, msg) +@unittest.skipIf(_interpreters is None, 'subinterpreters required') +class AllowingAllExtensionsTests(unittest.TestCase): + + ERROR = re.compile("^: module (.*) does not support loading in subinterpreters") + + def run_with_own_gil(self, script): + interpid = _interpreters.create(isolated=True) + try: + _interpreters.run_string(interpid, script) + except _interpreters.RunFailedError as exc: + if m := self.ERROR.match(str(exc)): + modname, = m.groups() + raise ImportError(modname) + + def run_with_shared_gil(self, script): + interpid = _interpreters.create(isolated=False) + try: + _interpreters.run_string(interpid, script) + except _interpreters.RunFailedError as exc: + if m := self.ERROR.match(str(exc)): + modname, = m.groups() + raise ImportError(modname) + + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_single_phase_init_module(self): + script = textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(): + import _testsinglephase + ''') + with self.subTest('check disabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check disabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + script = textwrap.dedent(f''' + import importlib.util + with importlib.util.allowing_all_extensions(False): + import _testsinglephase + ''') + with self.subTest('check enabled, shared GIL'): + with self.assertRaises(ImportError): + self.run_with_shared_gil(script) + with self.subTest('check enabled, per-interpreter GIL'): + with self.assertRaises(ImportError): + self.run_with_own_gil(script) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_incomplete_multi_phase_init_module(self): + prescript = textwrap.dedent(f''' + from importlib.util import spec_from_loader, module_from_spec + from importlib.machinery import ExtensionFileLoader + + name = '_test_shared_gil_only' + filename = {_testmultiphase.__file__!r} + loader = ExtensionFileLoader(name, filename) + spec = spec_from_loader(name, loader) + + ''') + + script = prescript + textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(): + module = module_from_spec(spec) + loader.exec_module(module) + ''') + with self.subTest('check disabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check disabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + script = prescript + textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(False): + module = module_from_spec(spec) + loader.exec_module(module) + ''') + with self.subTest('check enabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check enabled, per-interpreter GIL'): + with self.assertRaises(ImportError): + self.run_with_own_gil(script) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_complete_multi_phase_init_module(self): + script = textwrap.dedent(''' + import importlib.util + with importlib.util.allowing_all_extensions(): + import _testmultiphase + ''') + with self.subTest('check disabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check disabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + script = textwrap.dedent(f''' + import importlib.util + with importlib.util.allowing_all_extensions(False): + import _testmultiphase + ''') + with self.subTest('check enabled, shared GIL'): + self.run_with_shared_gil(script) + with self.subTest('check enabled, per-interpreter GIL'): + self.run_with_own_gil(script) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index e118114186dec3..364f75db908b05 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2111,6 +2111,28 @@ def __dict__(self): self.assertEqual(inspect.getattr_static(foo, 'a'), 3) self.assertFalse(test.called) + def test_mutated_mro(self): + test = self + test.called = False + + class Foo(dict): + a = 3 + @property + def __dict__(self): + test.called = True + return {} + + class Bar(dict): + a = 4 + + class Baz(Bar): pass + + baz = Baz() + self.assertEqual(inspect.getattr_static(baz, 'a'), 4) + Baz.__bases__ = (Foo,) + self.assertEqual(inspect.getattr_static(baz, 'a'), 3) + self.assertFalse(test.called) + def test_custom_object_dict(self): test = self test.called = False @@ -2165,6 +2187,35 @@ class Thing(metaclass=Meta): inspect.getattr_static(Thing, "spam") self.assertFalse(Thing.executed) + def test_custom___getattr__(self): + test = self + test.called = False + + class Foo: + def __getattr__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'whatever') + + self.assertFalse(test.called) + + def test_custom___getattribute__(self): + test = self + test.called = False + + class Foo: + def __getattribute__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'really_could_be_anything') + + self.assertFalse(test.called) + + class TestGetGeneratorState(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 7586610833b063..ee0ef9a34c385c 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1655,7 +1655,8 @@ class P(_BasePurePathSubclass, self.cls): p = P(BASE, session_id=42) self.assertEqual(42, p.absolute().session_id) self.assertEqual(42, p.resolve().session_id) - self.assertEqual(42, p.with_segments('~').expanduser().session_id) + if not is_wasi: # WASI has no user accounts. + self.assertEqual(42, p.with_segments('~').expanduser().session_id) self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) if os_helper.can_symlink(): @@ -1852,13 +1853,14 @@ def _check(path, pattern, case_sensitive, expected): def test_rglob_common(self): def _check(glob, expected): - self.assertEqual(set(glob), { P(BASE, q) for q in expected }) + self.assertEqual(sorted(glob), sorted(P(BASE, q) for q in expected)) P = self.cls p = P(BASE) it = p.rglob("fileA") self.assertIsInstance(it, collections.abc.Iterator) _check(it, ["fileA"]) _check(p.rglob("fileB"), ["dirB/fileB"]) + _check(p.rglob("**/fileB"), ["dirB/fileB"]) _check(p.rglob("*/fileA"), []) if not os_helper.can_symlink(): _check(p.rglob("*/fileB"), ["dirB/fileB"]) @@ -1882,9 +1884,12 @@ def _check(glob, expected): _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p.rglob("dir*/**"), ["dirC/dirD"]) _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) _check(p.rglob("*/"), ["dirC/dirD"]) _check(p.rglob(""), ["dirC", "dirC/dirD"]) + _check(p.rglob("**"), ["dirC", "dirC/dirD"]) # gh-91616, a re module regression _check(p.rglob("*.txt"), ["dirC/novel.txt"]) _check(p.rglob("*.*"), ["dirC/novel.txt"]) diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 2b977d78d39324..5418f9f35485f8 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -1,4 +1,5 @@ import unittest +import string import subprocess import sys import sysconfig @@ -70,9 +71,14 @@ def baz(): perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") self.assertTrue(perf_file.exists()) perf_file_contents = perf_file.read_text() - self.assertIn(f"py::foo:{script}", perf_file_contents) - self.assertIn(f"py::bar:{script}", perf_file_contents) - self.assertIn(f"py::baz:{script}", perf_file_contents) + perf_lines = perf_file_contents.splitlines(); + expected_symbols = [f"py::foo:{script}", f"py::bar:{script}", f"py::baz:{script}"] + for expected_symbol in expected_symbols: + perf_line = next((line for line in perf_lines if expected_symbol in line), None) + self.assertIsNotNone(perf_line, f"Could not find {expected_symbol} in perf file") + perf_addr = perf_line.split(" ")[0] + self.assertFalse(perf_addr.startswith("0x"), "Address should not be prefixed with 0x") + self.assertTrue(set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters") def test_trampoline_works_with_forks(self): code = """if 1: diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 1bb0e13e356e78..328b0467e7fa3d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1495,6 +1495,14 @@ def test_blob_closed_db_read(self): "Cannot operate on a closed database", blob.read) + def test_blob_32bit_rowid(self): + # gh-100370: we should not get an OverflowError for 32-bit rowids + with memory_database() as cx: + rowid = 2**32 + cx.execute("create table t(t blob)") + cx.execute("insert into t(rowid, t) values (?, zeroblob(1))", (rowid,)) + cx.blobopen('t', 't', rowid) + @threading_helper.requires_working_threading() class ThreadTests(unittest.TestCase): diff --git a/Lib/turtle.py b/Lib/turtle.py index 2de406e0f517af..cf111158b7c149 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -21,7 +21,6 @@ # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. - """ Turtle graphics is a popular way for introducing programming to kids. It was part of the original Logo programming language developed @@ -97,13 +96,8 @@ Behind the scenes there are some features included with possible extensions in mind. These will be commented and documented elsewhere. - """ -_ver = "turtle 1.1b- - for Python 3.1 - 4. 5. 2009" - -# print(_ver) - import tkinter as TK import types import math @@ -141,7 +135,7 @@ _tg_utilities = ['write_docstringdict', 'done'] __all__ = (_tg_classes + _tg_screen_functions + _tg_turtle_functions + - _tg_utilities + ['Terminator']) # + _math_functions) + _tg_utilities + ['Terminator']) _alias_list = ['addshape', 'backward', 'bk', 'fd', 'ht', 'lt', 'pd', 'pos', 'pu', 'rt', 'seth', 'setpos', 'setposition', 'st', @@ -598,9 +592,6 @@ def _write(self, pos, txt, align, font, pencolor): x0, y0, x1, y1 = self.cv.bbox(item) return item, x1-1 -## def _dot(self, pos, size, color): -## """may be implemented for some other graphics toolkit""" - def _onclick(self, item, fun, num=1, add=None): """Bind fun to mouse-click event on turtle. fun must be a function with two arguments, the coordinates @@ -2726,7 +2717,7 @@ def _cc(self, args): if not ((0 <= r <= 255) and (0 <= g <= 255) and (0 <= b <= 255)): raise TurtleGraphicsError("bad color sequence: %s" % str(args)) return "#%02x%02x%02x" % (r, g, b) - + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: """Instantly move turtle to an absolute position. @@ -2738,14 +2729,14 @@ def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: call: teleport(x, y) # two coordinates --or: teleport(x) # teleport to x position, keeping y as is --or: teleport(y=y) # teleport to y position, keeping x as is - --or: teleport(x, y, fill_gap=True) + --or: teleport(x, y, fill_gap=True) # teleport but fill the gap in between Move turtle to an absolute position. Unlike goto(x, y), a line will not be drawn. The turtle's orientation does not change. If currently filling, the polygon(s) teleported from will be filled after leaving, and filling will begin again after teleporting. This can be disabled - with fill_gap=True, which makes the imaginary line traveled during + with fill_gap=True, which makes the imaginary line traveled during teleporting act as a fill barrier like in goto(x, y). Example (for a Turtle instance named turtle): @@ -2773,7 +2764,7 @@ def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: self._position = Vec2D(new_x, new_y) self.pen(pendown=pendown) if was_filling and not fill_gap: - self.begin_fill() + self.begin_fill() def clone(self): """Create and return a clone of the turtle. @@ -3455,27 +3446,22 @@ def dot(self, size=None, *color): if size is None: size = self._pensize + max(self._pensize, 4) color = self._colorstr(color) - if hasattr(self.screen, "_dot"): - item = self.screen._dot(self._position, size, color) - self.items.append(item) - if self.undobuffer: - self.undobuffer.push(("dot", item)) - else: - pen = self.pen() - if self.undobuffer: - self.undobuffer.push(["seq"]) - self.undobuffer.cumulate = True - try: - if self.resizemode() == 'auto': - self.ht() - self.pendown() - self.pensize(size) - self.pencolor(color) - self.forward(0) - finally: - self.pen(pen) - if self.undobuffer: - self.undobuffer.cumulate = False + # If screen were to gain a dot function, see GH #104218. + pen = self.pen() + if self.undobuffer: + self.undobuffer.push(["seq"]) + self.undobuffer.cumulate = True + try: + if self.resizemode() == 'auto': + self.ht() + self.pendown() + self.pensize(size) + self.pencolor(color) + self.forward(0) + finally: + self.pen(pen) + if self.undobuffer: + self.undobuffer.cumulate = False def _write(self, txt, align, font): """Performs the writing for write() @@ -3751,11 +3737,6 @@ class _Screen(TurtleScreen): _title = _CFG["title"] def __init__(self): - # XXX there is no need for this code to be conditional, - # as there will be only a single _Screen instance, anyway - # XXX actually, the turtle demo is injecting root window, - # so perhaps the conditional creation of a root should be - # preserved (perhaps by passing it as an optional parameter) if _Screen._root is None: _Screen._root = self._root = _Root() self._root.title(_Screen._title) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index caea022da4a688..f6c9d6aa6f9a32 100755 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -203,10 +203,10 @@ def __init__(self, filename=None): def onResize(self, event): - cwidth = self._canvas.winfo_width() - cheight = self._canvas.winfo_height() - self._canvas.xview_moveto(0.5*(self.canvwidth-cwidth)/self.canvwidth) - self._canvas.yview_moveto(0.5*(self.canvheight-cheight)/self.canvheight) + cwidth = self.canvas.winfo_width() + cheight = self.canvas.winfo_height() + self.canvas.xview_moveto(0.5*(self.canvwidth-cwidth)/self.canvwidth) + self.canvas.yview_moveto(0.5*(self.canvheight-cheight)/self.canvheight) def makeTextFrame(self, root): self.text_frame = text_frame = Frame(root) @@ -237,19 +237,23 @@ def makeTextFrame(self, root): return text_frame def makeGraphFrame(self, root): + # t._Screen is a singleton class instantiated or retrieved + # by calling Screen. Since tdemo canvas needs a different + # configuration, we manually set class attributes before + # calling Screen and manually call superclass init after. turtle._Screen._root = root + self.canvwidth = 1000 self.canvheight = 800 - turtle._Screen._canvas = self._canvas = canvas = turtle.ScrolledCanvas( + turtle._Screen._canvas = self.canvas = canvas = turtle.ScrolledCanvas( root, 800, 600, self.canvwidth, self.canvheight) canvas.adjustScrolls() canvas._rootwindow.bind('', self.onResize) canvas._canvas['borderwidth'] = 0 - self.screen = _s_ = turtle.Screen() - turtle.TurtleScreen.__init__(_s_, _s_._canvas) - self.scanvas = _s_._canvas - turtle.RawTurtle.screens = [_s_] + self.screen = screen = turtle.Screen() + turtle.TurtleScreen.__init__(screen, canvas) + turtle.RawTurtle.screens = [screen] return canvas def set_txtsize(self, size): @@ -373,7 +377,7 @@ def startDemo(self): def clearCanvas(self): self.refreshCanvas() self.screen._delete("all") - self.scanvas.config(cursor="") + self.canvas.config(cursor="") self.configGUI(NORMAL, DISABLED, DISABLED) def stopIt(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index 736a520d0e8fb6..329466580b9cb0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2698,7 +2698,7 @@ MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_MD5.h Modules/_hacl/Hacl_Hash_MD5.c MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA1.h Modules/_hacl/Hacl_Hash_SHA1.c MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) -MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h +MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst new file mode 100644 index 00000000000000..5434660e9d6ffb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst @@ -0,0 +1 @@ +Change the perf map format to remove the '0x' prefix from the addresses diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst new file mode 100644 index 00000000000000..42e26cb27b6e01 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst @@ -0,0 +1,6 @@ +The GIL is now (optionally) per-interpreter. This is the fundamental change +for PEP 684. This is all made possible by virtue of the isolated state of +each interpreter in the process. The behavior of the main interpreter +remains unchanged. Likewise, interpreters created using +``Py_NewInterpreter()`` are not affected. To get an interpreter with its +own GIL, call ``Py_NewInterpreterFromConfig()``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst new file mode 100644 index 00000000000000..afd26750846167 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst @@ -0,0 +1,11 @@ +Multi-phase init extension modules may now indicate that they support +running in subinterpreters that have their own GIL. This is done by using +``Py_MOD_PER_INTERPRETER_GIL_SUPPORTED`` as the value for the +``Py_mod_multiple_interpreters`` module def slot. Otherwise the module, by +default, cannot be imported in such subinterpreters. (This does not affect +the main interpreter or subinterpreters that do not have their own GIL.) In +addition to the isolation that multi-phase init already normally requires, +support for per-interpreter GIL involves one additional constraint: +thread-safety. If the module has external (linked) dependencies and those +libraries have any state that isn't thread-safe then the module must do the +additional work to add thread-safety. This should be an uncommon case. diff --git a/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst b/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst new file mode 100644 index 00000000000000..6539efbc9d0eb0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst @@ -0,0 +1,4 @@ +Deprecation warnings are now emitted for :class:`!ast.Num`, +:class:`!ast.Bytes`, :class:`!ast.Str`, :class:`!ast.NameConstant` and +:class:`!ast.Ellipsis`. These have been documented as deprecated since Python +3.8, and will be removed in Python 3.14. diff --git a/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst b/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst new file mode 100644 index 00000000000000..556db0eae00c0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst @@ -0,0 +1,2 @@ +Decrease execution time of some :mod:`gzip` file writes by 15% by +adding more appropriate buffering. diff --git a/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst b/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst new file mode 100644 index 00000000000000..917cf0f97b9e06 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst @@ -0,0 +1 @@ +:class:`argparse.ArgumentParser` now catches errors when writing messages, such as when :data:`sys.stderr` is ``None``. Patch by Oleg Iarygin. diff --git a/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst b/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst new file mode 100644 index 00000000000000..238f2868867472 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst @@ -0,0 +1,4 @@ +Improve performance of :meth:`pathlib.Path.absolute` and +:meth:`~pathlib.Path.cwd` by joining paths only when necessary. Also improve +performance of :meth:`pathlib.PurePath.is_absolute` on Posix by skipping path +parsing and normalization. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst b/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst new file mode 100644 index 00000000000000..9022d55c48cb11 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst @@ -0,0 +1,2 @@ +Fix potential :exc:`OverflowError` in :meth:`sqlite3.Connection.blobopen` +for 32-bit builds. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst new file mode 100644 index 00000000000000..b975d48ed3385c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst @@ -0,0 +1,3 @@ +Optimize :func:`asyncio.gather` when using :func:`asyncio.eager_task_factory` +to complete eagerly if all fututres completed eagerly. +Avoid scheduling done callbacks for futures that complete eagerly. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst new file mode 100644 index 00000000000000..ced3b7cea04954 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst @@ -0,0 +1,2 @@ +Optimize :class:`asyncio.TaskGroup` when using :func:`asyncio.eager_task_factory`. +Skip scheduling a done callback if a TaskGroup task completes eagerly. diff --git a/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst b/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst new file mode 100644 index 00000000000000..54b0190192863c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst @@ -0,0 +1 @@ +Remove unneeded comments and code in turtle.py. diff --git a/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst b/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst new file mode 100644 index 00000000000000..01f8b948d2cb65 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst @@ -0,0 +1,3 @@ +Improve performance of :meth:`pathlib.Path.glob` when expanding recursive +wildcards ("``**``") by merging adjacent wildcards and de-duplicating +results only when necessary. diff --git a/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst b/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst new file mode 100644 index 00000000000000..9c582844bf909b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst @@ -0,0 +1,4 @@ +Prevent possible crash by disallowing instantiation of the +:class:`!_csv.Reader` and :class:`!_csv.Writer` types. +The regression was introduced in 3.10.0a4 with PR 23224 (:issue:`14935`). +Patch by Radislav Chugunov. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst b/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst new file mode 100644 index 00000000000000..3743d569995f2e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst @@ -0,0 +1,3 @@ +Users may now use ``importlib.util.allowing_all_extensions()`` (a context +manager) to temporarily disable the strict compatibility checks for +importing extension modules in subinterpreters. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst b/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst new file mode 100644 index 00000000000000..03775845450caa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst @@ -0,0 +1 @@ +:func:`socket.getnameinfo` now releases the GIL while contacting the DNS server diff --git a/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst b/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst new file mode 100644 index 00000000000000..f259acf753831c --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst @@ -0,0 +1,2 @@ +Upgrade built-in :mod:`hashlib` SHA3 implementation to a verified implementation +from the ``HACL*`` project. Used when OpenSSL is not present or lacks SHA3. diff --git a/Modules/Setup b/Modules/Setup index 1c6f2f7ea5182d..312e99fea530dc 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -166,7 +166,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a -#_sha3 _sha3/sha3module.c +#_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE # text encodings and unicode #_codecs_cn cjkcodecs/_codecs_cn.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index a7803cf7c00e68..8e66576b5c5f00 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -80,7 +80,7 @@ @MODULE__MD5_TRUE@_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA1_TRUE@_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a -@MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c +@MODULE__SHA3_TRUE@_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c ############################################################################ diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 822d5f2a41de33..3830245abe87b3 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -8,7 +8,6 @@ #include "pycore_runtime_init.h" // _Py_ID() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef -#include "cpython/context.h" #include // offsetof() @@ -2064,21 +2063,6 @@ swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) return prev_task; } -static int -is_loop_running(PyObject *loop) -{ - PyObject *func = PyObject_GetAttr(loop, &_Py_ID(is_running)); - if (func == NULL) { - PyErr_Format(PyExc_TypeError, "Loop missing is_running()"); - return -1; - } - PyObject *res = PyObject_CallNoArgs(func); - int retval = Py_IsTrue(res); - Py_DECREF(func); - Py_DECREF(res); - return !!retval; -} - /* ----- Task */ /*[clinic input] @@ -2149,11 +2133,13 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, } if (eager_start) { - int loop_running = is_loop_running(self->task_loop); - if (loop_running == -1) { + PyObject *res = PyObject_CallMethodNoArgs(loop, &_Py_ID(is_running)); + if (res == NULL) { return -1; } - if (loop_running) { + int is_loop_running = Py_IsTrue(res); + Py_DECREF(res); + if (is_loop_running) { if (task_eager_start(state, self)) { return -1; } diff --git a/Modules/_blake2/blake2module.h b/Modules/_blake2/blake2module.h index aa8f281178eadc..c8144ec9d48d29 100644 --- a/Modules/_blake2/blake2module.h +++ b/Modules/_blake2/blake2module.h @@ -38,6 +38,6 @@ #endif // HAVE_LIBB2 // for secure_zero_memory(), store32(), store48(), and store64() -#include "impl/blake2-impl.h" +#include "impl/blake2-impl.h" #endif // Py_BLAKE2MODULE_H diff --git a/Modules/_blake2/impl/blake2b-round.h b/Modules/_blake2/impl/blake2b-round.h index cebc22550da4cd..5b452c4d63babe 100644 --- a/Modules/_blake2/impl/blake2b-round.h +++ b/Modules/_blake2/impl/blake2b-round.h @@ -62,7 +62,7 @@ \ row2l = _mm_roti_epi64(row2l, -24); \ row2h = _mm_roti_epi64(row2h, -24); \ - + #define G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \ row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \ row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \ @@ -81,7 +81,7 @@ \ row2l = _mm_roti_epi64(row2l, -63); \ row2h = _mm_roti_epi64(row2h, -63); \ - + #if defined(HAVE_SSSE3) #define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ t0 = _mm_alignr_epi8(row2h, row2l, 8); \ diff --git a/Modules/_blake2/impl/blake2s-load-xop.h b/Modules/_blake2/impl/blake2s-load-xop.h index ac591a77d191a7..14d9e7f7640672 100644 --- a/Modules/_blake2/impl/blake2s-load-xop.h +++ b/Modules/_blake2/impl/blake2s-load-xop.h @@ -166,7 +166,7 @@ buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(7)) ); #define LOAD_MSG_8_3(buf) \ t0 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(6),TOB(1),TOB(0),TOB(0)) ); \ buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(5),TOB(4)) ); \ - + #define LOAD_MSG_8_4(buf) \ buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(5),TOB(4),TOB(7),TOB(2)) ); diff --git a/Modules/_blake2/impl/blake2s-round.h b/Modules/_blake2/impl/blake2s-round.h index 1e2f2b7f59bd6c..3af4be35bee5d4 100644 --- a/Modules/_blake2/impl/blake2s-round.h +++ b/Modules/_blake2/impl/blake2s-round.h @@ -86,6 +86,6 @@ LOAD_MSG_ ##r ##_4(buf4); \ G2(row1,row2,row3,row4,buf4); \ UNDIAGONALIZE(row1,row2,row3,row4); \ - + #endif diff --git a/Modules/_csv.c b/Modules/_csv.c index 0cde5c5a8bdc68..9ab2ad266c2739 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1000,7 +1000,7 @@ PyType_Spec Reader_Type_spec = { .name = "_csv.reader", .basicsize = sizeof(ReaderObj), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), .slots = Reader_Type_slots }; @@ -1431,7 +1431,7 @@ PyType_Spec Writer_Type_spec = { .name = "_csv.writer", .basicsize = sizeof(WriterObj), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), .slots = Writer_Type_slots, }; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index ce652b362d5bb3..ddfb2c8a332a9e 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1036,7 +1036,7 @@ EXPORT (HRESULT) KeepObject(IUnknown *punk) #ifdef MS_WIN32 -// i38748: c stub for testing stack corruption +// i38748: c stub for testing stack corruption // When executing a Python callback with a long and a long long typedef long(__stdcall *_test_i38748_funcType)(long, long long); diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c new file mode 100644 index 00000000000000..100afe7c2c6d1f --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -0,0 +1,826 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_SHA3.h" + +static uint32_t block_len(Spec_Hash_Definitions_hash_alg a) +{ + switch (a) + { + case Spec_Hash_Definitions_SHA3_224: + { + return (uint32_t)144U; + } + case Spec_Hash_Definitions_SHA3_256: + { + return (uint32_t)136U; + } + case Spec_Hash_Definitions_SHA3_384: + { + return (uint32_t)104U; + } + case Spec_Hash_Definitions_SHA3_512: + { + return (uint32_t)72U; + } + case Spec_Hash_Definitions_Shake128: + { + return (uint32_t)168U; + } + case Spec_Hash_Definitions_Shake256: + { + return (uint32_t)136U; + } + default: + { + KRML_HOST_EPRINTF("KaRaMeL incomplete match at %s:%d\n", __FILE__, __LINE__); + KRML_HOST_EXIT(253U); + } + } +} + +static uint32_t hash_len(Spec_Hash_Definitions_hash_alg a) +{ + switch (a) + { + case Spec_Hash_Definitions_SHA3_224: + { + return (uint32_t)28U; + } + case Spec_Hash_Definitions_SHA3_256: + { + return (uint32_t)32U; + } + case Spec_Hash_Definitions_SHA3_384: + { + return (uint32_t)48U; + } + case Spec_Hash_Definitions_SHA3_512: + { + return (uint32_t)64U; + } + default: + { + KRML_HOST_EPRINTF("KaRaMeL incomplete match at %s:%d\n", __FILE__, __LINE__); + KRML_HOST_EXIT(253U); + } + } +} + +void +Hacl_Hash_SHA3_update_multi_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *blocks, + uint32_t n_blocks +) +{ + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint8_t *block = blocks + i * block_len(a); + Hacl_Impl_SHA3_absorb_inner(block_len(a), block, s); + } +} + +void +Hacl_Hash_SHA3_update_last_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *input, + uint32_t input_len +) +{ + uint8_t suffix; + if (a == Spec_Hash_Definitions_Shake128 || a == Spec_Hash_Definitions_Shake256) + { + suffix = (uint8_t)0x1fU; + } + else + { + suffix = (uint8_t)0x06U; + } + uint32_t len = block_len(a); + if (input_len == len) + { + Hacl_Impl_SHA3_absorb_inner(len, input, s); + uint8_t *uu____0 = input + input_len; + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, uu____0, (uint32_t)0U * sizeof (uint8_t)); + lastBlock[0U] = suffix; + Hacl_Impl_SHA3_loadState(len, lastBlock, s); + if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && (uint32_t)0U == len - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(len, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); + return; + } + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, input, input_len * sizeof (uint8_t)); + lastBlock[input_len] = suffix; + Hacl_Impl_SHA3_loadState(len, lastBlock, s); + if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && input_len == len - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(len, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); +} + +typedef struct hash_buf2_s +{ + Hacl_Streaming_Keccak_hash_buf fst; + Hacl_Streaming_Keccak_hash_buf snd; +} +hash_buf2; + +Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + return block_state.fst; +} + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a) +{ + KRML_CHECK_SIZE(sizeof (uint8_t), block_len(a)); + uint8_t *buf0 = (uint8_t *)KRML_HOST_CALLOC(block_len(a), sizeof (uint8_t)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); + Hacl_Streaming_Keccak_hash_buf block_state = { .fst = a, .snd = buf }; + Hacl_Streaming_Keccak_state + s = { .block_state = block_state, .buf = buf0, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_Keccak_state + *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + p[0U] = s; + uint64_t *s1 = block_state.snd; + for (uint32_t _i = 0U; _i < (uint32_t)25U; ++_i) + ((void **)s1)[_i] = (void *)(uint64_t)0U; + return p; +} + +void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + uint8_t *buf = scrut.buf; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + uint64_t *s1 = block_state.snd; + KRML_HOST_FREE(s1); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(s); +} + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0) +{ + Hacl_Streaming_Keccak_state scrut0 = *s0; + Hacl_Streaming_Keccak_hash_buf block_state0 = scrut0.block_state; + uint8_t *buf0 = scrut0.buf; + uint64_t total_len0 = scrut0.total_len; + Spec_Hash_Definitions_hash_alg i = block_state0.fst; + KRML_CHECK_SIZE(sizeof (uint8_t), block_len(i)); + uint8_t *buf1 = (uint8_t *)KRML_HOST_CALLOC(block_len(i), sizeof (uint8_t)); + memcpy(buf1, buf0, block_len(i) * sizeof (uint8_t)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); + Hacl_Streaming_Keccak_hash_buf block_state = { .fst = i, .snd = buf }; + hash_buf2 scrut = { .fst = block_state0, .snd = block_state }; + uint64_t *s_dst = scrut.snd.snd; + uint64_t *s_src = scrut.fst.snd; + memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); + Hacl_Streaming_Keccak_state + s = { .block_state = block_state, .buf = buf1, .total_len = total_len0 }; + Hacl_Streaming_Keccak_state + *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + p[0U] = s; + return p; +} + +void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s) +{ + Hacl_Streaming_Keccak_state scrut = *s; + uint8_t *buf = scrut.buf; + Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + uint64_t *s1 = block_state.snd; + for (uint32_t _i = 0U; _i < (uint32_t)25U; ++_i) + ((void **)s1)[_i] = (void *)(uint64_t)0U; + Hacl_Streaming_Keccak_state + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +uint32_t +Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len) +{ + Hacl_Streaming_Keccak_state s = *p; + Hacl_Streaming_Keccak_hash_buf block_state = s.block_state; + uint64_t total_len = s.total_len; + Spec_Hash_Definitions_hash_alg i = block_state.fst; + if ((uint64_t)len > (uint64_t)0xffffffffU - total_len) + { + return (uint32_t)1U; + } + uint32_t sz; + if (total_len % (uint64_t)block_len(i) == (uint64_t)0U && total_len > (uint64_t)0U) + { + sz = block_len(i); + } + else + { + sz = (uint32_t)(total_len % (uint64_t)block_len(i)); + } + if (len <= block_len(i) - sz) + { + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, data, len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)len; + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == (uint32_t)0U) + { + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + if (!(sz1 == (uint32_t)0U)) + { + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, buf, block_len(i) / block_len(a1)); + } + uint32_t ite; + if ((uint64_t)len % (uint64_t)block_len(i) == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + { + ite = block_len(i); + } + else + { + ite = (uint32_t)((uint64_t)len % (uint64_t)block_len(i)); + } + uint32_t n_blocks = (len - ite) / block_len(i); + uint32_t data1_len = n_blocks * block_len(i); + uint32_t data2_len = len - data1_len; + uint8_t *data1 = data; + uint8_t *data2 = data + data1_len; + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)len + } + ); + } + else + { + uint32_t diff = block_len(i) - sz; + uint8_t *data1 = data; + uint8_t *data2 = data + diff; + Hacl_Streaming_Keccak_state s1 = *p; + Hacl_Streaming_Keccak_hash_buf block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)block_len(i) == (uint64_t)0U && total_len10 > (uint64_t)0U) + { + sz10 = block_len(i); + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)block_len(i)); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, data1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Streaming_Keccak_state s10 = *p; + Hacl_Streaming_Keccak_hash_buf block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = block_len(i); + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); + } + if (!(sz1 == (uint32_t)0U)) + { + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, buf, block_len(i) / block_len(a1)); + } + uint32_t ite; + if + ( + (uint64_t)(len - diff) + % (uint64_t)block_len(i) + == (uint64_t)0U + && (uint64_t)(len - diff) > (uint64_t)0U + ) + { + ite = block_len(i); + } + else + { + ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)block_len(i)); + } + uint32_t n_blocks = (len - diff - ite) / block_len(i); + uint32_t data1_len = n_blocks * block_len(i); + uint32_t data2_len = len - diff - data1_len; + uint8_t *data11 = data2; + uint8_t *data21 = data2 + data1_len; + Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; + uint64_t *s2 = block_state1.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data11, data1_len / block_len(a1)); + uint8_t *dst = buf; + memcpy(dst, data21, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_Keccak_state){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(len - diff) + } + ); + } + return (uint32_t)0U; +} + +static void +finish_( + Spec_Hash_Definitions_hash_alg a, + Hacl_Streaming_Keccak_state *p, + uint8_t *dst, + uint32_t l +) +{ + Hacl_Streaming_Keccak_state scrut0 = *p; + Hacl_Streaming_Keccak_hash_buf block_state = scrut0.block_state; + uint8_t *buf_ = scrut0.buf; + uint64_t total_len = scrut0.total_len; + uint32_t r; + if (total_len % (uint64_t)block_len(a) == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = block_len(a); + } + else + { + r = (uint32_t)(total_len % (uint64_t)block_len(a)); + } + uint8_t *buf_1 = buf_; + uint64_t buf[25U] = { 0U }; + Hacl_Streaming_Keccak_hash_buf tmp_block_state = { .fst = a, .snd = buf }; + hash_buf2 scrut = { .fst = block_state, .snd = tmp_block_state }; + uint64_t *s_dst = scrut.snd.snd; + uint64_t *s_src = scrut.fst.snd; + memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); + uint32_t ite0; + if (r % block_len(a) == (uint32_t)0U && r > (uint32_t)0U) + { + ite0 = block_len(a); + } + else + { + ite0 = r % block_len(a); + } + uint8_t *buf_last = buf_1 + r - ite0; + uint8_t *buf_multi = buf_1; + Spec_Hash_Definitions_hash_alg a1 = tmp_block_state.fst; + uint64_t *s0 = tmp_block_state.snd; + Hacl_Hash_SHA3_update_multi_sha3(a1, s0, buf_multi, (uint32_t)0U / block_len(a1)); + Spec_Hash_Definitions_hash_alg a10 = tmp_block_state.fst; + uint64_t *s1 = tmp_block_state.snd; + Hacl_Hash_SHA3_update_last_sha3(a10, s1, buf_last, r); + Spec_Hash_Definitions_hash_alg a11 = tmp_block_state.fst; + uint64_t *s = tmp_block_state.snd; + if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) + { + uint32_t ite; + if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) + { + ite = l; + } + else + { + ite = hash_len(a11); + } + Hacl_Impl_SHA3_squeeze(s, block_len(a11), ite, dst); + return; + } + Hacl_Impl_SHA3_squeeze(s, block_len(a11), hash_len(a11), dst); +} + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + if (a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256) + { + return Hacl_Streaming_Keccak_InvalidAlgorithm; + } + finish_(a1, s, dst, hash_len(a1)); + return Hacl_Streaming_Keccak_Success; +} + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + if (!(a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256)) + { + return Hacl_Streaming_Keccak_InvalidAlgorithm; + } + if (l == (uint32_t)0U) + { + return Hacl_Streaming_Keccak_InvalidLength; + } + finish_(a1, s, dst, l); + return Hacl_Streaming_Keccak_Success; +} + +uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + return block_len(a1); +} + +uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + return hash_len(a1); +} + +bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s) +{ + Spec_Hash_Definitions_hash_alg uu____0 = Hacl_Streaming_Keccak_get_alg(s); + return uu____0 == Spec_Hash_Definitions_Shake128 || uu____0 == Spec_Hash_Definitions_Shake256; +} + +void +Hacl_SHA3_shake128_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1344U, + (uint32_t)256U, + inputByteLen, + input, + (uint8_t)0x1FU, + outputByteLen, + output); +} + +void +Hacl_SHA3_shake256_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1088U, + (uint32_t)512U, + inputByteLen, + input, + (uint8_t)0x1FU, + outputByteLen, + output); +} + +void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1152U, + (uint32_t)448U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)28U, + output); +} + +void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)1088U, + (uint32_t)512U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)32U, + output); +} + +void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)832U, + (uint32_t)768U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)48U, + output); +} + +void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +{ + Hacl_Impl_SHA3_keccak((uint32_t)576U, + (uint32_t)1024U, + inputByteLen, + input, + (uint8_t)0x06U, + (uint32_t)64U, + output); +} + +static const +uint32_t +keccak_rotc[24U] = + { + (uint32_t)1U, (uint32_t)3U, (uint32_t)6U, (uint32_t)10U, (uint32_t)15U, (uint32_t)21U, + (uint32_t)28U, (uint32_t)36U, (uint32_t)45U, (uint32_t)55U, (uint32_t)2U, (uint32_t)14U, + (uint32_t)27U, (uint32_t)41U, (uint32_t)56U, (uint32_t)8U, (uint32_t)25U, (uint32_t)43U, + (uint32_t)62U, (uint32_t)18U, (uint32_t)39U, (uint32_t)61U, (uint32_t)20U, (uint32_t)44U + }; + +static const +uint32_t +keccak_piln[24U] = + { + (uint32_t)10U, (uint32_t)7U, (uint32_t)11U, (uint32_t)17U, (uint32_t)18U, (uint32_t)3U, + (uint32_t)5U, (uint32_t)16U, (uint32_t)8U, (uint32_t)21U, (uint32_t)24U, (uint32_t)4U, + (uint32_t)15U, (uint32_t)23U, (uint32_t)19U, (uint32_t)13U, (uint32_t)12U, (uint32_t)2U, + (uint32_t)20U, (uint32_t)14U, (uint32_t)22U, (uint32_t)9U, (uint32_t)6U, (uint32_t)1U + }; + +static const +uint64_t +keccak_rndc[24U] = + { + (uint64_t)0x0000000000000001U, (uint64_t)0x0000000000008082U, (uint64_t)0x800000000000808aU, + (uint64_t)0x8000000080008000U, (uint64_t)0x000000000000808bU, (uint64_t)0x0000000080000001U, + (uint64_t)0x8000000080008081U, (uint64_t)0x8000000000008009U, (uint64_t)0x000000000000008aU, + (uint64_t)0x0000000000000088U, (uint64_t)0x0000000080008009U, (uint64_t)0x000000008000000aU, + (uint64_t)0x000000008000808bU, (uint64_t)0x800000000000008bU, (uint64_t)0x8000000000008089U, + (uint64_t)0x8000000000008003U, (uint64_t)0x8000000000008002U, (uint64_t)0x8000000000000080U, + (uint64_t)0x000000000000800aU, (uint64_t)0x800000008000000aU, (uint64_t)0x8000000080008081U, + (uint64_t)0x8000000000008080U, (uint64_t)0x0000000080000001U, (uint64_t)0x8000000080008008U + }; + +void Hacl_Impl_SHA3_state_permute(uint64_t *s) +{ + for (uint32_t i0 = (uint32_t)0U; i0 < (uint32_t)24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + _C[i] = + s[i + + (uint32_t)0U] + ^ + (s[i + + (uint32_t)5U] + ^ (s[i + (uint32_t)10U] ^ (s[i + (uint32_t)15U] ^ s[i + (uint32_t)20U])));); + KRML_MAYBE_FOR5(i1, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + uint64_t uu____0 = _C[(i1 + (uint32_t)1U) % (uint32_t)5U]; + uint64_t + _D = + _C[(i1 + (uint32_t)4U) + % (uint32_t)5U] + ^ (uu____0 << (uint32_t)1U | uu____0 >> (uint32_t)63U); + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + s[i1 + (uint32_t)5U * i] = s[i1 + (uint32_t)5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = (uint32_t)0U; i < (uint32_t)24U; i++) + { + uint32_t _Y = keccak_piln[i]; + uint32_t r = keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> ((uint32_t)64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + uint64_t + v0 = + s[(uint32_t)0U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)1U + (uint32_t)5U * i] & s[(uint32_t)2U + (uint32_t)5U * i]); + uint64_t + v1 = + s[(uint32_t)1U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)2U + (uint32_t)5U * i] & s[(uint32_t)3U + (uint32_t)5U * i]); + uint64_t + v2 = + s[(uint32_t)2U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)3U + (uint32_t)5U * i] & s[(uint32_t)4U + (uint32_t)5U * i]); + uint64_t + v3 = + s[(uint32_t)3U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)4U + (uint32_t)5U * i] & s[(uint32_t)0U + (uint32_t)5U * i]); + uint64_t + v4 = + s[(uint32_t)4U + + (uint32_t)5U * i] + ^ (~s[(uint32_t)0U + (uint32_t)5U * i] & s[(uint32_t)1U + (uint32_t)5U * i]); + s[(uint32_t)0U + (uint32_t)5U * i] = v0; + s[(uint32_t)1U + (uint32_t)5U * i] = v1; + s[(uint32_t)2U + (uint32_t)5U * i] = v2; + s[(uint32_t)3U + (uint32_t)5U * i] = v3; + s[(uint32_t)4U + (uint32_t)5U * i] = v4;); + uint64_t c = keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } +} + +void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s) +{ + uint8_t block[200U] = { 0U }; + memcpy(block, input, rateInBytes * sizeof (uint8_t)); + for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + { + uint64_t u = load64_le(block + i * (uint32_t)8U); + uint64_t x = u; + s[i] = s[i] ^ x; + } +} + +static void storeState(uint32_t rateInBytes, uint64_t *s, uint8_t *res) +{ + uint8_t block[200U] = { 0U }; + for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + { + uint64_t sj = s[i]; + store64_le(block + i * (uint32_t)8U, sj); + } + memcpy(res, block, rateInBytes * sizeof (uint8_t)); +} + +void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s) +{ + Hacl_Impl_SHA3_loadState(rateInBytes, block, s); + Hacl_Impl_SHA3_state_permute(s); +} + +static void +absorb( + uint64_t *s, + uint32_t rateInBytes, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix +) +{ + uint32_t n_blocks = inputByteLen / rateInBytes; + uint32_t rem = inputByteLen % rateInBytes; + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint8_t *block = input + i * rateInBytes; + Hacl_Impl_SHA3_absorb_inner(rateInBytes, block, s); + } + uint8_t *last = input + n_blocks * rateInBytes; + uint8_t lastBlock_[200U] = { 0U }; + uint8_t *lastBlock = lastBlock_; + memcpy(lastBlock, last, rem * sizeof (uint8_t)); + lastBlock[rem] = delimitedSuffix; + Hacl_Impl_SHA3_loadState(rateInBytes, lastBlock, s); + if (!((delimitedSuffix & (uint8_t)0x80U) == (uint8_t)0U) && rem == rateInBytes - (uint32_t)1U) + { + Hacl_Impl_SHA3_state_permute(s); + } + uint8_t nextBlock_[200U] = { 0U }; + uint8_t *nextBlock = nextBlock_; + nextBlock[rateInBytes - (uint32_t)1U] = (uint8_t)0x80U; + Hacl_Impl_SHA3_loadState(rateInBytes, nextBlock, s); + Hacl_Impl_SHA3_state_permute(s); +} + +void +Hacl_Impl_SHA3_squeeze( + uint64_t *s, + uint32_t rateInBytes, + uint32_t outputByteLen, + uint8_t *output +) +{ + uint32_t outBlocks = outputByteLen / rateInBytes; + uint32_t remOut = outputByteLen % rateInBytes; + uint8_t *last = output + outputByteLen - remOut; + uint8_t *blocks = output; + for (uint32_t i = (uint32_t)0U; i < outBlocks; i++) + { + storeState(rateInBytes, s, blocks + i * rateInBytes); + Hacl_Impl_SHA3_state_permute(s); + } + storeState(remOut, s, last); +} + +void +Hacl_Impl_SHA3_keccak( + uint32_t rate, + uint32_t capacity, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix, + uint32_t outputByteLen, + uint8_t *output +) +{ + uint32_t rateInBytes = rate / (uint32_t)8U; + uint64_t s[25U] = { 0U }; + absorb(s, rateInBytes, inputByteLen, input, delimitedSuffix); + Hacl_Impl_SHA3_squeeze(s, rateInBytes, outputByteLen, output); +} + diff --git a/Modules/_hacl/Hacl_Hash_SHA3.h b/Modules/_hacl/Hacl_Hash_SHA3.h new file mode 100644 index 00000000000000..2a5cf4b1844b9d --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA3.h @@ -0,0 +1,136 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_SHA3_H +#define __Hacl_Hash_SHA3_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + +typedef struct Hacl_Streaming_Keccak_hash_buf_s +{ + Spec_Hash_Definitions_hash_alg fst; + uint64_t *snd; +} +Hacl_Streaming_Keccak_hash_buf; + +typedef struct Hacl_Streaming_Keccak_state_s +{ + Hacl_Streaming_Keccak_hash_buf block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Streaming_Keccak_state; + +Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s); + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a); + +void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s); + +Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0); + +void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s); + +uint32_t +Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len); + +#define Hacl_Streaming_Keccak_Success 0 +#define Hacl_Streaming_Keccak_InvalidAlgorithm 1 +#define Hacl_Streaming_Keccak_InvalidLength 2 + +typedef uint8_t Hacl_Streaming_Keccak_error_code; + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst); + +Hacl_Streaming_Keccak_error_code +Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l); + +uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s); + +uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s); + +bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s); + +void +Hacl_SHA3_shake128_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +); + +void +Hacl_SHA3_shake256_hacl( + uint32_t inputByteLen, + uint8_t *input, + uint32_t outputByteLen, + uint8_t *output +); + +void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output); + +void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s); + +void +Hacl_Impl_SHA3_squeeze( + uint64_t *s, + uint32_t rateInBytes, + uint32_t outputByteLen, + uint8_t *output +); + +void +Hacl_Impl_SHA3_keccak( + uint32_t rate, + uint32_t capacity, + uint32_t inputByteLen, + uint8_t *input, + uint8_t delimitedSuffix, + uint32_t outputByteLen, + uint8_t *output +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_SHA3_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Streaming_Types.h b/Modules/_hacl/Hacl_Streaming_Types.h index 51057611ca978d..8a60b707bc4958 100644 --- a/Modules/_hacl/Hacl_Streaming_Types.h +++ b/Modules/_hacl/Hacl_Streaming_Types.h @@ -35,6 +35,23 @@ extern "C" { #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" +#define Spec_Hash_Definitions_SHA2_224 0 +#define Spec_Hash_Definitions_SHA2_256 1 +#define Spec_Hash_Definitions_SHA2_384 2 +#define Spec_Hash_Definitions_SHA2_512 3 +#define Spec_Hash_Definitions_SHA1 4 +#define Spec_Hash_Definitions_MD5 5 +#define Spec_Hash_Definitions_Blake2S 6 +#define Spec_Hash_Definitions_Blake2B 7 +#define Spec_Hash_Definitions_SHA3_256 8 +#define Spec_Hash_Definitions_SHA3_224 9 +#define Spec_Hash_Definitions_SHA3_384 10 +#define Spec_Hash_Definitions_SHA3_512 11 +#define Spec_Hash_Definitions_Shake128 12 +#define Spec_Hash_Definitions_Shake256 13 + +typedef uint8_t Spec_Hash_Definitions_hash_alg; + typedef struct Hacl_Streaming_MD_state_32_s { uint32_t *block_state; diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index dcbe7007b17be8..5a2f94eb2ec8da 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -19,6 +19,28 @@ # define inline __inline__ #endif +/******************************************************************************/ +/* Macros that KaRaMeL will generate. */ +/******************************************************************************/ + +/* For "bare" targets that do not have a C stdlib, the user might want to use + * [-add-early-include '"mydefinitions.h"'] and override these. */ +#ifndef KRML_HOST_PRINTF +# define KRML_HOST_PRINTF printf +#endif + +#if ( \ + (defined __STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + (!(defined KRML_HOST_EPRINTF))) +# define KRML_HOST_EPRINTF(...) fprintf(stderr, __VA_ARGS__) +#elif !(defined KRML_HOST_EPRINTF) && defined(_MSC_VER) +# define KRML_HOST_EPRINTF(...) fprintf(stderr, __VA_ARGS__) +#endif + +#ifndef KRML_HOST_EXIT +# define KRML_HOST_EXIT exit +#endif + #ifndef KRML_HOST_MALLOC # define KRML_HOST_MALLOC malloc #endif @@ -35,6 +57,28 @@ # define KRML_HOST_IGNORE(x) (void)(x) #endif +/* In FStar.Buffer.fst, the size of arrays is uint32_t, but it's a number of + * *elements*. Do an ugly, run-time check (some of which KaRaMeL can eliminate). + */ +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 4)) +# define _KRML_CHECK_SIZE_PRAGMA \ + _Pragma("GCC diagnostic ignored \"-Wtype-limits\"") +#else +# define _KRML_CHECK_SIZE_PRAGMA +#endif + +#define KRML_CHECK_SIZE(size_elt, sz) \ + do { \ + _KRML_CHECK_SIZE_PRAGMA \ + if (((size_t)(sz)) > ((size_t)(SIZE_MAX / (size_elt)))) { \ + KRML_HOST_PRINTF( \ + "Maximum allocatable size exceeded, aborting before overflow at " \ + "%s:%d\n", \ + __FILE__, __LINE__); \ + KRML_HOST_EXIT(253); \ + } \ + } while (0) + /* Macros for prettier unrolling of loops */ #define KRML_LOOP1(i, n, x) { \ x \ diff --git a/Modules/_hacl/include/krml/lowstar_endianness.h b/Modules/_hacl/include/krml/lowstar_endianness.h index 32a7391e817ebb..1aa2ccd644c06f 100644 --- a/Modules/_hacl/include/krml/lowstar_endianness.h +++ b/Modules/_hacl/include/krml/lowstar_endianness.h @@ -77,7 +77,7 @@ # define le64toh(x) (x) /* ... for Windows (GCC-like, e.g. mingw or clang) */ -#elif (defined(_WIN32) || defined(_WIN64)) && \ +#elif (defined(_WIN32) || defined(_WIN64) || defined(__EMSCRIPTEN__)) && \ (defined(__GNUC__) || defined(__clang__)) # define htobe16(x) __builtin_bswap16(x) @@ -96,7 +96,8 @@ # define le64toh(x) (x) /* ... generic big-endian fallback code */ -#elif defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +/* ... AIX doesn't have __BYTE_ORDER__ (with XLC compiler) & is always big-endian */ +#elif (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || defined(_AIX) /* byte swapping code inspired by: * https://github.com/rweather/arduinolibs/blob/master/libraries/Crypto/utility/EndianUtil.h diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA3.h b/Modules/_hacl/internal/Hacl_Hash_SHA3.h new file mode 100644 index 00000000000000..1c9808b8dd497c --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_SHA3.h @@ -0,0 +1,65 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_SHA3_H +#define __internal_Hacl_Hash_SHA3_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "../Hacl_Hash_SHA3.h" + +void +Hacl_Hash_SHA3_update_multi_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *blocks, + uint32_t n_blocks +); + +void +Hacl_Hash_SHA3_update_last_sha3( + Spec_Hash_Definitions_hash_alg a, + uint64_t *s, + uint8_t *input, + uint32_t input_len +); + +void Hacl_Impl_SHA3_state_permute(uint64_t *s); + +void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_SHA3_H_DEFINED +#endif diff --git a/Modules/_hacl/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h index ee28f244266b85..0df236282ac509 100644 --- a/Modules/_hacl/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -59,5 +59,28 @@ #define Hacl_Streaming_SHA1_legacy_copy python_hashlib_Hacl_Streaming_SHA1_legacy_copy #define Hacl_Streaming_SHA1_legacy_hash python_hashlib_Hacl_Streaming_SHA1_legacy_hash +#define Hacl_Hash_SHA3_update_last_sha3 python_hashlib_Hacl_Hash_SHA3_update_last_sha3 +#define Hacl_Hash_SHA3_update_multi_sha3 python_hashlib_Hacl_Hash_SHA3_update_multi_sha3 +#define Hacl_Impl_SHA3_absorb_inner python_hashlib_Hacl_Impl_SHA3_absorb_inner +#define Hacl_Impl_SHA3_keccak python_hashlib_Hacl_Impl_SHA3_keccak +#define Hacl_Impl_SHA3_loadState python_hashlib_Hacl_Impl_SHA3_loadState +#define Hacl_Impl_SHA3_squeeze python_hashlib_Hacl_Impl_SHA3_squeeze +#define Hacl_Impl_SHA3_state_permute python_hashlib_Hacl_Impl_SHA3_state_permute +#define Hacl_SHA3_sha3_224 python_hashlib_Hacl_SHA3_sha3_224 +#define Hacl_SHA3_sha3_256 python_hashlib_Hacl_SHA3_sha3_256 +#define Hacl_SHA3_sha3_384 python_hashlib_Hacl_SHA3_sha3_384 +#define Hacl_SHA3_sha3_512 python_hashlib_Hacl_SHA3_sha3_512 +#define Hacl_SHA3_shake128_hacl python_hashlib_Hacl_SHA3_shake128_hacl +#define Hacl_SHA3_shake256_hacl python_hashlib_Hacl_SHA3_shake256_hacl +#define Hacl_Streaming_Keccak_block_len python_hashlib_Hacl_Streaming_Keccak_block_len +#define Hacl_Streaming_Keccak_copy python_hashlib_Hacl_Streaming_Keccak_copy +#define Hacl_Streaming_Keccak_finish python_hashlib_Hacl_Streaming_Keccak_finish +#define Hacl_Streaming_Keccak_free python_hashlib_Hacl_Streaming_Keccak_free +#define Hacl_Streaming_Keccak_get_alg python_hashlib_Hacl_Streaming_Keccak_get_alg +#define Hacl_Streaming_Keccak_hash_len python_hashlib_Hacl_Streaming_Keccak_hash_len +#define Hacl_Streaming_Keccak_is_shake python_hashlib_Hacl_Streaming_Keccak_is_shake +#define Hacl_Streaming_Keccak_malloc python_hashlib_Hacl_Streaming_Keccak_malloc +#define Hacl_Streaming_Keccak_reset python_hashlib_Hacl_Streaming_Keccak_reset +#define Hacl_Streaming_Keccak_update python_hashlib_Hacl_Streaming_Keccak_update #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index 76b92ec4599102..220ebbe5561341 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=13e0c6721ac9206c4249ecc1dc04ed617ad1e262 +expected_hacl_star_rev=363eae2c2eb60e46f182ddd4bd1cd3f1d00b35c9 hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" @@ -45,11 +45,14 @@ dist_files=( Hacl_Hash_SHA1.h internal/Hacl_Hash_SHA1.h Hacl_Hash_MD5.h + Hacl_Hash_SHA3.h internal/Hacl_Hash_MD5.h + internal/Hacl_Hash_SHA3.h internal/Hacl_SHA2_Generic.h Hacl_Streaming_SHA2.c Hacl_Hash_SHA1.c Hacl_Hash_MD5.c + Hacl_Hash_SHA3.c ) declare -a include_files @@ -134,9 +137,9 @@ $sed -i -z 's!#include \n!#include \n#include "python_hacl_n # Finally, we remove a bunch of ifdefs from target.h that are, again, useful in # the general case, but not exercised by the subset of HACL* that we vendor. -$sed -z -i 's!#ifndef KRML_\(HOST_PRINTF\|HOST_EXIT\|PRE_ALIGN\|POST_ALIGN\|ALIGNED_MALLOC\|ALIGNED_FREE\|HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h -$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#define KRML_\(EABORT\|EXIT\|CHECK_SIZE\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h +$sed -z -i 's!#ifndef KRML_\(PRE_ALIGN\|POST_ALIGN\|ALIGNED_MALLOC\|ALIGNED_FREE\|HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h +$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#define KRML_\(EABORT\|EXIT\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h $sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*#define KRML_\(EABORT\|EXIT\|CHECK_SIZE\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h -$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*# define _\?KRML_\(DEPRECATED\|CHECK_SIZE_PRAGMA\|HOST_EPRINTF\|HOST_SNPRINTF\)[^\n]*\n\([^#][^\n]*\n\|#el[^\n]*\n\|# [^\n]*\n\)*#endif!!g' include/krml/internal/target.h +$sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*# define _\?KRML_\(DEPRECATED\|HOST_SNPRINTF\)[^\n]*\n\([^#][^\n]*\n\|#el[^\n]*\n\|# [^\n]*\n\)*#endif!!g' include/krml/internal/target.h echo "Updated; verify all is okay using git diff and git status." diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 8ec3a6081c98d9..c05407b5d61815 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -321,7 +321,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, #ifdef HAVE_WINDOWS_CONSOLE_IO const PyConfig *config = _Py_GetConfig(); if (!config->legacy_windows_stdio && _PyIO_get_console_type(path_or_fd) != '\0') { - RawIO_class = (PyObject *)&PyWindowsConsoleIO_Type; + RawIO_class = (PyObject *)state->PyWindowsConsoleIO_Type; encoding = "utf-8"; } #endif @@ -580,17 +580,24 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { _PyIO_State *state = get_io_state(mod); if (!state->initialized) return 0; - Py_VISIT(state->locale_module); Py_VISIT(state->unsupported_operation); + Py_VISIT(state->PyIncrementalNewlineDecoder_Type); + Py_VISIT(state->PyRawIOBase_Type); + Py_VISIT(state->PyBufferedIOBase_Type); Py_VISIT(state->PyBufferedRWPair_Type); Py_VISIT(state->PyBufferedRandom_Type); Py_VISIT(state->PyBufferedReader_Type); Py_VISIT(state->PyBufferedWriter_Type); + Py_VISIT(state->PyBytesIOBuffer_Type); Py_VISIT(state->PyBytesIO_Type); Py_VISIT(state->PyFileIO_Type); Py_VISIT(state->PyStringIO_Type); + Py_VISIT(state->PyTextIOBase_Type); Py_VISIT(state->PyTextIOWrapper_Type); +#ifdef HAVE_WINDOWS_CONSOLE_IO + Py_VISIT(state->PyWindowsConsoleIO_Type); +#endif return 0; } @@ -600,18 +607,24 @@ iomodule_clear(PyObject *mod) { _PyIO_State *state = get_io_state(mod); if (!state->initialized) return 0; - if (state->locale_module != NULL) - Py_CLEAR(state->locale_module); Py_CLEAR(state->unsupported_operation); + Py_CLEAR(state->PyIncrementalNewlineDecoder_Type); + Py_CLEAR(state->PyRawIOBase_Type); + Py_CLEAR(state->PyBufferedIOBase_Type); Py_CLEAR(state->PyBufferedRWPair_Type); Py_CLEAR(state->PyBufferedRandom_Type); Py_CLEAR(state->PyBufferedReader_Type); Py_CLEAR(state->PyBufferedWriter_Type); + Py_CLEAR(state->PyBytesIOBuffer_Type); Py_CLEAR(state->PyBytesIO_Type); Py_CLEAR(state->PyFileIO_Type); Py_CLEAR(state->PyStringIO_Type); + Py_CLEAR(state->PyTextIOBase_Type); Py_CLEAR(state->PyTextIOWrapper_Type); +#ifdef HAVE_WINDOWS_CONSOLE_IO + Py_CLEAR(state->PyWindowsConsoleIO_Type); +#endif return 0; } @@ -653,31 +666,17 @@ struct PyModuleDef _PyIO_Module = { static PyTypeObject* static_types[] = { // Base classes &PyIOBase_Type, - &PyIncrementalNewlineDecoder_Type, // PyIOBase_Type subclasses &PyBufferedIOBase_Type, &PyRawIOBase_Type, &PyTextIOBase_Type, - - // PyRawIOBase_Type(PyIOBase_Type) subclasses - &_PyBytesIOBuffer_Type, -#ifdef HAVE_WINDOWS_CONSOLE_IO - &PyWindowsConsoleIO_Type, -#endif }; PyStatus _PyIO_InitTypes(PyInterpreterState *interp) { -#ifdef HAVE_WINDOWS_CONSOLE_IO - if (_Py_IsMainInterpreter(interp)) { - // Set type base classes - PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type; - } -#endif - for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; if (_PyStaticType_InitBuiltin(interp, type) < 0) { @@ -749,24 +748,37 @@ PyInit__io(void) } } + // Base classes + ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); + ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, NULL); + + // PyIOBase_Type subclasses + state->PyRawIOBase_Type = (PyTypeObject *)Py_NewRef(&PyRawIOBase_Type); + state->PyBufferedIOBase_Type = (PyTypeObject *)Py_NewRef(&PyBufferedIOBase_Type); + state->PyTextIOBase_Type = (PyTypeObject *)Py_NewRef(&PyTextIOBase_Type); + // PyBufferedIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, &PyBufferedIOBase_Type); + ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedWriter_Type, &bufferedwriter_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedReader_Type, &bufferedreader_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedRWPair_Type, &bufferedrwpair_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); ADD_TYPE(m, state->PyBufferedRandom_Type, &bufferedrandom_spec, - &PyBufferedIOBase_Type); + state->PyBufferedIOBase_Type); // PyRawIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, &PyRawIOBase_Type); + ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, state->PyRawIOBase_Type); +#ifdef MS_WINDOWS + ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &winconsoleio_spec, + state->PyRawIOBase_Type); +#endif // PyTextIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, &PyTextIOBase_Type); + ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, state->PyTextIOBase_Type); ADD_TYPE(m, state->PyTextIOWrapper_Type, &textiowrapper_spec, - &PyTextIOBase_Type); + state->PyTextIOBase_Type); state->initialized = 1; diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index d7224e56f9a722..1bf301c9cf0a94 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -5,6 +5,7 @@ #include "exports.h" #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "structmember.h" /* ABCs */ @@ -13,22 +14,21 @@ extern PyTypeObject PyRawIOBase_Type; extern PyTypeObject PyBufferedIOBase_Type; extern PyTypeObject PyTextIOBase_Type; -/* Concrete classes */ -extern PyTypeObject PyIncrementalNewlineDecoder_Type; - /* Type specs */ extern PyType_Spec bufferedrandom_spec; extern PyType_Spec bufferedreader_spec; extern PyType_Spec bufferedrwpair_spec; extern PyType_Spec bufferedwriter_spec; extern PyType_Spec bytesio_spec; +extern PyType_Spec bytesiobuf_spec; extern PyType_Spec fileio_spec; +extern PyType_Spec nldecoder_spec; extern PyType_Spec stringio_spec; extern PyType_Spec textiowrapper_spec; #ifdef HAVE_WINDOWS_CONSOLE_IO -extern PyTypeObject PyWindowsConsoleIO_Type; -#endif /* HAVE_WINDOWS_CONSOLE_IO */ +extern PyType_Spec winconsoleio_spec; +#endif /* These functions are used as METH_NOARGS methods, are normally called * with args=NULL, and return a new reference. @@ -142,19 +142,25 @@ extern PyModuleDef _PyIO_Module; typedef struct { int initialized; - PyObject *locale_module; - PyObject *unsupported_operation; /* Types */ + PyTypeObject *PyIncrementalNewlineDecoder_Type; + PyTypeObject *PyRawIOBase_Type; + PyTypeObject *PyBufferedIOBase_Type; PyTypeObject *PyBufferedRWPair_Type; PyTypeObject *PyBufferedRandom_Type; PyTypeObject *PyBufferedReader_Type; PyTypeObject *PyBufferedWriter_Type; + PyTypeObject *PyBytesIOBuffer_Type; PyTypeObject *PyBytesIO_Type; PyTypeObject *PyFileIO_Type; PyTypeObject *PyStringIO_Type; + PyTypeObject *PyTextIOBase_Type; PyTypeObject *PyTextIOWrapper_Type; +#ifdef MS_WINDOWS + PyTypeObject *PyWindowsConsoleIO_Type; +#endif } _PyIO_State; #define IO_MOD_STATE(mod) ((_PyIO_State *)PyModule_GetState(mod)) @@ -168,6 +174,14 @@ get_io_state(PyObject *module) return (_PyIO_State *)state; } +static inline _PyIO_State * +get_io_state_by_cls(PyTypeObject *cls) +{ + void *state = _PyType_GetModuleState(cls); + assert(state != NULL); + return (_PyIO_State *)state; +} + static inline _PyIO_State * find_io_state_by_def(PyTypeObject *type) { @@ -181,5 +195,3 @@ extern _PyIO_State *_PyIO_get_module_state(void); #ifdef HAVE_WINDOWS_CONSOLE_IO extern char _PyIO_get_console_type(PyObject *); #endif - -extern Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 2c71768be97870..723d16b47fef9f 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -2231,7 +2231,7 @@ bufferedrwpair_close(rwpair *self, PyObject *Py_UNUSED(ignored)) } else { Py_DECREF(ret); - } + } ret = _forward_call(self->reader, &_Py_ID(close), NULL); if (exc != NULL) { _PyErr_ChainExceptions1(exc); diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 7e9d28b3b9655c..3fddfc2ed0bc9c 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -308,14 +308,18 @@ _io_BytesIO_flush_impl(bytesio *self) /*[clinic input] _io.BytesIO.getbuffer + cls: defining_class + / + Get a read-write view over the contents of the BytesIO object. [clinic start generated code]*/ static PyObject * -_io_BytesIO_getbuffer_impl(bytesio *self) -/*[clinic end generated code: output=72cd7c6e13aa09ed input=8f738ef615865176]*/ +_io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls) +/*[clinic end generated code: output=045091d7ce87fe4e input=0668fbb48f95dffa]*/ { - PyTypeObject *type = &_PyBytesIOBuffer_Type; + _PyIO_State *state = get_io_state_by_cls(cls); + PyTypeObject *type = state->PyBytesIOBuffer_Type; bytesiobuf *buf; PyObject *view; @@ -1086,9 +1090,17 @@ bytesiobuf_releasebuffer(bytesiobuf *obj, Py_buffer *view) b->exports--; } +static int +bytesiobuf_clear(bytesiobuf *self) +{ + Py_CLEAR(self->source); + return 0; +} + static int bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->source); return 0; } @@ -1096,54 +1108,29 @@ bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg) static void bytesiobuf_dealloc(bytesiobuf *self) { + PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - Py_CLEAR(self->source); - Py_TYPE(self)->tp_free(self); + (void)bytesiobuf_clear(self); + tp->tp_free(self); + Py_DECREF(tp); } -static PyBufferProcs bytesiobuf_as_buffer = { - (getbufferproc) bytesiobuf_getbuffer, - (releasebufferproc) bytesiobuf_releasebuffer, +static PyType_Slot bytesiobuf_slots[] = { + {Py_tp_dealloc, bytesiobuf_dealloc}, + {Py_tp_traverse, bytesiobuf_traverse}, + {Py_tp_clear, bytesiobuf_clear}, + + // Buffer protocol + {Py_bf_getbuffer, bytesiobuf_getbuffer}, + {Py_bf_releasebuffer, bytesiobuf_releasebuffer}, + {0, NULL}, }; -Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._BytesIOBuffer", /*tp_name*/ - sizeof(bytesiobuf), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)bytesiobuf_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - &bytesiobuf_as_buffer, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - 0, /*tp_doc*/ - (traverseproc)bytesiobuf_traverse, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - 0, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - 0, /*tp_new*/ +PyType_Spec bytesiobuf_spec = { + .name = "_io._BytesIOBuffer", + .basicsize = sizeof(bytesiobuf), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), + .slots = bytesiobuf_slots, }; diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 84b58db6c7a702..9550c8728c251e 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -87,15 +87,19 @@ PyDoc_STRVAR(_io_BytesIO_getbuffer__doc__, "Get a read-write view over the contents of the BytesIO object."); #define _IO_BYTESIO_GETBUFFER_METHODDEF \ - {"getbuffer", (PyCFunction)_io_BytesIO_getbuffer, METH_NOARGS, _io_BytesIO_getbuffer__doc__}, + {"getbuffer", _PyCFunction_CAST(_io_BytesIO_getbuffer), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_BytesIO_getbuffer__doc__}, static PyObject * -_io_BytesIO_getbuffer_impl(bytesio *self); +_io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * -_io_BytesIO_getbuffer(bytesio *self, PyObject *Py_UNUSED(ignored)) +_io_BytesIO_getbuffer(bytesio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io_BytesIO_getbuffer_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); + return NULL; + } + return _io_BytesIO_getbuffer_impl(self, cls); } PyDoc_STRVAR(_io_BytesIO_getvalue__doc__, @@ -534,4 +538,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=a44770efbaeb80dd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=098584d485420b65 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index b6e9bd5b65a029..dfad8a58c4723e 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -18,15 +18,19 @@ PyDoc_STRVAR(_io_FileIO_close__doc__, "called more than once without error."); #define _IO_FILEIO_CLOSE_METHODDEF \ - {"close", (PyCFunction)_io_FileIO_close, METH_NOARGS, _io_FileIO_close__doc__}, + {"close", _PyCFunction_CAST(_io_FileIO_close), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_close__doc__}, static PyObject * -_io_FileIO_close_impl(fileio *self); +_io_FileIO_close_impl(fileio *self, PyTypeObject *cls); static PyObject * -_io_FileIO_close(fileio *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_close(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io_FileIO_close_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); + return NULL; + } + return _io_FileIO_close_impl(self, cls); } PyDoc_STRVAR(_io_FileIO___init____doc__, @@ -466,4 +470,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=27f883807a6c29ae input=a9049054013a1b77]*/ +/*[clinic end generated code: output=29ed2ae6c451c139 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 1118d86e6c9a10..cc0e7307b9da77 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -130,6 +130,9 @@ internal_close(fileio *self) /*[clinic input] _io.FileIO.close + cls: defining_class + / + Close the file. A closed file cannot be used for further I/O operations. close() may be @@ -137,18 +140,20 @@ called more than once without error. [clinic start generated code]*/ static PyObject * -_io_FileIO_close_impl(fileio *self) -/*[clinic end generated code: output=7737a319ef3bad0b input=f35231760d54a522]*/ +_io_FileIO_close_impl(fileio *self, PyTypeObject *cls) +/*[clinic end generated code: output=c30cbe9d1f23ca58 input=70da49e63db7c64d]*/ { PyObject *res; - PyObject *exc; int rc; - res = PyObject_CallMethodOneArg((PyObject*)&PyRawIOBase_Type, + _PyIO_State *state = get_io_state_by_cls(cls); + res = PyObject_CallMethodOneArg((PyObject*)state->PyRawIOBase_Type, &_Py_ID(close), (PyObject *)self); if (!self->closefd) { self->fd = -1; return res; } + + PyObject *exc; if (res == NULL) { exc = PyErr_GetRaisedException(); } diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 54c050f0be4688..13d3b870b39a81 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -716,9 +716,10 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, self->writenl = Py_NewRef(self->readnl); } + _PyIO_State *module_state = find_io_state_by_def(Py_TYPE(self)); if (self->readuniversal) { self->decoder = PyObject_CallFunctionObjArgs( - (PyObject *)&PyIncrementalNewlineDecoder_Type, + (PyObject *)module_state->PyIncrementalNewlineDecoder_Type, Py_None, self->readtranslate ? Py_True : Py_False, NULL); if (self->decoder == NULL) return -1; @@ -750,7 +751,7 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, self->state = STATE_ACCUMULATING; } self->pos = 0; - self->module_state = find_io_state_by_def(Py_TYPE(self)); + self->module_state = module_state; self->closed = 0; self->ok = 1; return 0; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 3ff84cb623af74..070687a83d1bc2 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -18,10 +18,10 @@ /*[clinic input] module _io -class _io.IncrementalNewlineDecoder "nldecoder_object *" "&PyIncrementalNewlineDecoder_Type" +class _io.IncrementalNewlineDecoder "nldecoder_object *" "clinic_state()->PyIncrementalNewlineDecoder_Type" class _io.TextIOWrapper "textio *" "clinic_state()->TextIOWrapper_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d3f032e90f74c8f2]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=81f67cf54eaa6001]*/ /* TextIOBase */ @@ -248,12 +248,32 @@ _io_IncrementalNewlineDecoder___init___impl(nldecoder_object *self, return 0; } -static void -incrementalnewlinedecoder_dealloc(nldecoder_object *self) +static int +incrementalnewlinedecoder_traverse(nldecoder_object *self, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->decoder); + Py_VISIT(self->errors); + return 0; +} + +static int +incrementalnewlinedecoder_clear(nldecoder_object *self) { Py_CLEAR(self->decoder); Py_CLEAR(self->errors); - Py_TYPE(self)->tp_free((PyObject *)self); + return 0; +} + +static void +incrementalnewlinedecoder_dealloc(nldecoder_object *self) +{ + PyTypeObject *tp = Py_TYPE(self); + _PyObject_GC_UNTRACK(self); + (void)incrementalnewlinedecoder_clear(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } static int @@ -872,8 +892,9 @@ _textiowrapper_set_decoder(textio *self, PyObject *codec_info, return -1; if (self->readuniversal) { + _PyIO_State *state = self->state; PyObject *incrementalDecoder = PyObject_CallFunctionObjArgs( - (PyObject *)&PyIncrementalNewlineDecoder_Type, + (PyObject *)state->PyIncrementalNewlineDecoder_Type, self->decoder, self->readtranslate ? Py_True : Py_False, NULL); if (incrementalDecoder == NULL) return -1; @@ -884,11 +905,12 @@ _textiowrapper_set_decoder(textio *self, PyObject *codec_info, } static PyObject* -_textiowrapper_decode(PyObject *decoder, PyObject *bytes, int eof) +_textiowrapper_decode(_PyIO_State *state, PyObject *decoder, PyObject *bytes, + int eof) { PyObject *chars; - if (Py_IS_TYPE(decoder, &PyIncrementalNewlineDecoder_Type)) + if (Py_IS_TYPE(decoder, state->PyIncrementalNewlineDecoder_Type)) chars = _PyIncrementalNewlineDecoder_decode(decoder, bytes, eof); else chars = PyObject_CallMethodObjArgs(decoder, &_Py_ID(decode), bytes, @@ -1167,6 +1189,8 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, self->buffer = Py_NewRef(buffer); /* Build the decoder object */ + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + self->state = state; if (_textiowrapper_set_decoder(self, codec_info, PyUnicode_AsUTF8(errors)) != 0) goto error; @@ -1177,7 +1201,6 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, /* Finished sorting out the codec details */ Py_CLEAR(codec_info); - _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); if (Py_IS_TYPE(buffer, state->PyBufferedReader_Type) || Py_IS_TYPE(buffer, state->PyBufferedWriter_Type) || Py_IS_TYPE(buffer, state->PyBufferedRandom_Type)) @@ -1214,7 +1237,6 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, goto error; } - self->state = state; self->ok = 1; return 0; @@ -1843,7 +1865,8 @@ textiowrapper_read_chunk(textio *self, Py_ssize_t size_hint) nbytes = input_chunk_buf.len; eof = (nbytes == 0); - decoded_chars = _textiowrapper_decode(self->decoder, input_chunk, eof); + decoded_chars = _textiowrapper_decode(self->state, self->decoder, + input_chunk, eof); PyBuffer_Release(&input_chunk_buf); if (decoded_chars == NULL) goto fail; @@ -1913,7 +1936,8 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) if (bytes == NULL) goto fail; - if (Py_IS_TYPE(self->decoder, &PyIncrementalNewlineDecoder_Type)) + _PyIO_State *state = self->state; + if (Py_IS_TYPE(self->decoder, state->PyIncrementalNewlineDecoder_Type)) decoded = _PyIncrementalNewlineDecoder_decode(self->decoder, bytes, 1); else @@ -3172,45 +3196,23 @@ static PyGetSetDef incrementalnewlinedecoder_getset[] = { {NULL} }; -PyTypeObject PyIncrementalNewlineDecoder_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.IncrementalNewlineDecoder", /*tp_name*/ - sizeof(nldecoder_object), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)incrementalnewlinedecoder_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - _io_IncrementalNewlineDecoder___init____doc__, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /*tp_weaklistoffset*/ - 0, /* tp_iter */ - 0, /* tp_iternext */ - incrementalnewlinedecoder_methods, /* tp_methods */ - 0, /* tp_members */ - incrementalnewlinedecoder_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - _io_IncrementalNewlineDecoder___init__, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ +static PyType_Slot nldecoder_slots[] = { + {Py_tp_dealloc, incrementalnewlinedecoder_dealloc}, + {Py_tp_doc, (void *)_io_IncrementalNewlineDecoder___init____doc__}, + {Py_tp_methods, incrementalnewlinedecoder_methods}, + {Py_tp_getset, incrementalnewlinedecoder_getset}, + {Py_tp_traverse, incrementalnewlinedecoder_traverse}, + {Py_tp_clear, incrementalnewlinedecoder_clear}, + {Py_tp_init, _io_IncrementalNewlineDecoder___init__}, + {0, NULL}, +}; + +PyType_Spec nldecoder_spec = { + .name = "_io.IncrementalNewlineDecoder", + .basicsize = sizeof(nldecoder_object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = nldecoder_slots, }; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index f836e230243020..fdb57cff7c04d6 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -137,9 +137,9 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { /*[clinic input] module _io -class _io._WindowsConsoleIO "winconsoleio *" "&PyWindowsConsoleIO_Type" +class _io._WindowsConsoleIO "winconsoleio *" "clinic_state()->PyWindowsConsoleIO_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e897fdc1fba4e131]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=05526e723011ab36]*/ typedef struct { PyObject_HEAD @@ -156,8 +156,6 @@ typedef struct { wchar_t wbuf; } winconsoleio; -PyTypeObject PyWindowsConsoleIO_Type; - int _PyWindowsConsoleIO_closed(PyObject *self) { @@ -265,7 +263,10 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, int fd_is_own = 0; HANDLE handle = NULL; - assert(PyObject_TypeCheck(self, (PyTypeObject *)&PyWindowsConsoleIO_Type)); +#ifdef Py_DEBUG + _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); + assert(PyObject_TypeCheck(self, state->PyWindowsConsoleIO_Type)); +#endif if (self->fd >= 0) { if (self->closefd) { /* Have to close the existing file first. */ @@ -417,6 +418,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, static int winconsoleio_traverse(winconsoleio *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } @@ -431,6 +433,7 @@ winconsoleio_clear(winconsoleio *self) static void winconsoleio_dealloc(winconsoleio *self) { + PyTypeObject *tp = Py_TYPE(self); self->finalizing = 1; if (_PyIOBase_finalize((PyObject *) self) < 0) return; @@ -438,7 +441,8 @@ winconsoleio_dealloc(winconsoleio *self) if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *)self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } static PyObject * @@ -1078,7 +1082,9 @@ _io__WindowsConsoleIO_isatty_impl(winconsoleio *self) Py_RETURN_TRUE; } +#define clinic_state() (IO_STATE()) #include "clinic/winconsoleio.c.h" +#undef clinic_state static PyMethodDef winconsoleio_methods[] = { _IO__WINDOWSCONSOLEIO_READ_METHODDEF @@ -1124,59 +1130,32 @@ static PyGetSetDef winconsoleio_getsetlist[] = { static PyMemberDef winconsoleio_members[] = { {"_blksize", T_UINT, offsetof(winconsoleio, blksize), 0}, {"_finalizing", T_BOOL, offsetof(winconsoleio, finalizing), 0}, + {"__weaklistoffset__", T_PYSSIZET, offsetof(winconsoleio, weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(winconsoleio, dict), READONLY}, {NULL} }; -PyTypeObject PyWindowsConsoleIO_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._WindowsConsoleIO", - sizeof(winconsoleio), - 0, - (destructor)winconsoleio_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)winconsoleio_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - _io__WindowsConsoleIO___init____doc__, /* tp_doc */ - (traverseproc)winconsoleio_traverse, /* tp_traverse */ - (inquiry)winconsoleio_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(winconsoleio, weakreflist), /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - winconsoleio_methods, /* tp_methods */ - winconsoleio_members, /* tp_members */ - winconsoleio_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(winconsoleio, dict), /* tp_dictoffset */ - _io__WindowsConsoleIO___init__, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - winconsoleio_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ +static PyType_Slot winconsoleio_slots[] = { + {Py_tp_dealloc, winconsoleio_dealloc}, + {Py_tp_repr, winconsoleio_repr}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_doc, (void *)_io__WindowsConsoleIO___init____doc__}, + {Py_tp_traverse, winconsoleio_traverse}, + {Py_tp_clear, winconsoleio_clear}, + {Py_tp_methods, winconsoleio_methods}, + {Py_tp_members, winconsoleio_members}, + {Py_tp_getset, winconsoleio_getsetlist}, + {Py_tp_init, _io__WindowsConsoleIO___init__}, + {Py_tp_new, winconsoleio_new}, + {0, NULL}, +}; + +PyType_Spec winconsoleio_spec = { + .name = "_io._WindowsConsoleIO", + .basicsize = sizeof(winconsoleio), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = winconsoleio_slots, }; #endif /* HAVE_WINDOWS_CONSOLE_IO */ diff --git a/Modules/_sha3/LICENSE b/Modules/_sha3/LICENSE deleted file mode 100644 index d2d484d8820dcf..00000000000000 --- a/Modules/_sha3/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Markku-Juhani O. Saarinen - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/Modules/_sha3/README.txt b/Modules/_sha3/README.txt deleted file mode 100644 index b35919b01677d3..00000000000000 --- a/Modules/_sha3/README.txt +++ /dev/null @@ -1,8 +0,0 @@ -tiny_sha3 -========= - -https://github.com/mjosaarinen/tiny_sha3 -commit dcbb3192047c2a721f5f851db591871d428036a9 - -- All functions have been converted to static functions. -- sha3() function is commented out. diff --git a/Modules/_sha3/sha3.c b/Modules/_sha3/sha3.c deleted file mode 100644 index e2d3fd7b8ad855..00000000000000 --- a/Modules/_sha3/sha3.c +++ /dev/null @@ -1,193 +0,0 @@ -// sha3.c -// 19-Nov-11 Markku-Juhani O. Saarinen - -// Revised 07-Aug-15 to match with official release of FIPS PUB 202 "SHA3" -// Revised 03-Sep-15 for portability + OpenSSL - style API - -#include "sha3.h" - -// update the state with given number of rounds - -static void sha3_keccakf(uint64_t st[25]) -{ - // constants - const uint64_t keccakf_rndc[24] = { - 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, - 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, - 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, - 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, - 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, - 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, - 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, - 0x8000000000008080, 0x0000000080000001, 0x8000000080008008 - }; - const int keccakf_rotc[24] = { - 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, - 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 - }; - const int keccakf_piln[24] = { - 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, - 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 - }; - - // variables - int i, j, r; - uint64_t t, bc[5]; - -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ - uint8_t *v; - - // endianess conversion. this is redundant on little-endian targets - for (i = 0; i < 25; i++) { - v = (uint8_t *) &st[i]; - st[i] = ((uint64_t) v[0]) | (((uint64_t) v[1]) << 8) | - (((uint64_t) v[2]) << 16) | (((uint64_t) v[3]) << 24) | - (((uint64_t) v[4]) << 32) | (((uint64_t) v[5]) << 40) | - (((uint64_t) v[6]) << 48) | (((uint64_t) v[7]) << 56); - } -#endif - - // actual iteration - for (r = 0; r < KECCAKF_ROUNDS; r++) { - - // Theta - for (i = 0; i < 5; i++) - bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15] ^ st[i + 20]; - - for (i = 0; i < 5; i++) { - t = bc[(i + 4) % 5] ^ ROTL64(bc[(i + 1) % 5], 1); - for (j = 0; j < 25; j += 5) - st[j + i] ^= t; - } - - // Rho Pi - t = st[1]; - for (i = 0; i < 24; i++) { - j = keccakf_piln[i]; - bc[0] = st[j]; - st[j] = ROTL64(t, keccakf_rotc[i]); - t = bc[0]; - } - - // Chi - for (j = 0; j < 25; j += 5) { - for (i = 0; i < 5; i++) - bc[i] = st[j + i]; - for (i = 0; i < 5; i++) - st[j + i] ^= (~bc[(i + 1) % 5]) & bc[(i + 2) % 5]; - } - - // Iota - st[0] ^= keccakf_rndc[r]; - } - -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ - // endianess conversion. this is redundant on little-endian targets - for (i = 0; i < 25; i++) { - v = (uint8_t *) &st[i]; - t = st[i]; - v[0] = t & 0xFF; - v[1] = (t >> 8) & 0xFF; - v[2] = (t >> 16) & 0xFF; - v[3] = (t >> 24) & 0xFF; - v[4] = (t >> 32) & 0xFF; - v[5] = (t >> 40) & 0xFF; - v[6] = (t >> 48) & 0xFF; - v[7] = (t >> 56) & 0xFF; - } -#endif -} - -// Initialize the context for SHA3 - -static int sha3_init(sha3_ctx_t *c, int mdlen) -{ - int i; - - for (i = 0; i < 25; i++) - c->st.q[i] = 0; - c->mdlen = mdlen; - c->rsiz = 200 - 2 * mdlen; - c->pt = 0; - - return 1; -} - -// update state with more data - -static int sha3_update(sha3_ctx_t *c, const void *data, size_t len) -{ - size_t i; - int j; - - j = c->pt; - for (i = 0; i < len; i++) { - c->st.b[j++] ^= ((const uint8_t *) data)[i]; - if (j >= c->rsiz) { - sha3_keccakf(c->st.q); - j = 0; - } - } - c->pt = j; - - return 1; -} - -// finalize and output a hash - -static int sha3_final(void *md, sha3_ctx_t *c) -{ - int i; - - c->st.b[c->pt] ^= 0x06; - c->st.b[c->rsiz - 1] ^= 0x80; - sha3_keccakf(c->st.q); - - for (i = 0; i < c->mdlen; i++) { - ((uint8_t *) md)[i] = c->st.b[i]; - } - - return 1; -} - -#if 0 -// compute a SHA-3 hash (md) of given byte length from "in" - -void *sha3(const void *in, size_t inlen, void *md, int mdlen) -{ - sha3_ctx_t sha3; - - sha3_init(&sha3, mdlen); - sha3_update(&sha3, in, inlen); - sha3_final(md, &sha3); - - return md; -} -#endif - -// SHAKE128 and SHAKE256 extensible-output functionality - -static void shake_xof(sha3_ctx_t *c) -{ - c->st.b[c->pt] ^= 0x1F; - c->st.b[c->rsiz - 1] ^= 0x80; - sha3_keccakf(c->st.q); - c->pt = 0; -} - -static void shake_out(sha3_ctx_t *c, void *out, size_t len) -{ - size_t i; - int j; - - j = c->pt; - for (i = 0; i < len; i++) { - if (j >= c->rsiz) { - sha3_keccakf(c->st.q); - j = 0; - } - ((uint8_t *) out)[i] = c->st.b[j++]; - } - c->pt = j; -} - diff --git a/Modules/_sha3/sha3.h b/Modules/_sha3/sha3.h deleted file mode 100644 index f973d6733ec2cc..00000000000000 --- a/Modules/_sha3/sha3.h +++ /dev/null @@ -1,49 +0,0 @@ -// sha3.h -// 19-Nov-11 Markku-Juhani O. Saarinen - -#ifndef SHA3_H -#define SHA3_H - -#include -#include - -#ifndef KECCAKF_ROUNDS -#define KECCAKF_ROUNDS 24 -#endif - -#ifndef ROTL64 -#define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y)))) -#endif - -// state context -typedef struct { - union { // state: - uint8_t b[200]; // 8-bit bytes - uint64_t q[25]; // 64-bit words - } st; - int pt, rsiz, mdlen; // these don't overflow -} sha3_ctx_t; - -// Compression function. -static void sha3_keccakf(uint64_t st[25]); - -// OpenSSL - like interfece -static int sha3_init(sha3_ctx_t *c, int mdlen); // mdlen = hash output in bytes -static int sha3_update(sha3_ctx_t *c, const void *data, size_t len); -static int sha3_final(void *md, sha3_ctx_t *c); // digest goes to md - -// compute a sha3 hash (md) of given byte length from "in" -#if 0 -static void *sha3(const void *in, size_t inlen, void *md, int mdlen); -#endif - -// SHAKE128 and SHAKE256 extensible-output functions -#define shake128_init(c) sha3_init(c, 16) -#define shake256_init(c) sha3_init(c, 32) -#define shake_update sha3_update - -static void shake_xof(sha3_ctx_t *c); -static void shake_out(sha3_ctx_t *c, void *out, size_t len); - -#endif - diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 182754cca36d61..417abcc4626170 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -228,7 +228,7 @@ PyDoc_STRVAR(blobopen__doc__, static PyObject * blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, - int row, int readonly, const char *name); + sqlite3_int64 row, int readonly, const char *name); static PyObject * blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -263,7 +263,7 @@ blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyO Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; const char *table; const char *col; - int row; + sqlite3_int64 row; int readonly = 0; const char *name = "main"; @@ -297,8 +297,7 @@ blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyO PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - row = _PyLong_AsInt(args[2]); - if (row == -1 && PyErr_Occurred()) { + if (!sqlite3_int64_converter(args[2], &row)) { goto exit; } if (!noptargs) { @@ -1666,4 +1665,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=8b03149c115ee6da input=a9049054013a1b77]*/ +/*[clinic end generated code: output=834a99827555bf1a input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index aec3aa8bbf4ed8..7bbb462ed54dfa 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -118,6 +118,20 @@ autocommit_converter(PyObject *val, enum autocommit_mode *result) return 0; } +static int +sqlite3_int64_converter(PyObject *obj, sqlite3_int64 *result) +{ + if (!PyLong_Check(obj)) { + PyErr_SetString(PyExc_TypeError, "expected 'int'"); + return 0; + } + *result = _pysqlite_long_as_int64(obj); + if (PyErr_Occurred()) { + return 0; + } + return 1; +} + #define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/connection.c.h" #undef clinic_state @@ -188,8 +202,12 @@ class Autocommit_converter(CConverter): type = "enum autocommit_mode" converter = "autocommit_converter" +class sqlite3_int64_converter(CConverter): + type = "sqlite3_int64" + converter = "sqlite3_int64_converter" + [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=bc2aa6c7ba0c5f8f]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=dff8760fb1eba6a1]*/ // NB: This needs to be in sync with the sqlite3.connect docstring /*[clinic input] @@ -483,7 +501,7 @@ _sqlite3.Connection.blobopen as blobopen Table name. column as col: str Column name. - row: int + row: sqlite3_int64 Row index. / * @@ -497,8 +515,8 @@ Open and return a BLOB object. static PyObject * blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, - int row, int readonly, const char *name) -/*[clinic end generated code: output=0c8e2e58516d0b5c input=fa73c83aa7a7ddee]*/ + sqlite3_int64 row, int readonly, const char *name) +/*[clinic end generated code: output=6a02d43efb885d1c input=23576bd1108d8774]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; diff --git a/Modules/_testcapi/immortal.c b/Modules/_testcapi/immortal.c index 10e1733d08a9ea..9f81389811c645 100644 --- a/Modules/_testcapi/immortal.c +++ b/Modules/_testcapi/immortal.c @@ -1,6 +1,6 @@ #include "parts.h" -int verify_immortality(PyObject *object) +int verify_immortality(PyObject *object) { assert(_Py_IsImmortal(object)); Py_ssize_t old_count = Py_REFCNT(object); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index fbe0c2bbdf8d32..70ef17e38a1f9c 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -593,17 +593,19 @@ _testinternalcapi.compiler_codegen -> object ast: object filename: object optimize: int + compile_mode: int = 0 Apply compiler code generation to an AST. [clinic start generated code]*/ static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, - PyObject *filename, int optimize) -/*[clinic end generated code: output=fbbbbfb34700c804 input=e9fbe6562f7f75e4]*/ + PyObject *filename, int optimize, + int compile_mode) +/*[clinic end generated code: output=40a68f6e13951cc8 input=a0e00784f1517cd7]*/ { PyCompilerFlags *flags = NULL; - return _PyCompile_CodeGen(ast, filename, flags, optimize); + return _PyCompile_CodeGen(ast, filename, flags, optimize, compile_mode); } diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 58b064bb17cd87..ca71b6156b005d 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -681,6 +681,27 @@ PyInit__testmultiphase_export_unreported_exception(void) return PyModuleDef_Init(&main_def); } +static PyObject* +createfunc_noop(PyObject *spec, PyModuleDef *def) +{ + return PyModule_New("spam"); +} + +static PyModuleDef_Slot slots_multiple_create_slots[] = { + {Py_mod_create, createfunc_noop}, + {Py_mod_create, createfunc_noop}, + {0, NULL}, +}; + +static PyModuleDef def_multiple_create_slots = TEST_MODULE_DEF( + "_testmultiphase_multiple_create_slots", slots_multiple_create_slots, NULL); + +PyMODINIT_FUNC +PyInit__testmultiphase_multiple_create_slots(void) +{ + return PyModuleDef_Init(&def_multiple_create_slots); +} + static PyObject* createfunc_null(PyObject *spec, PyModuleDef *def) { @@ -892,7 +913,24 @@ PyInit__test_module_state_shared(void) } -/* multiple interpreters supports */ +/* multiple interpreters support */ + +static PyModuleDef_Slot slots_multiple_multiple_interpreters_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL}, +}; + +static PyModuleDef def_multiple_multiple_interpreters_slots = TEST_MODULE_DEF( + "_testmultiphase_multiple_multiple_interpreters_slots", + slots_multiple_multiple_interpreters_slots, + NULL); + +PyMODINIT_FUNC +PyInit__testmultiphase_multiple_multiple_interpreters_slots(void) +{ + return PyModuleDef_Init(&def_multiple_multiple_interpreters_slots); +} static PyModuleDef_Slot non_isolated_slots[] = { {Py_mod_exec, execfunc}, @@ -909,3 +947,23 @@ PyInit__test_non_isolated(void) { return PyModuleDef_Init(&non_isolated_def); } + + +static PyModuleDef_Slot shared_gil_only_slots[] = { + {Py_mod_exec, execfunc}, + /* Note that Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED is the default. + We put it here explicitly to draw attention to the contrast + with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. */ + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, + {0, NULL}, +}; + +static PyModuleDef shared_gil_only_def = TEST_MODULE_DEF("_test_shared_gil_only", + shared_gil_only_slots, + testexport_methods); + +PyMODINIT_FUNC +PyInit__test_shared_gil_only(void) +{ + return PyModuleDef_Init(&shared_gil_only_def); +} diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 89573222572594..41dd50437956c4 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -9,7 +9,7 @@ preserve PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, -"compiler_codegen($module, /, ast, filename, optimize)\n" +"compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" "\n" "Apply compiler code generation to an AST."); @@ -19,7 +19,8 @@ PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, - PyObject *filename, int optimize); + PyObject *filename, int optimize, + int compile_mode); static PyObject * _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -27,14 +28,14 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(ast), &_Py_ID(filename), &_Py_ID(optimize), }, + .ob_item = { &_Py_ID(ast), &_Py_ID(filename), &_Py_ID(optimize), &_Py_ID(compile_mode), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -43,19 +44,21 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"ast", "filename", "optimize", NULL}; + static const char * const _keywords[] = {"ast", "filename", "optimize", "compile_mode", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "compiler_codegen", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *ast; PyObject *filename; int optimize; + int compile_mode = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 4, 0, argsbuf); if (!args) { goto exit; } @@ -65,7 +68,15 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s if (optimize == -1 && PyErr_Occurred()) { goto exit; } - return_value = _testinternalcapi_compiler_codegen_impl(module, ast, filename, optimize); + if (!noptargs) { + goto skip_optional_pos; + } + compile_mode = _PyLong_AsInt(args[3]); + if (compile_mode == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = _testinternalcapi_compiler_codegen_impl(module, ast, filename, optimize, compile_mode); exit: return return_value; @@ -190,4 +201,4 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=d5e08c9d67f9721f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ab661d56a14b1a1c input=a9049054013a1b77]*/ diff --git a/Modules/_sha3/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h similarity index 98% rename from Modules/_sha3/clinic/sha3module.c.h rename to Modules/clinic/sha3module.c.h index a0c7c1c043e515..299803a3420bf6 100644 --- a/Modules/_sha3/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -12,7 +12,7 @@ PyDoc_STRVAR(py_sha3_new__doc__, "sha3_224(data=b\'\', /, *, usedforsecurity=True)\n" "--\n" "\n" -"Return a new BLAKE2b hash object."); +"Return a new SHA3 hash object."); static PyObject * py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity); @@ -193,4 +193,4 @@ _sha3_shake_128_hexdigest(SHA3object *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=747c3f34ddd14063 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=907cb475f3dc9ee0 input=a9049054013a1b77]*/ diff --git a/Modules/_sha3/sha3module.c b/Modules/sha3module.c similarity index 75% rename from Modules/_sha3/sha3module.c rename to Modules/sha3module.c index 93abc3b2710e55..f05187498a19b3 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/sha3module.c @@ -22,23 +22,9 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "../hashlib.h" - -#include "sha3.c" +#include "hashlib.h" #define SHA3_MAX_DIGESTSIZE 64 /* 64 Bytes (512 Bits) for 224 to 512 */ -#define SHA3_LANESIZE 0 -#define SHA3_state sha3_ctx_t -#define SHA3_init sha3_init -#define SHA3_process sha3_update -#define SHA3_done(state, digest) sha3_final(digest, state) -#define SHA3_squeeze(state, out, len) shake_xof(state), shake_out(state, out, len) -#define SHA3_copystate(dest, src) memcpy(&(dest), &(src), sizeof(SHA3_state)) - -// no optimization -#define KeccakOpt 0 - -typedef enum { SUCCESS = 1, FAIL = 0, BAD_HASHLEN = 2 } HashReturn; typedef struct { PyTypeObject *sha3_224_type; @@ -70,10 +56,11 @@ class _sha3.shake_256 "SHA3object *" "&SHAKE256type" /* The structure for storing SHA3 info */ +#include "_hacl/Hacl_Hash_SHA3.h" + typedef struct { PyObject_HEAD - SHA3_state hash_state; - PyThread_type_lock lock; + Hacl_Streaming_Keccak_state *hash_state; } SHA3object; #include "clinic/sha3module.c.h" @@ -86,10 +73,23 @@ newSHA3object(PyTypeObject *type) if (newobj == NULL) { return NULL; } - newobj->lock = NULL; return newobj; } +static void sha3_update(Hacl_Streaming_Keccak_state *state, uint8_t *buf, Py_ssize_t len) { + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to hash more than 2^64 bytes. */ +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_Keccak_update(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ + Hacl_Streaming_Keccak_update(state, buf, (uint32_t) len); +} + /*[clinic input] @classmethod _sha3.sha3_224.__new__ as py_sha3_new @@ -98,14 +98,13 @@ _sha3.sha3_224.__new__ as py_sha3_new * usedforsecurity: bool = True -Return a new BLAKE2b hash object. +Return a new SHA3 hash object. [clinic start generated code]*/ static PyObject * py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) -/*[clinic end generated code: output=90409addc5d5e8b0 input=bcfcdf2e4368347a]*/ +/*[clinic end generated code: output=90409addc5d5e8b0 input=637e5f8f6a93982a]*/ { - HashReturn res; Py_buffer buf = {NULL, NULL}; SHA3State *state = _PyType_GetModuleState(type); SHA3object *self = newSHA3object(type); @@ -116,49 +115,29 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) assert(state != NULL); if (type == state->sha3_224_type) { - res = sha3_init(&self->hash_state, 28); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_224); } else if (type == state->sha3_256_type) { - res = sha3_init(&self->hash_state, 32); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_256); } else if (type == state->sha3_384_type) { - res = sha3_init(&self->hash_state, 48); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_384); } else if (type == state->sha3_512_type) { - res = sha3_init(&self->hash_state, 64); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_512); } else if (type == state->shake_128_type) { - res = sha3_init(&self->hash_state, 16); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake128); } else if (type == state->shake_256_type) { - res = sha3_init(&self->hash_state, 32); + self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake256); } else { PyErr_BadInternalCall(); goto error; } - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 initialize()"); - goto error; - } - if (data) { GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* invariant: New objects can't be accessed by other code yet, - * thus it's safe to release the GIL without locking the object. - */ - Py_BEGIN_ALLOW_THREADS - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - } - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 Update()"); - goto error; - } - PyBuffer_Release(&buf); + sha3_update(self->hash_state, buf.buf, buf.len); } + PyBuffer_Release(&buf); + return (PyObject *)self; error: @@ -177,10 +156,7 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) static void SHA3_dealloc(SHA3object *self) { - if (self->lock) { - PyThread_free_lock(self->lock); - } - + Hacl_Streaming_Keccak_free(self->hash_state); PyTypeObject *tp = Py_TYPE(self); PyObject_Free(self); Py_DECREF(tp); @@ -205,9 +181,7 @@ _sha3_sha3_224_copy_impl(SHA3object *self) if ((newobj = newSHA3object(Py_TYPE(self))) == NULL) { return NULL; } - ENTER_HASHLIB(self); - SHA3_copystate(newobj->hash_state, self->hash_state); - LEAVE_HASHLIB(self); + newobj->hash_state = Hacl_Streaming_Keccak_copy(self->hash_state); return (PyObject *)newobj; } @@ -222,20 +196,12 @@ static PyObject * _sha3_sha3_224_digest_impl(SHA3object *self) /*[clinic end generated code: output=fd531842e20b2d5b input=5b2a659536bbd248]*/ { - unsigned char digest[SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE]; - SHA3_state temp; - HashReturn res; - - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - res = SHA3_done(&temp, digest); - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, "internal error in SHA3 Final()"); - return NULL; - } + unsigned char digest[SHA3_MAX_DIGESTSIZE]; + // This function errors out if the algorithm is Shake. Here, we know this + // not to be the case, and therefore do not perform error checking. + Hacl_Streaming_Keccak_finish(self->hash_state, digest); return PyBytes_FromStringAndSize((const char *)digest, - self->hash_state.mdlen); + Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -249,21 +215,10 @@ static PyObject * _sha3_sha3_224_hexdigest_impl(SHA3object *self) /*[clinic end generated code: output=75ad03257906918d input=2d91bb6e0d114ee3]*/ { - unsigned char digest[SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE]; - SHA3_state temp; - HashReturn res; - - /* Get the raw (binary) digest value */ - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - res = SHA3_done(&temp, digest); - if (res != SUCCESS) { - PyErr_SetString(PyExc_RuntimeError, "internal error in SHA3 Final()"); - return NULL; - } + unsigned char digest[SHA3_MAX_DIGESTSIZE]; + Hacl_Streaming_Keccak_finish(self->hash_state, digest); return _Py_strhex((const char *)digest, - self->hash_state.mdlen); + Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -281,36 +236,8 @@ _sha3_sha3_224_update(SHA3object *self, PyObject *data) /*[clinic end generated code: output=d3223352286ed357 input=a887f54dcc4ae227]*/ { Py_buffer buf; - HashReturn res; - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - - /* add new data, the function takes the length in bits not bytes */ - if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { - self->lock = PyThread_allocate_lock(); - } - /* Once a lock exists all code paths must be synchronized. We have to - * release the GIL even for small buffers as acquiring the lock may take - * an unlimited amount of time when another thread updates this object - * with lots of data. */ - if (self->lock) { - Py_BEGIN_ALLOW_THREADS - PyThread_acquire_lock(self->lock, 1); - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - PyThread_release_lock(self->lock); - Py_END_ALLOW_THREADS - } - else { - res = SHA3_process(&self->hash_state, buf.buf, buf.len); - } - - if (res != SUCCESS) { - PyBuffer_Release(&buf); - PyErr_SetString(PyExc_RuntimeError, - "internal error in SHA3 Update()"); - return NULL; - } - + sha3_update(self->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); Py_RETURN_NONE; } @@ -328,7 +255,7 @@ static PyMethodDef SHA3_methods[] = { static PyObject * SHA3_get_block_size(SHA3object *self, void *closure) { - int rate = self->hash_state.rsiz; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state); return PyLong_FromLong(rate); } @@ -363,14 +290,19 @@ SHA3_get_name(SHA3object *self, void *closure) static PyObject * SHA3_get_digest_size(SHA3object *self, void *closure) { - return PyLong_FromLong(self->hash_state.mdlen); + // Preserving previous behavior: variable-length algorithms return 0 + if (Hacl_Streaming_Keccak_is_shake(self->hash_state)) + return PyLong_FromLong(0); + else + return PyLong_FromLong(Hacl_Streaming_Keccak_hash_len(self->hash_state)); } static PyObject * SHA3_get_capacity_bits(SHA3object *self, void *closure) { - int capacity = 1600 - self->hash_state.rsiz * 8; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; + int capacity = 1600 - rate; return PyLong_FromLong(capacity); } @@ -378,7 +310,7 @@ SHA3_get_capacity_bits(SHA3object *self, void *closure) static PyObject * SHA3_get_rate_bits(SHA3object *self, void *closure) { - unsigned int rate = self->hash_state.rsiz * 8; + uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; return PyLong_FromLong(rate); } @@ -455,28 +387,26 @@ static PyObject * _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) { unsigned char *digest = NULL; - SHA3_state temp; PyObject *result = NULL; if (digestlen >= (1 << 29)) { PyErr_SetString(PyExc_ValueError, "length is too large"); return NULL; } - /* ExtractLane needs at least SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE and - * SHA3_LANESIZE extra space. - */ - digest = (unsigned char*)PyMem_Malloc(digestlen + SHA3_LANESIZE); + digest = (unsigned char*)PyMem_Malloc(digestlen); if (digest == NULL) { return PyErr_NoMemory(); } - /* Get the raw (binary) digest value */ - ENTER_HASHLIB(self); - SHA3_copystate(temp, self->hash_state); - LEAVE_HASHLIB(self); - SHA3_squeeze(&temp, digest, digestlen); + /* Get the raw (binary) digest value. The HACL functions errors out if: + * - the algorith is not shake -- not the case here + * - the output length is zero -- we follow the existing behavior and return + * an empty digest, without raising an error */ + if (digestlen > 0) { + Hacl_Streaming_Keccak_squeeze(self->hash_state, digest, digestlen); + } if (hex) { - result = _Py_strhex((const char *)digest, digestlen); + result = _Py_strhex((const char *)digest, digestlen); } else { result = PyBytes_FromStringAndSize((const char *)digest, digestlen); @@ -628,11 +558,8 @@ _sha3_exec(PyObject *m) init_sha3type(shake_256_type, SHAKE256_spec); #undef init_sha3type - if (PyModule_AddIntConstant(m, "keccakopt", KeccakOpt) < 0) { - return -1; - } if (PyModule_AddStringConstant(m, "implementation", - "tiny_sha3") < 0) { + "HACL") < 0) { return -1; } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 60219593be61e2..c11fb4400eab2f 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -6883,8 +6883,10 @@ socket_getnameinfo(PyObject *self, PyObject *args) } #endif } + Py_BEGIN_ALLOW_THREADS error = getnameinfo(res->ai_addr, (socklen_t) res->ai_addrlen, hbuf, sizeof(hbuf), pbuf, sizeof(pbuf), flags); + Py_END_ALLOW_THREADS if (error) { socket_state *state = get_module_state(self); set_gaierror(state, error); diff --git a/Modules/termios.c b/Modules/termios.c index 169a36fc6477d8..6dc8200572bc0c 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -85,7 +85,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) int r; Py_BEGIN_ALLOW_THREADS - r = tcgetattr(fd, &mode); + r = tcgetattr(fd, &mode); Py_END_ALLOW_THREADS if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); @@ -372,7 +372,7 @@ termios_tcgetwinsize_impl(PyObject *module, int fd) #if defined(TIOCGWINSZ) termiosmodulestate *state = PyModule_GetState(module); struct winsize w; - int r; + int r; Py_BEGIN_ALLOW_THREADS r = ioctl(fd, TIOCGWINSZ, &w); diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 49d4dd524696a5..c36db59baaa10d 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -61,6 +61,7 @@ static void bytearray_releasebuffer(PyByteArrayObject *obj, Py_buffer *view) { obj->ob_exports--; + assert(obj->ob_exports >= 0); } static int diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index e7e85cc19cda75..abbf3eeb16c35c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -423,9 +423,6 @@ formatfloat(PyObject *v, int flags, int prec, int type, if (flags & F_ALT) { dtoa_flags |= Py_DTSF_ALT; } - if (flags & F_NO_NEG_0) { - dtoa_flags |= Py_DTSF_NO_NEG_0; - } p = PyOS_double_to_string(x, type, prec, dtoa_flags, NULL); if (p == NULL) diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index f008a8cc3e0474..b0168044d9f85a 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -193,6 +193,20 @@ PyTypeObject _PyManagedBuffer_Type = { return -1; \ } +#define CHECK_RESTRICTED(mv) \ + if (((PyMemoryViewObject *)(mv))->flags & _Py_MEMORYVIEW_RESTRICTED) { \ + PyErr_SetString(PyExc_ValueError, \ + "cannot create new view on restricted memoryview"); \ + return NULL; \ + } + +#define CHECK_RESTRICTED_INT(mv) \ + if (((PyMemoryViewObject *)(mv))->flags & _Py_MEMORYVIEW_RESTRICTED) { \ + PyErr_SetString(PyExc_ValueError, \ + "cannot create new view on restricted memoryview"); \ + return -1; \ + } + /* See gh-92888. These macros signal that we need to check the memoryview again due to possible read after frees. */ #define CHECK_RELEASED_AGAIN(mv) CHECK_RELEASED(mv) @@ -781,7 +795,7 @@ PyMemoryView_FromBuffer(const Py_buffer *info) using the given flags. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ -PyObject * +static PyObject * PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) { _PyManagedBufferObject *mbuf; @@ -789,6 +803,7 @@ PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) if (PyMemoryView_Check(v)) { PyMemoryViewObject *mv = (PyMemoryViewObject *)v; CHECK_RELEASED(mv); + CHECK_RESTRICTED(mv); return mbuf_add_view(mv->mbuf, &mv->view); } else if (PyObject_CheckBuffer(v)) { @@ -806,6 +821,30 @@ PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) Py_TYPE(v)->tp_name); return NULL; } + +/* Create a memoryview from an object that implements the buffer protocol, + using the given flags. + If the object is a memoryview, the new memoryview must be registered + with the same managed buffer. Otherwise, a new managed buffer is created. */ +PyObject * +_PyMemoryView_FromBufferProc(PyObject *v, int flags, getbufferproc bufferproc) +{ + _PyManagedBufferObject *mbuf = mbuf_alloc(); + if (mbuf == NULL) + return NULL; + + int res = bufferproc(v, &mbuf->master, flags); + if (res < 0) { + mbuf->master.obj = NULL; + Py_DECREF(mbuf); + return NULL; + } + + PyObject *ret = mbuf_add_view(mbuf, NULL); + Py_DECREF(mbuf); + return ret; +} + /* Create a memoryview from an object that implements the buffer protocol. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ @@ -1397,6 +1436,7 @@ memoryview_cast_impl(PyMemoryViewObject *self, PyObject *format, Py_ssize_t ndim = 1; CHECK_RELEASED(self); + CHECK_RESTRICTED(self); if (!MV_C_CONTIGUOUS(self->flags)) { PyErr_SetString(PyExc_TypeError, @@ -1452,6 +1492,7 @@ memoryview_toreadonly_impl(PyMemoryViewObject *self) /*[clinic end generated code: output=2c7e056f04c99e62 input=dc06d20f19ba236f]*/ { CHECK_RELEASED(self); + CHECK_RESTRICTED(self); /* Even if self is already readonly, we still need to create a new * object for .release() to work correctly. */ @@ -1474,6 +1515,7 @@ memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) int baseflags = self->flags; CHECK_RELEASED_INT(self); + CHECK_RESTRICTED_INT(self); /* start with complete information */ *view = *base; @@ -2535,6 +2577,7 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) return memory_item(self, index); } else if (PySlice_Check(key)) { + CHECK_RESTRICTED(self); PyMemoryViewObject *sliced; sliced = (PyMemoryViewObject *)mbuf_add_view(self->mbuf, view); diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index c100d018d3f0df..985be58d02c784 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -323,7 +323,13 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } } - // XXX Do a similar check once we have PyInterpreterState.ceval.own_gil. + else if (multiple_interpreters != Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + && interp->ceval.own_gil + && !_Py_IsMainInterpreter(interp) + && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) + { + goto error; + } if (create) { m = create(spec, def); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 456b10ee01d6bc..98fac276a873e1 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6,7 +6,7 @@ #include "pycore_symtable.h" // _Py_Mangle() #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_memoryobject.h" // PyMemoryView_FromObjectAndFlags() +#include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_long.h" // _PyLong_IsNegative() @@ -56,6 +56,11 @@ typedef struct PySlot_Offset { short slot_offset; } PySlot_Offset; +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer); + +static void +releasebuffer_call_python(PyObject *self, Py_buffer *buffer); static PyObject * slot_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds); @@ -8078,7 +8083,8 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) return NULL; } - return PyMemoryView_FromObjectAndFlags(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int)); + return _PyMemoryView_FromBufferProc(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int), + (getbufferproc)wrapped); } static PyObject * @@ -8094,6 +8100,10 @@ wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) return NULL; } PyMemoryViewObject *mview = (PyMemoryViewObject *)arg; + if (mview->view.obj == NULL) { + // Already released, ignore + Py_RETURN_NONE; + } if (mview->view.obj != self) { PyErr_SetString(PyExc_ValueError, "memoryview's buffer is not this object"); @@ -8978,12 +8988,26 @@ bufferwrapper_releasebuf(PyObject *self, Py_buffer *view) { PyBufferWrapper *bw = (PyBufferWrapper *)self; - assert(PyMemoryView_Check(bw->mv)); - Py_TYPE(bw->mv)->tp_as_buffer->bf_releasebuffer(bw->mv, view); - if (Py_TYPE(bw->obj)->tp_as_buffer != NULL - && Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer != NULL) { - Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer(bw->obj, view); + if (bw->mv == NULL || bw->obj == NULL) { + // Already released + return; + } + + PyObject *mv = bw->mv; + PyObject *obj = bw->obj; + + assert(PyMemoryView_Check(mv)); + Py_TYPE(mv)->tp_as_buffer->bf_releasebuffer(mv, view); + // We only need to call bf_releasebuffer if it's a Python function. If it's a C + // bf_releasebuf, it will be called when the memoryview is released. + if (((PyMemoryViewObject *)mv)->view.obj != obj + && Py_TYPE(obj)->tp_as_buffer != NULL + && Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer == slot_bf_releasebuffer) { + releasebuffer_call_python(obj, view); } + + Py_CLEAR(bw->mv); + Py_CLEAR(bw->obj); } static PyBufferProcs bufferwrapper_as_buffer = { @@ -9047,31 +9071,112 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) return -1; } +static int +releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) +{ + PyTypeObject *self_type = Py_TYPE(self); + PyObject *mro = lookup_tp_mro(self_type); + if (mro == NULL) { + return -1; + } + + assert(PyTuple_Check(mro)); + Py_ssize_t n = PyTuple_GET_SIZE(mro); + Py_ssize_t i; + + /* No need to check the last one: it's gonna be skipped anyway. */ + for (i = 0; i < n -1; i++) { + if ((PyObject *)(self_type) == PyTuple_GET_ITEM(mro, i)) + break; + } + i++; /* skip self_type */ + if (i >= n) + return -1; + + releasebufferproc base_releasebuffer = NULL; + for (; i < n; i++) { + PyObject *obj = PyTuple_GET_ITEM(mro, i); + if (!PyType_Check(obj)) { + continue; + } + PyTypeObject *base_type = (PyTypeObject *)obj; + if (base_type->tp_as_buffer != NULL + && base_type->tp_as_buffer->bf_releasebuffer != NULL + && base_type->tp_as_buffer->bf_releasebuffer != slot_bf_releasebuffer) { + base_releasebuffer = base_type->tp_as_buffer->bf_releasebuffer; + break; + } + } + + if (base_releasebuffer != NULL) { + base_releasebuffer(self, buffer); + } + return 0; +} + static void -slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +releasebuffer_call_python(PyObject *self, Py_buffer *buffer) { PyObject *mv; - if (Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type) { + bool is_buffer_wrapper = Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type; + if (is_buffer_wrapper) { // Make sure we pass the same memoryview to // __release_buffer__() that __buffer__() returned. - mv = Py_NewRef(((PyBufferWrapper *)buffer->obj)->mv); + PyBufferWrapper *bw = (PyBufferWrapper *)buffer->obj; + if (bw->mv == NULL) { + return; + } + mv = Py_NewRef(bw->mv); } else { + // This means we are not dealing with a memoryview returned + // from a Python __buffer__ function. mv = PyMemoryView_FromBuffer(buffer); if (mv == NULL) { PyErr_WriteUnraisable(self); return; } + // Set the memoryview to restricted mode, which forbids + // users from saving any reference to the underlying buffer + // (e.g., by doing .cast()). This is necessary to ensure + // no Python code retains a reference to the to-be-released + // buffer. + ((PyMemoryViewObject *)mv)->flags |= _Py_MEMORYVIEW_RESTRICTED; } PyObject *stack[2] = {self, mv}; PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); - Py_DECREF(mv); if (ret == NULL) { PyErr_WriteUnraisable(self); } else { Py_DECREF(ret); } + if (!is_buffer_wrapper) { + PyObject_CallMethodNoArgs(mv, &_Py_ID(release)); + } + Py_DECREF(mv); +} + +/* + * bf_releasebuffer is very delicate, because we need to ensure that + * C bf_releasebuffer slots are called correctly (or we'll leak memory), + * but we cannot trust any __release_buffer__ implemented in Python to + * do so correctly. Therefore, if a base class has a C bf_releasebuffer + * slot, we call it directly here. That is safe because this function + * only gets called from C callers of the bf_releasebuffer slot. Python + * code that calls __release_buffer__ directly instead goes through + * wrap_releasebuffer(), which doesn't call the bf_releasebuffer slot + * directly but instead simply releases the associated memoryview. + */ +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +{ + releasebuffer_call_python(self, buffer); + if (releasebuffer_maybe_call_super(self, buffer) < 0) { + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + } } static PyObject * diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1585a582f00aad..7726f2fb17afde 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13452,8 +13452,6 @@ formatfloat(PyObject *v, struct unicode_format_arg_t *arg, if (arg->flags & F_ALT) dtoa_flags |= Py_DTSF_ALT; - if (arg->flags & F_NO_NEG_0) - dtoa_flags |= Py_DTSF_NO_NEG_0; p = PyOS_double_to_string(x, arg->ch, prec, dtoa_flags, NULL); if (p == NULL) return -1; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 8aafcb786a6064..28b1517c6f6b3a 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -375,13 +375,16 @@ + + + + - @@ -404,17 +407,15 @@ - - - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 07476f30833372..75e6fbb13f98ba 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -776,6 +776,15 @@ Modules + + Modules + + + Modules + + + Modules + Modules @@ -794,9 +803,6 @@ Modules - - Modules - Modules @@ -875,16 +881,13 @@ Modules - - Modules - Modules - + Modules - + Modules diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 52d0d9a534cb6a..91ffabac56c7b3 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -48,7 +48,7 @@ static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) { } static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) { assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); + assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]); } #else diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 8270fa8e372d93..3883ec9e21c765 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -317,7 +317,6 @@ simple_format_arg_parse(PyObject *fmt, Py_ssize_t *ppos, case ' ': *flags |= F_BLANK; continue; case '#': *flags |= F_ALT; continue; case '0': *flags |= F_ZERO; continue; - case 'z': *flags |= F_NO_NEG_0; continue; } break; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 8840bbabe4b584..ddddc03ca316e0 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3014,9 +3014,16 @@ static PyMethodDef builtin_methods[] = { }; PyDoc_STRVAR(builtin_doc, -"Built-in functions, exceptions, and other objects.\n\ +"Built-in functions, types, exceptions, and other objects.\n\ \n\ -Noteworthy: None is the `nil' object; Ellipsis represents `...' in slices."); +This module provides direct access to all 'built-in'\n\ +identifiers of Python; for example, builtins.len is\n\ +the full name for the built-in function len().\n\ +\n\ +This module is not normally accessed explicitly by most\n\ +applications, but can be useful in modules that provide\n\ +objects with the same name as a built-in value, but in\n\ +which the built-in of that name is also needed."); static struct PyModuleDef builtinsmodule = { PyModuleDef_HEAD_INIT, diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index a390bec80d556c..42e1436bc9130d 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -464,8 +464,7 @@ take_gil(PyThreadState *tstate) void _PyEval_SetSwitchInterval(unsigned long microseconds) { - /* XXX per-interpreter GIL */ - PyInterpreterState *interp = _PyInterpreterState_Main(); + PyInterpreterState *interp = _PyInterpreterState_Get(); struct _gil_runtime_state *gil = interp->ceval.gil; assert(gil != NULL); gil->interval = microseconds; @@ -473,8 +472,7 @@ void _PyEval_SetSwitchInterval(unsigned long microseconds) unsigned long _PyEval_GetSwitchInterval(void) { - /* XXX per-interpreter GIL */ - PyInterpreterState *interp = _PyInterpreterState_Main(); + PyInterpreterState *interp = _PyInterpreterState_Get(); struct _gil_runtime_state *gil = interp->ceval.gil; assert(gil != NULL); return gil->interval; @@ -484,7 +482,9 @@ unsigned long _PyEval_GetSwitchInterval(void) int _PyEval_ThreadsInitialized(void) { - /* XXX per-interpreter GIL */ + /* XXX This is only needed for an assert in PyGILState_Ensure(), + * which currently does not work with subinterpreters. + * Thus we only use the main interpreter. */ PyInterpreterState *interp = _PyInterpreterState_Main(); if (interp == NULL) { return 0; @@ -499,68 +499,73 @@ PyEval_ThreadsInitialized(void) return _PyEval_ThreadsInitialized(); } +static inline int +current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) +{ + if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) != tstate) { + return 0; + } + return _Py_atomic_load_relaxed(&gil->locked); +} + +static void +init_shared_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) +{ + assert(gil_created(gil)); + interp->ceval.gil = gil; + interp->ceval.own_gil = 0; +} + +static void +init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) +{ + assert(!gil_created(gil)); + create_gil(gil); + assert(gil_created(gil)); + interp->ceval.gil = gil; + interp->ceval.own_gil = 1; +} + PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil) { assert(tstate->interp->ceval.gil == NULL); + int locked; if (!own_gil) { + /* The interpreter will share the main interpreter's instead. */ PyInterpreterState *main_interp = _PyInterpreterState_Main(); assert(tstate->interp != main_interp); struct _gil_runtime_state *gil = main_interp->ceval.gil; - assert(gil_created(gil)); - tstate->interp->ceval.gil = gil; - tstate->interp->ceval.own_gil = 0; - return _PyStatus_OK(); + init_shared_gil(tstate->interp, gil); + locked = current_thread_holds_gil(gil, tstate); } - - /* XXX per-interpreter GIL */ - struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; - if (!_Py_IsMainInterpreter(tstate->interp)) { - /* Currently, the GIL is shared by all interpreters, - and only the main interpreter is responsible to create - and destroy it. */ - assert(gil_created(gil)); - tstate->interp->ceval.gil = gil; - // XXX For now we lie. - tstate->interp->ceval.own_gil = 1; - return _PyStatus_OK(); + else { + PyThread_init_thread(); + init_own_gil(tstate->interp, &tstate->interp->_gil); + locked = 0; + } + if (!locked) { + take_gil(tstate); } - assert(own_gil); - - assert(!gil_created(gil)); - PyThread_init_thread(); - create_gil(gil); - assert(gil_created(gil)); - tstate->interp->ceval.gil = gil; - tstate->interp->ceval.own_gil = 1; - take_gil(tstate); return _PyStatus_OK(); } void _PyEval_FiniGIL(PyInterpreterState *interp) { - if (interp->ceval.gil == NULL) { + struct _gil_runtime_state *gil = interp->ceval.gil; + if (gil == NULL) { /* It was already finalized (or hasn't been initialized yet). */ assert(!interp->ceval.own_gil); return; } else if (!interp->ceval.own_gil) { +#ifdef Py_DEBUG PyInterpreterState *main_interp = _PyInterpreterState_Main(); - assert(interp != main_interp); + assert(main_interp != NULL && interp != main_interp); assert(interp->ceval.gil == main_interp->ceval.gil); - interp->ceval.gil = NULL; - return; - } - - /* XXX per-interpreter GIL */ - struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; - if (!_Py_IsMainInterpreter(interp)) { - /* Currently, the GIL is shared by all interpreters, - and only the main interpreter is responsible to create - and destroy it. */ - assert(interp->ceval.gil == gil); +#endif interp->ceval.gil = NULL; return; } @@ -609,9 +614,17 @@ PyEval_ReleaseLock(void) drop_gil(ceval, tstate); } +void +_PyEval_AcquireLock(PyThreadState *tstate) +{ + _Py_EnsureTstateNotNULL(tstate); + take_gil(tstate); +} + void _PyEval_ReleaseLock(PyThreadState *tstate) { + _Py_EnsureTstateNotNULL(tstate); struct _ceval_state *ceval = &tstate->interp->ceval; drop_gil(ceval, tstate); } @@ -623,7 +636,7 @@ PyEval_AcquireThread(PyThreadState *tstate) take_gil(tstate); - if (_PyThreadState_Swap(tstate->interp->runtime, tstate) != NULL) { + if (_PyThreadState_SwapNoGIL(tstate) != NULL) { Py_FatalError("non-NULL old thread state"); } } @@ -633,8 +646,7 @@ PyEval_ReleaseThread(PyThreadState *tstate) { assert(is_tstate_valid(tstate)); - _PyRuntimeState *runtime = tstate->interp->runtime; - PyThreadState *new_tstate = _PyThreadState_Swap(runtime, NULL); + PyThreadState *new_tstate = _PyThreadState_SwapNoGIL(NULL); if (new_tstate != tstate) { Py_FatalError("wrong thread state"); } @@ -682,8 +694,7 @@ _PyEval_SignalAsyncExc(PyInterpreterState *interp) PyThreadState * PyEval_SaveThread(void) { - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyThreadState_Swap(runtime, NULL); + PyThreadState *tstate = _PyThreadState_SwapNoGIL(NULL); _Py_EnsureTstateNotNULL(tstate); struct _ceval_state *ceval = &tstate->interp->ceval; @@ -699,7 +710,7 @@ PyEval_RestoreThread(PyThreadState *tstate) take_gil(tstate); - _PyThreadState_Swap(tstate->interp->runtime, tstate); + _PyThreadState_SwapNoGIL(tstate); } @@ -942,21 +953,13 @@ Py_MakePendingCalls(void) return 0; } -/* The interpreter's recursion limit */ - void -_PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval) +_PyEval_InitState(PyInterpreterState *interp, PyThread_type_lock pending_lock) { - /* XXX per-interpreter GIL */ - _gil_initialize(&ceval->gil); -} + _gil_initialize(&interp->_gil); -void -_PyEval_InitState(struct _ceval_state *ceval, PyThread_type_lock pending_lock) -{ - struct _pending_calls *pending = &ceval->pending; + struct _pending_calls *pending = &interp->ceval.pending; assert(pending->lock == NULL); - pending->lock = pending_lock; } @@ -1003,7 +1006,7 @@ _Py_HandlePending(PyThreadState *tstate) /* GIL drop request */ if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gil_drop_request)) { /* Give another thread a chance */ - if (_PyThreadState_Swap(runtime, NULL) != tstate) { + if (_PyThreadState_SwapNoGIL(NULL) != tstate) { Py_FatalError("tstate mix-up"); } drop_gil(interp_ceval_state, tstate); @@ -1012,7 +1015,7 @@ _Py_HandlePending(PyThreadState *tstate) take_gil(tstate); - if (_PyThreadState_Swap(runtime, tstate) != NULL) { + if (_PyThreadState_SwapNoGIL(tstate) != NULL) { Py_FatalError("orphan tstate"); } } diff --git a/Python/compile.c b/Python/compile.c index a9fdb19d962096..60cc1f5d67ba87 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7469,7 +7469,7 @@ cfg_to_instructions(cfg_builder *g) PyObject * _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, - int optimize) + int optimize, int compile_mode) { PyObject *res = NULL; @@ -7483,7 +7483,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - mod_ty mod = PyAST_obj2mod(ast, arena, 0 /* exec */); + mod_ty mod = PyAST_obj2mod(ast, arena, compile_mode); if (mod == NULL || !_PyAST_Validate(mod)) { _PyArena_Free(arena); return NULL; @@ -7498,6 +7498,10 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, if (compiler_codegen(c, mod) < 0) { goto finally; } + int addNone = mod->kind != Expression_kind; + if (add_return_at_end(c, addNone) < 0) { + return NULL; + } res = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 1957ab82c33951..3b183280e1f24c 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -253,7 +253,7 @@ perf_map_write_entry(void *state, const void *code_addr, NULL); return; } - fprintf(method_file, "%p %x py::%s:%s\n", code_addr, code_size, entry, + fprintf(method_file, "%" PRIxPTR " %x py::%s:%s\n", (uintptr_t) code_addr, code_size, entry, filename); fflush(method_file); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 705708698c6a8b..61f87c5eba60ed 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -591,6 +591,7 @@ init_interp_create_gil(PyThreadState *tstate, int own_gil) /* finalize_interp_delete() comment explains why _PyEval_FiniGIL() is only called here. */ + // XXX This is broken with a per-interpreter GIL. _PyEval_FiniGIL(tstate->interp); /* Auto-thread-state API */ @@ -645,7 +646,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return _PyStatus_ERR("can't make first thread"); } _PyThreadState_Bind(tstate); - (void) PyThreadState_Swap(tstate); + // XXX For now we do this before the GIL is created. + (void) _PyThreadState_SwapNoGIL(tstate); status = init_interp_create_gil(tstate, config.own_gil); if (_PyStatus_EXCEPTION(status)) { @@ -2025,11 +2027,20 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) } _PyThreadState_Bind(tstate); - PyThreadState *save_tstate = PyThreadState_Swap(tstate); + // XXX For now we do this before the GIL is created. + PyThreadState *save_tstate = _PyThreadState_SwapNoGIL(tstate); + int has_gil = 0; + + /* From this point until the init_interp_create_gil() call, + we must not do anything that requires that the GIL be held + (or otherwise exist). That applies whether or not the new + interpreter has its own GIL (e.g. the main interpreter). */ /* Copy the current interpreter config into the new interpreter */ const PyConfig *src_config; if (save_tstate != NULL) { + // XXX Might new_interpreter() have been called without the GIL held? + _PyEval_ReleaseLock(save_tstate); src_config = _PyInterpreterState_GetConfig(save_tstate->interp); } else @@ -2039,11 +2050,13 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) src_config = _PyInterpreterState_GetConfig(main_interp); } + /* This does not require that the GIL be held. */ status = _PyConfig_Copy(&interp->config, src_config); if (_PyStatus_EXCEPTION(status)) { goto error; } + /* This does not require that the GIL be held. */ status = init_interp_settings(interp, config); if (_PyStatus_EXCEPTION(status)) { goto error; @@ -2053,6 +2066,7 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) if (_PyStatus_EXCEPTION(status)) { goto error; } + has_gil = 1; status = pycore_interp_init(tstate); if (_PyStatus_EXCEPTION(status)) { @@ -2072,7 +2086,12 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) /* Oops, it didn't work. Undo it all. */ PyErr_PrintEx(0); - PyThreadState_Swap(save_tstate); + if (has_gil) { + PyThreadState_Swap(save_tstate); + } + else { + _PyThreadState_SwapNoGIL(save_tstate); + } PyThreadState_Clear(tstate); PyThreadState_Delete(tstate); PyInterpreterState_Delete(interp); diff --git a/Python/pystate.c b/Python/pystate.c index 75bd9f41e301a3..26debf1f88b94a 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -425,8 +425,6 @@ init_runtime(_PyRuntimeState *runtime, runtime->open_code_userdata = open_code_userdata; runtime->audit_hook_head = audit_hook_head; - _PyEval_InitRuntimeState(&runtime->ceval); - PyPreConfig_InitPythonConfig(&runtime->preconfig); PyThread_type_lock *lockptrs[NUMLOCKS] = { @@ -682,7 +680,7 @@ init_interpreter(PyInterpreterState *interp, memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); } - _PyEval_InitState(&interp->ceval, pending_lock); + _PyEval_InitState(interp, pending_lock); _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); @@ -1863,17 +1861,11 @@ PyThreadState_Get(void) } -PyThreadState * -_PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) +static void +_swap_thread_states(_PyRuntimeState *runtime, + PyThreadState *oldts, PyThreadState *newts) { -#if defined(Py_DEBUG) - /* This can be called from PyEval_RestoreThread(). Similar - to it, we need to ensure errno doesn't change. - */ - int err = errno; -#endif - PyThreadState *oldts = current_fast_get(runtime); - + // XXX Do this only if oldts != NULL? current_fast_clear(runtime); if (oldts != NULL) { @@ -1887,6 +1879,20 @@ _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) current_fast_set(runtime, newts); tstate_activate(newts); } +} + +PyThreadState * +_PyThreadState_SwapNoGIL(PyThreadState *newts) +{ +#if defined(Py_DEBUG) + /* This can be called from PyEval_RestoreThread(). Similar + to it, we need to ensure errno doesn't change. + */ + int err = errno; +#endif + + PyThreadState *oldts = current_fast_get(&_PyRuntime); + _swap_thread_states(&_PyRuntime, oldts, newts); #if defined(Py_DEBUG) errno = err; @@ -1894,6 +1900,20 @@ _PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) return oldts; } +PyThreadState * +_PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) +{ + PyThreadState *oldts = current_fast_get(runtime); + if (oldts != NULL) { + _PyEval_ReleaseLock(oldts); + } + _swap_thread_states(runtime, oldts, newts); + if (newts != NULL) { + _PyEval_AcquireLock(newts); + } + return oldts; +} + PyThreadState * PyThreadState_Swap(PyThreadState *newts) { diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 5924ab7860d8d5..9bd54db0f59c51 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -114,6 +114,7 @@ def clean_lines(text): Modules/md5module.c Modules/_hacl/include Modules/sha1module.c Modules/_hacl/include Modules/sha2module.c Modules/_hacl/include +Modules/sha3module.c Modules/_hacl/include Objects/stringlib/*.h Objects # possible system-installed headers, just in case @@ -271,13 +272,6 @@ def clean_lines(text): Modules/expat/xmlparse.c XML_POOR_ENTROPY 1 Modules/_dbmmodule.c HAVE_GDBM_DASH_NDBM_H 1 -# from Modules/_sha3/sha3module.c -Modules/_sha3/kcp/KeccakP-1600-inplace32BI.c PLATFORM_BYTE_ORDER 4321 # force big-endian -Modules/_sha3/kcp/*.c KeccakOpt 64 -Modules/_sha3/kcp/*.c KeccakP200_excluded 1 -Modules/_sha3/kcp/*.c KeccakP400_excluded 1 -Modules/_sha3/kcp/*.c KeccakP800_excluded 1 - # others Modules/_sre/sre_lib.h LOCAL(type) static inline type Modules/_sre/sre_lib.h SRE(F) sre_ucs2_##F diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 165bd74587d7c7..453f63ec3f1cfb 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -318,12 +318,9 @@ Python/instrumentation.c - _PyInstrumentation_MISSING - ## static types Modules/_io/bufferedio.c - PyBufferedIOBase_Type - -Modules/_io/bytesio.c - _PyBytesIOBuffer_Type - Modules/_io/iobase.c - PyIOBase_Type - Modules/_io/iobase.c - PyRawIOBase_Type - -Modules/_io/textio.c - PyIncrementalNewlineDecoder_Type - Modules/_io/textio.c - PyTextIOBase_Type - -Modules/_io/winconsoleio.c - PyWindowsConsoleIO_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorBase_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorDerived_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorNopGet_Type - diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 704325670bc994..a6f330d1502dad 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -356,6 +356,7 @@ def __init__(self): # you should check the _return_value for errors, and # "goto exit" if there are any. self.return_conversion = [] + self.converter_retval = "_return_value" # The C statements required to do some operations # after the end of parsing but before cleaning up. @@ -3893,15 +3894,15 @@ def __init__(self, *, py_default=None, **kwargs): def return_converter_init(self): pass - def declare(self, data, name="_return_value"): + def declare(self, data): line = [] add = line.append add(self.type) if not self.type.endswith('*'): add(' ') - add(name + ';') + add(data.converter_retval + ';') data.declarations.append(''.join(line)) - data.return_value = name + data.return_value = data.converter_retval def err_occurred_if(self, expr, data): data.return_conversion.append('if (({}) && PyErr_Occurred()) {{\n goto exit;\n}}\n'.format(expr)) @@ -3923,8 +3924,10 @@ class bool_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == -1", data) - data.return_conversion.append('return_value = PyBool_FromLong((long)_return_value);\n') + self.err_occurred_if(f"{data.converter_retval} == -1", data) + data.return_conversion.append( + f'return_value = PyBool_FromLong((long){data.converter_retval});\n' + ) class long_return_converter(CReturnConverter): type = 'long' @@ -3934,9 +3937,10 @@ class long_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == {}-1".format(self.unsigned_cast), data) + self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) data.return_conversion.append( - ''.join(('return_value = ', self.conversion_fn, '(', self.cast, '_return_value);\n'))) + f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' + ) class int_return_converter(long_return_converter): type = 'int' @@ -3978,9 +3982,10 @@ class double_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) - self.err_occurred_if("_return_value == -1.0", data) + self.err_occurred_if(f"{data.converter_retval} == -1.0", data) data.return_conversion.append( - 'return_value = PyFloat_FromDouble(' + self.cast + '_return_value);\n') + f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' + ) class float_return_converter(double_return_converter): type = 'float' diff --git a/Tools/msi/bundle/bootstrap/pch.h b/Tools/msi/bundle/bootstrap/pch.h index b0aa5111dabd0d..6d0974b34c61e7 100644 --- a/Tools/msi/bundle/bootstrap/pch.h +++ b/Tools/msi/bundle/bootstrap/pch.h @@ -5,7 +5,7 @@ // The license and further copyright text can be found in the file // LICENSE.TXT at the root directory of the distribution. // -// +// // // Precompiled header for standard bootstrapper application. // diff --git a/Tools/msi/bundle/bootstrap/resource.h b/Tools/msi/bundle/bootstrap/resource.h index 53c03c319f091f..d951e651f6d20d 100644 --- a/Tools/msi/bundle/bootstrap/resource.h +++ b/Tools/msi/bundle/bootstrap/resource.h @@ -14,7 +14,7 @@ // Next default values for new objects -// +// #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS #define _APS_NEXT_RESOURCE_VALUE 102