Merge branch 'master' into feature/34632-importlib-metadata

.azure-pipelines/posix-steps.yml

@@ -10,6 +10,10 @@ steps:
   clean: true
   fetchDepth: 5
 
+# Work around a known issue affecting Ubuntu VMs on Pipelines
+- script: sudo setfacl -Rb /home/vsts
+  displayName: 'Workaround ACL issue'
+
 - script: ${{ parameters.sudo_dependencies }} ./.azure-pipelines/posix-deps-${{ parameters.dependencies }}.sh $(openssl_version)
   displayName: 'Install dependencies'
 

.github/CODEOWNERS

@@ -11,6 +11,7 @@
 **/*context*                  @1st1
 **/*genobject*                @1st1
 **/*hamt*                     @1st1
+Objects/dict*                 @methane
 
 # Hashing
 **/*hashlib*                  @python/crypto-team
@@ -25,8 +26,8 @@
 # Ignoring importlib.h so as to not get flagged on
 # all pull requests that change the emitted
 # bytecode.
-**/*import*.c                 @python/import-team
-**/*import*.py                @python/import-team
+**/*import*.c                 @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw
+**/*import*.py                @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw
 
 
 # SSL
@@ -42,13 +43,22 @@ Python/bootstrap_hash.c       @python/crypto-team
 **/*imap*                     @python/email-team
 **/*poplib*                   @python/email-team
 
+# Parser/Pgen
+/Parser/pgen/                 @pablogsal
+
+# SQLite 3
+**/*sqlite*                   @berkerpeksag
+
 # subprocess
 **/*subprocess*               @gpshead
 
 # Windows
 /PC/                          @python/windows-team
 /PCbuild/                     @python/windows-team
 
+# Urllib
+**/*robotparser*              @berkerpeksag
+
 # Windows installer packages
 /Tools/msi/                   @python/windows-team
 /Tools/nuget/                 @python/windows-team

Doc/Makefile

@@ -48,11 +48,19 @@ build:
 	@if [ -f  ../Misc/NEWS ] ; then \
 		echo "Using existing Misc/NEWS file"; \
 		cp ../Misc/NEWS build/NEWS; \
-	elif [ -d ../Misc/NEWS.d ]; then \
-		echo "Building NEWS from Misc/NEWS.d with blurb"; \
-		$(BLURB) merge -f build/NEWS; \
+	elif $(BLURB) help >/dev/null 2>&1 && $(SPHINXBUILD) --version >/dev/null 2>&1; then \
+		if [ -d ../Misc/NEWS.d ]; then \
+			echo "Building NEWS from Misc/NEWS.d with blurb"; \
+			$(BLURB) merge -f build/NEWS; \
+		else \
+			echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \
+			exit 1; \
+		fi \
 	else \
-		echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \
+		echo ""; \
+		echo "Missing the required blurb or sphinx-build tools."; \
+		echo "Please run 'make venv' to install local copies."; \
+		echo ""; \
 		exit 1; \
 	fi
 	$(SPHINXBUILD) $(ALLSPHINXOPTS)

Doc/bugs.rst

@@ -25,7 +25,15 @@ docs@python.org (behavioral bugs can be sent to python-list@python.org).
 though it may take a while to be processed.
 
 .. seealso::
-   `Documentation bugs`_ on the Python issue tracker
+
+   `Documentation bugs`_
+      A list of documentation bugs that have been submitted to the Python issue tracker.
+
+   `Issue Tracking <https://devguide.python.org/tracker/>`_
+      Overview of the process involved in reporting an improvement on the tracker.
+
+   `Helping with Documentation <https://devguide.python.org/docquality/#helping-with-documentation>`_
+      Comprehensive guide for individuals that are interested in contributing to Python documentation.
 
 .. _using-the-tracker:
 

Doc/c-api/datetime.rst

@@ -98,6 +98,22 @@ Macros to create objects:
    minute, second and microsecond.
 
 
+.. c:function:: PyObject* PyDateTime_FromDateAndTimeAndFold(int year, int month, int day, int hour, int minute, int second, int usecond, int fold)
+
+   Return a :class:`datetime.datetime` object with the specified year, month, day, hour,
+   minute, second, microsecond and fold.
+
+   .. versionadded:: 3.6
+
+
+.. c:function:: PyObject* PyTime_FromTimeAndFold(int hour, int minute, int second, int usecond, int fold)
+
+   Return a :class:`datetime.time` object with the specified hour, minute, second,
+   microsecond and fold.
+
+   .. versionadded:: 3.6
+
+
 .. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond)
 
    Return a :class:`datetime.time` object with the specified hour, minute, second and

Doc/c-api/exceptions.rst

@@ -53,8 +53,8 @@ Printing and clearing
 .. c:function:: void PyErr_PrintEx(int set_sys_last_vars)
 
    Print a standard traceback to ``sys.stderr`` and clear the error indicator.
-   **Unless** the error is a ``SystemExit``.  In that case the no traceback
-   is printed and Python process will exit with the error code specified by
+   **Unless** the error is a ``SystemExit``, in that case no traceback is
+   printed and the Python process will exit with the error code specified by
    the ``SystemExit`` instance.
 
    Call this function **only** when the error indicator is set.  Otherwise it

Doc/c-api/init.rst

@@ -37,6 +37,7 @@ The following functions can be safely called before Python is initialized:
 
 * Informative functions:
 
+  * :c:func:`Py_IsInitialized`
   * :c:func:`PyMem_GetAllocator`
   * :c:func:`PyObject_GetArenaAllocator`
   * :c:func:`Py_GetBuildInfo`
@@ -855,6 +856,12 @@ code, or when embedding the Python interpreter:
    created, the current thread must not have acquired it, otherwise deadlock
    ensues.
 
+   .. note::
+      Calling this function from a thread when the runtime is finalizing
+      will terminate the thread, even if the thread was not created by Python.
+      You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to
+      check if the interpreter is in process of being finalized before calling
+      this function to avoid unwanted termination.
 
 .. c:function:: PyThreadState* PyThreadState_Get()
 
@@ -902,6 +909,12 @@ with sub-interpreters:
    When the function returns, the current thread will hold the GIL and be able
    to call arbitrary Python code.  Failure is a fatal error.
 
+   .. note::
+      Calling this function from a thread when the runtime is finalizing
+      will terminate the thread, even if the thread was not created by Python.
+      You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to
+      check if the interpreter is in process of being finalized before calling
+      this function to avoid unwanted termination.
 
 .. c:function:: void PyGILState_Release(PyGILState_STATE)
 
@@ -1067,6 +1080,18 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
    *tstate*, which should not be *NULL*.  The lock must have been created earlier.
    If this thread already has the lock, deadlock ensues.
 
+   .. note::
+      Calling this function from a thread when the runtime is finalizing
+      will terminate the thread, even if the thread was not created by Python.
+      You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to
+      check if the interpreter is in process of being finalized before calling
+      this function to avoid unwanted termination.
+
+   .. versionchanged:: 3.8
+      Updated to be consistent with :c:func:`PyEval_RestoreThread`,
+      :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`,
+      and terminate the current thread if called while the interpreter is finalizing.
+
    :c:func:`PyEval_RestoreThread` is a higher-level function which is always
    available (even when threads have not been initialized).
 
@@ -1093,6 +1118,18 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
       :c:func:`PyEval_RestoreThread` or :c:func:`PyEval_AcquireThread`
       instead.
 
+   .. note::
+      Calling this function from a thread when the runtime is finalizing
+      will terminate the thread, even if the thread was not created by Python.
+      You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to
+      check if the interpreter is in process of being finalized before calling
+      this function to avoid unwanted termination.
+
+   .. versionchanged:: 3.8
+      Updated to be consistent with :c:func:`PyEval_RestoreThread`,
+      :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`,
+      and terminate the current thread if called while the interpreter is finalizing.
+
 
 .. c:function:: void PyEval_ReleaseLock()
 
@@ -1394,6 +1431,11 @@ These functions are only intended to be used by advanced debugging tools.
    Return the interpreter state object at the head of the list of all such objects.
 
 
+.. c:function:: PyInterpreterState* PyInterpreterState_Main()
+
+   Return the main interpreter state object.
+
+
 .. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp)
 
    Return the next interpreter state object after *interp* from the list of all

Doc/c-api/intro.rst

@@ -48,7 +48,8 @@ Include Files
 All function, type and macro definitions needed to use the Python/C API are
 included in your code by the following line::
 
-   #include "Python.h"
+   #define PY_SSIZE_T_CLEAN
+   #include <Python.h>
 
 This implies inclusion of the following standard headers: ``<stdio.h>``,
 ``<string.h>``, ``<errno.h>``, ``<limits.h>``, ``<assert.h>`` and ``<stdlib.h>``
@@ -60,6 +61,9 @@ This implies inclusion of the following standard headers: ``<stdio.h>``,
    headers on some systems, you *must* include :file:`Python.h` before any standard
    headers are included.
 
+   It is recommended to always define ``PY_SSIZE_T_CLEAN`` before including
+   ``Python.h``.  See :ref:`arg-parsing` for a description of this macro.
+
 All user visible names defined by Python.h (except those defined by the included
 standard headers) have one of the prefixes ``Py`` or ``_Py``.  Names beginning
 with ``_Py`` are for internal use by the Python implementation and should not be

Doc/c-api/memory.rst

@@ -440,8 +440,9 @@ Customize Memory Allocators
 
    Setup hooks to detect bugs in the Python memory allocator functions.
 
-   Newly allocated memory is filled with the byte ``0xCB``, freed memory is
-   filled with the byte ``0xDB``.
+   Newly allocated memory is filled with the byte ``0xCD`` (``CLEANBYTE``),
+   freed memory is filled with the byte ``0xDD`` (``DEADBYTE``). Memory blocks
+   are surrounded by "forbidden bytes" (``FORBIDDENBYTE``: byte ``0xFD``).
 
    Runtime checks:
 
@@ -471,6 +472,12 @@ Customize Memory Allocators
       if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and
       :c:data:`PYMEM_DOMAIN_MEM` domains are called.
 
+   .. versionchanged:: 3.8.0
+      Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and
+      ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD``
+      and ``0xFD`` to use the same values than Windows CRT debug ``malloc()``
+      and ``free()``.
+
 
 .. _pymalloc:
 

Doc/c-api/typeobj.rst

@@ -2052,7 +2052,7 @@ Sequence Object Structures
    signature.  It should modify its first operand, and return it.  This slot
    may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceConcat`
    will fall back to :c:func:`PySequence_Concat`.  It is also used by the
-   augmented assignment ``+=``, after trying numeric inplace addition
+   augmented assignment ``+=``, after trying numeric in-place addition
    via the :c:member:`~PyNumberMethods.nb_inplace_add` slot.
 
 .. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat
@@ -2061,7 +2061,7 @@ Sequence Object Structures
    signature.  It should modify its first operand, and return it.  This slot
    may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceRepeat`
    will fall back to :c:func:`PySequence_Repeat`.  It is also used by the
-   augmented assignment ``*=``, after trying numeric inplace multiplication
+   augmented assignment ``*=``, after trying numeric in-place multiplication
    via the :c:member:`~PyNumberMethods.nb_inplace_multiply` slot.
 
 

Doc/c-api/veryhigh.rst

@@ -109,6 +109,10 @@ the same library that the Python runtime is using.
    (:func:`sys.getfilesystemencoding`).  If *closeit* is true, the file is
    closed before PyRun_SimpleFileExFlags returns.
 
+   .. note::
+      On Windows, *fp* should be opened as binary mode (e.g. ``fopen(filename, "rb")``.
+      Otherwise, Python may not handle script file with LF line ending correctly.
+
 
 .. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename)
 

Doc/distutils/apiref.rst

@@ -277,6 +277,11 @@ the full reference.
    |                        | simply skip the extension.     |                           |
    +------------------------+--------------------------------+---------------------------+
 
+   .. versionchanged:: 3.8
+
+      On Unix, C extensions are no longer linked to libpython except on
+      Android.
+
 
 .. class:: Distribution
 

Doc/distutils/setupscript.rst

@@ -523,7 +523,7 @@ following way::
 
     setup(...,
           data_files=[('bitmaps', ['bm/b1.gif', 'bm/b2.gif']),
-                      ('config', ['cfg/data.cfg']),
+                      ('config', ['cfg/data.cfg'])],
          )
 
 Each (*directory*, *files*) pair in the sequence specifies the installation

Doc/extending/building.rst

@@ -145,7 +145,7 @@ that distutils gets the invocations right.
 Distributing your extension modules
 ===================================
 
-When an extension has been successfully build, there are three ways to use it.
+When an extension has been successfully built, there are three ways to use it.
 
 End-users will typically want to install the module, they do so by running ::
 
@@ -158,7 +158,7 @@ Module maintainers should produce source packages; to do so, they run ::
 In some cases, additional files need to be included in a source distribution;
 this is done through a :file:`MANIFEST.in` file; see :ref:`manifest` for details.
 
-If the source distribution has been build successfully, maintainers can also
+If the source distribution has been built successfully, maintainers can also
 create binary distributions. Depending on the platform, one of the following
 commands can be used to do so. ::
 

Doc/extending/embedding.rst

@@ -53,6 +53,7 @@ interface. This interface is intended to execute a Python script without needing
 to interact with the application directly. This can for example be used to
 perform some operation on a file. ::
 
+   #define PY_SSIZE_T_CLEAN
    #include <Python.h>
 
    int

Doc/extending/extending.rst

@@ -55,8 +55,9 @@ called ``spam``, the C file containing its implementation is called
 :file:`spammodule.c`; if the module name is very long, like ``spammify``, the
 module name can be just :file:`spammify.c`.)
 
-The first line of our file can be::
+The first two lines of our file can be::
 
+   #define PY_SSIZE_T_CLEAN
    #include <Python.h>
 
 which pulls in the Python API (you can add a comment describing the purpose of
@@ -68,6 +69,9 @@ the module and a copyright notice if you like).
    headers on some systems, you *must* include :file:`Python.h` before any standard
    headers are included.
 
+   It is recommended to always define ``PY_SSIZE_T_CLEAN`` before including
+   ``Python.h``.  See :ref:`parsetuple` for a description of this macro.
+
 All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or
 ``PY``, except those defined in standard header files. For convenience, and
 since they are used extensively by the Python interpreter, ``"Python.h"``
@@ -729,7 +733,8 @@ it returns false and raises an appropriate exception.
 Here is an example module which uses keywords, based on an example by Geoff
 Philbrick (philbrick@hks.com)::
 
-   #include "Python.h"
+   #define PY_SSIZE_T_CLEAN  /* Make "s#" use Py_ssize_t rather than int. */
+   #include <Python.h>
 
    static PyObject *
    keywdarg_parrot(PyObject *self, PyObject *args, PyObject *keywds)
@@ -1228,7 +1233,7 @@ The function :c:func:`spam_system` is modified in a trivial way::
 
 In the beginning of the module, right after the line ::
 
-   #include "Python.h"
+   #include <Python.h>
 
 two more lines must be added::
 

Doc/extending/newtypes_tutorial.rst

@@ -92,6 +92,7 @@ The second bit is the definition of the type object. ::
        .tp_doc = "Custom objects",
        .tp_basicsize = sizeof(CustomObject),
        .tp_itemsize = 0,
+       .tp_flags = Py_TPFLAGS_DEFAULT,
        .tp_new = PyType_GenericNew,
    };
 

Doc/faq/extending.rst

@@ -280,6 +280,7 @@ solution then is to call :c:func:`PyParser_ParseString` and test for ``e.error``
 equal to ``E_EOF``, which means the input is incomplete.  Here's a sample code
 fragment, untested, inspired by code from Alex Farber::
 
+   #define PY_SSIZE_T_CLEAN
    #include <Python.h>
    #include <node.h>
    #include <errcode.h>
@@ -318,6 +319,7 @@ complete example using the GNU readline library (you may want to ignore
    #include <stdio.h>
    #include <readline.h>
 
+   #define PY_SSIZE_T_CLEAN
    #include <Python.h>
    #include <object.h>
    #include <compile.h>