From 58395759b04273edccf3d199606088e0703ae6b1 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Thu, 9 Feb 2023 11:40:52 +0300 Subject: [PATCH 01/83] gh-101678: refactor the math module to use special functions from c11 (GH-101679) Shouldn't affect users, hence no news. Automerge-Triggered-By: GH:mdickinson --- Modules/_math.h | 5 +- Modules/mathmodule.c | 187 ++----------------------------------------- 2 files changed, 8 insertions(+), 184 deletions(-) diff --git a/Modules/_math.h b/Modules/_math.h index 4a6bc223ef5fb5..2285b64747c0bd 100644 --- a/Modules/_math.h +++ b/Modules/_math.h @@ -7,8 +7,9 @@ static double _Py_log1p(double x) { - /* Some platforms supply a log1p function but don't respect the sign of - zero: log1p(-0.0) gives 0.0 instead of the correct result of -0.0. + /* Some platforms (e.g. MacOS X 10.8, see gh-59682) supply a log1p function + but don't respect the sign of zero: log1p(-0.0) gives 0.0 instead of + the correct result of -0.0. To save fiddling with configure tests and platform checks, we handle the special case of zero input directly on all platforms. diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 992957e675a7a3..939954c95d9ff2 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -101,10 +101,6 @@ get_math_module_state(PyObject *module) static const double pi = 3.141592653589793238462643383279502884197; static const double logpi = 1.144729885849400174143427351353058711647; -#if !defined(HAVE_ERF) || !defined(HAVE_ERFC) -static const double sqrtpi = 1.772453850905516027298167483341145182798; -#endif /* !defined(HAVE_ERF) || !defined(HAVE_ERFC) */ - /* Version of PyFloat_AsDouble() with in-line fast paths for exact floats and integers. Gives a substantial @@ -162,7 +158,9 @@ m_sinpi(double x) return copysign(1.0, x)*r; } -/* Implementation of the real gamma function. In extensive but non-exhaustive +/* Implementation of the real gamma function. Kept here to work around + issues (see e.g. gh-70309) with quality of libm's tgamma/lgamma implementations + on various platforms (Windows, MacOS). In extensive but non-exhaustive random tests, this function proved accurate to within <= 10 ulps across the entire float domain. Note that accuracy may depend on the quality of the system math functions, the pow function in particular. Special cases @@ -458,163 +456,6 @@ m_lgamma(double x) return r; } -#if !defined(HAVE_ERF) || !defined(HAVE_ERFC) - -/* - Implementations of the error function erf(x) and the complementary error - function erfc(x). - - Method: we use a series approximation for erf for small x, and a continued - fraction approximation for erfc(x) for larger x; - combined with the relations erf(-x) = -erf(x) and erfc(x) = 1.0 - erf(x), - this gives us erf(x) and erfc(x) for all x. - - The series expansion used is: - - erf(x) = x*exp(-x*x)/sqrt(pi) * [ - 2/1 + 4/3 x**2 + 8/15 x**4 + 16/105 x**6 + ...] - - The coefficient of x**(2k-2) here is 4**k*factorial(k)/factorial(2*k). - This series converges well for smallish x, but slowly for larger x. - - The continued fraction expansion used is: - - erfc(x) = x*exp(-x*x)/sqrt(pi) * [1/(0.5 + x**2 -) 0.5/(2.5 + x**2 - ) - 3.0/(4.5 + x**2 - ) 7.5/(6.5 + x**2 - ) ...] - - after the first term, the general term has the form: - - k*(k-0.5)/(2*k+0.5 + x**2 - ...). - - This expansion converges fast for larger x, but convergence becomes - infinitely slow as x approaches 0.0. The (somewhat naive) continued - fraction evaluation algorithm used below also risks overflow for large x; - but for large x, erfc(x) == 0.0 to within machine precision. (For - example, erfc(30.0) is approximately 2.56e-393). - - Parameters: use series expansion for abs(x) < ERF_SERIES_CUTOFF and - continued fraction expansion for ERF_SERIES_CUTOFF <= abs(x) < - ERFC_CONTFRAC_CUTOFF. ERFC_SERIES_TERMS and ERFC_CONTFRAC_TERMS are the - numbers of terms to use for the relevant expansions. */ - -#define ERF_SERIES_CUTOFF 1.5 -#define ERF_SERIES_TERMS 25 -#define ERFC_CONTFRAC_CUTOFF 30.0 -#define ERFC_CONTFRAC_TERMS 50 - -/* - Error function, via power series. - - Given a finite float x, return an approximation to erf(x). - Converges reasonably fast for small x. -*/ - -static double -m_erf_series(double x) -{ - double x2, acc, fk, result; - int i, saved_errno; - - x2 = x * x; - acc = 0.0; - fk = (double)ERF_SERIES_TERMS + 0.5; - for (i = 0; i < ERF_SERIES_TERMS; i++) { - acc = 2.0 + x2 * acc / fk; - fk -= 1.0; - } - /* Make sure the exp call doesn't affect errno; - see m_erfc_contfrac for more. */ - saved_errno = errno; - result = acc * x * exp(-x2) / sqrtpi; - errno = saved_errno; - return result; -} - -/* - Complementary error function, via continued fraction expansion. - - Given a positive float x, return an approximation to erfc(x). Converges - reasonably fast for x large (say, x > 2.0), and should be safe from - overflow if x and nterms are not too large. On an IEEE 754 machine, with x - <= 30.0, we're safe up to nterms = 100. For x >= 30.0, erfc(x) is smaller - than the smallest representable nonzero float. */ - -static double -m_erfc_contfrac(double x) -{ - double x2, a, da, p, p_last, q, q_last, b, result; - int i, saved_errno; - - if (x >= ERFC_CONTFRAC_CUTOFF) - return 0.0; - - x2 = x*x; - a = 0.0; - da = 0.5; - p = 1.0; p_last = 0.0; - q = da + x2; q_last = 1.0; - for (i = 0; i < ERFC_CONTFRAC_TERMS; i++) { - double temp; - a += da; - da += 2.0; - b = da + x2; - temp = p; p = b*p - a*p_last; p_last = temp; - temp = q; q = b*q - a*q_last; q_last = temp; - } - /* Issue #8986: On some platforms, exp sets errno on underflow to zero; - save the current errno value so that we can restore it later. */ - saved_errno = errno; - result = p / q * x * exp(-x2) / sqrtpi; - errno = saved_errno; - return result; -} - -#endif /* !defined(HAVE_ERF) || !defined(HAVE_ERFC) */ - -/* Error function erf(x), for general x */ - -static double -m_erf(double x) -{ -#ifdef HAVE_ERF - return erf(x); -#else - double absx, cf; - - if (Py_IS_NAN(x)) - return x; - absx = fabs(x); - if (absx < ERF_SERIES_CUTOFF) - return m_erf_series(x); - else { - cf = m_erfc_contfrac(absx); - return x > 0.0 ? 1.0 - cf : cf - 1.0; - } -#endif -} - -/* Complementary error function erfc(x), for general x. */ - -static double -m_erfc(double x) -{ -#ifdef HAVE_ERFC - return erfc(x); -#else - double absx, cf; - - if (Py_IS_NAN(x)) - return x; - absx = fabs(x); - if (absx < ERF_SERIES_CUTOFF) - return 1.0 - m_erf_series(x); - else { - cf = m_erfc_contfrac(absx); - return x > 0.0 ? cf : 2.0 - cf; - } -#endif -} - /* wrapper for atan2 that deals directly with special cases before delegating to the platform libm for the remaining cases. This @@ -801,25 +642,7 @@ m_log2(double x) } if (x > 0.0) { -#ifdef HAVE_LOG2 return log2(x); -#else - double m; - int e; - m = frexp(x, &e); - /* We want log2(m * 2**e) == log(m) / log(2) + e. Care is needed when - * x is just greater than 1.0: in that case e is 1, log(m) is negative, - * and we get significant cancellation error from the addition of - * log(m) / log(2) to e. The slight rewrite of the expression below - * avoids this problem. - */ - if (x >= 1.0) { - return log(2.0 * m) / log(2.0) + (e - 1); - } - else { - return log(m) / log(2.0) + e; - } -#endif } else if (x == 0.0) { errno = EDOM; @@ -1261,10 +1084,10 @@ FUNC1(cos, cos, 0, FUNC1(cosh, cosh, 1, "cosh($module, x, /)\n--\n\n" "Return the hyperbolic cosine of x.") -FUNC1A(erf, m_erf, +FUNC1A(erf, erf, "erf($module, x, /)\n--\n\n" "Error function at x.") -FUNC1A(erfc, m_erfc, +FUNC1A(erfc, erfc, "erfc($module, x, /)\n--\n\n" "Complementary error function at x.") FUNC1(exp, exp, 1, From 45fa12aec8f508c224a1521cfe3ae597f1026264 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Thu, 9 Feb 2023 12:40:13 +0300 Subject: [PATCH 02/83] gh-101678: Merge math_1_to_whatever() and math_1() (#101730) `math_1_to_whatever()` is no longer useful, since all existing uses of it convert to `float`. Earlier versions of Python used `math_1_to_whatever` with an integer target; see gh-16991 for the PR where that use was removed. --- Modules/mathmodule.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 939954c95d9ff2..544560e8322c72 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -875,9 +875,7 @@ is_error(double x) */ static PyObject * -math_1_to_whatever(PyObject *arg, double (*func) (double), - PyObject *(*from_double_func) (double), - int can_overflow) +math_1(PyObject *arg, double (*func) (double), int can_overflow) { double x, r; x = PyFloat_AsDouble(arg); @@ -903,7 +901,7 @@ math_1_to_whatever(PyObject *arg, double (*func) (double), /* this branch unnecessary on most platforms */ return NULL; - return (*from_double_func)(r); + return PyFloat_FromDouble(r); } /* variant of math_1, to be used when the function being wrapped is known to @@ -951,12 +949,6 @@ math_1a(PyObject *arg, double (*func) (double)) OverflowError. */ -static PyObject * -math_1(PyObject *arg, double (*func) (double), int can_overflow) -{ - return math_1_to_whatever(arg, func, PyFloat_FromDouble, can_overflow); -} - static PyObject * math_2(PyObject *const *args, Py_ssize_t nargs, double (*func) (double, double), const char *funcname) From 1c49e61b9b18d550b9c5cff69a1dd3bb218e544a Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Thu, 9 Feb 2023 21:01:32 +0900 Subject: [PATCH 03/83] no-issue: Add Dong-hee Na as the cjkcodecs codeowner (gh-101731) --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8dd07d911f5b18..fc1bb3388976d5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -151,6 +151,8 @@ Lib/ast.py @isidentical **/*sysconfig* @FFY00 +**/*cjkcodecs* @corona10 + # macOS /Mac/ @python/macos-team **/*osx_support* @python/macos-team From ecfd2d37c529c1952dc11fabe1758156924de67a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 Feb 2023 13:05:53 +0000 Subject: [PATCH 04/83] GH-99293: Document that `Py_TPFLAGS_VALID_VERSION_TAG` shouldn't be used. (#GH-101736) Document that Py_TPFLAGS_VALID_VERSION_TAG shouldn't be used. --- Doc/c-api/typeobj.rst | 10 ++++++++++ .../2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst | 2 ++ 2 files changed, 12 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 644830b940b417..fd8f49ccb1caab 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1313,6 +1313,16 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.10 + .. data:: Py_TPFLAGS_VALID_VERSION_TAG + + Internal. Do not set or unset this flag. + To indicate that a class has changed call :c:func:`PyType_Modified` + + .. warning:: + This flag is present in header files, but is an internal feature and should + not be used. It will be removed in a future version of CPython + + .. c:member:: const char* PyTypeObject.tp_doc An optional pointer to a NUL-terminated C string giving the docstring for this diff --git a/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst b/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst new file mode 100644 index 00000000000000..8c0f05543747dc --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst @@ -0,0 +1,2 @@ +Document that the Py_TPFLAGS_VALID_VERSION_TAG is an internal feature, +should not be used, and will be removed. From cb2411886a181d25e0cff2c870f331d2949874d5 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Thu, 9 Feb 2023 19:49:02 +0300 Subject: [PATCH 05/83] gh-101670: typo fix in PyImport_ExtendInittab() (#101723) Co-authored-by: Eric Snow --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index 795d368966481e..302255d76edcd5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2651,7 +2651,7 @@ PyImport_ExtendInittab(struct _inittab *newtab) int res = 0; if (_PyRuntime.imports.inittab != NULL) { - Py_FatalError("PyImport_ExtendInittab() may be be called after Py_Initialize()"); + Py_FatalError("PyImport_ExtendInittab() may not be called after Py_Initialize()"); } /* Count the number of entries in both tables */ From f23371fbc9bb283207ecebf8efd81a22538b4327 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 9 Feb 2023 17:15:19 +0000 Subject: [PATCH 06/83] LibFFI build requires x64 Cygwin, and skip the ARM build (GH-101743) --- PCbuild/prepare_libffi.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PCbuild/prepare_libffi.bat b/PCbuild/prepare_libffi.bat index 7e7842a2fc97a4..ef36c36e058a15 100644 --- a/PCbuild/prepare_libffi.bat +++ b/PCbuild/prepare_libffi.bat @@ -60,7 +60,7 @@ goto :Usage if NOT DEFINED BUILD_X64 if NOT DEFINED BUILD_X86 if NOT DEFINED BUILD_ARM32 if NOT DEFINED BUILD_ARM64 ( set BUILD_X64=1 set BUILD_X86=1 - set BUILD_ARM32=1 + set BUILD_ARM32=0 set BUILD_ARM64=1 set COPY_LICENSE=1 ) @@ -204,7 +204,7 @@ if NOT DEFINED CYG_CACHE (set CYG_CACHE=C:/cygwin/var/cache/setup) if NOT DEFINED CYG_MIRROR (set CYG_MIRROR=http://mirrors.kernel.org/sourceware/cygwin/) powershell -c "md $env:CYG_ROOT -ErrorAction SilentlyContinue" -powershell -c "$setup = $env:CYG_ROOT+'/setup.exe'; if (!(Test-Path $setup)){invoke-webrequest https://cygwin.com/setup-x86.exe -outfile $setup} +powershell -c "$setup = $env:CYG_ROOT+'/setup.exe'; if (!(Test-Path $setup)){invoke-webrequest https://cygwin.com/setup-x86_64.exe -outfile $setup} %CYG_ROOT%/setup.exe -qnNdO -R "%CYG_ROOT%" -s "%CYG_MIRROR%" -l "%CYG_CACHE%" -P make -P autoconf -P automake -P libtool -P dejagnu endlocal From 6d92373f500eb81a175516b3abb16e21f0806c1f Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Thu, 9 Feb 2023 21:36:24 +0400 Subject: [PATCH 07/83] gh-101283: Fix 'versionchanged' for the shell=True fallback on Windows in 3.12 (GH-101728) --- Doc/library/subprocess.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index d792a43eeb271f..ccc431b2d92e07 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -111,7 +111,7 @@ underlying :class:`Popen` interface can be used directly. Added the *text* parameter, as a more understandable alias of *universal_newlines*. Added the *capture_output* parameter. - .. versionchanged:: 3.11.3 + .. versionchanged:: 3.12 Changed Windows shell search order for ``shell=True``. The current directory and ``%PATH%`` are replaced with ``%COMSPEC%`` and @@ -495,7 +495,7 @@ functions. *executable* parameter accepts a bytes and :term:`path-like object` on Windows. - .. versionchanged:: 3.11.3 + .. versionchanged:: 3.12 Changed Windows shell search order for ``shell=True``. The current directory and ``%PATH%`` are replaced with ``%COMSPEC%`` and @@ -1174,7 +1174,7 @@ calls these functions. .. versionchanged:: 3.3 *timeout* was added. - .. versionchanged:: 3.11.3 + .. versionchanged:: 3.12 Changed Windows shell search order for ``shell=True``. The current directory and ``%PATH%`` are replaced with ``%COMSPEC%`` and @@ -1214,7 +1214,7 @@ calls these functions. .. versionchanged:: 3.3 *timeout* was added. - .. versionchanged:: 3.11.3 + .. versionchanged:: 3.12 Changed Windows shell search order for ``shell=True``. The current directory and ``%PATH%`` are replaced with ``%COMSPEC%`` and @@ -1277,7 +1277,7 @@ calls these functions. .. versionadded:: 3.7 *text* was added as a more readable alias for *universal_newlines*. - .. versionchanged:: 3.11.3 + .. versionchanged:: 3.12 Changed Windows shell search order for ``shell=True``. The current directory and ``%PATH%`` are replaced with ``%COMSPEC%`` and From b41c47cd0606e8273aef4813e83fe2deaf9ab33b Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 9 Feb 2023 09:40:51 -0800 Subject: [PATCH 08/83] gh-101726: Update the OpenSSL version to 1.1.1t (GH-101727) Fixes CVE-2023-0286 (High) and a couple of Medium security issues. https://www.openssl.org/news/secadv/20230207.txt --- .azure-pipelines/ci.yml | 4 ++-- .azure-pipelines/pr.yml | 4 ++-- .github/workflows/build.yml | 6 +++--- Mac/BuildScript/build-installer.py | 6 +++--- .../Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst | 4 ++++ PCbuild/get_externals.bat | 4 ++-- PCbuild/python.props | 4 ++-- PCbuild/readme.txt | 2 +- Tools/ssl/multissltests.py | 4 ++-- 9 files changed, 21 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index e45dc2d4365999..6302b547982118 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -57,7 +57,7 @@ jobs: variables: testRunTitle: '$(build.sourceBranchName)-linux' testRunPlatform: linux - openssl_version: 1.1.1q + openssl_version: 1.1.1t steps: - template: ./posix-steps.yml @@ -83,7 +83,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1q + openssl_version: 1.1.1t steps: - template: ./posix-steps.yml diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index af94ebf78c8488..5f7218768c18af 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -57,7 +57,7 @@ jobs: variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.1q + openssl_version: 1.1.1t steps: - template: ./posix-steps.yml @@ -83,7 +83,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1q + openssl_version: 1.1.1t steps: - template: ./posix-steps.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f798992d8af61c..97ea2d94598e2c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -176,7 +176,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1s + OPENSSL_VER: 1.1.1t PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v3 @@ -235,7 +235,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1s, 3.0.7, 3.1.0-beta1] + openssl_ver: [1.1.1t, 3.0.8, 3.1.0-beta1] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -282,7 +282,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1s + OPENSSL_VER: 1.1.1t PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index cf97b5558c2ddc..048cb01379607e 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.1.1s", - url="https://www.openssl.org/source/openssl-1.1.1s.tar.gz", - checksum='c5ac01e760ee6ff0dab61d6b2bbd30146724d063eb322180c6f18a6f74e4b6aa', + name="OpenSSL 1.1.1t", + url="https://www.openssl.org/source/openssl-1.1.1t.tar.gz", + checksum='8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b', buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst b/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst new file mode 100644 index 00000000000000..43acc82063fd7a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst @@ -0,0 +1,4 @@ +Updated the OpenSSL version used in Windows and macOS binary release builds +to 1.1.1t to address CVE-2023-0286, CVE-2022-4303, and CVE-2022-4303 per +`the OpenSSL 2023-02-07 security advisory +`_. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 0a41d131a3e887..d4d96bd49d72c6 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,7 +53,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.3 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1s +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1t set libraries=%libraries% sqlite-3.39.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 @@ -77,7 +77,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.3 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1s +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1t if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/python.props b/PCbuild/python.props index 57360e57baba66..5926c7ded4708d 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -74,8 +74,8 @@ $(ExternalsDir)libffi-3.4.3\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include - $(ExternalsDir)openssl-1.1.1s\ - $(ExternalsDir)openssl-bin-1.1.1s\$(ArchName)\ + $(ExternalsDir)openssl-1.1.1t\ + $(ExternalsDir)openssl-bin-1.1.1t\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.2.13\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 3ed26a47b066b9..347be8aeeca398 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -169,7 +169,7 @@ _lzma Homepage: https://tukaani.org/xz/ _ssl - Python wrapper for version 1.1.1q of the OpenSSL secure sockets + Python wrapper for version 1.1.1t of the OpenSSL secure sockets library, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 5ad597c8347e56..c0fbee9ca6f98f 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -46,8 +46,8 @@ ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1s", - "3.0.7" + "1.1.1t", + "3.0.8" ] LIBRESSL_OLD_VERSIONS = [ From 272da55affe6f2b3b73ff5474e1246089517d051 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Fri, 10 Feb 2023 02:45:58 +0900 Subject: [PATCH 09/83] Fix typo in `test_fstring.py` (#101600) --- Lib/test/test_fstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 318f38a6ed5b14..a50056da116e32 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -667,7 +667,7 @@ def test_missing_expression(self): "f'''{\t\f\r\n}'''", ]) - # Different error messeges are raised when a specfier ('!', ':' or '=') is used after an empty expression + # Different error messages are raised when a specfier ('!', ':' or '=') is used after an empty expression self.assertAllRaise(SyntaxError, "f-string: expression required before '!'", ["f'{!r}'", "f'{ !r}'", From f1f3af7b8245e61a2e0abef03b2c6c5902ed7df8 Mon Sep 17 00:00:00 2001 From: "Partha P. Mukherjee" Date: Thu, 9 Feb 2023 12:46:40 -0500 Subject: [PATCH 10/83] GH-101228: Fix typo in docstring for read method of `_io.TextIOWrapper` class (#101227) --- Modules/_io/textio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 32ab8a44c62151..ea2ea32c336954 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -56,10 +56,10 @@ textiobase_detach(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(textiobase_read_doc, - "Read at most n characters from stream.\n" + "Read at most size characters from stream.\n" "\n" - "Read from underlying buffer until we have n characters or we hit EOF.\n" - "If n is negative or omitted, read until EOF.\n" + "Read from underlying buffer until we have size characters or we hit EOF.\n" + "If size is negative or omitted, read until EOF.\n" ); static PyObject * From 5b946d371979a772120e6ee7d37f9b735769d433 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 10 Feb 2023 08:30:03 +0900 Subject: [PATCH 11/83] gh-101430: Update tracemalloc to handle presize properly. (gh-101745) --- ...-02-10-01-15-57.gh-issue-101430.T3Gegb.rst | 2 ++ Modules/_tracemalloc.c | 24 +++++++------------ Objects/object.c | 9 ++----- 3 files changed, 12 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst new file mode 100644 index 00000000000000..e617d85242144e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst @@ -0,0 +1,2 @@ +Update :mod:`tracemalloc` to handle presize of object properly. Patch by +Dong-hee Na. diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 9826ad2935beaa..d69c5636486da9 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -2,6 +2,7 @@ #include "pycore_fileutils.h" // _Py_write_noraise() #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_object.h" // _PyType_PreHeaderSize #include "pycore_pymem.h" // _Py_tracemalloc_config #include "pycore_runtime.h" // _Py_ID() #include "pycore_traceback.h" @@ -1400,20 +1401,16 @@ _tracemalloc__get_object_traceback(PyObject *module, PyObject *obj) /*[clinic end generated code: output=41ee0553a658b0aa input=29495f1b21c53212]*/ { PyTypeObject *type; - void *ptr; traceback_t *traceback; type = Py_TYPE(obj); - if (PyType_IS_GC(type)) { - ptr = (void *)((char *)obj - sizeof(PyGC_Head)); - } - else { - ptr = (void *)obj; - } + const size_t presize = _PyType_PreHeaderSize(type); + uintptr_t ptr = (uintptr_t)((char *)obj - presize); - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (uintptr_t)ptr); - if (traceback == NULL) + traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); + if (traceback == NULL) { Py_RETURN_NONE; + } return traceback_to_pyobject(traceback, NULL); } @@ -1723,14 +1720,9 @@ _PyTraceMalloc_NewReference(PyObject *op) return -1; } - uintptr_t ptr; PyTypeObject *type = Py_TYPE(op); - if (PyType_IS_GC(type)) { - ptr = (uintptr_t)((char *)op - sizeof(PyGC_Head)); - } - else { - ptr = (uintptr_t)op; - } + const size_t presize = _PyType_PreHeaderSize(type); + uintptr_t ptr = (uintptr_t)((char *)op - presize); int res = -1; diff --git a/Objects/object.c b/Objects/object.c index 7817c04ef5f5be..446c7b1f5f0302 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2387,14 +2387,9 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, /* Display the traceback where the object has been allocated. Do it before dumping repr(obj), since repr() is more likely to crash than dumping the traceback. */ - void *ptr; PyTypeObject *type = Py_TYPE(obj); - if (_PyType_IS_GC(type)) { - ptr = (void *)((char *)obj - sizeof(PyGC_Head)); - } - else { - ptr = (void *)obj; - } + const size_t presize = _PyType_PreHeaderSize(type); + void *ptr = (void *)((char *)obj - presize); _PyMem_DumpTraceback(fileno(stderr), ptr); /* This might succeed or fail, but we're about to abort, so at least From 34c50ceb1e2d40f7faab673d2033ecaafd3c611a Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 10 Feb 2023 06:00:58 +0300 Subject: [PATCH 12/83] gh-101747: Fix refleak in new `OrderedDict` repr (GH-101748) --- Objects/odictobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/odictobject.c b/Objects/odictobject.c index ab2bbed35873de..215a8af54fb266 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1385,6 +1385,7 @@ odict_repr(PyODictObject *self) result = PyUnicode_FromFormat("%s(%R)", _PyType_Name(Py_TYPE(self)), dcopy); + Py_DECREF(dcopy); Done: Py_ReprLeave((PyObject *)self); From 448c7d154e72506158d0a7a766e9f1cb8adf3dec Mon Sep 17 00:00:00 2001 From: abel1502 <32196516+abel1502@users.noreply.github.com> Date: Fri, 10 Feb 2023 06:10:46 +0300 Subject: [PATCH 13/83] Fix some typos in asdl_c.py (GH-101757) --- Parser/asdl_c.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 3e307610b635f4..db0e597b7a5aa4 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -73,7 +73,7 @@ def reflow_c_string(s, depth): def is_simple(sum_type): """Return True if a sum is a simple. - A sum is simple if it's types have no fields and itself + A sum is simple if its types have no fields and itself doesn't have any attributes. Instances of these types are cached at C level, and they act like singletons when propagating parser generated nodes into Python level, e.g. @@ -352,7 +352,7 @@ def visitSum(self, sum, name): self.visit(t, name, sum.attributes) def get_args(self, fields): - """Return list of C argument into, one for each field. + """Return list of C argument info, one for each field. Argument info is 3-tuple of a C type, variable name, and flag that is true if type can be NULL. From d40a23c0a11060ba7fa076d50980c18a11a13a40 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Feb 2023 08:25:02 +0100 Subject: [PATCH 14/83] gh-101759: Update macOS installer to SQLite 3.40.1 (#101761) --- Mac/BuildScript/build-installer.py | 6 +++--- .../macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 048cb01379607e..5ea2cfc5183ef5 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.39.4", - url="https://sqlite.org/2022/sqlite-autoconf-3390400.tar.gz", - checksum="44b7e6691b0954086f717a6c43b622a5", + name="SQLite 3.40.1", + url="https://sqlite.org/2022/sqlite-autoconf-3400100.tar.gz", + checksum="5498af3a357753d473ee713e363fa5b7", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' diff --git a/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst b/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst new file mode 100644 index 00000000000000..fc53d08bffc4fd --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst @@ -0,0 +1 @@ +Update macOS installer to SQLite 3.40.1. From 826bf0e6957fd0addc321d1baee1fa846e9457eb Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Feb 2023 12:58:14 +0100 Subject: [PATCH 15/83] gh-101277: Finalise isolating itertools (GH-101305) Add repeat, islice, chain, tee, teedataobject, and batched types to module state. Automerge-Triggered-By: GH:erlend-aasland --- ...-01-25-00-14-52.gh-issue-101277.FceHX7.rst | 2 + Modules/clinic/itertoolsmodule.c.h | 6 +- Modules/itertoolsmodule.c | 534 +++++++----------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 22 - 4 files changed, 220 insertions(+), 344 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst diff --git a/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst b/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst new file mode 100644 index 00000000000000..e09c0e09fb388f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst @@ -0,0 +1,2 @@ +Remove global state from :mod:`itertools` module (:pep:`687`). Patches by +Erlend E. Aasland. diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index d15d5f0890ca98..32278bf715aa98 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -232,7 +232,7 @@ static PyObject * itertools_teedataobject(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - PyTypeObject *base_tp = &teedataobject_type; + PyTypeObject *base_tp = clinic_state()->teedataobject_type; PyObject *it; PyObject *values; PyObject *next; @@ -270,7 +270,7 @@ static PyObject * itertools__tee(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - PyTypeObject *base_tp = &tee_type; + PyTypeObject *base_tp = clinic_state()->tee_type; PyObject *iterable; if ((type == base_tp || type->tp_init == base_tp->tp_init) && @@ -913,4 +913,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=a08b58d7dac825da input=a9049054013a1b77]*/ +/*[clinic end generated code: output=111cbd102c2a23c9 input=a9049054013a1b77]*/ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ce8720d0fd9228..6986695e47b1ae 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -5,6 +5,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "structmember.h" // PyMemberDef #include // offsetof() /* Itertools module written and maintained @@ -13,6 +14,8 @@ typedef struct { PyTypeObject *accumulate_type; + PyTypeObject *batched_type; + PyTypeObject *chain_type; PyTypeObject *combinations_type; PyTypeObject *compress_type; PyTypeObject *count_type; @@ -22,11 +25,15 @@ typedef struct { PyTypeObject *filterfalse_type; PyTypeObject *groupby_type; PyTypeObject *_grouper_type; + PyTypeObject *islice_type; PyTypeObject *pairwise_type; PyTypeObject *permutations_type; PyTypeObject *product_type; + PyTypeObject *repeat_type; PyTypeObject *starmap_type; PyTypeObject *takewhile_type; + PyTypeObject *tee_type; + PyTypeObject *teedataobject_type; PyTypeObject *ziplongest_type; } itertools_state; @@ -60,14 +67,14 @@ find_state_by_type(PyTypeObject *tp) module itertools class itertools.groupby "groupbyobject *" "clinic_state()->groupby_type" class itertools._grouper "_grouperobject *" "clinic_state()->_grouper_type" -class itertools.teedataobject "teedataobject *" "&teedataobject_type" -class itertools._tee "teeobject *" "&tee_type" -class itertools.batched "batchedobject *" "&batched_type" +class itertools.teedataobject "teedataobject *" "clinic_state()->teedataobject_type" +class itertools._tee "teeobject *" "clinic_state()->tee_type" +class itertools.batched "batchedobject *" "clinic_state()->batched_type" class itertools.cycle "cycleobject *" "clinic_state()->cycle_type" class itertools.dropwhile "dropwhileobject *" "clinic_state()->dropwhile_type" class itertools.takewhile "takewhileobject *" "clinic_state()->takewhile_type" class itertools.starmap "starmapobject *" "clinic_state()->starmap_type" -class itertools.chain "chainobject *" "&chain_type" +class itertools.chain "chainobject *" "clinic_state()->chain_type" class itertools.combinations "combinationsobject *" "clinic_state()->combinations_type" class itertools.combinations_with_replacement "cwr_object *" "clinic_state()->cwr_type" class itertools.permutations "permutationsobject *" "clinic_state()->permutations_type" @@ -77,11 +84,7 @@ class itertools.filterfalse "filterfalseobject *" "clinic_state()->filterfalse_t class itertools.count "countobject *" "clinic_state()->count_type" class itertools.pairwise "pairwiseobject *" "clinic_state()->pairwise_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=28ffff5c0c93eed7]*/ - -static PyTypeObject teedataobject_type; -static PyTypeObject tee_type; -static PyTypeObject batched_type; +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=aa48fe4de9d4080f]*/ #define clinic_state() (find_state_by_type(type)) #define clinic_state_by_cls() (get_module_state_by_cls(base_tp)) @@ -162,17 +165,18 @@ batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n) static void batched_dealloc(batchedobject *bo) { + PyTypeObject *tp = Py_TYPE(bo); PyObject_GC_UnTrack(bo); Py_XDECREF(bo->it); - Py_TYPE(bo)->tp_free(bo); + tp->tp_free(bo); + Py_DECREF(tp); } static int batched_traverse(batchedobject *bo, visitproc visit, void *arg) { - if (bo->it != NULL) { - Py_VISIT(bo->it); - } + Py_VISIT(Py_TYPE(bo)); + Py_VISIT(bo->it); return 0; } @@ -222,48 +226,25 @@ batched_next(batchedobject *bo) return result; } -static PyTypeObject batched_type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "itertools.batched", /* tp_name */ - sizeof(batchedobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)batched_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - batched_new__doc__, /* tp_doc */ - (traverseproc)batched_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (iternextfunc)batched_next, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - batched_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ +static PyType_Slot batched_slots[] = { + {Py_tp_dealloc, batched_dealloc}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_doc, (void *)batched_new__doc__}, + {Py_tp_traverse, batched_traverse}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, batched_next}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_new, batched_new}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec batched_spec = { + .name = "itertools.batched", + .basicsize = sizeof(batchedobject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = batched_slots, }; @@ -737,14 +718,15 @@ typedef struct { teedataobject *dataobj; int index; /* 0 <= index <= LINKCELLS */ PyObject *weakreflist; + itertools_state *state; } teeobject; static PyObject * -teedataobject_newinternal(PyObject *it) +teedataobject_newinternal(itertools_state *state, PyObject *it) { teedataobject *tdo; - tdo = PyObject_GC_New(teedataobject, &teedataobject_type); + tdo = PyObject_GC_New(teedataobject, state->teedataobject_type); if (tdo == NULL) return NULL; @@ -757,10 +739,10 @@ teedataobject_newinternal(PyObject *it) } static PyObject * -teedataobject_jumplink(teedataobject *tdo) +teedataobject_jumplink(itertools_state *state, teedataobject *tdo) { if (tdo->nextlink == NULL) - tdo->nextlink = teedataobject_newinternal(tdo->it); + tdo->nextlink = teedataobject_newinternal(state, tdo->it); return Py_XNewRef(tdo->nextlink); } @@ -796,6 +778,7 @@ teedataobject_traverse(teedataobject *tdo, visitproc visit, void * arg) { int i; + Py_VISIT(Py_TYPE(tdo)); Py_VISIT(tdo->it); for (i = 0; i < tdo->numread; i++) Py_VISIT(tdo->values[i]); @@ -804,9 +787,9 @@ teedataobject_traverse(teedataobject *tdo, visitproc visit, void * arg) } static void -teedataobject_safe_decref(PyObject *obj) +teedataobject_safe_decref(PyObject *obj, PyTypeObject *tdo_type) { - while (obj && Py_IS_TYPE(obj, &teedataobject_type) && + while (obj && Py_IS_TYPE(obj, tdo_type) && Py_REFCNT(obj) == 1) { PyObject *nextlink = ((teedataobject *)obj)->nextlink; ((teedataobject *)obj)->nextlink = NULL; @@ -826,16 +809,19 @@ teedataobject_clear(teedataobject *tdo) Py_CLEAR(tdo->values[i]); tmp = tdo->nextlink; tdo->nextlink = NULL; - teedataobject_safe_decref(tmp); + itertools_state *state = get_module_state_by_cls(Py_TYPE(tdo)); + teedataobject_safe_decref(tmp, state->teedataobject_type); return 0; } static void teedataobject_dealloc(teedataobject *tdo) { + PyTypeObject *tp = Py_TYPE(tdo); PyObject_GC_UnTrack(tdo); teedataobject_clear(tdo); PyObject_GC_Del(tdo); + Py_DECREF(tp); } static PyObject * @@ -874,9 +860,10 @@ itertools_teedataobject_impl(PyTypeObject *type, PyObject *it, teedataobject *tdo; Py_ssize_t i, len; - assert(type == &teedataobject_type); + itertools_state *state = get_module_state_by_cls(type); + assert(type == state->teedataobject_type); - tdo = (teedataobject *)teedataobject_newinternal(it); + tdo = (teedataobject *)teedataobject_newinternal(state, it); if (!tdo) return NULL; @@ -892,7 +879,7 @@ itertools_teedataobject_impl(PyTypeObject *type, PyObject *it, if (len == LINKCELLS) { if (next != Py_None) { - if (!Py_IS_TYPE(next, &teedataobject_type)) + if (!Py_IS_TYPE(next, state->teedataobject_type)) goto err; assert(tdo->nextlink == NULL); tdo->nextlink = Py_NewRef(next); @@ -915,47 +902,24 @@ static PyMethodDef teedataobject_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyTypeObject teedataobject_type = { - PyVarObject_HEAD_INIT(0, 0) /* Must fill in type value later */ - "itertools._tee_dataobject", /* tp_name */ - sizeof(teedataobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)teedataobject_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - itertools_teedataobject__doc__, /* tp_doc */ - (traverseproc)teedataobject_traverse, /* tp_traverse */ - (inquiry)teedataobject_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - teedataobject_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - itertools_teedataobject, /* tp_new */ - PyObject_GC_Del, /* tp_free */ +static PyType_Slot teedataobject_slots[] = { + {Py_tp_dealloc, teedataobject_dealloc}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_doc, (void *)itertools_teedataobject__doc__}, + {Py_tp_traverse, teedataobject_traverse}, + {Py_tp_clear, teedataobject_clear}, + {Py_tp_methods, teedataobject_methods}, + {Py_tp_new, itertools_teedataobject}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec teedataobject_spec = { + .name = "itertools._tee_dataobject", + .basicsize = sizeof(teedataobject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = teedataobject_slots, }; @@ -965,7 +929,7 @@ tee_next(teeobject *to) PyObject *value, *link; if (to->index >= LINKCELLS) { - link = teedataobject_jumplink(to->dataobj); + link = teedataobject_jumplink(to->state, to->dataobj); if (link == NULL) return NULL; Py_SETREF(to->dataobj, (teedataobject *)link); @@ -981,6 +945,7 @@ tee_next(teeobject *to) static int tee_traverse(teeobject *to, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(to)); Py_VISIT((PyObject *)to->dataobj); return 0; } @@ -990,12 +955,13 @@ tee_copy(teeobject *to, PyObject *Py_UNUSED(ignored)) { teeobject *newto; - newto = PyObject_GC_New(teeobject, &tee_type); + newto = PyObject_GC_New(teeobject, Py_TYPE(to)); if (newto == NULL) return NULL; newto->dataobj = (teedataobject*)Py_NewRef(to->dataobj); newto->index = to->index; newto->weakreflist = NULL; + newto->state = to->state; PyObject_GC_Track(newto); return (PyObject *)newto; } @@ -1003,7 +969,7 @@ tee_copy(teeobject *to, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(teecopy_doc, "Returns an independent iterator."); static PyObject * -tee_fromiterable(PyObject *iterable) +tee_fromiterable(itertools_state *state, PyObject *iterable) { teeobject *to; PyObject *it; @@ -1011,17 +977,17 @@ tee_fromiterable(PyObject *iterable) it = PyObject_GetIter(iterable); if (it == NULL) return NULL; - if (PyObject_TypeCheck(it, &tee_type)) { + if (PyObject_TypeCheck(it, state->tee_type)) { to = (teeobject *)tee_copy((teeobject *)it, NULL); goto done; } - PyObject *dataobj = teedataobject_newinternal(it); + PyObject *dataobj = teedataobject_newinternal(state, it); if (!dataobj) { to = NULL; goto done; } - to = PyObject_GC_New(teeobject, &tee_type); + to = PyObject_GC_New(teeobject, state->tee_type); if (to == NULL) { Py_DECREF(dataobj); goto done; @@ -1029,6 +995,7 @@ tee_fromiterable(PyObject *iterable) to->dataobj = (teedataobject *)dataobj; to->index = 0; to->weakreflist = NULL; + to->state = state; PyObject_GC_Track(to); done: Py_DECREF(it); @@ -1047,7 +1014,8 @@ static PyObject * itertools__tee_impl(PyTypeObject *type, PyObject *iterable) /*[clinic end generated code: output=b02d3fd26c810c3f input=adc0779d2afe37a2]*/ { - return tee_fromiterable(iterable); + itertools_state *state = get_module_state_by_cls(type); + return tee_fromiterable(state, iterable); } static int @@ -1062,9 +1030,11 @@ tee_clear(teeobject *to) static void tee_dealloc(teeobject *to) { + PyTypeObject *tp = Py_TYPE(to); PyObject_GC_UnTrack(to); tee_clear(to); PyObject_GC_Del(to); + Py_DECREF(tp); } static PyObject * @@ -1082,7 +1052,8 @@ tee_setstate(teeobject *to, PyObject *state) PyErr_SetString(PyExc_TypeError, "state is not a tuple"); return NULL; } - if (!PyArg_ParseTuple(state, "O!i", &teedataobject_type, &tdo, &index)) { + PyTypeObject *tdo_type = to->state->teedataobject_type; + if (!PyArg_ParseTuple(state, "O!i", tdo_type, &tdo, &index)) { return NULL; } if (index < 0 || index > LINKCELLS) { @@ -1102,47 +1073,31 @@ static PyMethodDef tee_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyTypeObject tee_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "itertools._tee", /* tp_name */ - sizeof(teeobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)tee_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - itertools__tee__doc__, /* tp_doc */ - (traverseproc)tee_traverse, /* tp_traverse */ - (inquiry)tee_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(teeobject, weakreflist), /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (iternextfunc)tee_next, /* tp_iternext */ - tee_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - itertools__tee, /* tp_new */ - PyObject_GC_Del, /* tp_free */ +static PyMemberDef tee_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(teeobject, weakreflist), READONLY}, + {NULL}, +}; + +static PyType_Slot tee_slots[] = { + {Py_tp_dealloc, tee_dealloc}, + {Py_tp_doc, (void *)itertools__tee__doc__}, + {Py_tp_traverse, tee_traverse}, + {Py_tp_clear, tee_clear}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, tee_next}, + {Py_tp_methods, tee_methods}, + {Py_tp_members, tee_members}, + {Py_tp_new, itertools__tee}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec tee_spec = { + .name = "itertools._tee", + .basicsize = sizeof(teeobject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = tee_slots, }; /*[clinic input] @@ -1184,7 +1139,8 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n) copyable = it; } else { - copyable = tee_fromiterable(it); + itertools_state *state = get_module_state(module); + copyable = tee_fromiterable(state, it); Py_DECREF(it); if (copyable == NULL) { Py_DECREF(result); @@ -1682,8 +1638,6 @@ typedef struct { Py_ssize_t cnt; } isliceobject; -static PyTypeObject islice_type; - static PyObject * islice_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -1693,7 +1647,9 @@ islice_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_ssize_t numargs; isliceobject *lz; - if ((type == &islice_type || type->tp_init == islice_type.tp_init) && + itertools_state *st = find_state_by_type(type); + PyTypeObject *islice_type = st->islice_type; + if ((type == islice_type || type->tp_init == islice_type->tp_init) && !_PyArg_NoKeywords("islice", kwds)) return NULL; @@ -1772,14 +1728,17 @@ islice_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static void islice_dealloc(isliceobject *lz) { + PyTypeObject *tp = Py_TYPE(lz); PyObject_GC_UnTrack(lz); Py_XDECREF(lz->it); - Py_TYPE(lz)->tp_free(lz); + tp->tp_free(lz); + Py_DECREF(tp); } static int islice_traverse(isliceobject *lz, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(lz)); Py_VISIT(lz->it); return 0; } @@ -1885,48 +1844,25 @@ specified as another value, step determines how many values are\n\ skipped between successive calls. Works like a slice() on a list\n\ but returns an iterator."); -static PyTypeObject islice_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "itertools.islice", /* tp_name */ - sizeof(isliceobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)islice_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - islice_doc, /* tp_doc */ - (traverseproc)islice_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (iternextfunc)islice_next, /* tp_iternext */ - islice_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - islice_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ +static PyType_Slot islice_slots[] = { + {Py_tp_dealloc, islice_dealloc}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_doc, (void *)islice_doc}, + {Py_tp_traverse, islice_traverse}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, islice_next}, + {Py_tp_methods, islice_methods}, + {Py_tp_new, islice_new}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec islice_spec = { + .name = "itertools.islice", + .basicsize = sizeof(isliceobject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = islice_slots, }; @@ -2056,8 +1992,6 @@ typedef struct { PyObject *active; /* Currently running input iterator */ } chainobject; -static PyTypeObject chain_type; - static PyObject * chain_new_internal(PyTypeObject *type, PyObject *source) { @@ -2079,7 +2013,9 @@ chain_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyObject *source; - if ((type == &chain_type || type->tp_init == chain_type.tp_init) && + itertools_state *state = find_state_by_type(type); + PyTypeObject *chain_type = state->chain_type; + if ((type == chain_type || type->tp_init == chain_type->tp_init) && !_PyArg_NoKeywords("chain", kwds)) return NULL; @@ -2114,15 +2050,18 @@ itertools_chain_from_iterable(PyTypeObject *type, PyObject *arg) static void chain_dealloc(chainobject *lz) { + PyTypeObject *tp = Py_TYPE(lz); PyObject_GC_UnTrack(lz); Py_XDECREF(lz->active); Py_XDECREF(lz->source); - Py_TYPE(lz)->tp_free(lz); + tp->tp_free(lz); + Py_DECREF(tp); } static int chain_traverse(chainobject *lz, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(lz)); Py_VISIT(lz->source); Py_VISIT(lz->active); return 0; @@ -2227,48 +2166,25 @@ static PyMethodDef chain_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyTypeObject chain_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "itertools.chain", /* tp_name */ - sizeof(chainobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)chain_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - chain_doc, /* tp_doc */ - (traverseproc)chain_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (iternextfunc)chain_next, /* tp_iternext */ - chain_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - chain_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ +static PyType_Slot chain_slots[] = { + {Py_tp_dealloc, chain_dealloc}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_doc, (void *)chain_doc}, + {Py_tp_traverse, chain_traverse}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, chain_next}, + {Py_tp_methods, chain_methods}, + {Py_tp_new, chain_new}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec chain_spec = { + .name = "itertools.chain", + .basicsize = sizeof(chainobject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = chain_slots, }; @@ -3574,6 +3490,7 @@ typedef struct { PyObject *it; PyObject *binop; PyObject *initial; + itertools_state *state; } accumulateobject; /*[clinic input] @@ -3612,6 +3529,7 @@ itertools_accumulate_impl(PyTypeObject *type, PyObject *iterable, lz->total = NULL; lz->it = it; lz->initial = Py_XNewRef(initial); + lz->state = find_state_by_type(type); return (PyObject *)lz; } @@ -3674,13 +3592,13 @@ accumulate_next(accumulateobject *lz) static PyObject * accumulate_reduce(accumulateobject *lz, PyObject *Py_UNUSED(ignored)) { + itertools_state *state = lz->state; + if (lz->initial != Py_None) { PyObject *it; assert(lz->total == NULL); - if (PyType_Ready(&chain_type) < 0) - return NULL; - it = PyObject_CallFunction((PyObject *)&chain_type, "(O)O", + it = PyObject_CallFunction((PyObject *)(state->chain_type), "(O)O", lz->initial, lz->it); if (it == NULL) return NULL; @@ -3690,11 +3608,7 @@ accumulate_reduce(accumulateobject *lz, PyObject *Py_UNUSED(ignored)) if (lz->total == Py_None) { PyObject *it; - if (PyType_Ready(&chain_type) < 0) - return NULL; - if (PyType_Ready(&islice_type) < 0) - return NULL; - it = PyObject_CallFunction((PyObject *)&chain_type, "(O)O", + it = PyObject_CallFunction((PyObject *)(state->chain_type), "(O)O", lz->total, lz->it); if (it == NULL) return NULL; @@ -3702,7 +3616,8 @@ accumulate_reduce(accumulateobject *lz, PyObject *Py_UNUSED(ignored)) it, lz->binop ? lz->binop : Py_None); if (it == NULL) return NULL; - return Py_BuildValue("O(NiO)", &islice_type, it, 1, Py_None); + + return Py_BuildValue("O(NiO)", state->islice_type, it, 1, Py_None); } return Py_BuildValue("O(OO)O", Py_TYPE(lz), lz->it, lz->binop?lz->binop:Py_None, @@ -4261,8 +4176,6 @@ typedef struct { Py_ssize_t cnt; } repeatobject; -static PyTypeObject repeat_type; - static PyObject * repeat_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -4292,14 +4205,17 @@ repeat_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static void repeat_dealloc(repeatobject *ro) { + PyTypeObject *tp = Py_TYPE(ro); PyObject_GC_UnTrack(ro); Py_XDECREF(ro->element); - Py_TYPE(ro)->tp_free(ro); + tp->tp_free(ro); + Py_DECREF(tp); } static int repeat_traverse(repeatobject *ro, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(ro)); Py_VISIT(ro->element); return 0; } @@ -4361,48 +4277,26 @@ PyDoc_STRVAR(repeat_doc, for the specified number of times. If not specified, returns the object\n\ endlessly."); -static PyTypeObject repeat_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "itertools.repeat", /* tp_name */ - sizeof(repeatobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)repeat_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)repeat_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - repeat_doc, /* tp_doc */ - (traverseproc)repeat_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (iternextfunc)repeat_next, /* tp_iternext */ - repeat_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - repeat_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ +static PyType_Slot repeat_slots[] = { + {Py_tp_dealloc, repeat_dealloc}, + {Py_tp_repr, repeat_repr}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_doc, (void *)repeat_doc}, + {Py_tp_traverse, repeat_traverse}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, repeat_next}, + {Py_tp_methods, repeat_methods}, + {Py_tp_new, repeat_new}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec repeat_spec = { + .name = "itertools.repeat", + .basicsize = sizeof(repeatobject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = repeat_slots, }; @@ -4695,6 +4589,8 @@ itertoolsmodule_traverse(PyObject *mod, visitproc visit, void *arg) { itertools_state *state = get_module_state(mod); Py_VISIT(state->accumulate_type); + Py_VISIT(state->batched_type); + Py_VISIT(state->chain_type); Py_VISIT(state->combinations_type); Py_VISIT(state->compress_type); Py_VISIT(state->count_type); @@ -4704,11 +4600,15 @@ itertoolsmodule_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->filterfalse_type); Py_VISIT(state->groupby_type); Py_VISIT(state->_grouper_type); + Py_VISIT(state->islice_type); Py_VISIT(state->pairwise_type); Py_VISIT(state->permutations_type); Py_VISIT(state->product_type); + Py_VISIT(state->repeat_type); Py_VISIT(state->starmap_type); Py_VISIT(state->takewhile_type); + Py_VISIT(state->tee_type); + Py_VISIT(state->teedataobject_type); Py_VISIT(state->ziplongest_type); return 0; } @@ -4718,6 +4618,8 @@ itertoolsmodule_clear(PyObject *mod) { itertools_state *state = get_module_state(mod); Py_CLEAR(state->accumulate_type); + Py_CLEAR(state->batched_type); + Py_CLEAR(state->chain_type); Py_CLEAR(state->combinations_type); Py_CLEAR(state->compress_type); Py_CLEAR(state->count_type); @@ -4727,11 +4629,15 @@ itertoolsmodule_clear(PyObject *mod) Py_CLEAR(state->filterfalse_type); Py_CLEAR(state->groupby_type); Py_CLEAR(state->_grouper_type); + Py_CLEAR(state->islice_type); Py_CLEAR(state->pairwise_type); Py_CLEAR(state->permutations_type); Py_CLEAR(state->product_type); + Py_CLEAR(state->repeat_type); Py_CLEAR(state->starmap_type); Py_CLEAR(state->takewhile_type); + Py_CLEAR(state->tee_type); + Py_CLEAR(state->teedataobject_type); Py_CLEAR(state->ziplongest_type); return 0; } @@ -4758,6 +4664,8 @@ itertoolsmodule_exec(PyObject *mod) { itertools_state *state = get_module_state(mod); ADD_TYPE(mod, state->accumulate_type, &accumulate_spec); + ADD_TYPE(mod, state->batched_type, &batched_spec); + ADD_TYPE(mod, state->chain_type, &chain_spec); ADD_TYPE(mod, state->combinations_type, &combinations_spec); ADD_TYPE(mod, state->compress_type, &compress_spec); ADD_TYPE(mod, state->count_type, &count_spec); @@ -4767,30 +4675,18 @@ itertoolsmodule_exec(PyObject *mod) ADD_TYPE(mod, state->filterfalse_type, &filterfalse_spec); ADD_TYPE(mod, state->groupby_type, &groupby_spec); ADD_TYPE(mod, state->_grouper_type, &_grouper_spec); + ADD_TYPE(mod, state->islice_type, &islice_spec); ADD_TYPE(mod, state->pairwise_type, &pairwise_spec); ADD_TYPE(mod, state->permutations_type, &permutations_spec); ADD_TYPE(mod, state->product_type, &product_spec); + ADD_TYPE(mod, state->repeat_type, &repeat_spec); ADD_TYPE(mod, state->starmap_type, &starmap_spec); ADD_TYPE(mod, state->takewhile_type, &takewhile_spec); + ADD_TYPE(mod, state->tee_type, &tee_spec); + ADD_TYPE(mod, state->teedataobject_type, &teedataobject_spec); ADD_TYPE(mod, state->ziplongest_type, &ziplongest_spec); - PyTypeObject *typelist[] = { - &batched_type, - &islice_type, - &chain_type, - &repeat_type, - &tee_type, - &teedataobject_type - }; - - Py_SET_TYPE(&teedataobject_type, &PyType_Type); - - for (size_t i = 0; i < Py_ARRAY_LENGTH(typelist); i++) { - if (PyModule_AddType(mod, typelist[i]) < 0) { - return -1; - } - } - + Py_SET_TYPE(state->teedataobject_type, &PyType_Type); return 0; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index cf2d5c368f1bda..52ea0b4901d4bb 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -335,28 +335,6 @@ Modules/_testcapi/vectorcall.c - MethodDescriptorBase_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorDerived_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorNopGet_Type - Modules/_testcapi/vectorcall.c - MethodDescriptor2_Type - -Modules/itertoolsmodule.c - _grouper_type - -Modules/itertoolsmodule.c - accumulate_type - -Modules/itertoolsmodule.c - batched_type - -Modules/itertoolsmodule.c - chain_type - -Modules/itertoolsmodule.c - combinations_type - -Modules/itertoolsmodule.c - compress_type - -Modules/itertoolsmodule.c - count_type - -Modules/itertoolsmodule.c - cwr_type - -Modules/itertoolsmodule.c - cycle_type - -Modules/itertoolsmodule.c - dropwhile_type - -Modules/itertoolsmodule.c - filterfalse_type - -Modules/itertoolsmodule.c - groupby_type - -Modules/itertoolsmodule.c - islice_type - -Modules/itertoolsmodule.c - pairwise_type - -Modules/itertoolsmodule.c - permutations_type - -Modules/itertoolsmodule.c - product_type - -Modules/itertoolsmodule.c - repeat_type - -Modules/itertoolsmodule.c - starmap_type - -Modules/itertoolsmodule.c - takewhile_type - -Modules/itertoolsmodule.c - tee_type - -Modules/itertoolsmodule.c - teedataobject_type - -Modules/itertoolsmodule.c - ziplongest_type - ################################## From 5d15224011217487e1a174c144af0e5f5826c17c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Feb 2023 17:38:26 +0100 Subject: [PATCH 16/83] gh-101759: Update Windows installer to SQLite 3.40.1 (#101762) --- .../next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst | 1 + PCbuild/get_externals.bat | 2 +- PCbuild/python.props | 2 +- PCbuild/readme.txt | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst diff --git a/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst b/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst new file mode 100644 index 00000000000000..62bcac34397d2e --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst @@ -0,0 +1 @@ +Update Windows installer to SQLite 3.40.1. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index d4d96bd49d72c6..2c424517eae4b1 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.3 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1t -set libraries=%libraries% sqlite-3.39.4.0 +set libraries=%libraries% sqlite-3.40.1.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tix-8.4.3.6 diff --git a/PCbuild/python.props b/PCbuild/python.props index 5926c7ded4708d..28ee74d8594759 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -68,7 +68,7 @@ - $(ExternalsDir)sqlite-3.39.4.0\ + $(ExternalsDir)sqlite-3.40.1.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.2.5\ $(ExternalsDir)libffi-3.4.3\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 347be8aeeca398..4c799b64c461c1 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -188,7 +188,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.39.4, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.40.1, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter From 366b94905869d680b3f1d4801fb497e78811e511 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 10 Feb 2023 16:49:29 +0000 Subject: [PATCH 17/83] gh-101517: make bdb avoid looking up in linecache with lineno=None (#101787) --- Lib/bdb.py | 7 ++++--- Lib/test/test_bdb.py | 6 ++++++ .../Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst diff --git a/Lib/bdb.py b/Lib/bdb.py index 81fbb8514acb6f..7f9b09514ffd00 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -570,9 +570,10 @@ def format_stack_entry(self, frame_lineno, lprefix=': '): rv = frame.f_locals['__return__'] s += '->' s += reprlib.repr(rv) - line = linecache.getline(filename, lineno, frame.f_globals) - if line: - s += lprefix + line.strip() + if lineno is not None: + line = linecache.getline(filename, lineno, frame.f_globals) + if line: + s += lprefix + line.strip() return s # The following methods can be called by clients to use diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index 87a5ac308a12df..042c2daea7f797 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -1203,5 +1203,11 @@ def main(): tracer.runcall(tfunc_import) +class TestRegressions(unittest.TestCase): + def test_format_stack_entry_no_lineno(self): + # See gh-101517 + Bdb().format_stack_entry((sys._getframe(), None)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst b/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst new file mode 100644 index 00000000000000..a5f6bdfa5ac2f0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst @@ -0,0 +1 @@ +Fixed bug where :mod:`bdb` looks up the source line with :mod:`linecache` with a ``lineno=None``, which causes it to fail with an unhandled exception. From e1aadedf099e645fd2eb1aa8bdcde5a105cee95d Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 Feb 2023 16:57:30 +0000 Subject: [PATCH 18/83] gh-101763: Update bundled copy of libffi to 3.4.4 on Windows (GH-101784) --- .../Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst | 1 + PCbuild/get_externals.bat | 4 ++-- PCbuild/python.props | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst diff --git a/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst b/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst new file mode 100644 index 00000000000000..e7e5a73afeb532 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst @@ -0,0 +1 @@ +Updates copy of libffi bundled with Windows installs to 3.4.4. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 2c424517eae4b1..128241393f9f09 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -52,7 +52,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 -if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.3 +if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1t set libraries=%libraries% sqlite-3.40.1.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 @@ -76,7 +76,7 @@ for %%e in (%libraries%) do ( echo.Fetching external binaries... set binaries= -if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.3 +if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1t if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/python.props b/PCbuild/python.props index 28ee74d8594759..7994fbe7cd5e0b 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -71,7 +71,7 @@ $(ExternalsDir)sqlite-3.40.1.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.2.5\ - $(ExternalsDir)libffi-3.4.3\ + $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include $(ExternalsDir)openssl-1.1.1t\ From 2037ebf81bd4bbe5421421b822bd57cfd665a1e9 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Feb 2023 18:54:04 +0100 Subject: [PATCH 19/83] Docs: use parameter list for sqlite3.Cursor.execute* (#101782) Co-authored-by: Alex Waygood --- Doc/library/sqlite3.rst | 45 ++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index bbdc891c930cf4..8ffc0aad91995c 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1418,15 +1418,22 @@ Cursor objects .. method:: execute(sql, parameters=(), /) - Execute SQL statement *sql*. - Bind values to the statement using :ref:`placeholders - ` that map to the :term:`sequence` or :class:`dict` - *parameters*. + Execute SQL a single SQL statement, + optionally binding Python values using + :ref:`placeholders `. - :meth:`execute` will only execute a single SQL statement. If you try to execute - more than one statement with it, it will raise a :exc:`ProgrammingError`. Use - :meth:`executescript` if you want to execute multiple SQL statements with one - call. + :param str sql: + A single SQL statement. + + :param parameters: + Python values to bind to placeholders in *sql*. + A :class:`!dict` if named placeholders are used. + A :term:`!sequence` if unnamed placeholders are used. + See :ref:`sqlite3-placeholders`. + :type parameters: :class:`dict` | :term:`sequence` + + :raises ProgrammingError: + If *sql* contains more than one SQL statement. If :attr:`~Connection.autocommit` is :data:`LEGACY_TRANSACTION_CONTROL`, @@ -1435,15 +1442,29 @@ Cursor objects and there is no open transaction, a transaction is implicitly opened before executing *sql*. + Use :meth:`executescript` to execute multiple SQL statements. .. method:: executemany(sql, parameters, /) - Execute :ref:`parameterized ` SQL statement *sql* - against all parameter sequences or mappings found in the sequence - *parameters*. It is also possible to use an - :term:`iterator` yielding parameters instead of a sequence. + For every item in *parameters*, + repeatedly execute the :ref:`parameterized ` + SQL statement *sql*. + Uses the same implicit transaction handling as :meth:`~Cursor.execute`. + :param str sql: + A single SQL :abbr:`DML (Data Manipulation Language)` statement. + + :param parameters: + An :term:`!iterable` of parameters to bind with + the placeholders in *sql*. + See :ref:`sqlite3-placeholders`. + :type parameters: :term:`iterable` + + :raises ProgrammingError: + If *sql* contains more than one SQL statement, + or is not a DML statment. + Example: .. testcode:: sqlite3.cursor From 61f2be08661949e2f6dfc94143436297e60d47de Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Fri, 10 Feb 2023 20:46:12 +0200 Subject: [PATCH 20/83] Docs: Fix getstatus() -> getcode() typos (#101296) --- Doc/library/http.client.rst | 2 +- Doc/library/urllib.request.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 48582219695b41..ad3416135e307b 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -532,7 +532,7 @@ statement. .. deprecated:: 3.9 Deprecated in favor of :attr:`~HTTPResponse.headers`. -.. method:: HTTPResponse.getstatus() +.. method:: HTTPResponse.getcode() .. deprecated:: 3.9 Deprecated in favor of :attr:`~HTTPResponse.status`. diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 59e1f2da828a83..64cc9c388ec30d 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -1630,7 +1630,7 @@ The typical response object is a :class:`urllib.response.addinfourl` instance: .. deprecated:: 3.9 Deprecated in favor of :attr:`~addinfourl.status`. - .. method:: getstatus() + .. method:: getcode() .. deprecated:: 3.9 Deprecated in favor of :attr:`~addinfourl.status`. From 17143e2c30ae5e51945e04eeaec7ebb0e1f07fb5 Mon Sep 17 00:00:00 2001 From: busywhitespace Date: Sat, 11 Feb 2023 00:29:24 +0100 Subject: [PATCH 21/83] gh-101390: Fix docs for `imporlib.util.LazyLoader.factory` to properly call it a class method (GH-101391) --- Doc/library/importlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 3fc1531c0cdf19..89efa64c6b5203 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1387,7 +1387,7 @@ an :term:`importer`. .. classmethod:: factory(loader) - A static method which returns a callable that creates a lazy loader. This + A class method which returns a callable that creates a lazy loader. This is meant to be used in situations where the loader is passed by class instead of by instance. :: From b652d40f1c88fcd8595cd401513f6b7f8e499471 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sat, 11 Feb 2023 14:07:39 +0530 Subject: [PATCH 22/83] GH-101797: allocate `PyExpat_CAPI` capsule on heap (#101798) --- Modules/pyexpat.c | 72 +++++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 63a3392d5efe7d..0a744998b6c514 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1878,6 +1878,18 @@ add_features(PyObject *mod) } #endif +static void +pyexpat_capsule_destructor(PyObject *capsule) +{ + void *p = PyCapsule_GetPointer(capsule, PyExpat_CAPSULE_NAME); + if (p == NULL) { + PyErr_WriteUnraisable(capsule); + return; + } + PyMem_Free(p); +} + + static int pyexpat_exec(PyObject *mod) { @@ -1965,40 +1977,46 @@ pyexpat_exec(PyObject *mod) MYCONST(XML_PARAM_ENTITY_PARSING_ALWAYS); #undef MYCONST - static struct PyExpat_CAPI capi; + struct PyExpat_CAPI *capi = PyMem_Malloc(sizeof(*capi)); + if (capi == NULL) { + PyErr_NoMemory(); + return -1; + } /* initialize pyexpat dispatch table */ - capi.size = sizeof(capi); - capi.magic = PyExpat_CAPI_MAGIC; - capi.MAJOR_VERSION = XML_MAJOR_VERSION; - capi.MINOR_VERSION = XML_MINOR_VERSION; - capi.MICRO_VERSION = XML_MICRO_VERSION; - capi.ErrorString = XML_ErrorString; - capi.GetErrorCode = XML_GetErrorCode; - capi.GetErrorColumnNumber = XML_GetErrorColumnNumber; - capi.GetErrorLineNumber = XML_GetErrorLineNumber; - capi.Parse = XML_Parse; - capi.ParserCreate_MM = XML_ParserCreate_MM; - capi.ParserFree = XML_ParserFree; - capi.SetCharacterDataHandler = XML_SetCharacterDataHandler; - capi.SetCommentHandler = XML_SetCommentHandler; - capi.SetDefaultHandlerExpand = XML_SetDefaultHandlerExpand; - capi.SetElementHandler = XML_SetElementHandler; - capi.SetNamespaceDeclHandler = XML_SetNamespaceDeclHandler; - capi.SetProcessingInstructionHandler = XML_SetProcessingInstructionHandler; - capi.SetUnknownEncodingHandler = XML_SetUnknownEncodingHandler; - capi.SetUserData = XML_SetUserData; - capi.SetStartDoctypeDeclHandler = XML_SetStartDoctypeDeclHandler; - capi.SetEncoding = XML_SetEncoding; - capi.DefaultUnknownEncodingHandler = PyUnknownEncodingHandler; + capi->size = sizeof(*capi); + capi->magic = PyExpat_CAPI_MAGIC; + capi->MAJOR_VERSION = XML_MAJOR_VERSION; + capi->MINOR_VERSION = XML_MINOR_VERSION; + capi->MICRO_VERSION = XML_MICRO_VERSION; + capi->ErrorString = XML_ErrorString; + capi->GetErrorCode = XML_GetErrorCode; + capi->GetErrorColumnNumber = XML_GetErrorColumnNumber; + capi->GetErrorLineNumber = XML_GetErrorLineNumber; + capi->Parse = XML_Parse; + capi->ParserCreate_MM = XML_ParserCreate_MM; + capi->ParserFree = XML_ParserFree; + capi->SetCharacterDataHandler = XML_SetCharacterDataHandler; + capi->SetCommentHandler = XML_SetCommentHandler; + capi->SetDefaultHandlerExpand = XML_SetDefaultHandlerExpand; + capi->SetElementHandler = XML_SetElementHandler; + capi->SetNamespaceDeclHandler = XML_SetNamespaceDeclHandler; + capi->SetProcessingInstructionHandler = XML_SetProcessingInstructionHandler; + capi->SetUnknownEncodingHandler = XML_SetUnknownEncodingHandler; + capi->SetUserData = XML_SetUserData; + capi->SetStartDoctypeDeclHandler = XML_SetStartDoctypeDeclHandler; + capi->SetEncoding = XML_SetEncoding; + capi->DefaultUnknownEncodingHandler = PyUnknownEncodingHandler; #if XML_COMBINED_VERSION >= 20100 - capi.SetHashSalt = XML_SetHashSalt; + capi->SetHashSalt = XML_SetHashSalt; #else - capi.SetHashSalt = NULL; + capi->SetHashSalt = NULL; #endif /* export using capsule */ - PyObject *capi_object = PyCapsule_New(&capi, PyExpat_CAPSULE_NAME, NULL); + PyObject *capi_object = PyCapsule_New(capi, PyExpat_CAPSULE_NAME, + pyexpat_capsule_destructor); if (capi_object == NULL) { + PyMem_Free(capi); return -1; } From 3eb12df8b526aa5a2ca6b43f21a1c5e7d38ee634 Mon Sep 17 00:00:00 2001 From: mjoerg Date: Sat, 11 Feb 2023 16:34:15 +0100 Subject: [PATCH 23/83] Fix typo in test_fstring.py (#101823) --- Lib/test/test_fstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index a50056da116e32..b3f6ef41d77b8f 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -667,7 +667,7 @@ def test_missing_expression(self): "f'''{\t\f\r\n}'''", ]) - # Different error messages are raised when a specfier ('!', ':' or '=') is used after an empty expression + # Different error messages are raised when a specifier ('!', ':' or '=') is used after an empty expression self.assertAllRaise(SyntaxError, "f-string: expression required before '!'", ["f'{!r}'", "f'{ !r}'", From 1d194235e4d5981b5fea25c75318d61189103a58 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 11 Feb 2023 20:54:28 -0800 Subject: [PATCH 24/83] gh-89792: Prevent test_tools from copying 1000M of "source" in freeze test (#101837) Prevent test_tools from copying 1000M of "source" It doesn't need a git repo, just the checkout. We skip .git metadata, Doc/build, Doc/venv, and `__pycache__` subdirs, that developers often have in their clients to reduce the size of the source tree copy ten-fold. This should significantly reduce IO and presumably time on buildbots during this long test. --- .../2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst | 3 +++ Tools/freeze/test/freeze.py | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst diff --git a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst b/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst new file mode 100644 index 00000000000000..a3a3070d7f3790 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst @@ -0,0 +1,3 @@ +``test_tools`` now copies up to 10x less source data to a temporary +directory during the ``freeze`` test by ignoring git metadata and other +artifacts. diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index ddbfd7fc9c2f41..0ae983b15c98f0 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -80,7 +80,19 @@ def copy_source_tree(newroot, oldroot): if newroot == SRCDIR: raise Exception('this probably isn\'t what you wanted') shutil.rmtree(newroot) - shutil.copytree(oldroot, newroot) + + def ignore_non_src(src, names): + """Turns what could be a 1000M copy into a 100M copy.""" + # Don't copy the ~600M+ of needless git repo metadata. + # source only, ignore cached .pyc files. + subdirs_to_skip = {'.git', '__pycache__'} + if os.path.basename(src) == 'Doc': + # Another potential ~250M+ of non test related data. + subdirs_to_skip.add('build') + subdirs_to_skip.add('venv') + return subdirs_to_skip + + shutil.copytree(oldroot, newroot, ignore=ignore_non_src) if os.path.exists(os.path.join(newroot, 'Makefile')): _run_quiet([MAKE, 'clean'], newroot) From da2fb9264315dc30ac3012b4dbf5ba76d3f34433 Mon Sep 17 00:00:00 2001 From: Soumendra Ganguly <67527439+8vasu@users.noreply.github.com> Date: Sat, 11 Feb 2023 23:24:43 -0600 Subject: [PATCH 25/83] gh-85984: Utilize new "winsize" functions from termios in pty tests. (#101831) Utilize new functions termios.tcgetwinsize() and termios.tcsetwinsize in test_pty.py. Signed-off-by: Soumendra Ganguly Co-authored-by: Gregory P. Smith --- Lib/test/test_pty.py | 96 ++++++------------- ...3-02-11-22-36-10.gh-issue-85984.EVXjT9.rst | 1 + 2 files changed, 28 insertions(+), 69 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index fa0dbcc16f3ce8..c723bb362c5d87 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -3,6 +3,8 @@ # Skip these tests if termios or fcntl are not available import_module('termios') +# fcntl is a proxy for not being one of the wasm32 platforms even though we +# don't use this module... a proper check for what crashes those is needed. import_module("fcntl") import errno @@ -15,20 +17,12 @@ import socket import io # readline import unittest - -import struct -import fcntl import warnings TEST_STRING_1 = b"I wish to buy a fish license.\n" TEST_STRING_2 = b"For my pet fish, Eric.\n" -try: - _TIOCGWINSZ = tty.TIOCGWINSZ - _TIOCSWINSZ = tty.TIOCSWINSZ - _HAVE_WINSZ = True -except AttributeError: - _HAVE_WINSZ = False +_HAVE_WINSZ = hasattr(tty, "TIOCGWINSZ") and hasattr(tty, "TIOCSWINSZ") if verbose: def debug(msg): @@ -82,14 +76,6 @@ def expectedFailureIfStdinIsTTY(fun): pass return fun -def _get_term_winsz(fd): - s = struct.pack("HHHH", 0, 0, 0, 0) - return fcntl.ioctl(fd, _TIOCGWINSZ, s) - -def _set_term_winsz(fd, winsz): - fcntl.ioctl(fd, _TIOCSWINSZ, winsz) - - # Marginal testing of pty suite. Cannot do extensive 'do or fail' testing # because pty code is not too portable. class PtyTest(unittest.TestCase): @@ -105,18 +91,14 @@ def setUp(self): self.addCleanup(signal.alarm, 0) signal.alarm(10) - # Save original stdin window size - self.stdin_rows = None - self.stdin_cols = None + # Save original stdin window size. + self.stdin_dim = None if _HAVE_WINSZ: try: - stdin_dim = os.get_terminal_size(pty.STDIN_FILENO) - self.stdin_rows = stdin_dim.lines - self.stdin_cols = stdin_dim.columns - old_stdin_winsz = struct.pack("HHHH", self.stdin_rows, - self.stdin_cols, 0, 0) - self.addCleanup(_set_term_winsz, pty.STDIN_FILENO, old_stdin_winsz) - except OSError: + self.stdin_dim = tty.tcgetwinsize(pty.STDIN_FILENO) + self.addCleanup(tty.tcsetwinsize, pty.STDIN_FILENO, + self.stdin_dim) + except tty.error: pass def handle_sig(self, sig, frame): @@ -131,41 +113,40 @@ def test_openpty(self): try: mode = tty.tcgetattr(pty.STDIN_FILENO) except tty.error: - # not a tty or bad/closed fd + # Not a tty or bad/closed fd. debug("tty.tcgetattr(pty.STDIN_FILENO) failed") mode = None - new_stdin_winsz = None - if self.stdin_rows is not None and self.stdin_cols is not None: + new_dim = None + if self.stdin_dim: try: # Modify pty.STDIN_FILENO window size; we need to # check if pty.openpty() is able to set pty slave # window size accordingly. - debug("Setting pty.STDIN_FILENO window size") - debug(f"original size: (rows={self.stdin_rows}, cols={self.stdin_cols})") - target_stdin_rows = self.stdin_rows + 1 - target_stdin_cols = self.stdin_cols + 1 - debug(f"target size: (rows={target_stdin_rows}, cols={target_stdin_cols})") - target_stdin_winsz = struct.pack("HHHH", target_stdin_rows, - target_stdin_cols, 0, 0) - _set_term_winsz(pty.STDIN_FILENO, target_stdin_winsz) + debug("Setting pty.STDIN_FILENO window size.") + debug(f"original size: (row, col) = {self.stdin_dim}") + target_dim = (self.stdin_dim[0] + 1, self.stdin_dim[1] + 1) + debug(f"target size: (row, col) = {target_dim}") + tty.tcsetwinsize(pty.STDIN_FILENO, target_dim) # Were we able to set the window size # of pty.STDIN_FILENO successfully? - new_stdin_winsz = _get_term_winsz(pty.STDIN_FILENO) - self.assertEqual(new_stdin_winsz, target_stdin_winsz, + new_dim = tty.tcgetwinsize(pty.STDIN_FILENO) + self.assertEqual(new_dim, target_dim, "pty.STDIN_FILENO window size unchanged") except OSError: - warnings.warn("Failed to set pty.STDIN_FILENO window size") + warnings.warn("Failed to set pty.STDIN_FILENO window size.") pass try: debug("Calling pty.openpty()") try: - master_fd, slave_fd = pty.openpty(mode, new_stdin_winsz) + master_fd, slave_fd, slave_name = pty.openpty(mode, new_dim, + True) except TypeError: master_fd, slave_fd = pty.openpty() - debug(f"Got master_fd '{master_fd}', slave_fd '{slave_fd}'") + slave_name = None + debug(f"Got {master_fd=}, {slave_fd=}, {slave_name=}") except OSError: # " An optional feature could not be imported " ... ? raise unittest.SkipTest("Pseudo-terminals (seemingly) not functional.") @@ -181,8 +162,8 @@ def test_openpty(self): if mode: self.assertEqual(tty.tcgetattr(slave_fd), mode, "openpty() failed to set slave termios") - if new_stdin_winsz: - self.assertEqual(_get_term_winsz(slave_fd), new_stdin_winsz, + if new_dim: + self.assertEqual(tty.tcgetwinsize(slave_fd), new_dim, "openpty() failed to set slave window size") # Ensure the fd is non-blocking in case there's nothing to read. @@ -367,9 +348,8 @@ def _socketpair(self): self.files.extend(socketpair) return socketpair - def _mock_select(self, rfds, wfds, xfds, timeout=0): + def _mock_select(self, rfds, wfds, xfds): # This will raise IndexError when no more expected calls exist. - # This ignores the timeout self.assertEqual(self.select_rfds_lengths.pop(0), len(rfds)) return self.select_rfds_results.pop(0), [], [] @@ -409,28 +389,6 @@ def test__copy_to_each(self): self.assertEqual(os.read(read_from_stdout_fd, 20), b'from master') self.assertEqual(os.read(masters[1], 20), b'from stdin') - def test__copy_eof_on_all(self): - """Test the empty read EOF case on both master_fd and stdin.""" - read_from_stdout_fd, mock_stdout_fd = self._pipe() - pty.STDOUT_FILENO = mock_stdout_fd - mock_stdin_fd, write_to_stdin_fd = self._pipe() - pty.STDIN_FILENO = mock_stdin_fd - socketpair = self._socketpair() - masters = [s.fileno() for s in socketpair] - - socketpair[1].close() - os.close(write_to_stdin_fd) - - pty.select = self._mock_select - self.select_rfds_lengths.append(2) - self.select_rfds_results.append([mock_stdin_fd, masters[0]]) - # We expect that both fds were removed from the fds list as they - # both encountered an EOF before the second select call. - self.select_rfds_lengths.append(0) - - # We expect the function to return without error. - self.assertEqual(pty._copy(masters[0]), None) - def test__restore_tty_mode_normal_return(self): """Test that spawn resets the tty mode no when _copy returns normally.""" diff --git a/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst b/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst new file mode 100644 index 00000000000000..402f99ea6c6ebf --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst @@ -0,0 +1 @@ +Utilize new "winsize" functions from termios in pty tests. From dfc2e065a2e71011017077e549cd2f9bf4944c54 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 11 Feb 2023 22:07:52 -0800 Subject: [PATCH 26/83] gh-89792: Limit test_tools freeze test build parallelism based on the number of cores (#101841) unhardcode freeze test build parallelism. base it on the number of cpus, don't use more than max(2, os.cpu_count()/3). --- .../2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst | 7 ++++--- Tools/freeze/test/freeze.py | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst b/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst index a3a3070d7f3790..9de278919ef2f8 100644 --- a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst +++ b/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst @@ -1,3 +1,4 @@ -``test_tools`` now copies up to 10x less source data to a temporary -directory during the ``freeze`` test by ignoring git metadata and other -artifacts. +``test_tools`` now copies up to 10x less source data to a temporary directory +during the ``freeze`` test by ignoring git metadata and other artifacts. It +also limits its python build parallelism based on os.cpu_count instead of hard +coding it as 8 cores. diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 0ae983b15c98f0..b4c76ff36a873b 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -163,16 +163,25 @@ def prepare(script=None, outdir=None): if not MAKE: raise UnsupportedError('make') + cores = os.cpu_count() + if cores and cores >= 3: + # this test is most often run as part of the whole suite with a lot + # of other tests running in parallel, from 1-2 vCPU systems up to + # people's NNN core beasts. Don't attempt to use it all. + parallel = f'-j{cores*2//3}' + else: + parallel = '-j2' + # Build python. - print(f'building python in {builddir}...') + print(f'building python {parallel=} in {builddir}...') if os.path.exists(os.path.join(srcdir, 'Makefile')): # Out-of-tree builds require a clean srcdir. _run_quiet([MAKE, '-C', srcdir, 'clean']) - _run_quiet([MAKE, '-C', builddir, '-j8']) + _run_quiet([MAKE, '-C', builddir, parallel]) # Install the build. print(f'installing python into {prefix}...') - _run_quiet([MAKE, '-C', builddir, '-j8', 'install']) + _run_quiet([MAKE, '-C', builddir, 'install']) python = os.path.join(prefix, 'bin', 'python3') return outdir, scriptfile, python From 6ef6915d3530e844243893f91bf4bd702dfef570 Mon Sep 17 00:00:00 2001 From: Jean Abou-Samra Date: Sun, 12 Feb 2023 15:20:11 +0100 Subject: [PATCH 27/83] gh-101845: pyspecific: Fix i18n for availability directive (GH-101846) pyspecific: Fix i18n for availability directive If the directive has content, the previous code would nest paragraph nodes from that content inside a general paragraph node, which confuses Sphinx and leads it to drop the content when translating. Instead, use a container node for the body. Also use set_source_info so that any warnings have location info. --- Doc/tools/extensions/pyspecific.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index db7bb3b44219d2..d659a4a54b9d11 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -28,6 +28,7 @@ from sphinx.environment import NoUri from sphinx.locale import _ as sphinx_gettext from sphinx.util import status_iterator, logging +from sphinx.util.docutils import SphinxDirective from sphinx.util.nodes import split_explicit_title from sphinx.writers.text import TextWriter, TextTranslator @@ -119,7 +120,7 @@ def run(self): # Support for documenting platform availability -class Availability(Directive): +class Availability(SphinxDirective): has_content = True required_arguments = 1 @@ -139,18 +140,19 @@ class Availability(Directive): def run(self): availability_ref = ':ref:`Availability `: ' + avail_nodes, avail_msgs = self.state.inline_text( + availability_ref + self.arguments[0], + self.lineno) pnode = nodes.paragraph(availability_ref + self.arguments[0], - classes=["availability"],) - n, m = self.state.inline_text(availability_ref, self.lineno) - pnode.extend(n + m) - n, m = self.state.inline_text(self.arguments[0], self.lineno) - pnode.extend(n + m) + '', *avail_nodes, *avail_msgs) + self.set_source_info(pnode) + cnode = nodes.container("", pnode, classes=["availability"]) + self.set_source_info(cnode) if self.content: - self.state.nested_parse(self.content, self.content_offset, pnode) - + self.state.nested_parse(self.content, self.content_offset, cnode) self.parse_platforms() - return [pnode] + return [cnode] def parse_platforms(self): """Parse platform information from arguments From a1f08f5f19753c7c9295f51b5ae1262c7a1c838f Mon Sep 17 00:00:00 2001 From: Steve Kowalik Date: Mon, 13 Feb 2023 20:11:43 +1100 Subject: [PATCH 28/83] Correct trivial grammar in reset_mock docs (#101861) --- Doc/library/unittest.mock.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index e009f303fef317..d6d8e5e9557d5c 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -406,7 +406,7 @@ the *new_callable* argument to :func:`patch`. False .. versionchanged:: 3.6 - Added two keyword only argument to the reset_mock function. + Added two keyword-only arguments to the reset_mock function. This can be useful where you want to make a series of assertions that reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the @@ -416,8 +416,8 @@ the *new_callable* argument to :func:`patch`. parameter as ``True``. Child mocks and the return value mock (if any) are reset as well. - .. note:: *return_value*, and :attr:`side_effect` are keyword only - argument. + .. note:: *return_value*, and :attr:`side_effect` are keyword-only + arguments. .. method:: mock_add_spec(spec, spec_set=False) From 160f2fe2b90ed5ec7838cb4141dd35768422891f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 Feb 2023 11:24:55 +0000 Subject: [PATCH 29/83] GH-87849: Simplify stack effect of SEND and specialize it for generators and coroutines. (GH-101788) --- Include/internal/pycore_code.h | 7 ++ Include/internal/pycore_opcode.h | 5 +- Include/opcode.h | 1 + Lib/dis.py | 9 +- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 8 +- Lib/test/test_dis.py | 25 +++--- ...3-02-10-15-54-57.gh-issue-87849.IUVvPz.rst | 3 + Objects/frameobject.c | 6 +- Python/bytecodes.c | 88 ++++++++++-------- Python/compile.c | 2 + Python/generated_cases.c.h | 89 +++++++++++-------- Python/opcode_metadata.h | 11 ++- Python/opcode_targets.h | 2 +- Python/specialize.c | 25 +++++- 15 files changed, 185 insertions(+), 99 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index a287250acc1912..10f1e320a12ff4 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -92,6 +92,12 @@ typedef struct { #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) +typedef struct { + uint16_t counter; +} _PySendCache; + +#define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache) + // Borrowed references to common callables: struct callable_cache { PyObject *isinstance; @@ -233,6 +239,7 @@ extern void _Py_Specialize_CompareAndBranch(PyObject *lhs, PyObject *rhs, extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg); +extern void _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr); /* Finalizer function for static codeobjects used in deepfreeze.py */ extern void _PyStaticCode_Fini(PyCodeObject *co); diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 47c84721335196..5e65adee9e00a5 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -50,6 +50,7 @@ const uint8_t _PyOpcode_Caches[256] = { [COMPARE_OP] = 1, [LOAD_GLOBAL] = 5, [BINARY_OP] = 1, + [SEND] = 1, [COMPARE_AND_BRANCH] = 1, [CALL] = 4, }; @@ -196,6 +197,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [RETURN_GENERATOR] = RETURN_GENERATOR, [RETURN_VALUE] = RETURN_VALUE, [SEND] = SEND, + [SEND_GEN] = SEND, [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, [SET_ADD] = SET_ADD, [SET_UPDATE] = SET_UPDATE, @@ -395,7 +397,7 @@ static const char *const _PyOpcode_OpName[263] = { [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", - [166] = "<166>", + [SEND_GEN] = "SEND_GEN", [167] = "<167>", [168] = "<168>", [169] = "<169>", @@ -496,7 +498,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 166: \ case 167: \ case 168: \ case 169: \ diff --git a/Include/opcode.h b/Include/opcode.h index 77ad7c22440d72..d643741c3c3aa0 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -187,6 +187,7 @@ extern "C" { #define UNPACK_SEQUENCE_LIST 159 #define UNPACK_SEQUENCE_TUPLE 160 #define UNPACK_SEQUENCE_TWO_TUPLE 161 +#define SEND_GEN 166 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/dis.py b/Lib/dis.py index a6921008d9d0e5..9edde6ae8258da 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -39,6 +39,7 @@ BINARY_OP = opmap['BINARY_OP'] JUMP_BACKWARD = opmap['JUMP_BACKWARD'] FOR_ITER = opmap['FOR_ITER'] +SEND = opmap['SEND'] LOAD_ATTR = opmap['LOAD_ATTR'] CACHE = opmap["CACHE"] @@ -453,6 +454,7 @@ def _get_instructions_bytes(code, varname_from_oparg=None, argrepr = '' positions = Positions(*next(co_positions, ())) deop = _deoptop(op) + caches = _inline_cache_entries[deop] if arg is not None: # Set argval to the dereferenced value of the argument when # available, and argrepr to the string representation of argval. @@ -478,8 +480,7 @@ def _get_instructions_bytes(code, varname_from_oparg=None, elif deop in hasjrel: signed_arg = -arg if _is_backward_jump(deop) else arg argval = offset + 2 + signed_arg*2 - if deop == FOR_ITER: - argval += 2 + argval += 2 * caches argrepr = "to " + repr(argval) elif deop in haslocal or deop in hasfree: argval, argrepr = _get_name_info(arg, varname_from_oparg) @@ -633,12 +634,12 @@ def findlabels(code): for offset, op, arg in _unpack_opargs(code): if arg is not None: deop = _deoptop(op) + caches = _inline_cache_entries[deop] if deop in hasjrel: if _is_backward_jump(deop): arg = -arg label = offset + 2 + arg*2 - if deop == FOR_ITER: - label += 2 + label += 2 * caches elif deop in hasjabs: label = arg*2 else: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 933c8c7d7e0590..38d4a384c2cc95 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -432,6 +432,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) # Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) # Python 3.12a5 3518 (Add RETURN_CONST instruction) +# Python 3.12a5 3519 (Modify SEND instruction) # Python 3.13 will start with 3550 @@ -444,7 +445,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3518).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3519).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 5f163d2ccb80df..b69cd1bbdd61ca 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -167,7 +167,7 @@ def pseudo_op(name, op, real_ops): def_op('RETURN_CONST', 121) hasconst.append(121) def_op('BINARY_OP', 122) -jrel_op('SEND', 123) # Number of bytes to skip +jrel_op('SEND', 123) # Number of words to skip def_op('LOAD_FAST', 124) # Local variable number, no null check haslocal.append(124) def_op('STORE_FAST', 125) # Local variable number @@ -370,6 +370,9 @@ def pseudo_op(name, op, real_ops): "UNPACK_SEQUENCE_TUPLE", "UNPACK_SEQUENCE_TWO_TUPLE", ], + "SEND": [ + "SEND_GEN", + ], } _specialized_instructions = [ opcode for family in _specializations.values() for opcode in family @@ -429,6 +432,9 @@ def pseudo_op(name, op, real_ops): "STORE_SUBSCR": { "counter": 1, }, + "SEND": { + "counter": 1, + }, } _inline_cache_entries = [ diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 1050b15e16eaaa..9086824dd6f40c 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -475,11 +475,13 @@ async def _asyncwith(c): BEFORE_ASYNC_WITH GET_AWAITABLE 1 LOAD_CONST 0 (None) - >> SEND 3 (to 22) + >> SEND 3 (to 24) YIELD_VALUE 2 RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 4 (to 14) - >> POP_TOP + JUMP_BACKWARD_NO_INTERRUPT 5 (to 14) + >> SWAP 2 + POP_TOP + POP_TOP %3d LOAD_CONST 1 (1) STORE_FAST 1 (x) @@ -490,30 +492,33 @@ async def _asyncwith(c): CALL 2 GET_AWAITABLE 2 LOAD_CONST 0 (None) - >> SEND 3 (to 56) + >> SEND 3 (to 64) YIELD_VALUE 2 RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 4 (to 48) + JUMP_BACKWARD_NO_INTERRUPT 5 (to 54) >> POP_TOP + POP_TOP %3d LOAD_CONST 2 (2) STORE_FAST 2 (y) RETURN_CONST 0 (None) %3d >> CLEANUP_THROW - JUMP_BACKWARD 23 (to 22) + JUMP_BACKWARD 27 (to 24) >> CLEANUP_THROW - JUMP_BACKWARD 8 (to 56) + JUMP_BACKWARD 9 (to 64) >> PUSH_EXC_INFO WITH_EXCEPT_START GET_AWAITABLE 2 LOAD_CONST 0 (None) - >> SEND 4 (to 90) + >> SEND 4 (to 102) YIELD_VALUE 3 RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 4 (to 80) + JUMP_BACKWARD_NO_INTERRUPT 5 (to 90) >> CLEANUP_THROW - >> POP_JUMP_IF_TRUE 1 (to 94) + >> SWAP 2 + POP_TOP + POP_JUMP_IF_TRUE 1 (to 110) RERAISE 2 >> POP_TOP POP_EXCEPT diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst new file mode 100644 index 00000000000000..da5f3ff79fd575 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst @@ -0,0 +1,3 @@ +Change the ``SEND`` instruction to leave the receiver on the stack. This +allows the specialized form of ``SEND`` to skip the chain of C calls and jump +directly to the ``RESUME`` in the generator or coroutine. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 0e52a3e2399c06..581ed2d214c4d9 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -334,10 +334,10 @@ mark_stacks(PyCodeObject *code_obj, int len) break; } case SEND: - j = get_arg(code, i) + i + 1; + j = get_arg(code, i) + i + INLINE_CACHE_ENTRIES_SEND + 1; assert(j < len); - assert(stacks[j] == UNINITIALIZED || stacks[j] == pop_value(next_stack)); - stacks[j] = pop_value(next_stack); + assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack); + stacks[j] = next_stack; stacks[i+1] = next_stack; break; case JUMP_FORWARD: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2b9f12fefa14e9..429cd7fdafa168 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -680,51 +680,66 @@ dummy_func( PREDICT(LOAD_CONST); } - inst(SEND, (receiver, v -- receiver if (!jump), retval)) { + family(for_iter, INLINE_CACHE_ENTRIES_FOR_ITER) = { + SEND, + SEND_GEN, + }; + + inst(SEND, (unused/1, receiver, v -- receiver, retval)) { + #if ENABLE_SPECIALIZATION + _PySendCache *cache = (_PySendCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + next_instr--; + _Py_Specialize_Send(receiver, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(SEND, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); + #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); - bool jump = false; - PySendResult gen_status; - if (tstate->c_tracefunc == NULL) { - gen_status = PyIter_Send(receiver, v, &retval); - } else { - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (tstate->c_tracefunc != NULL - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - gen_status = PYGEN_RETURN; - } - else { - gen_status = PYGEN_ERROR; - } + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); + } + else { + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (tstate->c_tracefunc != NULL + && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) + call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); } else { - gen_status = PYGEN_NEXT; + assert(retval == NULL); + goto error; } } - if (gen_status == PYGEN_ERROR) { - assert(retval == NULL); - goto error; - } - Py_DECREF(v); - if (gen_status == PYGEN_RETURN) { - assert(retval != NULL); - Py_DECREF(receiver); - JUMPBY(oparg); - jump = true; - } else { - assert(gen_status == PYGEN_NEXT); assert(retval != NULL); } } + inst(SEND_GEN, (unused/1, receiver, v -- receiver)) { + assert(cframe.use_tracing == 0); + PyGenObject *gen = (PyGenObject *)receiver; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && + Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->yield_offset = oparg; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); + DISPATCH_INLINED(gen_frame); + } + inst(YIELD_VALUE, (retval -- unused)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() @@ -796,12 +811,13 @@ dummy_func( } } - inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- value)) { + inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- none, value)) { assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); DECREF_INPUTS(); + none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); diff --git a/Python/compile.c b/Python/compile.c index a3c915c3c14a96..b49eda314eeef1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1789,6 +1789,8 @@ compiler_add_yield_from(struct compiler *c, location loc, int await) ADDOP(c, loc, CLEANUP_THROW); USE_LABEL(c, exit); + ADDOP_I(c, loc, SWAP, 2); + ADDOP(c, loc, POP_TOP); return SUCCESS; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a224d4eb892785..093ebff026b509 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -882,57 +882,69 @@ } TARGET(SEND) { + PREDICTED(SEND); PyObject *v = PEEK(1); PyObject *receiver = PEEK(2); PyObject *retval; + #if ENABLE_SPECIALIZATION + _PySendCache *cache = (_PySendCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + next_instr--; + _Py_Specialize_Send(receiver, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(SEND, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); + #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); - bool jump = false; - PySendResult gen_status; - if (tstate->c_tracefunc == NULL) { - gen_status = PyIter_Send(receiver, v, &retval); - } else { - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (tstate->c_tracefunc != NULL - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - gen_status = PYGEN_RETURN; - } - else { - gen_status = PYGEN_ERROR; - } + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); + } + else { + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (tstate->c_tracefunc != NULL + && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) + call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); } else { - gen_status = PYGEN_NEXT; + assert(retval == NULL); + goto error; } } - if (gen_status == PYGEN_ERROR) { - assert(retval == NULL); - goto error; - } - Py_DECREF(v); - if (gen_status == PYGEN_RETURN) { - assert(retval != NULL); - Py_DECREF(receiver); - JUMPBY(oparg); - jump = true; - } else { - assert(gen_status == PYGEN_NEXT); assert(retval != NULL); } - STACK_SHRINK(1); - STACK_GROW(((!jump) ? 1 : 0)); POKE(1, retval); + JUMPBY(1); DISPATCH(); } + TARGET(SEND_GEN) { + PyObject *v = PEEK(1); + PyObject *receiver = PEEK(2); + assert(cframe.use_tracing == 0); + PyGenObject *gen = (PyGenObject *)receiver; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && + Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->yield_offset = oparg; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); + DISPATCH_INLINED(gen_frame); + } + TARGET(YIELD_VALUE) { PyObject *retval = PEEK(1); // NOTE: It's important that YIELD_VALUE never raises an exception! @@ -1026,6 +1038,7 @@ PyObject *exc_value = PEEK(1); PyObject *last_sent_val = PEEK(2); PyObject *sub_iter = PEEK(3); + PyObject *none; PyObject *value; assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); @@ -1034,13 +1047,15 @@ Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); + none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } - STACK_SHRINK(2); + STACK_SHRINK(1); POKE(1, value); + POKE(2, none); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index db1dfd37a90132..d622eb12c8cb2d 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -104,6 +104,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case SEND: return 2; + case SEND_GEN: + return 2; case YIELD_VALUE: return 1; case POP_EXCEPT: @@ -453,7 +455,9 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case GET_AWAITABLE: return 1; case SEND: - return ((!jump) ? 1 : 0) + 1; + return 2; + case SEND_GEN: + return 1; case YIELD_VALUE: return 1; case POP_EXCEPT: @@ -465,7 +469,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case END_ASYNC_FOR: return 0; case CLEANUP_THROW: - return 1; + return 2; case LOAD_ASSERTION_ERROR: return 1; case LOAD_BUILD_CLASS: @@ -763,7 +767,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [GET_AITER] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [GET_ANEXT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [GET_AWAITABLE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [SEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [SEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, + [SEND_GEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, [YIELD_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [POP_EXCEPT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [RERAISE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index eceb246fac4909..301ec6e005dad6 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -165,7 +165,7 @@ static void *opcode_targets[256] = { &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&_unknown_opcode, + &&TARGET_SEND_GEN, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index 908ad6dceb57f3..4ede3122d38046 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -128,6 +128,7 @@ print_spec_stats(FILE *out, OpcodeStats *stats) fprintf(out, "opcode[%d].specializable : 1\n", BINARY_SLICE); fprintf(out, "opcode[%d].specializable : 1\n", COMPARE_OP); fprintf(out, "opcode[%d].specializable : 1\n", STORE_SLICE); + fprintf(out, "opcode[%d].specializable : 1\n", SEND); for (int i = 0; i < 256; i++) { if (_PyOpcode_Caches[i]) { fprintf(out, "opcode[%d].specializable : 1\n", i); @@ -1084,7 +1085,7 @@ PyObject *descr, DescriptorClassification kind) if (dict) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; - } + } assert(owner_cls->tp_dictoffset > 0); assert(owner_cls->tp_dictoffset <= INT16_MAX); _py_set_opcode(instr, LOAD_ATTR_METHOD_LAZY_DICT); @@ -2183,3 +2184,25 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) STAT_INC(FOR_ITER, success); cache->counter = adaptive_counter_cooldown(); } + +void +_Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) +{ + assert(ENABLE_SPECIALIZATION); + assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND); + _PySendCache *cache = (_PySendCache *)(instr + 1); + PyTypeObject *tp = Py_TYPE(receiver); + if (tp == &PyGen_Type || tp == &PyCoro_Type) { + _py_set_opcode(instr, SEND_GEN); + goto success; + } + SPECIALIZATION_FAIL(SEND, + _PySpecialization_ClassifyIterator(receiver)); + STAT_INC(SEND, failure); + _py_set_opcode(instr, SEND); + cache->counter = adaptive_counter_backoff(cache->counter); + return; +success: + STAT_INC(SEND, success); + cache->counter = adaptive_counter_cooldown(); +} From d9199175c7386a95aaac91822a2197b9365eb0e8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 13 Feb 2023 11:31:15 +0000 Subject: [PATCH 30/83] GH-100987: Refactor `_PyInterpreterFrame` a bit, to assist generator improvement. (GH-100988) Refactor _PyInterpreterFrame a bit, to assist generator improvement. --- Include/internal/pycore_frame.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index f12b225ebfccf2..81d16b219c305b 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -47,15 +47,13 @@ enum _frameowner { }; typedef struct _PyInterpreterFrame { - /* "Specials" section */ + PyCodeObject *f_code; /* Strong reference */ + struct _PyInterpreterFrame *previous; PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ - PyCodeObject *f_code; /* Strong reference */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ - /* Linkage section */ - struct _PyInterpreterFrame *previous; // NOTE: This is not necessarily the last instruction started in the given // frame. Rather, it is the code unit *prior to* the *next* instruction. For // example, it may be an inline CACHE entry, an instruction we just jumped From 2db2c4b45501eebef5b3ff89118554bd5eb92ed4 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 13 Feb 2023 13:36:42 +0100 Subject: [PATCH 31/83] gh-92547: Purge sqlite3_enable_shared_cache() detection from configure (#101873) --- configure | 51 --------------------------------------------------- configure.ac | 1 - 2 files changed, 52 deletions(-) diff --git a/configure b/configure index 97694c602d1cc8..35088f9e5cafd6 100755 --- a/configure +++ b/configure @@ -13689,57 +13689,6 @@ fi - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_enable_shared_cache in -lsqlite3" >&5 -$as_echo_n "checking for sqlite3_enable_shared_cache in -lsqlite3... " >&6; } -if ${ac_cv_lib_sqlite3_sqlite3_enable_shared_cache+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lsqlite3 $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char sqlite3_enable_shared_cache (); -int -main () -{ -return sqlite3_enable_shared_cache (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_sqlite3_sqlite3_enable_shared_cache=yes -else - ac_cv_lib_sqlite3_sqlite3_enable_shared_cache=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_enable_shared_cache" >&5 -$as_echo "$ac_cv_lib_sqlite3_sqlite3_enable_shared_cache" >&6; } -if test "x$ac_cv_lib_sqlite3_sqlite3_enable_shared_cache" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBSQLITE3 1 -_ACEOF - - LIBS="-lsqlite3 $LIBS" - -else - - have_supported_sqlite3=no - -fi - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_progress_handler in -lsqlite3" >&5 $as_echo_n "checking for sqlite3_progress_handler in -lsqlite3... " >&6; } if ${ac_cv_lib_sqlite3_sqlite3_progress_handler+:} false; then : diff --git a/configure.ac b/configure.ac index 09369b985b33f6..1ab48e0d1c160a 100644 --- a/configure.ac +++ b/configure.ac @@ -3867,7 +3867,6 @@ dnl hence CPPFLAGS instead of CFLAGS. PY_CHECK_SQLITE_FUNC([sqlite3_column_decltype]) PY_CHECK_SQLITE_FUNC([sqlite3_column_double]) PY_CHECK_SQLITE_FUNC([sqlite3_complete]) - PY_CHECK_SQLITE_FUNC([sqlite3_enable_shared_cache]) PY_CHECK_SQLITE_FUNC([sqlite3_progress_handler]) PY_CHECK_SQLITE_FUNC([sqlite3_result_double]) PY_CHECK_SQLITE_FUNC([sqlite3_set_authorizer]) From 95cbb3d908175ccd855078b3fab7f99e7d0bca88 Mon Sep 17 00:00:00 2001 From: James Lee <49257044+juria90@users.noreply.github.com> Date: Mon, 13 Feb 2023 05:49:44 -0800 Subject: [PATCH 32/83] gh-101810: Remove duplicated st_ino calculation (GH-101811) --- Python/fileutils.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/fileutils.c b/Python/fileutils.c index 244bd899b3bd24..22b2257a56d0ec 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1162,8 +1162,6 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) } _Py_attribute_data_to_stat(&info, 0, status); - /* specific to fstat() */ - status->st_ino = (((uint64_t)info.nFileIndexHigh) << 32) + info.nFileIndexLow; return 0; #else return fstat(fd, status); From 0c6fe81dce9d6bb1dce5e4503f1b42bc5355ba24 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Feb 2023 20:33:48 +0000 Subject: [PATCH 33/83] gh-101849: Add upgrade codes for old versions of launcher that ended up with later version numbers (GH-101877) --- ...-02-13-16-32-50.gh-issue-101849.7lm_53.rst | 1 + Tools/msi/common.wxs | 2 +- Tools/msi/launcher/launcher.wxs | 21 +++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst diff --git a/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst b/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst new file mode 100644 index 00000000000000..861d4de9f9a650 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst @@ -0,0 +1 @@ +Ensures installer will correctly upgrade existing ``py.exe`` launcher installs. diff --git a/Tools/msi/common.wxs b/Tools/msi/common.wxs index 55cb44860d02c0..54fa749ab17cdd 100644 --- a/Tools/msi/common.wxs +++ b/Tools/msi/common.wxs @@ -25,7 +25,6 @@ - @@ -42,6 +41,7 @@ UPGRADE + diff --git a/Tools/msi/launcher/launcher.wxs b/Tools/msi/launcher/launcher.wxs index b83058c63bf6d9..49f1f7b8c1762e 100644 --- a/Tools/msi/launcher/launcher.wxs +++ b/Tools/msi/launcher/launcher.wxs @@ -34,13 +34,34 @@ NOT Installed AND NOT ALLUSERS=1 NOT Installed AND ALLUSERS=1 + + UPGRADE or REMOVE_350_LAUNCHER or REMOVE_360A1_LAUNCHER or UPGRADE_3_11_0 or UPGRADE_3_11_1 + UPGRADE or REMOVE_350_LAUNCHER or REMOVE_360A1_LAUNCHER + + + Installed OR NOT DOWNGRADE OR UPGRADE_3_11_0 OR UPGRADE_3_11_1 + + Installed OR NOT DOWNGRADE + + + + + + + From 928752ce4c23f47d3175dd47ecacf08d86a99c9d Mon Sep 17 00:00:00 2001 From: Radek Smejkal Date: Tue, 14 Feb 2023 02:37:34 +0100 Subject: [PATCH 34/83] gh-74895: getaddrinfo no longer raises OverflowError (#2435) `socket.getaddrinfo()` no longer raises `OverflowError` based on the **port** argument. Error reporting (or not) for its value is left up to the underlying C library `getaddrinfo()` implementation. --- Lib/test/test_socket.py | 48 +++++++++++++++++++ Misc/ACKS | 1 + ...3-02-13-22-21-58.gh-issue-74895.esMNtq.rst | 5 ++ Modules/getaddrinfo.c | 6 ++- Modules/socketmodule.c | 14 ++++-- 5 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index f1b4018c265e18..a313da29b4a4fd 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1600,6 +1600,54 @@ def testGetaddrinfo(self): except socket.gaierror: pass + def test_getaddrinfo_int_port_overflow(self): + # gh-74895: Test that getaddrinfo does not raise OverflowError on port. + # + # POSIX getaddrinfo() never specify the valid range for "service" + # decimal port number values. For IPv4 and IPv6 they are technically + # unsigned 16-bit values, but the API is protocol agnostic. Which values + # trigger an error from the C library function varies by platform as + # they do not all perform validation. + + # The key here is that we don't want to produce OverflowError as Python + # prior to 3.12 did for ints outside of a [LONG_MIN, LONG_MAX] range. + # Leave the error up to the underlying string based platform C API. + + from _testcapi import ULONG_MAX, LONG_MAX, LONG_MIN + try: + socket.getaddrinfo(None, ULONG_MAX + 1) + except OverflowError: + # Platforms differ as to what values consitute a getaddrinfo() error + # return. Some fail for LONG_MAX+1, others ULONG_MAX+1, and Windows + # silently accepts such huge "port" aka "service" numeric values. + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MAX + 1) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MAX - 0xffff + 1) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MIN - 1) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + socket.getaddrinfo(None, 0) # No error expected. + socket.getaddrinfo(None, 0xffff) # No error expected. + def test_getnameinfo(self): # only IP addresses are allowed self.assertRaises(OSError, socket.getnameinfo, ('mail.python.org',0), 0) diff --git a/Misc/ACKS b/Misc/ACKS index e12cbea0ebd6ed..ca92608868f23f 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1688,6 +1688,7 @@ Roman Skurikhin Ville Skyttä Michael Sloan Nick Sloan +Radek Smejkal Václav Šmilauer Casper W. Smet Allen W. Smith diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst new file mode 100644 index 00000000000000..adbbb601634a60 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst @@ -0,0 +1,5 @@ +:mod:`socket.getaddrinfo` no longer raises :class:`OverflowError` for +:class:`int` **port** values outside of the C long range. Out of range values +are left up to the underlying string based C library API to report. A +:class:`socket.gaierror` ``SAI_SERVICE`` may occur instead, or no error at all +as not all platform C libraries generate an error. diff --git a/Modules/getaddrinfo.c b/Modules/getaddrinfo.c index 0b4620ed683de9..f1c28d7d9312ac 100644 --- a/Modules/getaddrinfo.c +++ b/Modules/getaddrinfo.c @@ -342,7 +342,11 @@ getaddrinfo(const char*hostname, const char*servname, pai->ai_socktype = SOCK_DGRAM; pai->ai_protocol = IPPROTO_UDP; } - port = htons((u_short)atoi(servname)); + long maybe_port = strtol(servname, NULL, 10); + if (maybe_port < 0 || maybe_port > 0xffff) { + ERR(EAI_SERVICE); + } + port = htons((u_short)maybe_port); } else { struct servent *sp; const char *proto; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 0a9e46512b157b..2d300f19436b1a 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -6650,7 +6650,7 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) struct addrinfo *res0 = NULL; PyObject *hobj = NULL; PyObject *pobj = (PyObject *)NULL; - char pbuf[30]; + PyObject *pstr = NULL; const char *hptr, *pptr; int family, socktype, protocol, flags; int error; @@ -6680,11 +6680,13 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) return NULL; } if (PyLong_CheckExact(pobj)) { - long value = PyLong_AsLong(pobj); - if (value == -1 && PyErr_Occurred()) + pstr = PyObject_Str(pobj); + if (pstr == NULL) + goto err; + assert(PyUnicode_Check(pstr)); + pptr = PyUnicode_AsUTF8(pstr); + if (pptr == NULL) goto err; - PyOS_snprintf(pbuf, sizeof(pbuf), "%ld", value); - pptr = pbuf; } else if (PyUnicode_Check(pobj)) { pptr = PyUnicode_AsUTF8(pobj); if (pptr == NULL) @@ -6750,12 +6752,14 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) Py_DECREF(single); } Py_XDECREF(idna); + Py_XDECREF(pstr); if (res0) freeaddrinfo(res0); return all; err: Py_XDECREF(all); Py_XDECREF(idna); + Py_XDECREF(pstr); if (res0) freeaddrinfo(res0); return (PyObject *)NULL; From 8be8101bca34b60481ec3d7ecaea4a3379fb7dbb Mon Sep 17 00:00:00 2001 From: Sam James Date: Tue, 14 Feb 2023 07:21:58 +0000 Subject: [PATCH 35/83] gh-101857: Allow xattr detection on musl libc (#101858) Previously, we checked exclusively for `__GLIBC__` (AND'd with some other conditions). Checking for `__linux__` instead should be fine. This fixes using e.g. `os.listxattr()` on systems using musl libc. Bug: https://bugs.gentoo.org/894130 Co-authored-by: Gregory P. Smith --- .../2023-02-12-22-40-22.gh-issue-101857._bribG.rst | 1 + Modules/posixmodule.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst new file mode 100644 index 00000000000000..832cc300fa9433 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst @@ -0,0 +1 @@ +Fix xattr support detection on Linux systems by widening the check to linux, not just glibc. This fixes support for musl. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index cba6cea48b77e1..d9e93473aeadaa 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -274,8 +274,9 @@ corresponding Unix manual entries for more information on calls."); # undef HAVE_SCHED_SETAFFINITY #endif -#if defined(HAVE_SYS_XATTR_H) && defined(__GLIBC__) && !defined(__FreeBSD_kernel__) && !defined(__GNU__) +#if defined(HAVE_SYS_XATTR_H) && defined(__linux__) && !defined(__FreeBSD_kernel__) && !defined(__GNU__) # define USE_XATTRS +# include // Needed for XATTR_SIZE_MAX on musl libc. #endif #ifdef USE_XATTRS From e5da9ab2c82c6b4e4f8ffa699a9a609ea1bea255 Mon Sep 17 00:00:00 2001 From: Jonathan Protzenko Date: Tue, 14 Feb 2023 01:25:16 -0800 Subject: [PATCH 36/83] gh-99108: Import SHA2-384/512 from HACL* (#101707) Replace the builtin hashlib implementations of SHA2-384 and SHA2-512 originally from LibTomCrypt with formally verified, side-channel resistant code from the [HACL*](https://github.com/hacl-star/hacl-star/) project. The builtins remain a fallback only used when OpenSSL does not provide them. --- Makefile.pre.in | 2 +- ...3-02-08-12-57-35.gh-issue-99108.6tnmhA.rst | 4 + Modules/Setup.stdlib.in | 2 +- Modules/_hacl/Hacl_Streaming_SHA2.c | 654 ++++++++++++++++++ Modules/_hacl/Hacl_Streaming_SHA2.h | 85 ++- .../include/krml/FStar_UInt128_Verified.h | 347 ++++++++++ .../include/krml/FStar_UInt_8_16_32_64.h | 2 +- .../krml/fstar_uint128_struct_endianness.h | 68 ++ Modules/_hacl/include/krml/types.h | 14 + .../_hacl/include/python_hacl_namespaces.h | 17 + Modules/_hacl/internal/Hacl_SHA2_Generic.h | 5 +- Modules/_hacl/refresh.sh | 45 +- Modules/sha256module.c | 1 + Modules/sha512module.c | 441 ++---------- configure | 2 +- configure.ac | 4 +- 16 files changed, 1259 insertions(+), 434 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst create mode 100644 Modules/_hacl/include/krml/FStar_UInt128_Verified.h create mode 100644 Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h create mode 100644 Modules/_hacl/include/krml/types.h diff --git a/Makefile.pre.in b/Makefile.pre.in index 7a84b953d97962..d42d4d8a3c1c9f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2608,7 +2608,7 @@ MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h MODULE__SHA256_DEPS=$(srcdir)/Modules/hashlib.h $(srcdir)/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h $(srcdir)/Modules/_hacl/include/krml/lowstar_endianness.h $(srcdir)/Modules/_hacl/include/krml/internal/target.h $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.h MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h -MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h +MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h $(srcdir)/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h $(srcdir)/Modules/_hacl/include/krml/lowstar_endianness.h $(srcdir)/Modules/_hacl/include/krml/internal/target.h $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.h MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h diff --git a/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst b/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst new file mode 100644 index 00000000000000..6a7a309dad5d8f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst @@ -0,0 +1,4 @@ +Replace the builtin :mod:`hashlib` implementations of SHA2-384 and SHA2-512 +originally from LibTomCrypt with formally verified, side-channel resistant +code from the `HACL* `_ project. +The builtins remain a fallback only used when OpenSSL does not provide them. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index f72783810f9415..b6d13e04d3fa87 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -80,7 +80,7 @@ @MODULE__MD5_TRUE@_md5 md5module.c @MODULE__SHA1_TRUE@_sha1 sha1module.c @MODULE__SHA256_TRUE@_sha256 sha256module.c _hacl/Hacl_Streaming_SHA2.c -@MODULE__SHA512_TRUE@_sha512 sha512module.c +@MODULE__SHA512_TRUE@_sha512 sha512module.c _hacl/Hacl_Streaming_SHA2.c @MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c diff --git a/Modules/_hacl/Hacl_Streaming_SHA2.c b/Modules/_hacl/Hacl_Streaming_SHA2.c index 84566571792a3c..8169c7a356731e 100644 --- a/Modules/_hacl/Hacl_Streaming_SHA2.c +++ b/Modules/_hacl/Hacl_Streaming_SHA2.c @@ -250,6 +250,229 @@ static inline void sha224_finish(uint32_t *st, uint8_t *h) memcpy(h, hbuf, (uint32_t)28U * sizeof (uint8_t)); } +void Hacl_SHA2_Scalar32_sha512_init(uint64_t *hash) +{ + KRML_MAYBE_FOR8(i, + (uint32_t)0U, + (uint32_t)8U, + (uint32_t)1U, + uint64_t *os = hash; + uint64_t x = Hacl_Impl_SHA2_Generic_h512[i]; + os[i] = x;); +} + +static inline void sha512_update(uint8_t *b, uint64_t *hash) +{ + uint64_t hash_old[8U] = { 0U }; + uint64_t ws[16U] = { 0U }; + memcpy(hash_old, hash, (uint32_t)8U * sizeof (uint64_t)); + uint8_t *b10 = b; + uint64_t u = load64_be(b10); + ws[0U] = u; + uint64_t u0 = load64_be(b10 + (uint32_t)8U); + ws[1U] = u0; + uint64_t u1 = load64_be(b10 + (uint32_t)16U); + ws[2U] = u1; + uint64_t u2 = load64_be(b10 + (uint32_t)24U); + ws[3U] = u2; + uint64_t u3 = load64_be(b10 + (uint32_t)32U); + ws[4U] = u3; + uint64_t u4 = load64_be(b10 + (uint32_t)40U); + ws[5U] = u4; + uint64_t u5 = load64_be(b10 + (uint32_t)48U); + ws[6U] = u5; + uint64_t u6 = load64_be(b10 + (uint32_t)56U); + ws[7U] = u6; + uint64_t u7 = load64_be(b10 + (uint32_t)64U); + ws[8U] = u7; + uint64_t u8 = load64_be(b10 + (uint32_t)72U); + ws[9U] = u8; + uint64_t u9 = load64_be(b10 + (uint32_t)80U); + ws[10U] = u9; + uint64_t u10 = load64_be(b10 + (uint32_t)88U); + ws[11U] = u10; + uint64_t u11 = load64_be(b10 + (uint32_t)96U); + ws[12U] = u11; + uint64_t u12 = load64_be(b10 + (uint32_t)104U); + ws[13U] = u12; + uint64_t u13 = load64_be(b10 + (uint32_t)112U); + ws[14U] = u13; + uint64_t u14 = load64_be(b10 + (uint32_t)120U); + ws[15U] = u14; + KRML_MAYBE_FOR5(i0, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + KRML_MAYBE_FOR16(i, + (uint32_t)0U, + (uint32_t)16U, + (uint32_t)1U, + uint64_t k_t = Hacl_Impl_SHA2_Generic_k384_512[(uint32_t)16U * i0 + i]; + uint64_t ws_t = ws[i]; + uint64_t a0 = hash[0U]; + uint64_t b0 = hash[1U]; + uint64_t c0 = hash[2U]; + uint64_t d0 = hash[3U]; + uint64_t e0 = hash[4U]; + uint64_t f0 = hash[5U]; + uint64_t g0 = hash[6U]; + uint64_t h02 = hash[7U]; + uint64_t k_e_t = k_t; + uint64_t + t1 = + h02 + + + ((e0 << (uint32_t)50U | e0 >> (uint32_t)14U) + ^ + ((e0 << (uint32_t)46U | e0 >> (uint32_t)18U) + ^ (e0 << (uint32_t)23U | e0 >> (uint32_t)41U))) + + ((e0 & f0) ^ (~e0 & g0)) + + k_e_t + + ws_t; + uint64_t + t2 = + ((a0 << (uint32_t)36U | a0 >> (uint32_t)28U) + ^ + ((a0 << (uint32_t)30U | a0 >> (uint32_t)34U) + ^ (a0 << (uint32_t)25U | a0 >> (uint32_t)39U))) + + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); + uint64_t a1 = t1 + t2; + uint64_t b1 = a0; + uint64_t c1 = b0; + uint64_t d1 = c0; + uint64_t e1 = d0 + t1; + uint64_t f1 = e0; + uint64_t g1 = f0; + uint64_t h12 = g0; + hash[0U] = a1; + hash[1U] = b1; + hash[2U] = c1; + hash[3U] = d1; + hash[4U] = e1; + hash[5U] = f1; + hash[6U] = g1; + hash[7U] = h12;); + if (i0 < (uint32_t)4U) + { + KRML_MAYBE_FOR16(i, + (uint32_t)0U, + (uint32_t)16U, + (uint32_t)1U, + uint64_t t16 = ws[i]; + uint64_t t15 = ws[(i + (uint32_t)1U) % (uint32_t)16U]; + uint64_t t7 = ws[(i + (uint32_t)9U) % (uint32_t)16U]; + uint64_t t2 = ws[(i + (uint32_t)14U) % (uint32_t)16U]; + uint64_t + s1 = + (t2 << (uint32_t)45U | t2 >> (uint32_t)19U) + ^ ((t2 << (uint32_t)3U | t2 >> (uint32_t)61U) ^ t2 >> (uint32_t)6U); + uint64_t + s0 = + (t15 << (uint32_t)63U | t15 >> (uint32_t)1U) + ^ ((t15 << (uint32_t)56U | t15 >> (uint32_t)8U) ^ t15 >> (uint32_t)7U); + ws[i] = s1 + t7 + s0 + t16;); + }); + KRML_MAYBE_FOR8(i, + (uint32_t)0U, + (uint32_t)8U, + (uint32_t)1U, + uint64_t *os = hash; + uint64_t x = hash[i] + hash_old[i]; + os[i] = x;); +} + +static inline void sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) +{ + uint32_t blocks = len / (uint32_t)128U; + for (uint32_t i = (uint32_t)0U; i < blocks; i++) + { + uint8_t *b0 = b; + uint8_t *mb = b0 + i * (uint32_t)128U; + sha512_update(mb, st); + } +} + +static inline void +sha512_update_last(FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *hash) +{ + uint32_t blocks; + if (len + (uint32_t)16U + (uint32_t)1U <= (uint32_t)128U) + { + blocks = (uint32_t)1U; + } + else + { + blocks = (uint32_t)2U; + } + uint32_t fin = blocks * (uint32_t)128U; + uint8_t last[256U] = { 0U }; + uint8_t totlen_buf[16U] = { 0U }; + FStar_UInt128_uint128 total_len_bits = FStar_UInt128_shift_left(totlen, (uint32_t)3U); + store128_be(totlen_buf, total_len_bits); + uint8_t *b0 = b; + memcpy(last, b0, len * sizeof (uint8_t)); + last[len] = (uint8_t)0x80U; + memcpy(last + fin - (uint32_t)16U, totlen_buf, (uint32_t)16U * sizeof (uint8_t)); + uint8_t *last00 = last; + uint8_t *last10 = last + (uint32_t)128U; + uint8_t *l0 = last00; + uint8_t *l1 = last10; + uint8_t *lb0 = l0; + uint8_t *lb1 = l1; + uint8_t *last0 = lb0; + uint8_t *last1 = lb1; + sha512_update(last0, hash); + if (blocks > (uint32_t)1U) + { + sha512_update(last1, hash); + return; + } +} + +static inline void sha512_finish(uint64_t *st, uint8_t *h) +{ + uint8_t hbuf[64U] = { 0U }; + KRML_MAYBE_FOR8(i, + (uint32_t)0U, + (uint32_t)8U, + (uint32_t)1U, + store64_be(hbuf + i * (uint32_t)8U, st[i]);); + memcpy(h, hbuf, (uint32_t)64U * sizeof (uint8_t)); +} + +static inline void sha384_init(uint64_t *hash) +{ + KRML_MAYBE_FOR8(i, + (uint32_t)0U, + (uint32_t)8U, + (uint32_t)1U, + uint64_t *os = hash; + uint64_t x = Hacl_Impl_SHA2_Generic_h384[i]; + os[i] = x;); +} + +static inline void sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) +{ + sha512_update_nblocks(len, b, st); +} + +static void +sha384_update_last(FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *st) +{ + sha512_update_last(totlen, len, b, st); +} + +static inline void sha384_finish(uint64_t *st, uint8_t *h) +{ + uint8_t hbuf[64U] = { 0U }; + KRML_MAYBE_FOR8(i, + (uint32_t)0U, + (uint32_t)8U, + (uint32_t)1U, + store64_be(hbuf + i * (uint32_t)8U, st[i]);); + memcpy(h, hbuf, (uint32_t)48U * sizeof (uint8_t)); +} + /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. @@ -680,3 +903,434 @@ void Hacl_Streaming_SHA2_sha224(uint8_t *input, uint32_t input_len, uint8_t *dst sha224_finish(st, rb); } +Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_512(void) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + Hacl_Streaming_SHA2_state_sha2_384 + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_SHA2_state_sha2_384 + *p = + (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( + Hacl_Streaming_SHA2_state_sha2_384 + )); + p[0U] = s; + Hacl_SHA2_Scalar32_sha512_init(block_state); + return p; +} + +/** +Copies the state passed as argument into a newly allocated state (deep copy). +The state is to be freed by calling `free_512`. Cloning the state this way is +useful, for instance, if your control-flow diverges and you need to feed +more (different) data into the hash in each branch. +*/ +Hacl_Streaming_SHA2_state_sha2_384 +*Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_SHA2_state_sha2_384 *s0) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *s0; + uint64_t *block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); + memcpy(buf, buf0, (uint32_t)128U * sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint64_t)); + Hacl_Streaming_SHA2_state_sha2_384 + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Streaming_SHA2_state_sha2_384 + *p = + (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( + Hacl_Streaming_SHA2_state_sha2_384 + )); + p[0U] = s; + return p; +} + +void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_SHA2_state_sha2_384 *s) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + uint8_t *buf = scrut.buf; + uint64_t *block_state = scrut.block_state; + Hacl_SHA2_Scalar32_sha512_init(block_state); + Hacl_Streaming_SHA2_state_sha2_384 + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +static inline uint32_t +update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t len) +{ + Hacl_Streaming_SHA2_state_sha2_384 s = *p; + uint64_t total_len = s.total_len; + if ((uint64_t)len > (uint64_t)18446744073709551615U - total_len) + { + return (uint32_t)1U; + } + uint32_t sz; + if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + { + sz = (uint32_t)128U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + } + if (len <= (uint32_t)128U - sz) + { + Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + uint64_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, data, len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)len; + *p + = + ( + (Hacl_Streaming_SHA2_state_sha2_384){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == (uint32_t)0U) + { + Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + uint64_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + } + if (!(sz1 == (uint32_t)0U)) + { + sha512_update_nblocks((uint32_t)128U, buf, block_state1); + } + uint32_t ite; + if ((uint64_t)len % (uint64_t)(uint32_t)128U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + { + ite = (uint32_t)128U; + } + else + { + ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)128U); + } + uint32_t n_blocks = (len - ite) / (uint32_t)128U; + uint32_t data1_len = n_blocks * (uint32_t)128U; + uint32_t data2_len = len - data1_len; + uint8_t *data1 = data; + uint8_t *data2 = data + data1_len; + sha512_update_nblocks(data1_len, data1, block_state1); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_SHA2_state_sha2_384){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)len + } + ); + } + else + { + uint32_t diff = (uint32_t)128U - sz; + uint8_t *data1 = data; + uint8_t *data2 = data + diff; + Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + uint64_t *block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len10 > (uint64_t)0U) + { + sz10 = (uint32_t)128U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)128U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, data1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *p + = + ( + (Hacl_Streaming_SHA2_state_sha2_384){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Streaming_SHA2_state_sha2_384 s10 = *p; + uint64_t *block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + } + if (!(sz1 == (uint32_t)0U)) + { + sha512_update_nblocks((uint32_t)128U, buf, block_state1); + } + uint32_t ite; + if + ( + (uint64_t)(len - diff) + % (uint64_t)(uint32_t)128U + == (uint64_t)0U + && (uint64_t)(len - diff) > (uint64_t)0U + ) + { + ite = (uint32_t)128U; + } + else + { + ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)128U); + } + uint32_t n_blocks = (len - diff - ite) / (uint32_t)128U; + uint32_t data1_len = n_blocks * (uint32_t)128U; + uint32_t data2_len = len - diff - data1_len; + uint8_t *data11 = data2; + uint8_t *data21 = data2 + data1_len; + sha512_update_nblocks(data1_len, data11, block_state1); + uint8_t *dst = buf; + memcpy(dst, data21, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_SHA2_state_sha2_384){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(len - diff) + } + ); + } + return (uint32_t)0U; +} + +/** +Feed an arbitrary amount of data into the hash. This function returns 0 for +success, or 1 if the combined length of all of the data passed to `update_512` +(since the last call to `init_512`) exceeds 2^125-1 bytes. + +This function is identical to the update function for SHA2_384. +*/ +uint32_t +Hacl_Streaming_SHA2_update_512( + Hacl_Streaming_SHA2_state_sha2_384 *p, + uint8_t *input, + uint32_t input_len +) +{ + return update_384_512(p, input, input_len); +} + +/** +Write the resulting hash into `dst`, an array of 64 bytes. The state remains +valid after a call to `finish_512`, meaning the user may feed more data into +the hash via `update_512`. (The finish_512 function operates on an internal copy of +the state and therefore does not invalidate the client-held state `p`.) +*/ +void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *p; + uint64_t *block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = (uint32_t)128U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + } + uint8_t *buf_1 = buf_; + uint64_t tmp_block_state[8U] = { 0U }; + memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint64_t)); + uint32_t ite; + if (r % (uint32_t)128U == (uint32_t)0U && r > (uint32_t)0U) + { + ite = (uint32_t)128U; + } + else + { + ite = r % (uint32_t)128U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + sha512_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + uint64_t prev_len_last = total_len - (uint64_t)r; + sha512_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), + FStar_UInt128_uint64_to_uint128((uint64_t)r)), + r, + buf_last, + tmp_block_state); + sha512_finish(tmp_block_state, dst); +} + +/** +Free a state allocated with `create_in_512`. + +This function is identical to the free function for SHA2_384. +*/ +void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_SHA2_state_sha2_384 *s) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + uint8_t *buf = scrut.buf; + uint64_t *block_state = scrut.block_state; + KRML_HOST_FREE(block_state); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(s); +} + +/** +Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. +*/ +void Hacl_Streaming_SHA2_sha512(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + uint8_t *ib = input; + uint8_t *rb = dst; + uint64_t st[8U] = { 0U }; + Hacl_SHA2_Scalar32_sha512_init(st); + uint32_t rem = input_len % (uint32_t)128U; + FStar_UInt128_uint128 len_ = FStar_UInt128_uint64_to_uint128((uint64_t)input_len); + sha512_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % (uint32_t)128U; + uint8_t *b0 = ib; + uint8_t *lb = b0 + input_len - rem1; + sha512_update_last(len_, rem, lb, st); + sha512_finish(st, rb); +} + +Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_384(void) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + Hacl_Streaming_SHA2_state_sha2_384 + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_SHA2_state_sha2_384 + *p = + (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( + Hacl_Streaming_SHA2_state_sha2_384 + )); + p[0U] = s; + sha384_init(block_state); + return p; +} + +void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_SHA2_state_sha2_384 *s) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + uint8_t *buf = scrut.buf; + uint64_t *block_state = scrut.block_state; + sha384_init(block_state); + Hacl_Streaming_SHA2_state_sha2_384 + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +uint32_t +Hacl_Streaming_SHA2_update_384( + Hacl_Streaming_SHA2_state_sha2_384 *p, + uint8_t *input, + uint32_t input_len +) +{ + return update_384_512(p, input, input_len); +} + +/** +Write the resulting hash into `dst`, an array of 48 bytes. The state remains +valid after a call to `finish_384`, meaning the user may feed more data into +the hash via `update_384`. +*/ +void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst) +{ + Hacl_Streaming_SHA2_state_sha2_384 scrut = *p; + uint64_t *block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = (uint32_t)128U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + } + uint8_t *buf_1 = buf_; + uint64_t tmp_block_state[8U] = { 0U }; + memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint64_t)); + uint32_t ite; + if (r % (uint32_t)128U == (uint32_t)0U && r > (uint32_t)0U) + { + ite = (uint32_t)128U; + } + else + { + ite = r % (uint32_t)128U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + sha384_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + uint64_t prev_len_last = total_len - (uint64_t)r; + sha384_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), + FStar_UInt128_uint64_to_uint128((uint64_t)r)), + r, + buf_last, + tmp_block_state); + sha384_finish(tmp_block_state, dst); +} + +void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_SHA2_state_sha2_384 *p) +{ + Hacl_Streaming_SHA2_free_512(p); +} + +/** +Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. +*/ +void Hacl_Streaming_SHA2_sha384(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + uint8_t *ib = input; + uint8_t *rb = dst; + uint64_t st[8U] = { 0U }; + sha384_init(st); + uint32_t rem = input_len % (uint32_t)128U; + FStar_UInt128_uint128 len_ = FStar_UInt128_uint64_to_uint128((uint64_t)input_len); + sha384_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % (uint32_t)128U; + uint8_t *b0 = ib; + uint8_t *lb = b0 + input_len - rem1; + sha384_update_last(len_, rem, lb, st); + sha384_finish(st, rb); +} + diff --git a/Modules/_hacl/Hacl_Streaming_SHA2.h b/Modules/_hacl/Hacl_Streaming_SHA2.h index c83a835afe70fd..2c905854f336fd 100644 --- a/Modules/_hacl/Hacl_Streaming_SHA2.h +++ b/Modules/_hacl/Hacl_Streaming_SHA2.h @@ -32,13 +32,12 @@ extern "C" { #include #include "python_hacl_namespaces.h" -#include "krml/FStar_UInt_8_16_32_64.h" +#include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" - typedef struct Hacl_Streaming_SHA2_state_sha2_224_s { uint32_t *block_state; @@ -49,6 +48,16 @@ Hacl_Streaming_SHA2_state_sha2_224; typedef Hacl_Streaming_SHA2_state_sha2_224 Hacl_Streaming_SHA2_state_sha2_256; +typedef struct Hacl_Streaming_SHA2_state_sha2_384_s +{ + uint64_t *block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Streaming_SHA2_state_sha2_384; + +typedef Hacl_Streaming_SHA2_state_sha2_384 Hacl_Streaming_SHA2_state_sha2_512; + /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. @@ -128,6 +137,78 @@ Hash `input`, of len `input_len`, into `dst`, an array of 28 bytes. */ void Hacl_Streaming_SHA2_sha224(uint8_t *input, uint32_t input_len, uint8_t *dst); +Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_512(void); + +/** +Copies the state passed as argument into a newly allocated state (deep copy). +The state is to be freed by calling `free_512`. Cloning the state this way is +useful, for instance, if your control-flow diverges and you need to feed +more (different) data into the hash in each branch. +*/ +Hacl_Streaming_SHA2_state_sha2_384 +*Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_SHA2_state_sha2_384 *s0); + +void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_SHA2_state_sha2_384 *s); + +/** +Feed an arbitrary amount of data into the hash. This function returns 0 for +success, or 1 if the combined length of all of the data passed to `update_512` +(since the last call to `init_512`) exceeds 2^125-1 bytes. + +This function is identical to the update function for SHA2_384. +*/ +uint32_t +Hacl_Streaming_SHA2_update_512( + Hacl_Streaming_SHA2_state_sha2_384 *p, + uint8_t *input, + uint32_t input_len +); + +/** +Write the resulting hash into `dst`, an array of 64 bytes. The state remains +valid after a call to `finish_512`, meaning the user may feed more data into +the hash via `update_512`. (The finish_512 function operates on an internal copy of +the state and therefore does not invalidate the client-held state `p`.) +*/ +void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst); + +/** +Free a state allocated with `create_in_512`. + +This function is identical to the free function for SHA2_384. +*/ +void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_SHA2_state_sha2_384 *s); + +/** +Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. +*/ +void Hacl_Streaming_SHA2_sha512(uint8_t *input, uint32_t input_len, uint8_t *dst); + +Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_384(void); + +void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_SHA2_state_sha2_384 *s); + +uint32_t +Hacl_Streaming_SHA2_update_384( + Hacl_Streaming_SHA2_state_sha2_384 *p, + uint8_t *input, + uint32_t input_len +); + +/** +Write the resulting hash into `dst`, an array of 48 bytes. The state remains +valid after a call to `finish_384`, meaning the user may feed more data into +the hash via `update_384`. +*/ +void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst); + +void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_SHA2_state_sha2_384 *p); + +/** +Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. +*/ +void Hacl_Streaming_SHA2_sha384(uint8_t *input, uint32_t input_len, uint8_t *dst); + #if defined(__cplusplus) } #endif diff --git a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h new file mode 100644 index 00000000000000..ee160193539e28 --- /dev/null +++ b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h @@ -0,0 +1,347 @@ +/* + Copyright (c) INRIA and Microsoft Corporation. All rights reserved. + Licensed under the Apache 2.0 License. +*/ + + +#ifndef __FStar_UInt128_Verified_H +#define __FStar_UInt128_Verified_H + + + +#include "FStar_UInt_8_16_32_64.h" +#include +#include +#include "krml/types.h" +#include "krml/internal/target.h" +static inline uint64_t FStar_UInt128_constant_time_carry(uint64_t a, uint64_t b) +{ + return (a ^ ((a ^ b) | ((a - b) ^ b))) >> (uint32_t)63U; +} + +static inline uint64_t FStar_UInt128_carry(uint64_t a, uint64_t b) +{ + return FStar_UInt128_constant_time_carry(a, b); +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_add(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low + b.low; + lit.high = a.high + b.high + FStar_UInt128_carry(a.low + b.low, b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_add_underspec(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low + b.low; + lit.high = a.high + b.high + FStar_UInt128_carry(a.low + b.low, b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_add_mod(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low + b.low; + lit.high = a.high + b.high + FStar_UInt128_carry(a.low + b.low, b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_sub(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low - b.low; + lit.high = a.high - b.high - FStar_UInt128_carry(a.low, a.low - b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_sub_underspec(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low - b.low; + lit.high = a.high - b.high - FStar_UInt128_carry(a.low, a.low - b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_sub_mod_impl(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low - b.low; + lit.high = a.high - b.high - FStar_UInt128_carry(a.low, a.low - b.low); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_sub_mod(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return FStar_UInt128_sub_mod_impl(a, b); +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_logand(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low & b.low; + lit.high = a.high & b.high; + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_logxor(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low ^ b.low; + lit.high = a.high ^ b.high; + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_logor(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = a.low | b.low; + lit.high = a.high | b.high; + return lit; +} + +static inline FStar_UInt128_uint128 FStar_UInt128_lognot(FStar_UInt128_uint128 a) +{ + FStar_UInt128_uint128 lit; + lit.low = ~a.low; + lit.high = ~a.high; + return lit; +} + +static uint32_t FStar_UInt128_u32_64 = (uint32_t)64U; + +static inline uint64_t FStar_UInt128_add_u64_shift_left(uint64_t hi, uint64_t lo, uint32_t s) +{ + return (hi << s) + (lo >> (FStar_UInt128_u32_64 - s)); +} + +static inline uint64_t +FStar_UInt128_add_u64_shift_left_respec(uint64_t hi, uint64_t lo, uint32_t s) +{ + return FStar_UInt128_add_u64_shift_left(hi, lo, s); +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_left_small(FStar_UInt128_uint128 a, uint32_t s) +{ + if (s == (uint32_t)0U) + { + return a; + } + else + { + FStar_UInt128_uint128 lit; + lit.low = a.low << s; + lit.high = FStar_UInt128_add_u64_shift_left_respec(a.high, a.low, s); + return lit; + } +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_left_large(FStar_UInt128_uint128 a, uint32_t s) +{ + FStar_UInt128_uint128 lit; + lit.low = (uint64_t)0U; + lit.high = a.low << (s - FStar_UInt128_u32_64); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_left(FStar_UInt128_uint128 a, uint32_t s) +{ + if (s < FStar_UInt128_u32_64) + { + return FStar_UInt128_shift_left_small(a, s); + } + else + { + return FStar_UInt128_shift_left_large(a, s); + } +} + +static inline uint64_t FStar_UInt128_add_u64_shift_right(uint64_t hi, uint64_t lo, uint32_t s) +{ + return (lo >> s) + (hi << (FStar_UInt128_u32_64 - s)); +} + +static inline uint64_t +FStar_UInt128_add_u64_shift_right_respec(uint64_t hi, uint64_t lo, uint32_t s) +{ + return FStar_UInt128_add_u64_shift_right(hi, lo, s); +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_right_small(FStar_UInt128_uint128 a, uint32_t s) +{ + if (s == (uint32_t)0U) + { + return a; + } + else + { + FStar_UInt128_uint128 lit; + lit.low = FStar_UInt128_add_u64_shift_right_respec(a.high, a.low, s); + lit.high = a.high >> s; + return lit; + } +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_right_large(FStar_UInt128_uint128 a, uint32_t s) +{ + FStar_UInt128_uint128 lit; + lit.low = a.high >> (s - FStar_UInt128_u32_64); + lit.high = (uint64_t)0U; + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_shift_right(FStar_UInt128_uint128 a, uint32_t s) +{ + if (s < FStar_UInt128_u32_64) + { + return FStar_UInt128_shift_right_small(a, s); + } + else + { + return FStar_UInt128_shift_right_large(a, s); + } +} + +static inline bool FStar_UInt128_eq(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return a.low == b.low && a.high == b.high; +} + +static inline bool FStar_UInt128_gt(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return a.high > b.high || (a.high == b.high && a.low > b.low); +} + +static inline bool FStar_UInt128_lt(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return a.high < b.high || (a.high == b.high && a.low < b.low); +} + +static inline bool FStar_UInt128_gte(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return a.high > b.high || (a.high == b.high && a.low >= b.low); +} + +static inline bool FStar_UInt128_lte(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + return a.high < b.high || (a.high == b.high && a.low <= b.low); +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_eq_mask(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = FStar_UInt64_eq_mask(a.low, b.low) & FStar_UInt64_eq_mask(a.high, b.high); + lit.high = FStar_UInt64_eq_mask(a.low, b.low) & FStar_UInt64_eq_mask(a.high, b.high); + return lit; +} + +static inline FStar_UInt128_uint128 +FStar_UInt128_gte_mask(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) +{ + FStar_UInt128_uint128 lit; + lit.low = + (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) + | (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); + lit.high = + (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) + | (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); + return lit; +} + +static inline FStar_UInt128_uint128 FStar_UInt128_uint64_to_uint128(uint64_t a) +{ + FStar_UInt128_uint128 lit; + lit.low = a; + lit.high = (uint64_t)0U; + return lit; +} + +static inline uint64_t FStar_UInt128_uint128_to_uint64(FStar_UInt128_uint128 a) +{ + return a.low; +} + +static inline uint64_t FStar_UInt128_u64_mod_32(uint64_t a) +{ + return a & (uint64_t)0xffffffffU; +} + +static uint32_t FStar_UInt128_u32_32 = (uint32_t)32U; + +static inline uint64_t FStar_UInt128_u32_combine(uint64_t hi, uint64_t lo) +{ + return lo + (hi << FStar_UInt128_u32_32); +} + +static inline FStar_UInt128_uint128 FStar_UInt128_mul32(uint64_t x, uint32_t y) +{ + FStar_UInt128_uint128 lit; + lit.low = + FStar_UInt128_u32_combine((x >> FStar_UInt128_u32_32) + * (uint64_t)y + + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32), + FStar_UInt128_u64_mod_32(FStar_UInt128_u64_mod_32(x) * (uint64_t)y)); + lit.high = + ((x >> FStar_UInt128_u32_32) + * (uint64_t)y + + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32)) + >> FStar_UInt128_u32_32; + return lit; +} + +static inline uint64_t FStar_UInt128_u32_combine_(uint64_t hi, uint64_t lo) +{ + return lo + (hi << FStar_UInt128_u32_32); +} + +static inline FStar_UInt128_uint128 FStar_UInt128_mul_wide(uint64_t x, uint64_t y) +{ + FStar_UInt128_uint128 lit; + lit.low = + FStar_UInt128_u32_combine_(FStar_UInt128_u64_mod_32(x) + * (y >> FStar_UInt128_u32_32) + + + FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) + * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)), + FStar_UInt128_u64_mod_32(FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y))); + lit.high = + (x >> FStar_UInt128_u32_32) + * (y >> FStar_UInt128_u32_32) + + + (((x >> FStar_UInt128_u32_32) + * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)) + >> FStar_UInt128_u32_32) + + + ((FStar_UInt128_u64_mod_32(x) + * (y >> FStar_UInt128_u32_32) + + + FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) + * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32))) + >> FStar_UInt128_u32_32); + return lit; +} + + +#define __FStar_UInt128_Verified_H_DEFINED +#endif diff --git a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h index 3e2e4b32b22f96..965afc836fd12b 100644 --- a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h +++ b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h @@ -14,7 +14,7 @@ #include #include "krml/lowstar_endianness.h" -#include "krml/FStar_UInt_8_16_32_64.h" +#include "krml/types.h" #include "krml/internal/target.h" static inline uint64_t FStar_UInt64_eq_mask(uint64_t a, uint64_t b) { diff --git a/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h b/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h new file mode 100644 index 00000000000000..e2b6d62859a5f1 --- /dev/null +++ b/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h @@ -0,0 +1,68 @@ +/* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. + Licensed under the Apache 2.0 License. */ + +#ifndef FSTAR_UINT128_STRUCT_ENDIANNESS_H +#define FSTAR_UINT128_STRUCT_ENDIANNESS_H + +/* Hand-written implementation of endianness-related uint128 functions + * for the extracted uint128 implementation */ + +/* Access 64-bit fields within the int128. */ +#define HIGH64_OF(x) ((x)->high) +#define LOW64_OF(x) ((x)->low) + +/* A series of definitions written using pointers. */ + +inline static void load128_le_(uint8_t *b, uint128_t *r) { + LOW64_OF(r) = load64_le(b); + HIGH64_OF(r) = load64_le(b + 8); +} + +inline static void store128_le_(uint8_t *b, uint128_t *n) { + store64_le(b, LOW64_OF(n)); + store64_le(b + 8, HIGH64_OF(n)); +} + +inline static void load128_be_(uint8_t *b, uint128_t *r) { + HIGH64_OF(r) = load64_be(b); + LOW64_OF(r) = load64_be(b + 8); +} + +inline static void store128_be_(uint8_t *b, uint128_t *n) { + store64_be(b, HIGH64_OF(n)); + store64_be(b + 8, LOW64_OF(n)); +} + +#ifndef KRML_NOSTRUCT_PASSING + +inline static uint128_t load128_le(uint8_t *b) { + uint128_t r; + load128_le_(b, &r); + return r; +} + +inline static void store128_le(uint8_t *b, uint128_t n) { + store128_le_(b, &n); +} + +inline static uint128_t load128_be(uint8_t *b) { + uint128_t r; + load128_be_(b, &r); + return r; +} + +inline static void store128_be(uint8_t *b, uint128_t n) { + store128_be_(b, &n); +} + +#else /* !defined(KRML_STRUCT_PASSING) */ + +# define print128 print128_ +# define load128_le load128_le_ +# define store128_le store128_le_ +# define load128_be load128_be_ +# define store128_be store128_be_ + +#endif /* KRML_STRUCT_PASSING */ + +#endif diff --git a/Modules/_hacl/include/krml/types.h b/Modules/_hacl/include/krml/types.h new file mode 100644 index 00000000000000..509f555536e4c6 --- /dev/null +++ b/Modules/_hacl/include/krml/types.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +typedef struct FStar_UInt128_uint128_s { + uint64_t low; + uint64_t high; +} FStar_UInt128_uint128, uint128_t; + +#define KRML_VERIFIED_UINT128 + +#include "krml/lowstar_endianness.h" +#include "krml/fstar_uint128_struct_endianness.h" +#include "krml/FStar_UInt128_Verified.h" diff --git a/Modules/_hacl/include/python_hacl_namespaces.h b/Modules/_hacl/include/python_hacl_namespaces.h index af390459311fe8..65608d1fd283c4 100644 --- a/Modules/_hacl/include/python_hacl_namespaces.h +++ b/Modules/_hacl/include/python_hacl_namespaces.h @@ -10,19 +10,36 @@ #define Hacl_Streaming_SHA2_state_sha2_224_s python_hashlib_Hacl_Streaming_SHA2_state_sha2_224_s #define Hacl_Streaming_SHA2_state_sha2_224 python_hashlib_Hacl_Streaming_SHA2_state_sha2_224 #define Hacl_Streaming_SHA2_state_sha2_256 python_hashlib_Hacl_Streaming_SHA2_state_sha2_256 +#define Hacl_Streaming_SHA2_state_sha2_384_s python_hashlib_Hacl_Streaming_SHA2_state_sha2_384_s +#define Hacl_Streaming_SHA2_state_sha2_384 python_hashlib_Hacl_Streaming_SHA2_state_sha2_384 +#define Hacl_Streaming_SHA2_state_sha2_512 python_hashlib_Hacl_Streaming_SHA2_state_sha2_512 #define Hacl_Streaming_SHA2_create_in_256 python_hashlib_Hacl_Streaming_SHA2_create_in_256 #define Hacl_Streaming_SHA2_create_in_224 python_hashlib_Hacl_Streaming_SHA2_create_in_224 +#define Hacl_Streaming_SHA2_create_in_512 python_hashlib_Hacl_Streaming_SHA2_create_in_512 +#define Hacl_Streaming_SHA2_create_in_384 python_hashlib_Hacl_Streaming_SHA2_create_in_384 #define Hacl_Streaming_SHA2_copy_256 python_hashlib_Hacl_Streaming_SHA2_copy_256 #define Hacl_Streaming_SHA2_copy_224 python_hashlib_Hacl_Streaming_SHA2_copy_224 +#define Hacl_Streaming_SHA2_copy_512 python_hashlib_Hacl_Streaming_SHA2_copy_512 +#define Hacl_Streaming_SHA2_copy_384 python_hashlib_Hacl_Streaming_SHA2_copy_384 #define Hacl_Streaming_SHA2_init_256 python_hashlib_Hacl_Streaming_SHA2_init_256 #define Hacl_Streaming_SHA2_init_224 python_hashlib_Hacl_Streaming_SHA2_init_224 +#define Hacl_Streaming_SHA2_init_512 python_hashlib_Hacl_Streaming_SHA2_init_512 +#define Hacl_Streaming_SHA2_init_384 python_hashlib_Hacl_Streaming_SHA2_init_384 #define Hacl_Streaming_SHA2_update_256 python_hashlib_Hacl_Streaming_SHA2_update_256 #define Hacl_Streaming_SHA2_update_224 python_hashlib_Hacl_Streaming_SHA2_update_224 +#define Hacl_Streaming_SHA2_update_512 python_hashlib_Hacl_Streaming_SHA2_update_512 +#define Hacl_Streaming_SHA2_update_384 python_hashlib_Hacl_Streaming_SHA2_update_384 #define Hacl_Streaming_SHA2_finish_256 python_hashlib_Hacl_Streaming_SHA2_finish_256 #define Hacl_Streaming_SHA2_finish_224 python_hashlib_Hacl_Streaming_SHA2_finish_224 +#define Hacl_Streaming_SHA2_finish_512 python_hashlib_Hacl_Streaming_SHA2_finish_512 +#define Hacl_Streaming_SHA2_finish_384 python_hashlib_Hacl_Streaming_SHA2_finish_384 #define Hacl_Streaming_SHA2_free_256 python_hashlib_Hacl_Streaming_SHA2_free_256 #define Hacl_Streaming_SHA2_free_224 python_hashlib_Hacl_Streaming_SHA2_free_224 +#define Hacl_Streaming_SHA2_free_512 python_hashlib_Hacl_Streaming_SHA2_free_512 +#define Hacl_Streaming_SHA2_free_384 python_hashlib_Hacl_Streaming_SHA2_free_384 #define Hacl_Streaming_SHA2_sha256 python_hashlib_Hacl_Streaming_SHA2_sha256 #define Hacl_Streaming_SHA2_sha224 python_hashlib_Hacl_Streaming_SHA2_sha224 +#define Hacl_Streaming_SHA2_sha512 python_hashlib_Hacl_Streaming_SHA2_sha512 +#define Hacl_Streaming_SHA2_sha384 python_hashlib_Hacl_Streaming_SHA2_sha384 #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/internal/Hacl_SHA2_Generic.h b/Modules/_hacl/internal/Hacl_SHA2_Generic.h index 23f7cea1eb3884..6ac47f3cf7ed36 100644 --- a/Modules/_hacl/internal/Hacl_SHA2_Generic.h +++ b/Modules/_hacl/internal/Hacl_SHA2_Generic.h @@ -31,13 +31,10 @@ extern "C" { #endif #include -#include "krml/FStar_UInt_8_16_32_64.h" +#include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" - - - static const uint32_t Hacl_Impl_SHA2_Generic_h224[8U] = diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index 594873862a2db0..dba8cb3972ea17 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=94aabbb4cf71347d3779a8db486c761403c6d036 +expected_hacl_star_rev=4751fc2b11639f651718abf8522fcc36902ca67c hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" @@ -54,6 +54,8 @@ include_files=( declare -a lib_files lib_files=( krmllib/dist/minimal/FStar_UInt_8_16_32_64.h + krmllib/dist/minimal/fstar_uint128_struct_endianness.h + krmllib/dist/minimal/FStar_UInt128_Verified.h ) # C files for the algorithms themselves: current directory @@ -82,10 +84,27 @@ fi readarray -t all_files < <(find . -name '*.h' -or -name '*.c') -# types.h is a simple wrapper that defines the uint128 type then proceeds to -# include FStar_UInt_8_16_32_64.h; we jump the types.h step since our current -# selection of algorithms does not necessitate the use of uint128 -$sed -i 's!#include.*types.h"!#include "krml/FStar_UInt_8_16_32_64.h"!g' "${all_files[@]}" +# types.h originally contains a complex series of if-defs and auxiliary type +# definitions; here, we just need a proper uint128 type in scope +# is a simple wrapper that defines the uint128 type +cat > include/krml/types.h < + +typedef struct FStar_UInt128_uint128_s { + uint64_t low; + uint64_t high; +} FStar_UInt128_uint128, uint128_t; + +#define KRML_VERIFIED_UINT128 + +#include "krml/lowstar_endianness.h" +#include "krml/fstar_uint128_struct_endianness.h" +#include "krml/FStar_UInt128_Verified.h" +EOF +# Adjust the include path to reflect the local directory structure +$sed -i 's!#include.*types.h"!#include "krml/types.h"!g' "${all_files[@]}" $sed -i 's!#include.*compat.h"!!g' "${all_files[@]}" # FStar_UInt_8_16_32_64 contains definitions useful in the general case, but not @@ -103,22 +122,6 @@ $sed -i 's!#include.*Hacl_Krmllib.h"!!g' "${all_files[@]}" # included in $dist_files). $sed -i 's!#include.*internal/Hacl_Streaming_SHA2.h"!#include "Hacl_Streaming_SHA2.h"!g' "${all_files[@]}" -# The SHA2 file contains all variants of SHA2. We strip 384 and 512 for the time -# being, to be included later. -# This regexp matches a separator (two new lines), followed by: -# -# * -# ... 384 or 512 ... { -# * -# } -# -# The first non-empty lines are the comment block. The second ... may spill over -# the next following lines if the arguments are printed in one-per-line mode. -$sed -i -z 's/\n\n\([^\n]\+\n\)*[^\n]*\(384\|512\)[^{]*{\n\?\( [^\n]*\n\)*}//g' Hacl_Streaming_SHA2.c - -# Same thing with function prototypes -$sed -i -z 's/\n\n\([^\n]\+\n\)*[^\n]*\(384\|512\)[^;]*;//g' Hacl_Streaming_SHA2.h - # Use globally unique names for the Hacl_ C APIs to avoid linkage conflicts. $sed -i -z 's!#include \n!#include \n#include "python_hacl_namespaces.h"\n!' Hacl_Streaming_SHA2.h diff --git a/Modules/sha256module.c b/Modules/sha256module.c index 630e4bf03bbe96..301c9837bb6720 100644 --- a/Modules/sha256module.c +++ b/Modules/sha256module.c @@ -8,6 +8,7 @@ Andrew Kuchling (amk@amk.ca) Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) + Jonathan Protzenko (jonathan@protzenko.fr) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. diff --git a/Modules/sha512module.c b/Modules/sha512module.c index bf4408b455f2c4..d7dfed4e5db03a 100644 --- a/Modules/sha512module.c +++ b/Modules/sha512module.c @@ -8,6 +8,7 @@ Andrew Kuchling (amk@amk.ca) Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) + Jonathan Protzenko (jonathan@protzenko.fr) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. @@ -31,400 +32,32 @@ class SHA512Type "SHAobject *" "&PyType_Type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=81a3ccde92bcfe8d]*/ -/* Some useful types */ - -typedef unsigned char SHA_BYTE; -typedef uint32_t SHA_INT32; /* 32-bit integer */ -typedef uint64_t SHA_INT64; /* 64-bit integer */ /* The SHA block size and message digest sizes, in bytes */ #define SHA_BLOCKSIZE 128 #define SHA_DIGESTSIZE 64 -/* The structure for storing SHA info */ +/* The SHA2-384 and SHA2-512 implementations defer to the HACL* verified + * library. */ + +#include "_hacl/Hacl_Streaming_SHA2.h" typedef struct { PyObject_HEAD - SHA_INT64 digest[8]; /* Message digest */ - SHA_INT32 count_lo, count_hi; /* 64-bit bit count */ - SHA_BYTE data[SHA_BLOCKSIZE]; /* SHA data buffer */ - int local; /* unprocessed amount in data */ int digestsize; + Hacl_Streaming_SHA2_state_sha2_512 *state; } SHAobject; #include "clinic/sha512module.c.h" -/* When run on a little-endian CPU we need to perform byte reversal on an - array of longwords. */ - -#if PY_LITTLE_ENDIAN -static void longReverse(SHA_INT64 *buffer, int byteCount) -{ - byteCount /= sizeof(*buffer); - for (; byteCount--; buffer++) { - *buffer = _Py_bswap64(*buffer); - } -} -#endif static void SHAcopy(SHAobject *src, SHAobject *dest) { - dest->local = src->local; dest->digestsize = src->digestsize; - dest->count_lo = src->count_lo; - dest->count_hi = src->count_hi; - memcpy(dest->digest, src->digest, sizeof(src->digest)); - memcpy(dest->data, src->data, sizeof(src->data)); -} - - -/* ------------------------------------------------------------------------ - * - * This code for the SHA-512 algorithm was noted as public domain. The - * original headers are pasted below. - * - * Several changes have been made to make it more compatible with the - * Python environment and desired interface. - * - */ - -/* LibTomCrypt, modular cryptographic library -- Tom St Denis - * - * LibTomCrypt is a library that provides various cryptographic - * algorithms in a highly modular and flexible manner. - * - * The library is free for all purposes without any express - * guarantee it works. - * - * Tom St Denis, tomstdenis@iahu.ca, https://www.libtom.net - */ - - -/* SHA512 by Tom St Denis */ - -/* Various logical functions */ -#define ROR64(x, y) \ - ( ((((x) & 0xFFFFFFFFFFFFFFFFULL)>>((unsigned long long)(y) & 63)) | \ - ((x)<<((unsigned long long)(64-((y) & 63))))) & 0xFFFFFFFFFFFFFFFFULL) -#define Ch(x,y,z) (z ^ (x & (y ^ z))) -#define Maj(x,y,z) (((x | y) & z) | (x & y)) -#define S(x, n) ROR64((x),(n)) -#define R(x, n) (((x) & 0xFFFFFFFFFFFFFFFFULL) >> ((unsigned long long)n)) -#define Sigma0(x) (S(x, 28) ^ S(x, 34) ^ S(x, 39)) -#define Sigma1(x) (S(x, 14) ^ S(x, 18) ^ S(x, 41)) -#define Gamma0(x) (S(x, 1) ^ S(x, 8) ^ R(x, 7)) -#define Gamma1(x) (S(x, 19) ^ S(x, 61) ^ R(x, 6)) - - -static void -sha512_transform(SHAobject *sha_info) -{ - int i; - SHA_INT64 S[8], W[80], t0, t1; - - memcpy(W, sha_info->data, sizeof(sha_info->data)); -#if PY_LITTLE_ENDIAN - longReverse(W, (int)sizeof(sha_info->data)); -#endif - - for (i = 16; i < 80; ++i) { - W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; - } - for (i = 0; i < 8; ++i) { - S[i] = sha_info->digest[i]; - } - - /* Compress */ -#define RND(a,b,c,d,e,f,g,h,i,ki) \ - t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; \ - t1 = Sigma0(a) + Maj(a, b, c); \ - d += t0; \ - h = t0 + t1; - - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],0,0x428a2f98d728ae22ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],1,0x7137449123ef65cdULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],2,0xb5c0fbcfec4d3b2fULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],3,0xe9b5dba58189dbbcULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],4,0x3956c25bf348b538ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],5,0x59f111f1b605d019ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],6,0x923f82a4af194f9bULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],7,0xab1c5ed5da6d8118ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],8,0xd807aa98a3030242ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],9,0x12835b0145706fbeULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],10,0x243185be4ee4b28cULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],11,0x550c7dc3d5ffb4e2ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],12,0x72be5d74f27b896fULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],13,0x80deb1fe3b1696b1ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],14,0x9bdc06a725c71235ULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],15,0xc19bf174cf692694ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],16,0xe49b69c19ef14ad2ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],17,0xefbe4786384f25e3ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],18,0x0fc19dc68b8cd5b5ULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],19,0x240ca1cc77ac9c65ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],20,0x2de92c6f592b0275ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],21,0x4a7484aa6ea6e483ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],22,0x5cb0a9dcbd41fbd4ULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],23,0x76f988da831153b5ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],24,0x983e5152ee66dfabULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],25,0xa831c66d2db43210ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],26,0xb00327c898fb213fULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],27,0xbf597fc7beef0ee4ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],28,0xc6e00bf33da88fc2ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],29,0xd5a79147930aa725ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],30,0x06ca6351e003826fULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],31,0x142929670a0e6e70ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],32,0x27b70a8546d22ffcULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],33,0x2e1b21385c26c926ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],34,0x4d2c6dfc5ac42aedULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],35,0x53380d139d95b3dfULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],36,0x650a73548baf63deULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],37,0x766a0abb3c77b2a8ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],38,0x81c2c92e47edaee6ULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],39,0x92722c851482353bULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],40,0xa2bfe8a14cf10364ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],41,0xa81a664bbc423001ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],42,0xc24b8b70d0f89791ULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],43,0xc76c51a30654be30ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],44,0xd192e819d6ef5218ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],45,0xd69906245565a910ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],46,0xf40e35855771202aULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],47,0x106aa07032bbd1b8ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],48,0x19a4c116b8d2d0c8ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],49,0x1e376c085141ab53ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],50,0x2748774cdf8eeb99ULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],51,0x34b0bcb5e19b48a8ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],52,0x391c0cb3c5c95a63ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],53,0x4ed8aa4ae3418acbULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],54,0x5b9cca4f7763e373ULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],55,0x682e6ff3d6b2b8a3ULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],56,0x748f82ee5defb2fcULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],57,0x78a5636f43172f60ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],58,0x84c87814a1f0ab72ULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],59,0x8cc702081a6439ecULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],60,0x90befffa23631e28ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],61,0xa4506cebde82bde9ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],62,0xbef9a3f7b2c67915ULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],63,0xc67178f2e372532bULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],64,0xca273eceea26619cULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],65,0xd186b8c721c0c207ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],66,0xeada7dd6cde0eb1eULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],67,0xf57d4f7fee6ed178ULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],68,0x06f067aa72176fbaULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],69,0x0a637dc5a2c898a6ULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],70,0x113f9804bef90daeULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],71,0x1b710b35131c471bULL); - RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],72,0x28db77f523047d84ULL); - RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],73,0x32caab7b40c72493ULL); - RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],74,0x3c9ebe0a15c9bebcULL); - RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],75,0x431d67c49c100d4cULL); - RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],76,0x4cc5d4becb3e42b6ULL); - RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],77,0x597f299cfc657e2aULL); - RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],78,0x5fcb6fab3ad6faecULL); - RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],79,0x6c44198c4a475817ULL); - -#undef RND - - /* feedback */ - for (i = 0; i < 8; i++) { - sha_info->digest[i] = sha_info->digest[i] + S[i]; - } - -} - - - -/* initialize the SHA digest */ - -static void -sha512_init(SHAobject *sha_info) -{ - sha_info->digest[0] = Py_ULL(0x6a09e667f3bcc908); - sha_info->digest[1] = Py_ULL(0xbb67ae8584caa73b); - sha_info->digest[2] = Py_ULL(0x3c6ef372fe94f82b); - sha_info->digest[3] = Py_ULL(0xa54ff53a5f1d36f1); - sha_info->digest[4] = Py_ULL(0x510e527fade682d1); - sha_info->digest[5] = Py_ULL(0x9b05688c2b3e6c1f); - sha_info->digest[6] = Py_ULL(0x1f83d9abfb41bd6b); - sha_info->digest[7] = Py_ULL(0x5be0cd19137e2179); - sha_info->count_lo = 0L; - sha_info->count_hi = 0L; - sha_info->local = 0; - sha_info->digestsize = 64; -} - -static void -sha384_init(SHAobject *sha_info) -{ - sha_info->digest[0] = Py_ULL(0xcbbb9d5dc1059ed8); - sha_info->digest[1] = Py_ULL(0x629a292a367cd507); - sha_info->digest[2] = Py_ULL(0x9159015a3070dd17); - sha_info->digest[3] = Py_ULL(0x152fecd8f70e5939); - sha_info->digest[4] = Py_ULL(0x67332667ffc00b31); - sha_info->digest[5] = Py_ULL(0x8eb44a8768581511); - sha_info->digest[6] = Py_ULL(0xdb0c2e0d64f98fa7); - sha_info->digest[7] = Py_ULL(0x47b5481dbefa4fa4); - sha_info->count_lo = 0L; - sha_info->count_hi = 0L; - sha_info->local = 0; - sha_info->digestsize = 48; -} - - -/* update the SHA digest */ - -static void -sha512_update(SHAobject *sha_info, SHA_BYTE *buffer, Py_ssize_t count) -{ - Py_ssize_t i; - SHA_INT32 clo; - - clo = sha_info->count_lo + ((SHA_INT32) count << 3); - if (clo < sha_info->count_lo) { - ++sha_info->count_hi; - } - sha_info->count_lo = clo; - sha_info->count_hi += (SHA_INT32) count >> 29; - if (sha_info->local) { - i = SHA_BLOCKSIZE - sha_info->local; - if (i > count) { - i = count; - } - memcpy(((SHA_BYTE *) sha_info->data) + sha_info->local, buffer, i); - count -= i; - buffer += i; - sha_info->local += (int)i; - if (sha_info->local == SHA_BLOCKSIZE) { - sha512_transform(sha_info); - } - else { - return; - } - } - while (count >= SHA_BLOCKSIZE) { - memcpy(sha_info->data, buffer, SHA_BLOCKSIZE); - buffer += SHA_BLOCKSIZE; - count -= SHA_BLOCKSIZE; - sha512_transform(sha_info); - } - memcpy(sha_info->data, buffer, count); - sha_info->local = (int)count; + dest->state = Hacl_Streaming_SHA2_copy_512(src->state); } -/* finish computing the SHA digest */ - -static void -sha512_final(unsigned char digest[SHA_DIGESTSIZE], SHAobject *sha_info) -{ - int count; - SHA_INT32 lo_bit_count, hi_bit_count; - - lo_bit_count = sha_info->count_lo; - hi_bit_count = sha_info->count_hi; - count = (int) ((lo_bit_count >> 3) & 0x7f); - ((SHA_BYTE *) sha_info->data)[count++] = 0x80; - if (count > SHA_BLOCKSIZE - 16) { - memset(((SHA_BYTE *) sha_info->data) + count, 0, - SHA_BLOCKSIZE - count); - sha512_transform(sha_info); - memset((SHA_BYTE *) sha_info->data, 0, SHA_BLOCKSIZE - 16); - } - else { - memset(((SHA_BYTE *) sha_info->data) + count, 0, - SHA_BLOCKSIZE - 16 - count); - } - - /* GJS: note that we add the hi/lo in big-endian. sha512_transform will - swap these values into host-order. */ - sha_info->data[112] = 0; - sha_info->data[113] = 0; - sha_info->data[114] = 0; - sha_info->data[115] = 0; - sha_info->data[116] = 0; - sha_info->data[117] = 0; - sha_info->data[118] = 0; - sha_info->data[119] = 0; - sha_info->data[120] = (hi_bit_count >> 24) & 0xff; - sha_info->data[121] = (hi_bit_count >> 16) & 0xff; - sha_info->data[122] = (hi_bit_count >> 8) & 0xff; - sha_info->data[123] = (hi_bit_count >> 0) & 0xff; - sha_info->data[124] = (lo_bit_count >> 24) & 0xff; - sha_info->data[125] = (lo_bit_count >> 16) & 0xff; - sha_info->data[126] = (lo_bit_count >> 8) & 0xff; - sha_info->data[127] = (lo_bit_count >> 0) & 0xff; - sha512_transform(sha_info); - digest[ 0] = (unsigned char) ((sha_info->digest[0] >> 56) & 0xff); - digest[ 1] = (unsigned char) ((sha_info->digest[0] >> 48) & 0xff); - digest[ 2] = (unsigned char) ((sha_info->digest[0] >> 40) & 0xff); - digest[ 3] = (unsigned char) ((sha_info->digest[0] >> 32) & 0xff); - digest[ 4] = (unsigned char) ((sha_info->digest[0] >> 24) & 0xff); - digest[ 5] = (unsigned char) ((sha_info->digest[0] >> 16) & 0xff); - digest[ 6] = (unsigned char) ((sha_info->digest[0] >> 8) & 0xff); - digest[ 7] = (unsigned char) ((sha_info->digest[0] ) & 0xff); - digest[ 8] = (unsigned char) ((sha_info->digest[1] >> 56) & 0xff); - digest[ 9] = (unsigned char) ((sha_info->digest[1] >> 48) & 0xff); - digest[10] = (unsigned char) ((sha_info->digest[1] >> 40) & 0xff); - digest[11] = (unsigned char) ((sha_info->digest[1] >> 32) & 0xff); - digest[12] = (unsigned char) ((sha_info->digest[1] >> 24) & 0xff); - digest[13] = (unsigned char) ((sha_info->digest[1] >> 16) & 0xff); - digest[14] = (unsigned char) ((sha_info->digest[1] >> 8) & 0xff); - digest[15] = (unsigned char) ((sha_info->digest[1] ) & 0xff); - digest[16] = (unsigned char) ((sha_info->digest[2] >> 56) & 0xff); - digest[17] = (unsigned char) ((sha_info->digest[2] >> 48) & 0xff); - digest[18] = (unsigned char) ((sha_info->digest[2] >> 40) & 0xff); - digest[19] = (unsigned char) ((sha_info->digest[2] >> 32) & 0xff); - digest[20] = (unsigned char) ((sha_info->digest[2] >> 24) & 0xff); - digest[21] = (unsigned char) ((sha_info->digest[2] >> 16) & 0xff); - digest[22] = (unsigned char) ((sha_info->digest[2] >> 8) & 0xff); - digest[23] = (unsigned char) ((sha_info->digest[2] ) & 0xff); - digest[24] = (unsigned char) ((sha_info->digest[3] >> 56) & 0xff); - digest[25] = (unsigned char) ((sha_info->digest[3] >> 48) & 0xff); - digest[26] = (unsigned char) ((sha_info->digest[3] >> 40) & 0xff); - digest[27] = (unsigned char) ((sha_info->digest[3] >> 32) & 0xff); - digest[28] = (unsigned char) ((sha_info->digest[3] >> 24) & 0xff); - digest[29] = (unsigned char) ((sha_info->digest[3] >> 16) & 0xff); - digest[30] = (unsigned char) ((sha_info->digest[3] >> 8) & 0xff); - digest[31] = (unsigned char) ((sha_info->digest[3] ) & 0xff); - digest[32] = (unsigned char) ((sha_info->digest[4] >> 56) & 0xff); - digest[33] = (unsigned char) ((sha_info->digest[4] >> 48) & 0xff); - digest[34] = (unsigned char) ((sha_info->digest[4] >> 40) & 0xff); - digest[35] = (unsigned char) ((sha_info->digest[4] >> 32) & 0xff); - digest[36] = (unsigned char) ((sha_info->digest[4] >> 24) & 0xff); - digest[37] = (unsigned char) ((sha_info->digest[4] >> 16) & 0xff); - digest[38] = (unsigned char) ((sha_info->digest[4] >> 8) & 0xff); - digest[39] = (unsigned char) ((sha_info->digest[4] ) & 0xff); - digest[40] = (unsigned char) ((sha_info->digest[5] >> 56) & 0xff); - digest[41] = (unsigned char) ((sha_info->digest[5] >> 48) & 0xff); - digest[42] = (unsigned char) ((sha_info->digest[5] >> 40) & 0xff); - digest[43] = (unsigned char) ((sha_info->digest[5] >> 32) & 0xff); - digest[44] = (unsigned char) ((sha_info->digest[5] >> 24) & 0xff); - digest[45] = (unsigned char) ((sha_info->digest[5] >> 16) & 0xff); - digest[46] = (unsigned char) ((sha_info->digest[5] >> 8) & 0xff); - digest[47] = (unsigned char) ((sha_info->digest[5] ) & 0xff); - digest[48] = (unsigned char) ((sha_info->digest[6] >> 56) & 0xff); - digest[49] = (unsigned char) ((sha_info->digest[6] >> 48) & 0xff); - digest[50] = (unsigned char) ((sha_info->digest[6] >> 40) & 0xff); - digest[51] = (unsigned char) ((sha_info->digest[6] >> 32) & 0xff); - digest[52] = (unsigned char) ((sha_info->digest[6] >> 24) & 0xff); - digest[53] = (unsigned char) ((sha_info->digest[6] >> 16) & 0xff); - digest[54] = (unsigned char) ((sha_info->digest[6] >> 8) & 0xff); - digest[55] = (unsigned char) ((sha_info->digest[6] ) & 0xff); - digest[56] = (unsigned char) ((sha_info->digest[7] >> 56) & 0xff); - digest[57] = (unsigned char) ((sha_info->digest[7] >> 48) & 0xff); - digest[58] = (unsigned char) ((sha_info->digest[7] >> 40) & 0xff); - digest[59] = (unsigned char) ((sha_info->digest[7] >> 32) & 0xff); - digest[60] = (unsigned char) ((sha_info->digest[7] >> 24) & 0xff); - digest[61] = (unsigned char) ((sha_info->digest[7] >> 16) & 0xff); - digest[62] = (unsigned char) ((sha_info->digest[7] >> 8) & 0xff); - digest[63] = (unsigned char) ((sha_info->digest[7] ) & 0xff); -} - -/* - * End of copied SHA code. - * - * ------------------------------------------------------------------------ - */ - typedef struct { PyTypeObject* sha384_type; PyTypeObject* sha512_type; @@ -463,14 +96,31 @@ SHA_traverse(PyObject *ptr, visitproc visit, void *arg) } static void -SHA512_dealloc(PyObject *ptr) +SHA512_dealloc(SHAobject *ptr) { + Hacl_Streaming_SHA2_free_512(ptr->state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); Py_DECREF(tp); } +/* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be + * 64 bits. */ +static void update_512(Hacl_Streaming_SHA2_state_sha2_512 *state, uint8_t *buf, Py_ssize_t len) { + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to overflow the maximum admissible length for this API + * (namely, 2^64-1 bytes). */ + while (len > UINT32_MAX) { + Hacl_Streaming_SHA2_update_512(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } + /* Cast to uint32_t is safe: upon exiting the loop, len <= UINT32_MAX, and + * therefore fits in a uint32_t */ + Hacl_Streaming_SHA2_update_512(state, buf, (uint32_t) len); +} + /* External methods for a hash object */ @@ -514,11 +164,10 @@ static PyObject * SHA512Type_digest_impl(SHAobject *self) /*[clinic end generated code: output=1080bbeeef7dde1b input=f6470dd359071f4b]*/ { - unsigned char digest[SHA_DIGESTSIZE]; - SHAobject temp; - - SHAcopy(self, &temp); - sha512_final(digest, &temp); + uint8_t digest[SHA_DIGESTSIZE]; + // HACL performs copies under the hood so that self->state remains valid + // after this call. + Hacl_Streaming_SHA2_finish_512(self->state, digest); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -532,13 +181,8 @@ static PyObject * SHA512Type_hexdigest_impl(SHAobject *self) /*[clinic end generated code: output=7373305b8601e18b input=498b877b25cbe0a2]*/ { - unsigned char digest[SHA_DIGESTSIZE]; - SHAobject temp; - - /* Get the raw (binary) digest value */ - SHAcopy(self, &temp); - sha512_final(digest, &temp); - + uint8_t digest[SHA_DIGESTSIZE]; + Hacl_Streaming_SHA2_finish_512(self->state, digest); return _Py_strhex((const char *)digest, self->digestsize); } @@ -559,7 +203,7 @@ SHA512Type_update(SHAobject *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - sha512_update(self, buf.buf, buf.len); + update_512(self->state, buf.buf, buf.len); PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -622,15 +266,6 @@ static PyType_Spec sha512_sha384_type_spec = { .slots = sha512_sha384_type_slots }; -static PyType_Slot sha512_sha512_type_slots[] = { - {Py_tp_dealloc, SHA512_dealloc}, - {Py_tp_methods, SHA_methods}, - {Py_tp_members, SHA_members}, - {Py_tp_getset, SHA_getseters}, - {Py_tp_traverse, SHA_traverse}, - {0,0} -}; - // Using PyType_GetModuleState() on this type is safe since // it cannot be subclassed: it does not have the Py_TPFLAGS_BASETYPE flag. static PyType_Spec sha512_sha512_type_spec = { @@ -638,7 +273,7 @@ static PyType_Spec sha512_sha512_type_spec = { .basicsize = sizeof(SHAobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), - .slots = sha512_sha512_type_slots + .slots = sha512_sha384_type_slots }; /* The single module-level function: new() */ @@ -671,7 +306,8 @@ _sha512_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - sha512_init(new); + new->state = Hacl_Streaming_SHA2_create_in_512(); + new->digestsize = 64; if (PyErr_Occurred()) { Py_DECREF(new); @@ -680,7 +316,7 @@ _sha512_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - sha512_update(new, buf.buf, buf.len); + update_512(new->state, buf.buf, buf.len); PyBuffer_Release(&buf); } @@ -715,7 +351,8 @@ _sha512_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - sha384_init(new); + new->state = Hacl_Streaming_SHA2_create_in_384(); + new->digestsize = 48; if (PyErr_Occurred()) { Py_DECREF(new); @@ -724,7 +361,7 @@ _sha512_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - sha512_update(new, buf.buf, buf.len); + update_512(new->state, buf.buf, buf.len); PyBuffer_Release(&buf); } diff --git a/configure b/configure index 35088f9e5cafd6..c00a1e1d2ec986 100755 --- a/configure +++ b/configure @@ -26950,7 +26950,7 @@ fi as_fn_append MODULE_BLOCK "MODULE__SHA512_STATE=$py_cv_module__sha512$as_nl" if test "x$py_cv_module__sha512" = xyes; then : - + as_fn_append MODULE_BLOCK "MODULE__SHA512_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" fi diff --git a/configure.ac b/configure.ac index 1ab48e0d1c160a..92a05c011026f2 100644 --- a/configure.ac +++ b/configure.ac @@ -7200,7 +7200,9 @@ PY_STDLIB_MOD([_sha1], [test "$with_builtin_sha1" = yes]) PY_STDLIB_MOD([_sha256], [test "$with_builtin_sha256" = yes], [], [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) -PY_STDLIB_MOD([_sha512], [test "$with_builtin_sha512" = yes]) +PY_STDLIB_MOD([_sha512], + [test "$with_builtin_sha512" = yes], [], + [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) PY_STDLIB_MOD([_sha3], [test "$with_builtin_sha3" = yes]) PY_STDLIB_MOD([_blake2], [test "$with_builtin_blake2" = yes], [], From 3690688149dca11589af59b7704541336613199a Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Tue, 14 Feb 2023 10:20:11 +0000 Subject: [PATCH 37/83] GH-101898: Fix missing term references for hashable definition (#101899) Fix missing term references for hashable definition --- Doc/c-api/dict.rst | 2 +- Doc/c-api/object.rst | 2 +- Doc/faq/programming.rst | 2 +- Doc/library/abc.rst | 2 +- Doc/library/collections.abc.rst | 4 ++-- Doc/library/collections.rst | 4 ++-- Doc/library/datetime.rst | 2 +- Doc/library/fractions.rst | 2 +- Doc/library/functools.rst | 2 +- Doc/library/graphlib.rst | 4 ++-- Doc/library/inspect.rst | 4 ++-- Doc/library/operator.rst | 2 +- Doc/library/pathlib.rst | 2 +- Doc/library/stdtypes.rst | 6 +++--- Doc/library/typing.rst | 2 +- Doc/reference/datamodel.rst | 2 +- 16 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index e5f28b59a701e0..34106ee6b1f30d 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -80,7 +80,7 @@ Dictionary Objects .. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key) - Remove the entry in dictionary *p* with key *key*. *key* must be hashable; + Remove the entry in dictionary *p* with key *key*. *key* must be :term:`hashable`; if it isn't, :exc:`TypeError` is raised. If *key* is not in the dictionary, :exc:`KeyError` is raised. Return ``0`` on success or ``-1`` on failure. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 5a25a2b6c9d3db..84c72e7e108b64 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -281,7 +281,7 @@ Object Protocol .. c:function:: Py_hash_t PyObject_HashNotImplemented(PyObject *o) - Set a :exc:`TypeError` indicating that ``type(o)`` is not hashable and return ``-1``. + Set a :exc:`TypeError` indicating that ``type(o)`` is not :term:`hashable` and return ``-1``. This function receives special treatment when stored in a ``tp_hash`` slot, allowing a type to explicitly indicate to the interpreter that it is not hashable. diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index ba42289f3466c2..38f9b171618b26 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1979,7 +1979,7 @@ method result will be released right away. The disadvantage is that if instances accumulate, so too will the accumulated method results. They can grow without bound. -The *lru_cache* approach works with methods that have hashable +The *lru_cache* approach works with methods that have :term:`hashable` arguments. It creates a reference to the instance unless special efforts are made to pass in weak references. diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 3b74622e7ff46c..274b2d69f0ab5c 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -21,7 +21,7 @@ The :mod:`collections` module has some concrete classes that derive from ABCs; these can, of course, be further derived. In addition, the :mod:`collections.abc` submodule has some ABCs that can be used to test whether a class or instance provides a particular interface, for example, if it is -hashable or if it is a mapping. +:term:`hashable` or if it is a mapping. This module provides the metaclass :class:`ABCMeta` for defining ABCs and diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 132b0ce7192ac1..1ada0d352a0cc6 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -22,7 +22,7 @@ This module provides :term:`abstract base classes ` that can be used to test whether a class provides a particular interface; for -example, whether it is hashable or whether it is a mapping. +example, whether it is :term:`hashable` or whether it is a mapping. An :func:`issubclass` or :func:`isinstance` test for an interface works in one of three ways. @@ -406,7 +406,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: (3) The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value for the set; however, :meth:`__hash__` is not defined because not all sets - are hashable or immutable. To add set hashability using mixins, + are :term:`hashable` or immutable. To add set hashability using mixins, inherit from both :meth:`Set` and :meth:`Hashable`, then define ``__hash__ = Set._hash``. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 2cffc2300a2298..bb46782c06e1c8 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -25,7 +25,7 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`, :func:`namedtuple` factory function for creating tuple subclasses with named fields :class:`deque` list-like container with fast appends and pops on either end :class:`ChainMap` dict-like class for creating a single view of multiple mappings -:class:`Counter` dict subclass for counting hashable objects +:class:`Counter` dict subclass for counting :term:`hashable` objects :class:`OrderedDict` dict subclass that remembers the order entries were added :class:`defaultdict` dict subclass that calls a factory function to supply missing values :class:`UserDict` wrapper around dictionary objects for easier dict subclassing @@ -242,7 +242,7 @@ For example:: .. class:: Counter([iterable-or-mapping]) - A :class:`Counter` is a :class:`dict` subclass for counting hashable objects. + A :class:`Counter` is a :class:`dict` subclass for counting :term:`hashable` objects. It is a collection where elements are stored as dictionary keys and their counts are stored as dictionary values. Counts are allowed to be any integer value including zero or negative counts. The :class:`Counter` diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 2f1ab7c3dd4b51..50827b27ebea04 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -160,7 +160,7 @@ The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` typ share these common features: - Objects of these types are immutable. -- Objects of these types are hashable, meaning that they can be used as +- Objects of these types are :term:`hashable`, meaning that they can be used as dictionary keys. - Objects of these types support efficient pickling via the :mod:`pickle` module. diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 06b0e038c89bd0..c61bbac892e0e4 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -77,7 +77,7 @@ another rational number, or from a string. The :class:`Fraction` class inherits from the abstract base class :class:`numbers.Rational`, and implements all of the methods and - operations from that class. :class:`Fraction` instances are hashable, + operations from that class. :class:`Fraction` instances are :term:`hashable`, and should be treated as immutable. In addition, :class:`Fraction` has the following properties and methods: diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 2f0a9bd8be8815..80a405e87d8d56 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -147,7 +147,7 @@ The :mod:`functools` module defines the following functions: threads. Since a dictionary is used to cache results, the positional and keyword - arguments to the function must be hashable. + arguments to the function must be :term:`hashable`. Distinct argument patterns may be considered to be distinct calls with separate cache entries. For example, ``f(a=1, b=2)`` and ``f(b=2, a=1)`` diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 2bc80da4ead2a2..fe7932e7a61cb5 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -17,7 +17,7 @@ .. class:: TopologicalSorter(graph=None) - Provides functionality to topologically sort a graph of hashable nodes. + Provides functionality to topologically sort a graph of :term:`hashable` nodes. A topological order is a linear ordering of the vertices in a graph such that for every directed edge u -> v from vertex u to vertex v, vertex u comes @@ -85,7 +85,7 @@ .. method:: add(node, *predecessors) Add a new node and its predecessors to the graph. Both the *node* and all - elements in *predecessors* must be hashable. + elements in *predecessors* must be :term:`hashable`. If called multiple times with the same node argument, the set of dependencies will be the union of all dependencies passed in. diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 58b84a35a890e3..9c3be5a250a67e 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -689,7 +689,7 @@ function. modified copy. .. versionchanged:: 3.5 - Signature objects are picklable and hashable. + Signature objects are picklable and :term:`hashable`. .. attribute:: Signature.empty @@ -768,7 +768,7 @@ function. you can use :meth:`Parameter.replace` to create a modified copy. .. versionchanged:: 3.5 - Parameter objects are picklable and hashable. + Parameter objects are picklable and :term:`hashable`. .. attribute:: Parameter.empty diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 35e9b49ea8bcca..6d90473f2367b5 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -327,7 +327,7 @@ expect a function argument. return g The items can be any type accepted by the operand's :meth:`__getitem__` - method. Dictionaries accept any hashable value. Lists, tuples, and + method. Dictionaries accept any :term:`hashable` value. Lists, tuples, and strings accept an index or a slice: >>> itemgetter(1)('ABCDEFG') diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index f222745a2c56bc..c8a734ecad8e7b 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -186,7 +186,7 @@ these classes, since they don't provide any operation that does system calls. General properties ^^^^^^^^^^^^^^^^^^ -Paths are immutable and hashable. Paths of a same flavour are comparable +Paths are immutable and :term:`hashable`. Paths of a same flavour are comparable and orderable. These properties respect the flavour's case-folding semantics:: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 0ef03035a572e5..98bda6ded30f79 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -3775,7 +3775,7 @@ copying. >>> data bytearray(b'z1spam') - One-dimensional memoryviews of hashable (read-only) types with formats + One-dimensional memoryviews of :term:`hashable` (read-only) types with formats 'B', 'b' or 'c' are also hashable. The hash is defined as ``hash(m) == hash(m.tobytes())``:: @@ -3789,7 +3789,7 @@ copying. .. versionchanged:: 3.3 One-dimensional memoryviews can now be sliced. - One-dimensional memoryviews with formats 'B', 'b' or 'c' are now hashable. + One-dimensional memoryviews with formats 'B', 'b' or 'c' are now :term:`hashable`. .. versionchanged:: 3.4 memoryview is now registered automatically with @@ -4710,7 +4710,7 @@ support membership tests: .. versionadded:: 3.10 -Keys views are set-like since their entries are unique and hashable. If all +Keys views are set-like since their entries are unique and :term:`hashable`. If all values are hashable, so that ``(key, value)`` pairs are unique and hashable, then the items view is also set-like. (Values views are not treated as set-like since the entries are generally not unique.) For set-like views, all of the diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 356f919a1897b2..169f7196a74ec6 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -439,7 +439,7 @@ are intended primarily for static type checking. A user-defined generic class can have ABCs as base classes without a metaclass conflict. Generic metaclasses are not supported. The outcome of parameterizing -generics is cached, and most types in the typing module are hashable and +generics is cached, and most types in the typing module are :term:`hashable` and comparable for equality. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 1d2ddf3507aee1..e4e471e50079ae 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1525,7 +1525,7 @@ Basic customization :meth:`__hash__`, its instances will not be usable as items in hashable collections. If a class defines mutable objects and implements an :meth:`__eq__` method, it should not implement :meth:`__hash__`, since the - implementation of hashable collections requires that a key's hash value is + implementation of :term:`hashable` collections requires that a key's hash value is immutable (if the object's hash value changes, it will be in the wrong hash bucket). From 81e3aa835c32363f4547b6566edf1125386f1f6d Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 14 Feb 2023 11:54:13 +0000 Subject: [PATCH 38/83] gh-101799: implement PREP_RERAISE_STAR as an intrinsic function (#101800) --- Doc/library/dis.rst | 26 +++++++++------- Include/internal/pycore_intrinsics.h | 13 ++++++++ Include/internal/pycore_opcode.h | 18 +++++------ Include/opcode.h | 20 ++++++------- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 2 +- ...-02-13-18-21-14.gh-issue-101799.wpHbCn.rst | 2 ++ Python/bytecodes.c | 20 ++++++------- Python/compile.c | 4 +-- Python/generated_cases.c.h | 30 +++++++++---------- Python/intrinsics.c | 18 +++++++++++ Python/opcode_metadata.h | 10 +++---- Python/opcode_targets.h | 14 ++++----- 13 files changed, 107 insertions(+), 73 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index b1e61d7e77b2f5..a5bc5e7fb6ea71 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -768,16 +768,6 @@ iterations of the loop. .. versionadded:: 3.11 -.. opcode:: PREP_RERAISE_STAR - - Combines the raised and reraised exceptions list from ``STACK[-1]``, into an - exception group to propagate from a try-except* block. Uses the original exception - group from ``STACK[-2]`` to reconstruct the structure of reraised exceptions. Pops - two items from the stack and pushes the exception to reraise or ``None`` - if there isn't one. - - .. versionadded:: 3.11 - .. opcode:: WITH_EXCEPT_START Calls the function in position 4 on the stack with arguments (type, val, tb) @@ -1515,7 +1505,8 @@ iterations of the loop. .. opcode:: CALL_INTRINSIC_1 Calls an intrinsic function with one argument. Passes ``STACK[-1]`` as the - argument and sets ``STACK[-1]`` to the result. Used to implement functionality that is necessary but not performance critical. + argument and sets ``STACK[-1]`` to the result. Used to implement + functionality that is necessary but not performance critical. The operand determines which intrinsic function is called: @@ -1529,6 +1520,19 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: CALL_INTRINSIC_2 + + Calls an intrinsic function with two arguments. Passes ``STACK[-2]``, ``STACK[-1]`` as the + arguments and sets ``STACK[-1]`` to the result. Used to implement functionality that is + necessary but not performance critical. + + The operand determines which intrinsic function is called: + + * ``0`` Not valid + * ``1`` Calculates the :exc:`ExceptionGroup` to raise from a ``try-except*``. + + .. versionadded:: 3.12 + **Pseudo-instructions** diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 1da618f2b4a548..deac145fff7627 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -1,4 +1,6 @@ +/* Unary Functions: */ + #define INTRINSIC_PRINT 1 #define INTRINSIC_IMPORT_STAR 2 #define INTRINSIC_STOPITERATION_ERROR 3 @@ -8,6 +10,17 @@ #define MAX_INTRINSIC_1 6 + +/* Binary Functions: */ + +#define INTRINSIC_PREP_RERAISE_STAR 1 + +#define MAX_INTRINSIC_2 1 + + typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value); +typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); extern instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; +extern instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; + diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 5e65adee9e00a5..f9ab95ca4bb9d3 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -87,6 +87,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_NO_KW_BUILTIN_FAST] = CALL, [CALL_NO_KW_BUILTIN_O] = CALL, @@ -187,7 +188,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, [POP_TOP] = POP_TOP, - [PREP_RERAISE_STAR] = PREP_RERAISE_STAR, [PUSH_EXC_INFO] = PUSH_EXC_INFO, [PUSH_NULL] = PUSH_NULL, [RAISE_VARARGS] = RAISE_VARARGS, @@ -319,7 +319,7 @@ static const char *const _PyOpcode_OpName[263] = { [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", [DELETE_NAME] = "DELETE_NAME", @@ -344,7 +344,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -374,7 +374,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_BACKWARD] = "JUMP_BACKWARD", [COMPARE_AND_BRANCH] = "COMPARE_AND_BRANCH", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -384,20 +384,20 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [SEND_GEN] = "SEND_GEN", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", - [SEND_GEN] = "SEND_GEN", + [166] = "<166>", [167] = "<167>", [168] = "<168>", [169] = "<169>", @@ -405,7 +405,7 @@ static const char *const _PyOpcode_OpName[263] = { [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", - [174] = "<174>", + [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", [175] = "<175>", [176] = "<176>", [177] = "<177>", @@ -498,11 +498,11 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ + case 166: \ case 167: \ case 168: \ case 169: \ case 170: \ - case 174: \ case 175: \ case 176: \ case 177: \ diff --git a/Include/opcode.h b/Include/opcode.h index d643741c3c3aa0..760ff945f31f9e 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -43,7 +43,6 @@ extern "C" { #define RETURN_GENERATOR 75 #define RETURN_VALUE 83 #define SETUP_ANNOTATIONS 85 -#define PREP_RERAISE_STAR 88 #define POP_EXCEPT 89 #define HAVE_ARGUMENT 90 #define STORE_NAME 90 @@ -117,6 +116,7 @@ extern "C" { #define CALL 171 #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 +#define CALL_INTRINSIC_2 174 #define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 @@ -179,15 +179,15 @@ extern "C" { #define LOAD_GLOBAL_MODULE 84 #define STORE_ATTR_INSTANCE_VALUE 86 #define STORE_ATTR_SLOT 87 -#define STORE_ATTR_WITH_HINT 113 -#define STORE_FAST__LOAD_FAST 143 -#define STORE_FAST__STORE_FAST 153 -#define STORE_SUBSCR_DICT 154 -#define STORE_SUBSCR_LIST_INT 158 -#define UNPACK_SEQUENCE_LIST 159 -#define UNPACK_SEQUENCE_TUPLE 160 -#define UNPACK_SEQUENCE_TWO_TUPLE 161 -#define SEND_GEN 166 +#define STORE_ATTR_WITH_HINT 88 +#define STORE_FAST__LOAD_FAST 113 +#define STORE_FAST__STORE_FAST 143 +#define STORE_SUBSCR_DICT 153 +#define STORE_SUBSCR_LIST_INT 154 +#define UNPACK_SEQUENCE_LIST 158 +#define UNPACK_SEQUENCE_TUPLE 159 +#define UNPACK_SEQUENCE_TWO_TUPLE 160 +#define SEND_GEN 161 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 38d4a384c2cc95..954401cfa85ed3 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -433,6 +433,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) # Python 3.12a5 3518 (Add RETURN_CONST instruction) # Python 3.12a5 3519 (Modify SEND instruction) +# Python 3.12a5 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) # Python 3.13 will start with 3550 @@ -445,7 +446,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3519).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3520).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index b69cd1bbdd61ca..809d24e51676bd 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -127,7 +127,6 @@ def pseudo_op(name, op, real_ops): def_op('SETUP_ANNOTATIONS', 85) -def_op('PREP_RERAISE_STAR', 88) def_op('POP_EXCEPT', 89) HAVE_ARGUMENT = 90 # real opcodes from here have an argument: @@ -224,6 +223,7 @@ def pseudo_op(name, op, real_ops): def_op('KW_NAMES', 172) hasconst.append(172) def_op('CALL_INTRINSIC_1', 173) +def_op('CALL_INTRINSIC_2', 174) hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst new file mode 100644 index 00000000000000..3233a573be7acd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst @@ -0,0 +1,2 @@ +Add :opcode:`CALL_INTRINSIC_2` and use it instead of +:opcode:`PREP_RERAISE_STAR`. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 429cd7fdafa168..be54e5f6f589eb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -501,7 +501,14 @@ dummy_func( inst(CALL_INTRINSIC_1, (value -- res)) { assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - Py_DECREF(value); + DECREF_INPUTS(); + ERROR_IF(res == NULL, error); + } + + inst(CALL_INTRINSIC_2, (value2, value1 -- res)) { + assert(oparg <= MAX_INTRINSIC_2); + res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -788,15 +795,6 @@ dummy_func( goto exception_unwind; } - inst(PREP_RERAISE_STAR, (orig, excs -- val)) { - assert(PyList_Check(excs)); - - val = _PyExc_PrepReraiseStar(orig, excs); - DECREF_INPUTS(); - - ERROR_IF(val == NULL, error); - } - inst(END_ASYNC_FOR, (awaitable, exc -- )) { assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { @@ -2383,7 +2381,7 @@ dummy_func( } // Cache layout: counter/1, func_version/2, min_args/1 - // Neither CALL_INTRINSIC_1 nor CALL_FUNCTION_EX are members! + // Neither CALL_INTRINSIC_1/2 nor CALL_FUNCTION_EX are members! family(call, INLINE_CACHE_ENTRIES_CALL) = { CALL, CALL_BOUND_METHOD_EXACT_ARGS, diff --git a/Python/compile.c b/Python/compile.c index b49eda314eeef1..0534b536e3d12e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3431,7 +3431,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) [orig, res, rest] Ln+1: LIST_APPEND 1 ) add unhandled exc to res (could be None) - [orig, res] PREP_RERAISE_STAR + [orig, res] CALL_INTRINSIC_2 PREP_RERAISE_STAR [exc] COPY 1 [exc, exc] POP_JUMP_IF_NOT_NONE RER [exc] POP_TOP @@ -3580,7 +3580,7 @@ compiler_try_star_except(struct compiler *c, stmt_ty s) NEW_JUMP_TARGET_LABEL(c, reraise); USE_LABEL(c, reraise_star); - ADDOP(c, NO_LOCATION, PREP_RERAISE_STAR); + ADDOP_I(c, NO_LOCATION, CALL_INTRINSIC_2, INTRINSIC_PREP_RERAISE_STAR); ADDOP_I(c, NO_LOCATION, COPY, 1); ADDOP_JUMP(c, NO_LOCATION, POP_JUMP_IF_NOT_NONE, reraise); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 093ebff026b509..beb797cbd233d7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -689,6 +689,20 @@ DISPATCH(); } + TARGET(CALL_INTRINSIC_2) { + PyObject *value1 = PEEK(1); + PyObject *value2 = PEEK(2); + PyObject *res; + assert(oparg <= MAX_INTRINSIC_2); + res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); + Py_DECREF(value2); + Py_DECREF(value1); + if (res == NULL) goto pop_2_error; + STACK_SHRINK(1); + POKE(1, res); + DISPATCH(); + } + TARGET(RAISE_VARARGS) { PyObject **args = &PEEK(oparg); PyObject *cause = NULL, *exc = NULL; @@ -999,22 +1013,6 @@ goto exception_unwind; } - TARGET(PREP_RERAISE_STAR) { - PyObject *excs = PEEK(1); - PyObject *orig = PEEK(2); - PyObject *val; - assert(PyList_Check(excs)); - - val = _PyExc_PrepReraiseStar(orig, excs); - Py_DECREF(orig); - Py_DECREF(excs); - - if (val == NULL) goto pop_2_error; - STACK_SHRINK(1); - POKE(1, val); - DISPATCH(); - } - TARGET(END_ASYNC_FOR) { PyObject *exc = PEEK(1); PyObject *awaitable = PEEK(2); diff --git a/Python/intrinsics.c b/Python/intrinsics.c index ae1775862d945d..9e90ef32130f1d 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -9,6 +9,7 @@ #include "pycore_pyerrors.h" +/******** Unary functions ********/ static PyObject * no_intrinsic(PyThreadState* tstate, PyObject *unused) @@ -208,3 +209,20 @@ _PyIntrinsics_UnaryFunctions[] = { [INTRINSIC_UNARY_POSITIVE] = unary_pos, [INTRINSIC_LIST_TO_TUPLE] = list_to_tuple, }; + + +/******** Binary functions ********/ + + +static PyObject * +prep_reraise_star(PyThreadState* unused, PyObject *orig, PyObject *excs) +{ + assert(PyList_Check(excs)); + return _PyExc_PrepReraiseStar(orig, excs); +} + +instrinsic_func2 +_PyIntrinsics_BinaryFunctions[] = { + [INTRINSIC_PREP_RERAISE_STAR] = prep_reraise_star, +}; + diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index d622eb12c8cb2d..f27906a3e349eb 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -88,6 +88,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case CALL_INTRINSIC_1: return 1; + case CALL_INTRINSIC_2: + return 2; case RAISE_VARARGS: return oparg; case INTERPRETER_EXIT: @@ -112,8 +114,6 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case RERAISE: return oparg + 1; - case PREP_RERAISE_STAR: - return 2; case END_ASYNC_FOR: return 2; case CLEANUP_THROW: @@ -440,6 +440,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case CALL_INTRINSIC_1: return 1; + case CALL_INTRINSIC_2: + return 1; case RAISE_VARARGS: return 0; case INTERPRETER_EXIT: @@ -464,8 +466,6 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case RERAISE: return oparg; - case PREP_RERAISE_STAR: - return 1; case END_ASYNC_FOR: return 0; case CLEANUP_THROW: @@ -760,6 +760,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [STORE_SUBSCR_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC }, [DELETE_SUBSCR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [CALL_INTRINSIC_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_INTRINSIC_2] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [RAISE_VARARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [INTERPRETER_EXIT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [RETURN_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, @@ -772,7 +773,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [YIELD_VALUE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [POP_EXCEPT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [RERAISE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [PREP_RERAISE_STAR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [END_ASYNC_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [CLEANUP_THROW] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [LOAD_ASSERTION_ERROR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 301ec6e005dad6..bc64bd582fd572 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -87,7 +87,7 @@ static void *opcode_targets[256] = { &&TARGET_SETUP_ANNOTATIONS, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, - &&TARGET_PREP_RERAISE_STAR, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -142,7 +142,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_BACKWARD, &&TARGET_COMPARE_AND_BRANCH, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,20 +152,20 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_SEND_GEN, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&TARGET_SEND_GEN, + &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -173,7 +173,7 @@ static void *opcode_targets[256] = { &&TARGET_CALL, &&TARGET_KW_NAMES, &&TARGET_CALL_INTRINSIC_1, - &&_unknown_opcode, + &&TARGET_CALL_INTRINSIC_2, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, From 096d0097a09e439a4564531b297a998e5d74c9b5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Feb 2023 14:26:03 -0700 Subject: [PATCH 39/83] gh-101758: Add a Test For Single-Phase Init Module Variants (gh-101891) The new test exercises the most important variants for single-phase init extension modules. We also add some explanation about those variants to import.c. https://github.com/python/cpython/issues/101758 --- Lib/test/test_imp.py | 199 +++++++++++++++++++ Modules/_testsinglephase.c | 394 ++++++++++++++++++++++++++++++++++--- Python/import.c | 105 +++++++++- 3 files changed, 660 insertions(+), 38 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 80abc720c3251a..31dce21587e2ca 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -251,6 +251,205 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) + @requires_load_dynamic + def test_singlephase_variants(self): + '''Exercise the most meaningful variants described in Python/import.c.''' + self.maxDiff = None + + basename = '_testsinglephase' + fileobj, pathname, _ = imp.find_module(basename) + fileobj.close() + + modules = {} + def load(name): + assert name not in modules + module = imp.load_dynamic(name, pathname) + self.assertNotIn(module, modules.values()) + modules[name] = module + return module + + def re_load(name, module): + assert sys.modules[name] is module + before = type(module)(module.__name__) + before.__dict__.update(vars(module)) + + reloaded = imp.load_dynamic(name, pathname) + + return before, reloaded + + def check_common(name, module): + summed = module.sum(1, 2) + lookedup = module.look_up_self() + initialized = module.initialized() + cached = sys.modules[name] + + # module.__name__ might not match, but the spec will. + self.assertEqual(module.__spec__.name, name) + if initialized is not None: + self.assertIsInstance(initialized, float) + self.assertGreater(initialized, 0) + self.assertEqual(summed, 3) + self.assertTrue(issubclass(module.error, Exception)) + self.assertEqual(module.int_const, 1969) + self.assertEqual(module.str_const, 'something different') + self.assertIs(cached, module) + + return lookedup, initialized, cached + + def check_direct(name, module, lookedup): + # The module has its own PyModuleDef, with a matching name. + self.assertEqual(module.__name__, name) + self.assertIs(lookedup, module) + + def check_indirect(name, module, lookedup, orig): + # The module re-uses another's PyModuleDef, with a different name. + assert orig is not module + assert orig.__name__ != name + self.assertNotEqual(module.__name__, name) + self.assertIs(lookedup, module) + + def check_basic(module, initialized): + init_count = module.initialized_count() + + self.assertIsNot(initialized, None) + self.assertIsInstance(init_count, int) + self.assertGreater(init_count, 0) + + return init_count + + def check_common_reloaded(name, module, cached, before, reloaded): + recached = sys.modules[name] + + self.assertEqual(reloaded.__spec__.name, name) + self.assertEqual(reloaded.__name__, before.__name__) + self.assertEqual(before.__dict__, module.__dict__) + self.assertIs(recached, reloaded) + + def check_basic_reloaded(module, lookedup, initialized, init_count, + before, reloaded): + relookedup = reloaded.look_up_self() + reinitialized = reloaded.initialized() + reinit_count = reloaded.initialized_count() + + self.assertIs(reloaded, module) + self.assertIs(reloaded.__dict__, module.__dict__) + # It only happens to be the same but that's good enough here. + # We really just want to verify that the re-loaded attrs + # didn't change. + self.assertIs(relookedup, lookedup) + self.assertEqual(reinitialized, initialized) + self.assertEqual(reinit_count, init_count) + + def check_with_reinit_reloaded(module, lookedup, initialized, + before, reloaded): + relookedup = reloaded.look_up_self() + reinitialized = reloaded.initialized() + + self.assertIsNot(reloaded, module) + self.assertIsNot(reloaded, module) + self.assertNotEqual(reloaded.__dict__, module.__dict__) + self.assertIs(relookedup, reloaded) + if initialized is None: + self.assertIs(reinitialized, None) + else: + self.assertGreater(reinitialized, initialized) + + # Check the "basic" module. + + name = basename + expected_init_count = 1 + with self.subTest(name): + mod = load(name) + lookedup, initialized, cached = check_common(name, mod) + check_direct(name, mod, lookedup) + init_count = check_basic(mod, initialized) + self.assertEqual(init_count, expected_init_count) + + before, reloaded = re_load(name, mod) + check_common_reloaded(name, mod, cached, before, reloaded) + check_basic_reloaded(mod, lookedup, initialized, init_count, + before, reloaded) + basic = mod + + # Check its indirect variants. + + name = f'{basename}_basic_wrapper' + expected_init_count += 1 + with self.subTest(name): + mod = load(name) + lookedup, initialized, cached = check_common(name, mod) + check_indirect(name, mod, lookedup, basic) + init_count = check_basic(mod, initialized) + self.assertEqual(init_count, expected_init_count) + + before, reloaded = re_load(name, mod) + check_common_reloaded(name, mod, cached, before, reloaded) + check_basic_reloaded(mod, lookedup, initialized, init_count, + before, reloaded) + + # Currently _PyState_AddModule() always replaces the cached module. + self.assertIs(basic.look_up_self(), mod) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # The cached module shouldn't be changed after this point. + basic_lookedup = mod + + # Check its direct variant. + + name = f'{basename}_basic_copy' + expected_init_count += 1 + with self.subTest(name): + mod = load(name) + lookedup, initialized, cached = check_common(name, mod) + check_direct(name, mod, lookedup) + init_count = check_basic(mod, initialized) + self.assertEqual(init_count, expected_init_count) + + before, reloaded = re_load(name, mod) + check_common_reloaded(name, mod, cached, before, reloaded) + check_basic_reloaded(mod, lookedup, initialized, init_count, + before, reloaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the non-basic variant that has no state. + + name = f'{basename}_with_reinit' + with self.subTest(name): + mod = load(name) + lookedup, initialized, cached = check_common(name, mod) + self.assertIs(initialized, None) + check_direct(name, mod, lookedup) + + before, reloaded = re_load(name, mod) + check_common_reloaded(name, mod, cached, before, reloaded) + check_with_reinit_reloaded(mod, lookedup, initialized, + before, reloaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the basic variant that has state. + + name = f'{basename}_with_state' + with self.subTest(name): + mod = load(name) + lookedup, initialized, cached = check_common(name, mod) + self.assertIsNot(initialized, None) + check_direct(name, mod, lookedup) + + before, reloaded = re_load(name, mod) + check_common_reloaded(name, mod, cached, before, reloaded) + check_with_reinit_reloaded(mod, lookedup, initialized, + before, reloaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + @requires_load_dynamic def test_load_dynamic_ImportError_path(self): # Issue #1559549 added `name` and `path` attributes to ImportError diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 3bfe159e54fe49..9e8dd647ee761a 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -5,74 +5,412 @@ # define Py_BUILD_CORE_MODULE 1 #endif +//#include #include "Python.h" #include "pycore_namespace.h" // _PyNamespace_New() +typedef struct { + _PyTime_t initialized; + PyObject *error; + PyObject *int_const; + PyObject *str_const; +} module_state; + +/* Process-global state is only used by _testsinglephase + since it's the only one that does not support re-init. */ +static struct { + int initialized_count; + module_state module; +} global_state = { .initialized_count = -1 }; + +static inline module_state * +get_module_state(PyObject *module) +{ + PyModuleDef *def = PyModule_GetDef(module); + if (def->m_size == -1) { + return &global_state.module; + } + else if (def->m_size == 0) { + return NULL; + } + else { + module_state *state = (module_state*)PyModule_GetState(module); + assert(state != NULL); + return state; + } +} + +static void +clear_state(module_state *state) +{ + state->initialized = 0; + Py_CLEAR(state->error); + Py_CLEAR(state->int_const); + Py_CLEAR(state->str_const); +} + +static int +_set_initialized(_PyTime_t *initialized) +{ + /* We go strictly monotonic to ensure each time is unique. */ + _PyTime_t prev; + if (_PyTime_GetMonotonicClockWithInfo(&prev, NULL) != 0) { + return -1; + } + /* We do a busy sleep since the interval should be super short. */ + _PyTime_t t; + do { + if (_PyTime_GetMonotonicClockWithInfo(&t, NULL) != 0) { + return -1; + } + } while (t == prev); + + *initialized = t; + return 0; +} + +static int +init_state(module_state *state) +{ + assert(state->initialized == 0 && + state->error == NULL && + state->int_const == NULL && + state->str_const == NULL); + + if (_set_initialized(&state->initialized) != 0) { + goto error; + } + assert(state->initialized > 0); + + /* Add an exception type */ + state->error = PyErr_NewException("_testsinglephase.error", NULL, NULL); + if (state->error == NULL) { + goto error; + } + + state->int_const = PyLong_FromLong(1969); + if (state->int_const == NULL) { + goto error; + } + + state->str_const = PyUnicode_FromString("something different"); + if (state->str_const == NULL) { + goto error; + } + + return 0; + +error: + clear_state(state); + return -1; +} + +static int +init_module(PyObject *module, module_state *state) +{ + if (PyModule_AddObjectRef(module, "error", state->error) != 0) { + return -1; + } + if (PyModule_AddObjectRef(module, "int_const", state->int_const) != 0) { + return -1; + } + if (PyModule_AddObjectRef(module, "str_const", state->str_const) != 0) { + return -1; + } + return 0; +} + + +PyDoc_STRVAR(common_initialized_doc, +"initialized()\n\ +\n\ +Return the seconds-since-epoch when the module was initialized."); + +static PyObject * +common_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + module_state *state = get_module_state(self); + if (state == NULL) { + Py_RETURN_NONE; + } + double d = _PyTime_AsSecondsDouble(state->initialized); + return PyFloat_FromDouble(d); +} + +#define INITIALIZED_METHODDEF \ + {"initialized", common_initialized, METH_NOARGS, \ + common_initialized_doc} + + +PyDoc_STRVAR(common_look_up_self_doc, +"look_up_self()\n\ +\n\ +Return the module associated with this module's def.m_base.m_index."); + +static PyObject * +common_look_up_self(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyModuleDef *def = PyModule_GetDef(self); + if (def == NULL) { + return NULL; + } + return Py_NewRef( + PyState_FindModule(def)); +} + +#define LOOK_UP_SELF_METHODDEF \ + {"look_up_self", common_look_up_self, METH_NOARGS, common_look_up_self_doc} + + /* Function of two integers returning integer */ -PyDoc_STRVAR(testexport_foo_doc, -"foo(i,j)\n\ +PyDoc_STRVAR(common_sum_doc, +"sum(i,j)\n\ \n\ Return the sum of i and j."); static PyObject * -testexport_foo(PyObject *self, PyObject *args) +common_sum(PyObject *self, PyObject *args) { long i, j; long res; - if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) + if (!PyArg_ParseTuple(args, "ll:sum", &i, &j)) return NULL; res = i + j; return PyLong_FromLong(res); } +#define SUM_METHODDEF \ + {"sum", common_sum, METH_VARARGS, common_sum_doc} -static PyMethodDef TestMethods[] = { - {"foo", testexport_foo, METH_VARARGS, - testexport_foo_doc}, - {NULL, NULL} /* sentinel */ -}; +PyDoc_STRVAR(basic_initialized_count_doc, +"initialized_count()\n\ +\n\ +Return how many times the module has been initialized."); + +static PyObject * +basic_initialized_count(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + assert(PyModule_GetDef(self)->m_size == -1); + return PyLong_FromLong(global_state.initialized_count); +} + +#define INITIALIZED_COUNT_METHODDEF \ + {"initialized_count", basic_initialized_count, METH_VARARGS, \ + basic_initialized_count_doc} + + +/*********************************************/ +/* the _testsinglephase module (and aliases) */ +/*********************************************/ + +/* This ia more typical of legacy extensions in the wild: + - single-phase init + - no module state + - does not support repeated initialization + (so m_copy is used) + - the module def is cached in _PyRuntime.extensions + (by name/filename) + + Also note that, because the module has single-phase init, + it is cached in interp->module_by_index (using mod->md_def->m_base.m_index). + */ -static struct PyModuleDef _testsinglephase = { +static PyMethodDef TestMethods_Basic[] = { + LOOK_UP_SELF_METHODDEF, + SUM_METHODDEF, + INITIALIZED_METHODDEF, + INITIALIZED_COUNT_METHODDEF, + {NULL, NULL} /* sentinel */ +}; + +static struct PyModuleDef _testsinglephase_basic = { PyModuleDef_HEAD_INIT, .m_name = "_testsinglephase", - .m_doc = PyDoc_STR("Test module _testsinglephase (main)"), + .m_doc = PyDoc_STR("Test module _testsinglephase"), .m_size = -1, // no module state - .m_methods = TestMethods, + .m_methods = TestMethods_Basic, }; +static PyObject * +init__testsinglephase_basic(PyModuleDef *def) +{ + if (global_state.initialized_count == -1) { + global_state.initialized_count = 0; + } + + PyObject *module = PyModule_Create(def); + if (module == NULL) { + return NULL; + } + + module_state *state = &global_state.module; + // It may have been set by a previous run or under a different name. + clear_state(state); + if (init_state(state) < 0) { + Py_CLEAR(module); + return NULL; + } + + if (init_module(module, state) < 0) { + Py_CLEAR(module); + goto finally; + } + + global_state.initialized_count++; + +finally: + return module; +} PyMODINIT_FUNC PyInit__testsinglephase(void) { - PyObject *module = PyModule_Create(&_testsinglephase); + return init__testsinglephase_basic(&_testsinglephase_basic); +} + + +PyMODINIT_FUNC +PyInit__testsinglephase_basic_wrapper(void) +{ + return PyInit__testsinglephase(); +} + + +PyMODINIT_FUNC +PyInit__testsinglephase_basic_copy(void) +{ + static struct PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_basic_copy", + .m_doc = PyDoc_STR("Test module _testsinglephase_basic_copy"), + .m_size = -1, // no module state + .m_methods = TestMethods_Basic, + }; + return init__testsinglephase_basic(&def); +} + + +/*******************************************/ +/* the _testsinglephase_with_reinit module */ +/*******************************************/ + +/* This ia less typical of legacy extensions in the wild: + - single-phase init (same as _testsinglephase above) + - no module state + - supports repeated initialization + (so m_copy is not used) + - the module def is not cached in _PyRuntime.extensions + + At this point most modules would reach for multi-phase init (PEP 489). + However, module state has been around a while (PEP 3121), + and most extensions predate multi-phase init. + + (This module is basically the same as _testsinglephase, + but supports repeated initialization.) + */ + +static PyMethodDef TestMethods_Reinit[] = { + LOOK_UP_SELF_METHODDEF, + SUM_METHODDEF, + INITIALIZED_METHODDEF, + {NULL, NULL} /* sentinel */ +}; + +static struct PyModuleDef _testsinglephase_with_reinit = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_reinit", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_reinit"), + .m_size = 0, + .m_methods = TestMethods_Reinit, +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_reinit(void) +{ + /* We purposefully do not try PyState_FindModule() first here + since we want to check the behavior of re-loading the module. */ + PyObject *module = PyModule_Create(&_testsinglephase_with_reinit); if (module == NULL) { return NULL; } - /* Add an exception type */ - PyObject *temp = PyErr_NewException("_testsinglephase.error", NULL, NULL); - if (temp == NULL) { - goto error; + assert(get_module_state(module) == NULL); + + module_state state = {0}; + if (init_state(&state) < 0) { + Py_CLEAR(module); + return NULL; } - if (PyModule_AddObject(module, "error", temp) != 0) { - Py_DECREF(temp); - goto error; + + if (init_module(module, &state) < 0) { + Py_CLEAR(module); + goto finally; } - if (PyModule_AddIntConstant(module, "int_const", 1969) != 0) { - goto error; +finally: + /* We only needed the module state for setting the module attrs. */ + clear_state(&state); + return module; +} + + +/******************************************/ +/* the _testsinglephase_with_state module */ +/******************************************/ + +/* This ia less typical of legacy extensions in the wild: + - single-phase init (same as _testsinglephase above) + - has some module state + - supports repeated initialization + (so m_copy is not used) + - the module def is not cached in _PyRuntime.extensions + + At this point most modules would reach for multi-phase init (PEP 489). + However, module state has been around a while (PEP 3121), + and most extensions predate multi-phase init. + */ + +static PyMethodDef TestMethods_WithState[] = { + LOOK_UP_SELF_METHODDEF, + SUM_METHODDEF, + INITIALIZED_METHODDEF, + {NULL, NULL} /* sentinel */ +}; + +static struct PyModuleDef _testsinglephase_with_state = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_state", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_state"), + .m_size = sizeof(module_state), + .m_methods = TestMethods_WithState, +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_state(void) +{ + /* We purposefully do not try PyState_FindModule() first here + since we want to check the behavior of re-loading the module. */ + PyObject *module = PyModule_Create(&_testsinglephase_with_state); + if (module == NULL) { + return NULL; } - if (PyModule_AddStringConstant(module, "str_const", "something different") != 0) { - goto error; + module_state *state = get_module_state(module); + assert(state != NULL); + if (init_state(state) < 0) { + Py_CLEAR(module); + return NULL; } - return module; + if (init_module(module, state) < 0) { + clear_state(state); + Py_CLEAR(module); + goto finally; + } -error: - Py_DECREF(module); - return NULL; +finally: + return module; } diff --git a/Python/import.c b/Python/import.c index 302255d76edcd5..63ed2443657b29 100644 --- a/Python/import.c +++ b/Python/import.c @@ -428,6 +428,71 @@ PyImport_GetMagicTag(void) } +/* +We support a number of kinds of single-phase init builtin/extension modules: + +* "basic" + * no module state (PyModuleDef.m_size == -1) + * does not support repeated init (we use PyModuleDef.m_base.m_copy) + * may have process-global state + * the module's def is cached in _PyRuntime.imports.extensions, + by (name, filename) +* "reinit" + * no module state (PyModuleDef.m_size == 0) + * supports repeated init (m_copy is never used) + * should not have any process-global state + * its def is never cached in _PyRuntime.imports.extensions + (except, currently, under the main interpreter, for some reason) +* "with state" (almost the same as reinit) + * has module state (PyModuleDef.m_size > 0) + * supports repeated init (m_copy is never used) + * should not have any process-global state + * its def is never cached in _PyRuntime.imports.extensions + (except, currently, under the main interpreter, for some reason) + +There are also variants within those classes: + +* two or more modules share a PyModuleDef + * a module's init func uses another module's PyModuleDef + * a module's init func calls another's module's init func + * a module's init "func" is actually a variable statically initialized + to another module's init func +* two or modules share "methods" + * a module's init func copies another module's PyModuleDef + (with a different name) +* (basic-only) two or modules share process-global state + +In the first case, where modules share a PyModuleDef, the following +notable weirdness happens: + +* the module's __name__ matches the def, not the requested name +* the last module (with the same def) to be imported for the first time wins + * returned by PyState_Find_Module() (via interp->modules_by_index) + * (non-basic-only) its init func is used when re-loading any of them + (via the def's m_init) + * (basic-only) the copy of its __dict__ is used when re-loading any of them + (via the def's m_copy) + +However, the following happens as expected: + +* a new module object (with its own __dict__) is created for each request +* the module's __spec__ has the requested name +* the loaded module is cached in sys.modules under the requested name +* the m_index field of the shared def is not changed, + so at least PyState_FindModule() will always look in the same place + +For "basic" modules there are other quirks: + +* (whether sharing a def or not) when loaded the first time, + m_copy is set before _init_module_attrs() is called + in importlib._bootstrap.module_from_spec(), + so when the module is re-loaded, the previous value + for __wpec__ (and others) is reset, possibly unexpectedly. + +Generally, when multiple interpreters are involved, some of the above +gets even messier. +*/ + /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than once, we keep a static dictionary 'extensions' keyed by the tuple @@ -489,9 +554,8 @@ _extensions_cache_clear(void) Py_CLEAR(_PyRuntime.imports.extensions); } -int -_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, - PyObject *filename, PyObject *modules) +static int +fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) { if (mod == NULL || !PyModule_Check(mod)) { PyErr_BadInternalCall(); @@ -505,16 +569,13 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, } PyThreadState *tstate = _PyThreadState_GET(); - if (PyObject_SetItem(modules, name, mod) < 0) { - return -1; - } if (_PyState_AddModule(tstate, mod, def) < 0) { - PyMapping_DelItem(modules, name); return -1; } // bpo-44050: Extensions and def->m_base.m_copy can be updated // when the extension module doesn't support sub-interpreters. + // XXX Why special-case the main interpreter? if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { if (def->m_size == -1) { if (def->m_base.m_copy) { @@ -541,15 +602,39 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, return 0; } +int +_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, + PyObject *filename, PyObject *modules) +{ + if (PyObject_SetItem(modules, name, mod) < 0) { + return -1; + } + if (fix_up_extension(mod, name, filename) < 0) { + PyMapping_DelItem(modules, name); + return -1; + } + return 0; +} + int _PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) { - int res; + int res = -1; PyObject *nameobj; nameobj = PyUnicode_InternFromString(name); - if (nameobj == NULL) + if (nameobj == NULL) { return -1; - res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj, modules); + } + if (PyObject_SetItem(modules, nameobj, mod) < 0) { + goto finally; + } + if (fix_up_extension(mod, nameobj, nameobj) < 0) { + PyMapping_DelItem(modules, nameobj); + goto finally; + } + res = 0; + +finally: Py_DECREF(nameobj); return res; } From d777790bab878b8d1a218a1a60894b2823485cca Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Tue, 14 Feb 2023 15:57:01 -0800 Subject: [PATCH 40/83] gh-99108: Build the hashlib HACL* code as a static library. (#101917) This builds HACL* as a library in one place. A followup to #101707 which broke some WASM builds. This fixes 2/4 of them, but the enscripten toolchain in the others don't deduplicate linker arguments and error out. A follow-on PR will address those. --- Makefile.pre.in | 33 +++++++++++++++++-- Modules/Setup.stdlib.in | 4 +-- .../{include => }/python_hacl_namespaces.h | 1 + 3 files changed, 34 insertions(+), 4 deletions(-) rename Modules/_hacl/{include => }/python_hacl_namespaces.h (97%) diff --git a/Makefile.pre.in b/Makefile.pre.in index d42d4d8a3c1c9f..ce3fed3d648536 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -207,6 +207,7 @@ ENSUREPIP= @ENSUREPIP@ # Internal static libraries LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a +LIBHACL_A= Modules/_hacl/libHacl_Streaming_SHA2.a # Module state, compiler flags and linker flags # Empty CFLAGS and LDFLAGS are omitted. @@ -571,6 +572,23 @@ LIBEXPAT_HEADERS= \ Modules/expat/xmltok.h \ Modules/expat/xmltok_impl.h +########################################################################## +# hashlib's HACL* library + +LIBHACL_OBJS= \ + Modules/_hacl/Hacl_Streaming_SHA2.o + +LIBHACL_HEADERS= \ + Modules/_hacl/Hacl_Streaming_SHA2.h \ + Modules/_hacl/include/krml/FStar_UInt128_Verified.h \ + Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h \ + Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h \ + Modules/_hacl/include/krml/internal/target.h \ + Modules/_hacl/include/krml/lowstar_endianness.h \ + Modules/_hacl/include/krml/types.h \ + Modules/_hacl/internal/Hacl_SHA2_Generic.h \ + Modules/_hacl/python_hacl_namespaces.h + ######################################################################### # Rules @@ -890,6 +908,17 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS) +########################################################################## +# Build HACL* static libraries for hashlib: libHacl_Streaming_SHA2.a +LIBHACL_CFLAGS=-I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED) + +Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c + +$(LIBHACL_A): $(LIBHACL_OBJS) + -rm -f $@ + $(AR) $(ARFLAGS) $@ $(LIBHACL_OBJS) + # create relative links from build/lib.platform/egg.so to Modules/egg.so # pybuilddir.txt is created too late. We cannot use it in Makefile # targets. ln --relative is not portable. @@ -2606,9 +2635,9 @@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h -MODULE__SHA256_DEPS=$(srcdir)/Modules/hashlib.h $(srcdir)/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h $(srcdir)/Modules/_hacl/include/krml/lowstar_endianness.h $(srcdir)/Modules/_hacl/include/krml/internal/target.h $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.h +MODULE__SHA256_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A) MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h -MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h $(srcdir)/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h $(srcdir)/Modules/_hacl/include/krml/lowstar_endianness.h $(srcdir)/Modules/_hacl/include/krml/internal/target.h $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.h +MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index b6d13e04d3fa87..22bcb423db233f 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -79,8 +79,8 @@ # hashing builtins, can be disabled with --without-builtin-hashlib-hashes @MODULE__MD5_TRUE@_md5 md5module.c @MODULE__SHA1_TRUE@_sha1 sha1module.c -@MODULE__SHA256_TRUE@_sha256 sha256module.c _hacl/Hacl_Streaming_SHA2.c -@MODULE__SHA512_TRUE@_sha512 sha512module.c _hacl/Hacl_Streaming_SHA2.c +@MODULE__SHA256_TRUE@_sha256 sha256module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a +@MODULE__SHA512_TRUE@_sha512 sha512module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a @MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c diff --git a/Modules/_hacl/include/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h similarity index 97% rename from Modules/_hacl/include/python_hacl_namespaces.h rename to Modules/_hacl/python_hacl_namespaces.h index 65608d1fd283c4..ac12f386257b19 100644 --- a/Modules/_hacl/include/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -25,6 +25,7 @@ #define Hacl_Streaming_SHA2_init_224 python_hashlib_Hacl_Streaming_SHA2_init_224 #define Hacl_Streaming_SHA2_init_512 python_hashlib_Hacl_Streaming_SHA2_init_512 #define Hacl_Streaming_SHA2_init_384 python_hashlib_Hacl_Streaming_SHA2_init_384 +#define Hacl_SHA2_Scalar32_sha512_init python_hashlib_Hacl_SHA2_Scalar32_sha512_init #define Hacl_Streaming_SHA2_update_256 python_hashlib_Hacl_Streaming_SHA2_update_256 #define Hacl_Streaming_SHA2_update_224 python_hashlib_Hacl_Streaming_SHA2_update_224 #define Hacl_Streaming_SHA2_update_512 python_hashlib_Hacl_Streaming_SHA2_update_512 From 8a2b7ee64d1bde762438b458ea7fe88f054a3a88 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 15 Feb 2023 06:27:16 +0100 Subject: [PATCH 41/83] gh-101693: In sqlite3, deprecate using named placeholders with parameters supplied as a sequence (#101698) --- Doc/library/sqlite3.rst | 19 ++++++++++++++++++- Doc/whatsnew/3.12.rst | 7 +++++++ Lib/test/test_sqlite3/test_dbapi.py | 15 +++++++++++++++ ...-02-08-18-20-58.gh-issue-101693.4_LPXj.rst | 6 ++++++ Modules/_sqlite/cursor.c | 13 +++++++++++++ 5 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 8ffc0aad91995c..18d0a5e630f6a9 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1442,6 +1442,14 @@ Cursor objects and there is no open transaction, a transaction is implicitly opened before executing *sql*. + .. deprecated-removed:: 3.12 3.14 + + :exc:`DeprecationWarning` is emitted if + :ref:`named placeholders ` are used + and *parameters* is a sequence instead of a :class:`dict`. + Starting with Python 3.14, :exc:`ProgrammingError` will + be raised instead. + Use :meth:`executescript` to execute multiple SQL statements. .. method:: executemany(sql, parameters, /) @@ -1476,6 +1484,15 @@ Cursor objects # cur is an sqlite3.Cursor object cur.executemany("INSERT INTO data VALUES(?)", rows) + .. deprecated-removed:: 3.12 3.14 + + :exc:`DeprecationWarning` is emitted if + :ref:`named placeholders ` are used + and the items in *parameters* are sequences + instead of :class:`dict`\s. + Starting with Python 3.14, :exc:`ProgrammingError` will + be raised instead. + .. method:: executescript(sql_script, /) Execute the SQL statements in *sql_script*. @@ -1971,7 +1988,7 @@ question marks (qmark style) or named placeholders (named style). For the qmark style, *parameters* must be a :term:`sequence` whose length must match the number of placeholders, or a :exc:`ProgrammingError` is raised. -For the named style, *parameters* should be +For the named style, *parameters* must be an instance of a :class:`dict` (or a subclass), which must contain keys for all named parameters; any extra items are ignored. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 45a5e5062d55b6..c62f462a19a2df 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -415,6 +415,13 @@ Deprecated and tailor them to your needs. (Contributed by Erlend E. Aasland in :gh:`90016`.) +* In :meth:`~sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted + when :ref:`named placeholders ` are used together with + parameters supplied as a :term:`sequence` instead of as a :class:`dict`. + Starting from Python 3.14, using named placeholders with parameters supplied + as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. + (Contributed by Erlend E. Aasland in :gh:`101698`.) + * The 3-arg signatures (type, value, traceback) of :meth:`~coroutine.throw`, :meth:`~generator.throw` and :meth:`~agen.athrow` are deprecated and may be removed in a future version of Python. Use the single-arg versions diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 363a308f3e5fec..695e213cdc7b75 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -861,6 +861,21 @@ def __getitem__(slf, x): with self.assertRaises(ZeroDivisionError): self.cu.execute("select name from test where name=?", L()) + def test_execute_named_param_and_sequence(self): + dataset = ( + ("select :a", (1,)), + ("select :a, ?, ?", (1, 2, 3)), + ("select ?, :b, ?", (1, 2, 3)), + ("select ?, ?, :c", (1, 2, 3)), + ("select :a, :b, ?", (1, 2, 3)), + ) + msg = "Binding.*is a named parameter" + for query, params in dataset: + with self.subTest(query=query, params=params): + with self.assertWarnsRegex(DeprecationWarning, msg) as cm: + self.cu.execute(query, params) + self.assertEqual(cm.filename, __file__) + def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER msg = "too many SQL variables" diff --git a/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst b/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst new file mode 100644 index 00000000000000..e436054b15b657 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst @@ -0,0 +1,6 @@ +In :meth:`sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted +when :ref:`named placeholders ` are used together with +parameters supplied as a :term:`sequence` instead of as a :class:`dict`. +Starting from Python 3.14, using named placeholders with parameters supplied +as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. +Patch by Erlend E. Aasland. diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index a4e22bb4a2b58d..6f7970cf8197a2 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -662,6 +662,19 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, return; } for (i = 0; i < num_params; i++) { + const char *name = sqlite3_bind_parameter_name(self->st, i+1); + if (name != NULL) { + int ret = PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "Binding %d ('%s') is a named parameter, but you " + "supplied a sequence which requires nameless (qmark) " + "placeholders. Starting with Python 3.14 an " + "sqlite3.ProgrammingError will be raised.", + i+1, name); + if (ret < 0) { + return; + } + } + if (PyTuple_CheckExact(parameters)) { PyObject *item = PyTuple_GET_ITEM(parameters, i); current_param = Py_NewRef(item); From e8b6aaad2faf11fe315410138a5c5943d610d8d8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 15 Feb 2023 11:18:27 +0100 Subject: [PATCH 42/83] gh-101819: Remove _testcapi dependencies on specific _io symbols (#101918) --- Modules/_io/_iomodule.c | 12 ++---------- Modules/_testcapimodule.c | 8 ++++++-- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 175fa97479d27d..811b1d221a0122 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -720,16 +720,8 @@ PyInit__io(void) // Add types for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; - // Private type not exposed in the _io module - if (type == &_PyBytesIOBuffer_Type) { - if (PyType_Ready(type) < 0) { - goto fail; - } - } - else { - if (PyModule_AddType(m, type) < 0) { - goto fail; - } + if (PyModule_AddType(m, type) < 0) { + goto fail; } } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3c411fa0d76358..5610a7689136f6 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1448,12 +1448,10 @@ test_from_contiguous(PyObject* self, PyObject *Py_UNUSED(ignored)) } #if (defined(__linux__) || defined(__FreeBSD__)) && defined(__GNUC__) -extern PyTypeObject _PyBytesIOBuffer_Type; static PyObject * test_pep3118_obsolete_write_locks(PyObject* self, PyObject *Py_UNUSED(ignored)) { - PyTypeObject *type = &_PyBytesIOBuffer_Type; PyObject *b; char *dummy[1]; int ret, match; @@ -1466,7 +1464,13 @@ test_pep3118_obsolete_write_locks(PyObject* self, PyObject *Py_UNUSED(ignored)) goto error; /* bytesiobuf_getbuffer() */ + PyTypeObject *type = (PyTypeObject *)_PyImport_GetModuleAttrString( + "_io", "_BytesIOBuffer"); + if (type == NULL) { + return NULL; + } b = type->tp_alloc(type, 0); + Py_DECREF(type); if (b == NULL) { return NULL; } From c7766245c14fa03b8afd3aff9be30b13d0069f95 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 15 Feb 2023 12:21:40 +0000 Subject: [PATCH 43/83] GH-87849: Fix refleak in SEND instruction. (GH-101908) Fix refleak in SEND instruction. --- Python/bytecodes.c | 1 + Python/generated_cases.c.h | 1 + 2 files changed, 2 insertions(+) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index be54e5f6f589eb..d5d5034cbfbf74 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -727,6 +727,7 @@ dummy_func( else { assert(retval != NULL); } + Py_DECREF(v); } inst(SEND_GEN, (unused/1, receiver, v -- receiver)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index beb797cbd233d7..8b8a7161ad898e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -934,6 +934,7 @@ else { assert(retval != NULL); } + Py_DECREF(v); POKE(1, retval); JUMPBY(1); DISPATCH(); From eb0c485b6c836abb71932537a5058344d11d7bc8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 15 Feb 2023 14:07:59 +0100 Subject: [PATCH 44/83] gh-101819: Remove _PyWindowsConsoleIO_Type from the Windows DLL (GH-101904) Automerge-Triggered-By: GH:erlend-aasland --- .../pycore_global_objects_fini_generated.h | 2 ++ Include/internal/pycore_global_strings.h | 2 ++ Include/internal/pycore_runtime_init_generated.h | 2 ++ Include/internal/pycore_unicodeobject_generated.h | 4 ++++ Modules/_io/_iomodule.h | 4 ---- Modules/_io/winconsoleio.c | 4 +--- PC/_testconsole.c | 11 +++++++++-- Python/pylifecycle.c | 14 +++++++++----- 8 files changed, 29 insertions(+), 14 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 8c210111b5899f..dc5cd58d853534 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -577,6 +577,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(True)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(WarningMessage)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_WindowsConsoleIO)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__IOBase_closed)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__abc_tpflags__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__abs__)); @@ -752,6 +753,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_handle_fromlist)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_io)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 6b1c8424424d96..8b23aa15479301 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -63,6 +63,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(True) STRUCT_FOR_ID(WarningMessage) STRUCT_FOR_ID(_) + STRUCT_FOR_ID(_WindowsConsoleIO) STRUCT_FOR_ID(__IOBase_closed) STRUCT_FOR_ID(__abc_tpflags__) STRUCT_FOR_ID(__abs__) @@ -238,6 +239,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_get_sourcefile) STRUCT_FOR_ID(_handle_fromlist) STRUCT_FOR_ID(_initializing) + STRUCT_FOR_ID(_io) STRUCT_FOR_ID(_is_text_encoding) STRUCT_FOR_ID(_length_) STRUCT_FOR_ID(_limbo) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index fcb613083ffe99..471efadb13bb4f 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -569,6 +569,7 @@ extern "C" { INIT_ID(True), \ INIT_ID(WarningMessage), \ INIT_ID(_), \ + INIT_ID(_WindowsConsoleIO), \ INIT_ID(__IOBase_closed), \ INIT_ID(__abc_tpflags__), \ INIT_ID(__abs__), \ @@ -744,6 +745,7 @@ extern "C" { INIT_ID(_get_sourcefile), \ INIT_ID(_handle_fromlist), \ INIT_ID(_initializing), \ + INIT_ID(_io), \ INIT_ID(_is_text_encoding), \ INIT_ID(_length_), \ INIT_ID(_limbo), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 301aee5210e799..b47d240e492ff9 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -32,6 +32,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_); PyUnicode_InternInPlace(&string); + string = &_Py_ID(_WindowsConsoleIO); + PyUnicode_InternInPlace(&string); string = &_Py_ID(__IOBase_closed); PyUnicode_InternInPlace(&string); string = &_Py_ID(__abc_tpflags__); @@ -382,6 +384,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_initializing); PyUnicode_InternInPlace(&string); + string = &_Py_ID(_io); + PyUnicode_InternInPlace(&string); string = &_Py_ID(_is_text_encoding); PyUnicode_InternInPlace(&string); string = &_Py_ID(_length_); diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index c260080f0e348b..7617cb8fb70e43 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -21,13 +21,9 @@ extern PyTypeObject PyBufferedRandom_Type; extern PyTypeObject PyTextIOWrapper_Type; extern PyTypeObject PyIncrementalNewlineDecoder_Type; -#ifndef Py_LIMITED_API #ifdef MS_WINDOWS extern PyTypeObject PyWindowsConsoleIO_Type; -PyAPI_DATA(PyObject *) _PyWindowsConsoleIO_Type; -#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), (PyTypeObject*)_PyWindowsConsoleIO_Type)) #endif /* MS_WINDOWS */ -#endif /* Py_LIMITED_API */ /* These functions are used as METH_NOARGS methods, are normally called * with args=NULL, and return a new reference. diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 4f41ab965e2e67..e913d831874617 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -260,7 +260,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, int fd_is_own = 0; HANDLE handle = NULL; - assert(PyWindowsConsoleIO_Check(self)); + assert(PyObject_TypeCheck(self, (PyTypeObject *)&PyWindowsConsoleIO_Type)); if (self->fd >= 0) { if (self->closefd) { /* Have to close the existing file first. */ @@ -1174,6 +1174,4 @@ PyTypeObject PyWindowsConsoleIO_Type = { 0, /* tp_finalize */ }; -PyObject * _PyWindowsConsoleIO_Type = (PyObject*)&PyWindowsConsoleIO_Type; - #endif /* MS_WINDOWS */ diff --git a/PC/_testconsole.c b/PC/_testconsole.c index a8308835d8f85d..f14a2d45b1be26 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -10,7 +10,7 @@ #ifdef MS_WINDOWS #include "pycore_fileutils.h" // _Py_get_osfhandle() -#include "..\modules\_io\_iomodule.h" +#include "pycore_runtime.h" // _Py_ID() #define WIN32_LEAN_AND_MEAN #include @@ -51,7 +51,14 @@ _testconsole_write_input_impl(PyObject *module, PyObject *file, { INPUT_RECORD *rec = NULL; - if (!PyWindowsConsoleIO_Check(file)) { + PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( + &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); + if (winconsoleio_type == NULL) { + return NULL; + } + int is_subclass = PyObject_TypeCheck(file, winconsoleio_type); + Py_DECREF(winconsoleio_type); + if (!is_subclass) { PyErr_SetString(PyExc_TypeError, "expected raw console object"); return NULL; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a8a8e7f3d84f21..045a2996e8988b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -54,10 +54,6 @@ extern void _PyIO_Fini(void); #ifdef MS_WINDOWS # undef BYTE - - extern PyTypeObject PyWindowsConsoleIO_Type; -# define PyWindowsConsoleIO_Check(op) \ - (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type)) #endif #define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) @@ -2358,8 +2354,16 @@ create_stdio(const PyConfig *config, PyObject* io, #ifdef MS_WINDOWS /* Windows console IO is always UTF-8 encoded */ - if (PyWindowsConsoleIO_Check(raw)) + PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( + &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); + if (winconsoleio_type == NULL) { + goto error; + } + int is_subclass = PyObject_TypeCheck(raw, winconsoleio_type); + Py_DECREF(winconsoleio_type); + if (is_subclass) { encoding = L"utf-8"; + } #endif text = PyUnicode_FromString(name); From c1ce0d178fe57b50f37578b285a343d77485ac02 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 15 Feb 2023 22:58:48 +0100 Subject: [PATCH 45/83] gh-99138: Isolate _zoneinfo (#99218) * Convert zone info type to heap type and add it to module state * Add global variables to module state --- Doc/library/zoneinfo.rst | 2 +- Lib/test/test_zoneinfo/test_zoneinfo.py | 4 +- ...3-01-02-22-41-44.gh-issue-99138.17hp9U.rst | 1 + Modules/_zoneinfo.c | 519 +++++++++++------- Modules/clinic/_zoneinfo.c.h | 215 +++++++- Tools/c-analyzer/cpython/globals-to-fix.tsv | 11 - 6 files changed, 539 insertions(+), 213 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index d2e5619e7e47c2..f8624da6e51dbb 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -241,7 +241,7 @@ The following class methods are also available: .. warning:: Invoking this function may change the semantics of datetimes using - ``ZoneInfo`` in surprising ways; this modifies process-wide global state + ``ZoneInfo`` in surprising ways; this modifies module state and thus may have wide-ranging effects. Only use it if you know that you need to. diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index fd0e3bc032ec0c..82041a2b488334 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -1797,12 +1797,10 @@ def test_cache_location(self): self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache")) def test_gc_tracked(self): - # The pure Python version is tracked by the GC but (for now) the C - # version is not. import gc self.assertTrue(gc.is_tracked(py_zoneinfo.ZoneInfo)) - self.assertFalse(gc.is_tracked(c_zoneinfo.ZoneInfo)) + self.assertTrue(gc.is_tracked(c_zoneinfo.ZoneInfo)) @dataclasses.dataclass(frozen=True) diff --git a/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst b/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst new file mode 100644 index 00000000000000..3dd4646f40e1e5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst @@ -0,0 +1 @@ +Apply :pep:`687` to :mod:`zoneinfo`. Patch by Erlend E. Aasland. diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 9d38589ea3d1b0..9f423559f51a43 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -19,10 +19,6 @@ class zoneinfo.ZoneInfo "PyObject *" "PyTypeObject *" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=d12c73c0eef36df8]*/ -// Imports -static PyObject *io_open = NULL; -static PyObject *_tzpath_find_tzfile = NULL; -static PyObject *_common_mod = NULL; typedef struct TransitionRuleType TransitionRuleType; typedef struct StrongCacheNode StrongCacheNode; @@ -90,15 +86,21 @@ struct StrongCacheNode { PyObject *zone; }; -static PyTypeObject PyZoneInfo_ZoneInfoType; +typedef struct { + PyTypeObject *ZoneInfoType; + + // Imports + PyObject *io_open; + PyObject *_tzpath_find_tzfile; + PyObject *_common_mod; -// Globals -static PyObject *TIMEDELTA_CACHE = NULL; -static PyObject *ZONEINFO_WEAK_CACHE = NULL; -static StrongCacheNode *ZONEINFO_STRONG_CACHE = NULL; -static size_t ZONEINFO_STRONG_CACHE_MAX_SIZE = 8; + // Caches + PyObject *TIMEDELTA_CACHE; + PyObject *ZONEINFO_WEAK_CACHE; + StrongCacheNode *ZONEINFO_STRONG_CACHE; -static _ttinfo NO_TTINFO = {NULL, NULL, NULL, 0}; + _ttinfo NO_TTINFO; +} zoneinfo_state; // Constants static const int EPOCHORDINAL = 719163; @@ -114,9 +116,12 @@ static const int SOURCE_NOCACHE = 0; static const int SOURCE_CACHE = 1; static const int SOURCE_FILE = 2; +static const size_t ZONEINFO_STRONG_CACHE_MAX_SIZE = 8; + // Forward declarations static int -load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj); +load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, + PyObject *file_obj); static void utcoff_to_dstoff(size_t *trans_idx, long *utcoffs, long *dstoffs, unsigned char *isdsts, size_t num_transitions, @@ -127,7 +132,7 @@ ts_to_local(size_t *trans_idx, int64_t *trans_utc, long *utcoff, size_t num_transitions); static int -parse_tz_str(PyObject *tz_str_obj, _tzrule *out); +parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out); static Py_ssize_t parse_abbr(const char *const p, PyObject **abbr); @@ -146,26 +151,27 @@ find_tzrule_ttinfo_fromutc(_tzrule *rule, int64_t ts, int year, unsigned char *fold); static int -build_ttinfo(long utcoffset, long dstoffset, PyObject *tzname, _ttinfo *out); +build_ttinfo(zoneinfo_state *state, long utcoffset, long dstoffset, + PyObject *tzname, _ttinfo *out); static void xdecref_ttinfo(_ttinfo *ttinfo); static int ttinfo_eq(const _ttinfo *const tti0, const _ttinfo *const tti1); static int -build_tzrule(PyObject *std_abbr, PyObject *dst_abbr, long std_offset, - long dst_offset, TransitionRuleType *start, +build_tzrule(zoneinfo_state *state, PyObject *std_abbr, PyObject *dst_abbr, + long std_offset, long dst_offset, TransitionRuleType *start, TransitionRuleType *end, _tzrule *out); static void free_tzrule(_tzrule *tzrule); static PyObject * -load_timedelta(long seconds); +load_timedelta(zoneinfo_state *state, long seconds); static int get_local_timestamp(PyObject *dt, int64_t *local_ts); static _ttinfo * -find_ttinfo(PyZoneInfo_ZoneInfo *self, PyObject *dt); +find_ttinfo(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *dt); static int ymd_to_ord(int y, int m, int d); @@ -176,27 +182,57 @@ static size_t _bisect(const int64_t value, const int64_t *arr, size_t size); static int -eject_from_strong_cache(const PyTypeObject *const type, PyObject *key); +eject_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *key); static void -clear_strong_cache(const PyTypeObject *const type); +clear_strong_cache(zoneinfo_state *state, const PyTypeObject *const type); static void -update_strong_cache(const PyTypeObject *const type, PyObject *key, - PyObject *zone); +update_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *key, PyObject *zone); static PyObject * -zone_from_strong_cache(const PyTypeObject *const type, PyObject *const key); +zone_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *const key); + +static inline zoneinfo_state * +zoneinfo_get_state(PyObject *mod) +{ + zoneinfo_state *state = (zoneinfo_state *)PyModule_GetState(mod); + assert(state != NULL); + return state; +} + +static inline zoneinfo_state * +zoneinfo_get_state_by_cls(PyTypeObject *cls) +{ + zoneinfo_state *state = (zoneinfo_state *)PyType_GetModuleState(cls); + assert(state != NULL); + return state; +} + +static struct PyModuleDef zoneinfomodule; + +static inline zoneinfo_state * +zoneinfo_get_state_by_self(PyTypeObject *self) +{ + PyObject *mod = PyType_GetModuleByDef(self, &zoneinfomodule); + assert(mod != NULL); + return zoneinfo_get_state(mod); +} static PyObject * -zoneinfo_new_instance(PyTypeObject *type, PyObject *key) +zoneinfo_new_instance(zoneinfo_state *state, PyTypeObject *type, PyObject *key) { PyObject *file_obj = NULL; PyObject *file_path = NULL; - file_path = PyObject_CallFunctionObjArgs(_tzpath_find_tzfile, key, NULL); + file_path = PyObject_CallFunctionObjArgs(state->_tzpath_find_tzfile, + key, NULL); if (file_path == NULL) { return NULL; } else if (file_path == Py_None) { - file_obj = PyObject_CallMethod(_common_mod, "load_tzdata", "O", key); + PyObject *meth = state->_common_mod; + file_obj = PyObject_CallMethod(meth, "load_tzdata", "O", key); if (file_obj == NULL) { Py_DECREF(file_path); return NULL; @@ -209,13 +245,14 @@ zoneinfo_new_instance(PyTypeObject *type, PyObject *key) } if (file_obj == NULL) { - file_obj = PyObject_CallFunction(io_open, "Os", file_path, "rb"); + PyObject *func = state->io_open; + file_obj = PyObject_CallFunction(func, "Os", file_path, "rb"); if (file_obj == NULL) { goto error; } } - if (load_data((PyZoneInfo_ZoneInfo *)self, file_obj)) { + if (load_data(state, (PyZoneInfo_ZoneInfo *)self, file_obj)) { goto error; } @@ -248,10 +285,10 @@ zoneinfo_new_instance(PyTypeObject *type, PyObject *key) } static PyObject * -get_weak_cache(PyTypeObject *type) +get_weak_cache(zoneinfo_state *state, PyTypeObject *type) { - if (type == &PyZoneInfo_ZoneInfoType) { - return ZONEINFO_WEAK_CACHE; + if (type == state->ZoneInfoType) { + return state->ZONEINFO_WEAK_CACHE; } else { PyObject *cache = @@ -273,12 +310,13 @@ zoneinfo_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - PyObject *instance = zone_from_strong_cache(type, key); + zoneinfo_state *state = zoneinfo_get_state_by_self(type); + PyObject *instance = zone_from_strong_cache(state, type, key); if (instance != NULL || PyErr_Occurred()) { return instance; } - PyObject *weak_cache = get_weak_cache(type); + PyObject *weak_cache = get_weak_cache(state, type); instance = PyObject_CallMethod(weak_cache, "get", "O", key, Py_None); if (instance == NULL) { return NULL; @@ -286,7 +324,7 @@ zoneinfo_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (instance == Py_None) { Py_DECREF(instance); - PyObject *tmp = zoneinfo_new_instance(type, key); + PyObject *tmp = zoneinfo_new_instance(state, type, key); if (tmp == NULL) { return NULL; } @@ -300,14 +338,32 @@ zoneinfo_new(PyTypeObject *type, PyObject *args, PyObject *kw) ((PyZoneInfo_ZoneInfo *)instance)->source = SOURCE_CACHE; } - update_strong_cache(type, key, instance); + update_strong_cache(state, type, key, instance); return instance; } +static int +zoneinfo_traverse(PyZoneInfo_ZoneInfo *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->key); + return 0; +} + +static int +zoneinfo_clear(PyZoneInfo_ZoneInfo *self) +{ + Py_CLEAR(self->key); + Py_CLEAR(self->file_repr); + return 0; +} + static void zoneinfo_dealloc(PyObject *obj_self) { PyZoneInfo_ZoneInfo *self = (PyZoneInfo_ZoneInfo *)obj_self; + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); if (self->weakreflist != NULL) { PyObject_ClearWeakRefs(obj_self); @@ -336,16 +392,16 @@ zoneinfo_dealloc(PyObject *obj_self) free_tzrule(&(self->tzrule_after)); - Py_XDECREF(self->key); - Py_XDECREF(self->file_repr); - - Py_TYPE(self)->tp_free((PyObject *)self); + zoneinfo_clear(self); + tp->tp_free(obj_self); + Py_DECREF(tp); } /*[clinic input] @classmethod zoneinfo.ZoneInfo.from_file + cls: defining_class file_obj: object / key: object = None @@ -354,9 +410,9 @@ Create a ZoneInfo file from a file object. [clinic start generated code]*/ static PyObject * -zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyObject *file_obj, - PyObject *key) -/*[clinic end generated code: output=68ed2022404ae5be input=ccfe73708133d2e4]*/ +zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *file_obj, PyObject *key) +/*[clinic end generated code: output=77887d1d56a48324 input=d26111f29eed6863]*/ { PyObject *file_repr = NULL; PyZoneInfo_ZoneInfo *self = NULL; @@ -372,7 +428,8 @@ zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyObject *file_obj, goto error; } - if (load_data(self, file_obj)) { + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + if (load_data(state, self, file_obj)) { goto error; } @@ -391,16 +448,20 @@ zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyObject *file_obj, @classmethod zoneinfo.ZoneInfo.no_cache + cls: defining_class + / key: object Get a new instance of ZoneInfo, bypassing the cache. [clinic start generated code]*/ static PyObject * -zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyObject *key) -/*[clinic end generated code: output=751c6894ad66f91b input=bb24afd84a80ba46]*/ +zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *key) +/*[clinic end generated code: output=b0b09b3344c171b7 input=0238f3d56b1ea3f1]*/ { - PyObject *out = zoneinfo_new_instance(type, key); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + PyObject *out = zoneinfo_new_instance(state, type, key); if (out != NULL) { ((PyZoneInfo_ZoneInfo *)out)->source = SOURCE_NOCACHE; } @@ -412,6 +473,8 @@ zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyObject *key) @classmethod zoneinfo.ZoneInfo.clear_cache + cls: defining_class + / * only_keys: object = None @@ -419,10 +482,12 @@ Clear the ZoneInfo cache. [clinic start generated code]*/ static PyObject * -zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyObject *only_keys) -/*[clinic end generated code: output=eec0a3276f07bd90 input=8cff0182a95f295b]*/ +zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *only_keys) +/*[clinic end generated code: output=114d9b7c8a22e660 input=e32ca3bb396788ba]*/ { - PyObject *weak_cache = get_weak_cache(type); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + PyObject *weak_cache = get_weak_cache(state, type); if (only_keys == NULL || only_keys == Py_None) { PyObject *rv = PyObject_CallMethod(weak_cache, "clear", NULL); @@ -430,7 +495,7 @@ zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyObject *only_keys) Py_DECREF(rv); } - clear_strong_cache(type); + clear_strong_cache(state, type); } else { PyObject *item = NULL; @@ -447,7 +512,7 @@ zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyObject *only_keys) while ((item = PyIter_Next(iter))) { // Remove from strong cache - if (eject_from_strong_cache(type, item) < 0) { + if (eject_from_strong_cache(state, type, item) < 0) { Py_DECREF(item); break; } @@ -473,30 +538,68 @@ zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyObject *only_keys) Py_RETURN_NONE; } +/*[clinic input] +zoneinfo.ZoneInfo.utcoffset + + cls: defining_class + dt: object + / + +Retrieve a timedelta representing the UTC offset in a zone at the given datetime. +[clinic start generated code]*/ + static PyObject * -zoneinfo_utcoffset(PyObject *self, PyObject *dt) +zoneinfo_ZoneInfo_utcoffset_impl(PyObject *self, PyTypeObject *cls, + PyObject *dt) +/*[clinic end generated code: output=b71016c319ba1f91 input=2bb6c5364938f19c]*/ { - _ttinfo *tti = find_ttinfo((PyZoneInfo_ZoneInfo *)self, dt); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + _ttinfo *tti = find_ttinfo(state, (PyZoneInfo_ZoneInfo *)self, dt); if (tti == NULL) { return NULL; } return Py_NewRef(tti->utcoff); } +/*[clinic input] +zoneinfo.ZoneInfo.dst + + cls: defining_class + dt: object + / + +Retrieve a timedelta representing the amount of DST applied in a zone at the given datetime. +[clinic start generated code]*/ + static PyObject * -zoneinfo_dst(PyObject *self, PyObject *dt) +zoneinfo_ZoneInfo_dst_impl(PyObject *self, PyTypeObject *cls, PyObject *dt) +/*[clinic end generated code: output=cb6168d7723a6ae6 input=2167fb80cf8645c6]*/ { - _ttinfo *tti = find_ttinfo((PyZoneInfo_ZoneInfo *)self, dt); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + _ttinfo *tti = find_ttinfo(state, (PyZoneInfo_ZoneInfo *)self, dt); if (tti == NULL) { return NULL; } return Py_NewRef(tti->dstoff); } +/*[clinic input] +zoneinfo.ZoneInfo.tzname + + cls: defining_class + dt: object + / + +Retrieve a string containing the abbreviation for the time zone that applies in a zone at a given datetime. +[clinic start generated code]*/ + static PyObject * -zoneinfo_tzname(PyObject *self, PyObject *dt) +zoneinfo_ZoneInfo_tzname_impl(PyObject *self, PyTypeObject *cls, + PyObject *dt) +/*[clinic end generated code: output=3b6ae6c3053ea75a input=15a59a4f92ed1f1f]*/ { - _ttinfo *tti = find_ttinfo((PyZoneInfo_ZoneInfo *)self, dt); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + _ttinfo *tti = find_ttinfo(state, (PyZoneInfo_ZoneInfo *)self, dt); if (tti == NULL) { return NULL; } @@ -693,28 +796,37 @@ zoneinfo_reduce(PyObject *obj_self, PyObject *unused) return rv; } +/*[clinic input] +@classmethod +zoneinfo.ZoneInfo._unpickle + + cls: defining_class + key: object + from_cache: unsigned_char(bitwise=True) + / + +Private method used in unpickling. +[clinic start generated code]*/ + static PyObject * -zoneinfo__unpickle(PyTypeObject *cls, PyObject *args) +zoneinfo_ZoneInfo__unpickle_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *key, unsigned char from_cache) +/*[clinic end generated code: output=556712fc709deecb input=6ac8c73eed3de316]*/ { - PyObject *key; - unsigned char from_cache; - if (!PyArg_ParseTuple(args, "OB", &key, &from_cache)) { - return NULL; - } - if (from_cache) { PyObject *val_args = Py_BuildValue("(O)", key); if (val_args == NULL) { return NULL; } - PyObject *rv = zoneinfo_new(cls, val_args, NULL); + PyObject *rv = zoneinfo_new(type, val_args, NULL); Py_DECREF(val_args); return rv; } else { - return zoneinfo_new_instance(cls, key); + zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); + return zoneinfo_new_instance(state, type, key); } } @@ -732,14 +844,14 @@ zoneinfo__unpickle(PyTypeObject *cls, PyObject *args) * This returns a new reference to the timedelta. */ static PyObject * -load_timedelta(long seconds) +load_timedelta(zoneinfo_state *state, long seconds) { PyObject *rv; PyObject *pyoffset = PyLong_FromLong(seconds); if (pyoffset == NULL) { return NULL; } - rv = PyDict_GetItemWithError(TIMEDELTA_CACHE, pyoffset); + rv = PyDict_GetItemWithError(state->TIMEDELTA_CACHE, pyoffset); if (rv == NULL) { if (PyErr_Occurred()) { goto error; @@ -751,7 +863,7 @@ load_timedelta(long seconds) goto error; } - rv = PyDict_SetDefault(TIMEDELTA_CACHE, pyoffset, tmp); + rv = PyDict_SetDefault(state->TIMEDELTA_CACHE, pyoffset, tmp); Py_DECREF(tmp); } @@ -768,19 +880,20 @@ load_timedelta(long seconds) * initialized _ttinfo objects. */ static int -build_ttinfo(long utcoffset, long dstoffset, PyObject *tzname, _ttinfo *out) +build_ttinfo(zoneinfo_state *state, long utcoffset, long dstoffset, + PyObject *tzname, _ttinfo *out) { out->utcoff = NULL; out->dstoff = NULL; out->tzname = NULL; out->utcoff_seconds = utcoffset; - out->utcoff = load_timedelta(utcoffset); + out->utcoff = load_timedelta(state, utcoffset); if (out->utcoff == NULL) { return -1; } - out->dstoff = load_timedelta(dstoffset); + out->dstoff = load_timedelta(state, dstoffset); if (out->dstoff == NULL) { return -1; } @@ -836,7 +949,7 @@ ttinfo_eq(const _ttinfo *const tti0, const _ttinfo *const tti1) * the object only needs to be freed / deallocated if this succeeds. */ static int -load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) +load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) { PyObject *data_tuple = NULL; @@ -854,7 +967,8 @@ load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) size_t ttinfos_allocated = 0; - data_tuple = PyObject_CallMethod(_common_mod, "load_data", "O", file_obj); + data_tuple = PyObject_CallMethod(state->_common_mod, "load_data", "O", + file_obj); if (data_tuple == NULL) { goto error; @@ -1012,7 +1126,9 @@ load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } ttinfos_allocated++; - if (build_ttinfo(utcoff[i], dstoff[i], tzname, &(self->_ttinfos[i]))) { + int rc = build_ttinfo(state, utcoff[i], dstoff[i], tzname, + &(self->_ttinfos[i])); + if (rc) { goto error; } } @@ -1044,7 +1160,7 @@ load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } if (tz_str != Py_None && PyObject_IsTrue(tz_str)) { - if (parse_tz_str(tz_str, &(self->tzrule_after))) { + if (parse_tz_str(state, tz_str, &(self->tzrule_after))) { goto error; } } @@ -1063,8 +1179,8 @@ load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } _ttinfo *tti = &(self->_ttinfos[idx]); - build_tzrule(tti->tzname, NULL, tti->utcoff_seconds, 0, NULL, NULL, - &(self->tzrule_after)); + build_tzrule(state, tti->tzname, NULL, tti->utcoff_seconds, 0, NULL, + NULL, &(self->tzrule_after)); // We've abused the build_tzrule constructor to construct an STD-only // rule mimicking whatever ttinfo we've picked up, but it's possible @@ -1463,7 +1579,7 @@ find_tzrule_ttinfo_fromutc(_tzrule *rule, int64_t ts, int year, * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html */ static int -parse_tz_str(PyObject *tz_str_obj, _tzrule *out) +parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out) { PyObject *std_abbr = NULL; PyObject *dst_abbr = NULL; @@ -1555,7 +1671,8 @@ parse_tz_str(PyObject *tz_str_obj, _tzrule *out) } complete: - build_tzrule(std_abbr, dst_abbr, std_offset, dst_offset, start, end, out); + build_tzrule(state, std_abbr, dst_abbr, std_offset, dst_offset, + start, end, out); Py_DECREF(std_abbr); Py_XDECREF(dst_abbr); @@ -1913,8 +2030,8 @@ parse_transition_time(const char *const p, int8_t *hour, int8_t *minute, * Returns 0 on success. */ static int -build_tzrule(PyObject *std_abbr, PyObject *dst_abbr, long std_offset, - long dst_offset, TransitionRuleType *start, +build_tzrule(zoneinfo_state *state, PyObject *std_abbr, PyObject *dst_abbr, + long std_offset, long dst_offset, TransitionRuleType *start, TransitionRuleType *end, _tzrule *out) { _tzrule rv = {{0}}; @@ -1922,13 +2039,13 @@ build_tzrule(PyObject *std_abbr, PyObject *dst_abbr, long std_offset, rv.start = start; rv.end = end; - if (build_ttinfo(std_offset, 0, std_abbr, &rv.std)) { + if (build_ttinfo(state, std_offset, 0, std_abbr, &rv.std)) { goto error; } if (dst_abbr != NULL) { rv.dst_diff = dst_offset - std_offset; - if (build_ttinfo(dst_offset, rv.dst_diff, dst_abbr, &rv.dst)) { + if (build_ttinfo(state, dst_offset, rv.dst_diff, dst_abbr, &rv.dst)) { goto error; } } @@ -2132,7 +2249,7 @@ _bisect(const int64_t value, const int64_t *arr, size_t size) /* Find the ttinfo rules that apply at a given local datetime. */ static _ttinfo * -find_ttinfo(PyZoneInfo_ZoneInfo *self, PyObject *dt) +find_ttinfo(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *dt) { // datetime.time has a .tzinfo attribute that passes None as the dt // argument; it only really has meaning for fixed-offset zones. @@ -2141,7 +2258,7 @@ find_ttinfo(PyZoneInfo_ZoneInfo *self, PyObject *dt) return &(self->tzrule_after.std); } else { - return &NO_TTINFO; + return &(state->NO_TTINFO); } } @@ -2317,10 +2434,10 @@ strong_cache_free(StrongCacheNode *root) * the front of the cache. */ static void -remove_from_strong_cache(StrongCacheNode *node) +remove_from_strong_cache(zoneinfo_state *state, StrongCacheNode *node) { - if (ZONEINFO_STRONG_CACHE == node) { - ZONEINFO_STRONG_CACHE = node->next; + if (state->ZONEINFO_STRONG_CACHE == node) { + state->ZONEINFO_STRONG_CACHE = node->next; } if (node->prev != NULL) { @@ -2366,15 +2483,17 @@ find_in_strong_cache(const StrongCacheNode *const root, PyObject *const key) * This function is used to enable the per-key functionality in clear_cache. */ static int -eject_from_strong_cache(const PyTypeObject *const type, PyObject *key) +eject_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *key) { - if (type != &PyZoneInfo_ZoneInfoType) { + if (type != state->ZoneInfoType) { return 0; } - StrongCacheNode *node = find_in_strong_cache(ZONEINFO_STRONG_CACHE, key); + StrongCacheNode *cache = state->ZONEINFO_STRONG_CACHE; + StrongCacheNode *node = find_in_strong_cache(cache, key); if (node != NULL) { - remove_from_strong_cache(node); + remove_from_strong_cache(state, node); strong_cache_node_free(node); } @@ -2390,14 +2509,15 @@ eject_from_strong_cache(const PyTypeObject *const type, PyObject *key) * it is not at the front of the cache, it needs to be moved there. */ static void -move_strong_cache_node_to_front(StrongCacheNode **root, StrongCacheNode *node) +move_strong_cache_node_to_front(zoneinfo_state *state, StrongCacheNode **root, + StrongCacheNode *node) { StrongCacheNode *root_p = *root; if (root_p == node) { return; } - remove_from_strong_cache(node); + remove_from_strong_cache(state, node); node->prev = NULL; node->next = root_p; @@ -2419,16 +2539,19 @@ move_strong_cache_node_to_front(StrongCacheNode **root, StrongCacheNode *node) * always returns a cache miss for subclasses. */ static PyObject * -zone_from_strong_cache(const PyTypeObject *const type, PyObject *const key) +zone_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *const key) { - if (type != &PyZoneInfo_ZoneInfoType) { + if (type != state->ZoneInfoType) { return NULL; // Strong cache currently only implemented for base class } - StrongCacheNode *node = find_in_strong_cache(ZONEINFO_STRONG_CACHE, key); + StrongCacheNode *cache = state->ZONEINFO_STRONG_CACHE; + StrongCacheNode *node = find_in_strong_cache(cache, key); if (node != NULL) { - move_strong_cache_node_to_front(&ZONEINFO_STRONG_CACHE, node); + StrongCacheNode **root = &(state->ZONEINFO_STRONG_CACHE); + move_strong_cache_node_to_front(state, root, node); return Py_NewRef(node->zone); } @@ -2442,16 +2565,16 @@ zone_from_strong_cache(const PyTypeObject *const type, PyObject *const key) * the cache to at most ZONEINFO_STRONG_CACHE_MAX_SIZE). */ static void -update_strong_cache(const PyTypeObject *const type, PyObject *key, - PyObject *zone) +update_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, + PyObject *key, PyObject *zone) { - if (type != &PyZoneInfo_ZoneInfoType) { + if (type != state->ZoneInfoType) { return; } StrongCacheNode *new_node = strong_cache_node_new(key, zone); - - move_strong_cache_node_to_front(&ZONEINFO_STRONG_CACHE, new_node); + StrongCacheNode **root = &(state->ZONEINFO_STRONG_CACHE); + move_strong_cache_node_to_front(state, root, new_node); StrongCacheNode *node = new_node->next; for (size_t i = 1; i < ZONEINFO_STRONG_CACHE_MAX_SIZE; ++i) { @@ -2476,14 +2599,14 @@ update_strong_cache(const PyTypeObject *const type, PyObject *key, * for everything except the base class. */ void -clear_strong_cache(const PyTypeObject *const type) +clear_strong_cache(zoneinfo_state *state, const PyTypeObject *const type) { - if (type != &PyZoneInfo_ZoneInfoType) { + if (type != state->ZoneInfoType) { return; } - strong_cache_free(ZONEINFO_STRONG_CACHE); - ZONEINFO_STRONG_CACHE = NULL; + strong_cache_free(state->ZONEINFO_STRONG_CACHE); + state->ZONEINFO_STRONG_CACHE = NULL; } static PyObject * @@ -2499,29 +2622,17 @@ new_weak_cache(void) return weak_cache; } +// This function is not idempotent and must be called on a new module object. static int -initialize_caches(void) +initialize_caches(zoneinfo_state *state) { - // TODO: Move to a PyModule_GetState / PEP 573 based caching system. - if (TIMEDELTA_CACHE == NULL) { - TIMEDELTA_CACHE = PyDict_New(); - } - else { - Py_INCREF(TIMEDELTA_CACHE); - } - - if (TIMEDELTA_CACHE == NULL) { + state->TIMEDELTA_CACHE = PyDict_New(); + if (state->TIMEDELTA_CACHE == NULL) { return -1; } - if (ZONEINFO_WEAK_CACHE == NULL) { - ZONEINFO_WEAK_CACHE = new_weak_cache(); - } - else { - Py_INCREF(ZONEINFO_WEAK_CACHE); - } - - if (ZONEINFO_WEAK_CACHE == NULL) { + state->ZONEINFO_WEAK_CACHE = new_weak_cache(); + if (state->ZONEINFO_WEAK_CACHE == NULL) { return -1; } @@ -2551,22 +2662,15 @@ static PyMethodDef zoneinfo_methods[] = { ZONEINFO_ZONEINFO_CLEAR_CACHE_METHODDEF ZONEINFO_ZONEINFO_NO_CACHE_METHODDEF ZONEINFO_ZONEINFO_FROM_FILE_METHODDEF - {"utcoffset", (PyCFunction)zoneinfo_utcoffset, METH_O, - PyDoc_STR("Retrieve a timedelta representing the UTC offset in a zone at " - "the given datetime.")}, - {"dst", (PyCFunction)zoneinfo_dst, METH_O, - PyDoc_STR("Retrieve a timedelta representing the amount of DST applied " - "in a zone at the given datetime.")}, - {"tzname", (PyCFunction)zoneinfo_tzname, METH_O, - PyDoc_STR("Retrieve a string containing the abbreviation for the time " - "zone that applies in a zone at a given datetime.")}, + ZONEINFO_ZONEINFO_UTCOFFSET_METHODDEF + ZONEINFO_ZONEINFO_DST_METHODDEF + ZONEINFO_ZONEINFO_TZNAME_METHODDEF {"fromutc", (PyCFunction)zoneinfo_fromutc, METH_O, PyDoc_STR("Given a datetime with local time in UTC, retrieve an adjusted " "datetime in local time.")}, {"__reduce__", (PyCFunction)zoneinfo_reduce, METH_NOARGS, PyDoc_STR("Function for serialization with the pickle protocol.")}, - {"_unpickle", (PyCFunction)zoneinfo__unpickle, METH_VARARGS | METH_CLASS, - PyDoc_STR("Private method used in unpickling.")}, + ZONEINFO_ZONEINFO__UNPICKLE_METHODDEF {"__init_subclass__", (PyCFunction)(void (*)(void))zoneinfo_init_subclass, METH_VARARGS | METH_KEYWORDS | METH_CLASS, PyDoc_STR("Function to initialize subclasses.")}, @@ -2579,50 +2683,88 @@ static PyMemberDef zoneinfo_members[] = { .type = T_OBJECT_EX, .flags = READONLY, .doc = NULL}, + {.name = "__weaklistoffset__", + .offset = offsetof(PyZoneInfo_ZoneInfo, weakreflist), + .type = T_PYSSIZET, + .flags = READONLY}, {NULL}, /* Sentinel */ }; -static PyTypeObject PyZoneInfo_ZoneInfoType = { - PyVarObject_HEAD_INIT(NULL, 0) // - .tp_name = "zoneinfo.ZoneInfo", - .tp_basicsize = sizeof(PyZoneInfo_ZoneInfo), - .tp_weaklistoffset = offsetof(PyZoneInfo_ZoneInfo, weakreflist), - .tp_repr = (reprfunc)zoneinfo_repr, - .tp_str = (reprfunc)zoneinfo_str, - .tp_getattro = PyObject_GenericGetAttr, - .tp_flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE), - /* .tp_doc = zoneinfo_doc, */ - .tp_methods = zoneinfo_methods, - .tp_members = zoneinfo_members, - .tp_new = zoneinfo_new, - .tp_dealloc = zoneinfo_dealloc, +static PyType_Slot zoneinfo_slots[] = { + {Py_tp_repr, zoneinfo_repr}, + {Py_tp_str, zoneinfo_str}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_methods, zoneinfo_methods}, + {Py_tp_members, zoneinfo_members}, + {Py_tp_new, zoneinfo_new}, + {Py_tp_dealloc, zoneinfo_dealloc}, + {Py_tp_traverse, zoneinfo_traverse}, + {Py_tp_clear, zoneinfo_clear}, + {0, NULL}, +}; + +static PyType_Spec zoneinfo_spec = { + .name = "zoneinfo.ZoneInfo", + .basicsize = sizeof(PyZoneInfo_ZoneInfo), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE), + .slots = zoneinfo_slots, }; ///// // Specify the _zoneinfo module static PyMethodDef module_methods[] = {{NULL, NULL}}; -static void -module_free(void *m) + +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) { - Py_CLEAR(_tzpath_find_tzfile); - Py_CLEAR(_common_mod); - Py_CLEAR(io_open); + zoneinfo_state *state = zoneinfo_get_state(mod); - xdecref_ttinfo(&NO_TTINFO); + Py_VISIT(state->ZoneInfoType); + Py_VISIT(state->io_open); + Py_VISIT(state->_tzpath_find_tzfile); + Py_VISIT(state->_common_mod); + Py_VISIT(state->TIMEDELTA_CACHE); + Py_VISIT(state->ZONEINFO_WEAK_CACHE); - if (TIMEDELTA_CACHE != NULL && Py_REFCNT(TIMEDELTA_CACHE) > 1) { - Py_DECREF(TIMEDELTA_CACHE); - } else { - Py_CLEAR(TIMEDELTA_CACHE); + StrongCacheNode *node = state->ZONEINFO_STRONG_CACHE; + while (node != NULL) { + StrongCacheNode *next = node->next; + Py_VISIT(node->key); + Py_VISIT(node->zone); + node = next; } - if (ZONEINFO_WEAK_CACHE != NULL && Py_REFCNT(ZONEINFO_WEAK_CACHE) > 1) { - Py_DECREF(ZONEINFO_WEAK_CACHE); - } else { - Py_CLEAR(ZONEINFO_WEAK_CACHE); - } + Py_VISIT(state->NO_TTINFO.utcoff); + Py_VISIT(state->NO_TTINFO.dstoff); + Py_VISIT(state->NO_TTINFO.tzname); - clear_strong_cache(&PyZoneInfo_ZoneInfoType); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + zoneinfo_state *state = zoneinfo_get_state(mod); + + Py_CLEAR(state->ZoneInfoType); + Py_CLEAR(state->io_open); + Py_CLEAR(state->_tzpath_find_tzfile); + Py_CLEAR(state->_common_mod); + Py_CLEAR(state->TIMEDELTA_CACHE); + Py_CLEAR(state->ZONEINFO_WEAK_CACHE); + clear_strong_cache(state, state->ZoneInfoType); + Py_CLEAR(state->NO_TTINFO.utcoff); + Py_CLEAR(state->NO_TTINFO.dstoff); + Py_CLEAR(state->NO_TTINFO.tzname); + + return 0; +} + +static void +module_free(void *mod) +{ + (void)module_clear((PyObject *)mod); } static int @@ -2632,39 +2774,45 @@ zoneinfomodule_exec(PyObject *m) if (PyDateTimeAPI == NULL) { goto error; } - PyZoneInfo_ZoneInfoType.tp_base = PyDateTimeAPI->TZInfoType; - if (PyType_Ready(&PyZoneInfo_ZoneInfoType) < 0) { + + zoneinfo_state *state = zoneinfo_get_state(m); + PyObject *base = (PyObject *)PyDateTimeAPI->TZInfoType; + state->ZoneInfoType = (PyTypeObject *)PyType_FromModuleAndSpec(m, + &zoneinfo_spec, base); + if (state->ZoneInfoType == NULL) { goto error; } - if (PyModule_AddObjectRef(m, "ZoneInfo", (PyObject *)&PyZoneInfo_ZoneInfoType) < 0) { + int rc = PyModule_AddObjectRef(m, "ZoneInfo", + (PyObject *)state->ZoneInfoType); + if (rc < 0) { goto error; } /* Populate imports */ - _tzpath_find_tzfile = + state->_tzpath_find_tzfile = _PyImport_GetModuleAttrString("zoneinfo._tzpath", "find_tzfile"); - if (_tzpath_find_tzfile == NULL) { + if (state->_tzpath_find_tzfile == NULL) { goto error; } - io_open = _PyImport_GetModuleAttrString("io", "open"); - if (io_open == NULL) { + state->io_open = _PyImport_GetModuleAttrString("io", "open"); + if (state->io_open == NULL) { goto error; } - _common_mod = PyImport_ImportModule("zoneinfo._common"); - if (_common_mod == NULL) { + state->_common_mod = PyImport_ImportModule("zoneinfo._common"); + if (state->_common_mod == NULL) { goto error; } - if (NO_TTINFO.utcoff == NULL) { - NO_TTINFO.utcoff = Py_NewRef(Py_None); - NO_TTINFO.dstoff = Py_NewRef(Py_None); - NO_TTINFO.tzname = Py_NewRef(Py_None); + if (state->NO_TTINFO.utcoff == NULL) { + state->NO_TTINFO.utcoff = Py_NewRef(Py_None); + state->NO_TTINFO.dstoff = Py_NewRef(Py_None); + state->NO_TTINFO.tzname = Py_NewRef(Py_None); } - if (initialize_caches()) { + if (initialize_caches(state)) { goto error; } @@ -2678,13 +2826,16 @@ static PyModuleDef_Slot zoneinfomodule_slots[] = { {Py_mod_exec, zoneinfomodule_exec}, {0, NULL}}; static struct PyModuleDef zoneinfomodule = { - PyModuleDef_HEAD_INIT, + .m_base = PyModuleDef_HEAD_INIT, .m_name = "_zoneinfo", .m_doc = "C implementation of the zoneinfo module", - .m_size = 0, + .m_size = sizeof(zoneinfo_state), .m_methods = module_methods, .m_slots = zoneinfomodule_slots, - .m_free = (freefunc)module_free}; + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free, +}; PyMODINIT_FUNC PyInit__zoneinfo(void) diff --git a/Modules/clinic/_zoneinfo.c.h b/Modules/clinic/_zoneinfo.c.h index 78fcbfa9411bb8..ae62865e0f67df 100644 --- a/Modules/clinic/_zoneinfo.c.h +++ b/Modules/clinic/_zoneinfo.c.h @@ -15,14 +15,14 @@ PyDoc_STRVAR(zoneinfo_ZoneInfo_from_file__doc__, "Create a ZoneInfo file from a file object."); #define ZONEINFO_ZONEINFO_FROM_FILE_METHODDEF \ - {"from_file", _PyCFunction_CAST(zoneinfo_ZoneInfo_from_file), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_from_file__doc__}, + {"from_file", _PyCFunction_CAST(zoneinfo_ZoneInfo_from_file), METH_METHOD|METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_from_file__doc__}, static PyObject * -zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyObject *file_obj, - PyObject *key); +zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *file_obj, PyObject *key); static PyObject * -zoneinfo_ZoneInfo_from_file(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zoneinfo_ZoneInfo_from_file(PyTypeObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -65,7 +65,7 @@ zoneinfo_ZoneInfo_from_file(PyTypeObject *type, PyObject *const *args, Py_ssize_ } key = args[1]; skip_optional_pos: - return_value = zoneinfo_ZoneInfo_from_file_impl(type, file_obj, key); + return_value = zoneinfo_ZoneInfo_from_file_impl(type, cls, file_obj, key); exit: return return_value; @@ -78,13 +78,14 @@ PyDoc_STRVAR(zoneinfo_ZoneInfo_no_cache__doc__, "Get a new instance of ZoneInfo, bypassing the cache."); #define ZONEINFO_ZONEINFO_NO_CACHE_METHODDEF \ - {"no_cache", _PyCFunction_CAST(zoneinfo_ZoneInfo_no_cache), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_no_cache__doc__}, + {"no_cache", _PyCFunction_CAST(zoneinfo_ZoneInfo_no_cache), METH_METHOD|METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_no_cache__doc__}, static PyObject * -zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyObject *key); +zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *key); static PyObject * -zoneinfo_ZoneInfo_no_cache(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zoneinfo_ZoneInfo_no_cache(PyTypeObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -120,7 +121,7 @@ zoneinfo_ZoneInfo_no_cache(PyTypeObject *type, PyObject *const *args, Py_ssize_t goto exit; } key = args[0]; - return_value = zoneinfo_ZoneInfo_no_cache_impl(type, key); + return_value = zoneinfo_ZoneInfo_no_cache_impl(type, cls, key); exit: return return_value; @@ -133,13 +134,14 @@ PyDoc_STRVAR(zoneinfo_ZoneInfo_clear_cache__doc__, "Clear the ZoneInfo cache."); #define ZONEINFO_ZONEINFO_CLEAR_CACHE_METHODDEF \ - {"clear_cache", _PyCFunction_CAST(zoneinfo_ZoneInfo_clear_cache), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_clear_cache__doc__}, + {"clear_cache", _PyCFunction_CAST(zoneinfo_ZoneInfo_clear_cache), METH_METHOD|METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo_clear_cache__doc__}, static PyObject * -zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyObject *only_keys); +zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *only_keys); static PyObject * -zoneinfo_ZoneInfo_clear_cache(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zoneinfo_ZoneInfo_clear_cache(PyTypeObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -180,9 +182,194 @@ zoneinfo_ZoneInfo_clear_cache(PyTypeObject *type, PyObject *const *args, Py_ssiz } only_keys = args[0]; skip_optional_kwonly: - return_value = zoneinfo_ZoneInfo_clear_cache_impl(type, only_keys); + return_value = zoneinfo_ZoneInfo_clear_cache_impl(type, cls, only_keys); exit: return return_value; } -/*[clinic end generated code: output=d2da73ef66146b83 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(zoneinfo_ZoneInfo_utcoffset__doc__, +"utcoffset($self, dt, /)\n" +"--\n" +"\n" +"Retrieve a timedelta representing the UTC offset in a zone at the given datetime."); + +#define ZONEINFO_ZONEINFO_UTCOFFSET_METHODDEF \ + {"utcoffset", _PyCFunction_CAST(zoneinfo_ZoneInfo_utcoffset), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, zoneinfo_ZoneInfo_utcoffset__doc__}, + +static PyObject * +zoneinfo_ZoneInfo_utcoffset_impl(PyObject *self, PyTypeObject *cls, + PyObject *dt); + +static PyObject * +zoneinfo_ZoneInfo_utcoffset(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "utcoffset", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *dt; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + dt = args[0]; + return_value = zoneinfo_ZoneInfo_utcoffset_impl(self, cls, dt); + +exit: + return return_value; +} + +PyDoc_STRVAR(zoneinfo_ZoneInfo_dst__doc__, +"dst($self, dt, /)\n" +"--\n" +"\n" +"Retrieve a timedelta representing the amount of DST applied in a zone at the given datetime."); + +#define ZONEINFO_ZONEINFO_DST_METHODDEF \ + {"dst", _PyCFunction_CAST(zoneinfo_ZoneInfo_dst), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, zoneinfo_ZoneInfo_dst__doc__}, + +static PyObject * +zoneinfo_ZoneInfo_dst_impl(PyObject *self, PyTypeObject *cls, PyObject *dt); + +static PyObject * +zoneinfo_ZoneInfo_dst(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "dst", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *dt; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + dt = args[0]; + return_value = zoneinfo_ZoneInfo_dst_impl(self, cls, dt); + +exit: + return return_value; +} + +PyDoc_STRVAR(zoneinfo_ZoneInfo_tzname__doc__, +"tzname($self, dt, /)\n" +"--\n" +"\n" +"Retrieve a string containing the abbreviation for the time zone that applies in a zone at a given datetime."); + +#define ZONEINFO_ZONEINFO_TZNAME_METHODDEF \ + {"tzname", _PyCFunction_CAST(zoneinfo_ZoneInfo_tzname), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, zoneinfo_ZoneInfo_tzname__doc__}, + +static PyObject * +zoneinfo_ZoneInfo_tzname_impl(PyObject *self, PyTypeObject *cls, + PyObject *dt); + +static PyObject * +zoneinfo_ZoneInfo_tzname(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "tzname", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *dt; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + dt = args[0]; + return_value = zoneinfo_ZoneInfo_tzname_impl(self, cls, dt); + +exit: + return return_value; +} + +PyDoc_STRVAR(zoneinfo_ZoneInfo__unpickle__doc__, +"_unpickle($type, key, from_cache, /)\n" +"--\n" +"\n" +"Private method used in unpickling."); + +#define ZONEINFO_ZONEINFO__UNPICKLE_METHODDEF \ + {"_unpickle", _PyCFunction_CAST(zoneinfo_ZoneInfo__unpickle), METH_METHOD|METH_FASTCALL|METH_KEYWORDS|METH_CLASS, zoneinfo_ZoneInfo__unpickle__doc__}, + +static PyObject * +zoneinfo_ZoneInfo__unpickle_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *key, unsigned char from_cache); + +static PyObject * +zoneinfo_ZoneInfo__unpickle(PyTypeObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_unpickle", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *key; + unsigned char from_cache; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + key = args[0]; + { + unsigned long ival = PyLong_AsUnsignedLongMask(args[1]); + if (ival == (unsigned long)-1 && PyErr_Occurred()) { + goto exit; + } + else { + from_cache = (unsigned char) ival; + } + } + return_value = zoneinfo_ZoneInfo__unpickle_impl(type, cls, key, from_cache); + +exit: + return return_value; +} +/*[clinic end generated code: output=54051388dfc408af input=a9049054013a1b77]*/ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 52ea0b4901d4bb..6011b1604508af 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -403,7 +403,6 @@ Modules/_pickle.c - PicklerMemoProxyType - Modules/_pickle.c - Pickler_Type - Modules/_pickle.c - UnpicklerMemoProxyType - Modules/_pickle.c - Unpickler_Type - -Modules/_zoneinfo.c - PyZoneInfo_ZoneInfoType - Modules/ossaudiodev.c - OSSAudioType - Modules/ossaudiodev.c - OSSMixerType - Modules/socketmodule.c - sock_type - @@ -442,11 +441,6 @@ Modules/xxmodule.c - ErrorObject - Modules/_ctypes/callproc.c _ctypes_get_errobj error_object_name - Modules/_ctypes/_ctypes.c CreateSwappedType suffix - -## other - during module init -Modules/_zoneinfo.c - io_open - -Modules/_zoneinfo.c - _tzpath_find_tzfile - -Modules/_zoneinfo.c - _common_mod - - ##----------------------- ## other @@ -481,8 +475,6 @@ Modules/_tkinter.c - tcl_lock - Modules/_tkinter.c - excInCmd - Modules/_tkinter.c - valInCmd - Modules/_tkinter.c - trbInCmd - -Modules/_zoneinfo.c - TIMEDELTA_CACHE - -Modules/_zoneinfo.c - ZONEINFO_WEAK_CACHE - ################################## @@ -556,9 +548,6 @@ Modules/_tkinter.c - HeadFHCD - Modules/_tkinter.c - stdin_ready - Modules/_tkinter.c - event_tstate - Modules/_xxsubinterpretersmodule.c - _globals - -Modules/_zoneinfo.c - ZONEINFO_STRONG_CACHE - -Modules/_zoneinfo.c - ZONEINFO_STRONG_CACHE_MAX_SIZE - -Modules/_zoneinfo.c - NO_TTINFO - Modules/readline.c - completer_word_break_characters - Modules/readline.c - _history_length - Modules/readline.c - should_auto_add_history - From b2fc5492789623d656953d458f3eeaac03c1ef56 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 15 Feb 2023 15:32:31 -0700 Subject: [PATCH 46/83] gh-101758: Clean Up Uses of Import State (gh-101919) This change is almost entirely moving code around and hiding import state behind internal API. We introduce no changes to behavior, nor to non-internal API. (Since there was already going to be a lot of churn, I took this as an opportunity to re-organize import.c into topically-grouped sections of code.) The motivation is to simplify a number of upcoming changes. Specific changes: * move existing import-related code to import.c, wherever possible * add internal API for interacting with import state (both global and per-interpreter) * use only API outside of import.c (to limit churn there when changing the location, etc.) * consolidate the import-related state of PyInterpreterState into a single struct field (this changes layout slightly) * add macros for import state in import.c (to simplify changing the location) * group code in import.c into sections *remove _PyState_AddModule() https://github.com/python/cpython/issues/101758 --- Include/internal/pycore_import.h | 103 +- Include/internal/pycore_interp.h | 35 +- Include/internal/pycore_pylifecycle.h | 4 - Include/internal/pycore_pystate.h | 6 - Include/internal/pycore_runtime_init.h | 15 +- Include/internal/pycore_sysmodule.h | 3 + Lib/test/test_imp.py | 2 +- Lib/test/test_stable_abi_ctypes.py | 1 - Misc/stable_abi.toml | 3 - Objects/moduleobject.c | 25 +- PC/python3dll.c | 1 - Python/_warnings.c | 11 +- Python/ceval.c | 2 +- Python/dynload_shlib.c | 2 +- Python/import.c | 2210 +++++++++++++++--------- Python/importdl.c | 17 +- Python/pylifecycle.c | 127 +- Python/pystate.c | 151 +- Python/pythonrun.c | 10 +- Python/sysmodule.c | 40 +- 20 files changed, 1586 insertions(+), 1182 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 9036dff6725330..da766253ef6b9c 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -36,11 +36,112 @@ struct _import_runtime_state { const char * pkgcontext; }; +struct _import_state { + /* cached sys.modules dictionary */ + PyObject *modules; + /* This is the list of module objects for all legacy (single-phase init) + extension modules ever loaded in this process (i.e. imported + in this interpreter or in any other). Py_None stands in for + modules that haven't actually been imported in this interpreter. + + A module's index (PyModuleDef.m_base.m_index) is used to look up + the corresponding module object for this interpreter, if any. + (See PyState_FindModule().) When any extension module + is initialized during import, its moduledef gets initialized by + PyModuleDef_Init(), and the first time that happens for each + PyModuleDef, its index gets set to the current value of + a global counter (see _PyRuntimeState.imports.last_module_index). + The entry for that index in this interpreter remains unset until + the module is actually imported here. (Py_None is used as + a placeholder.) Note that multi-phase init modules always get + an index for which there will never be a module set. + + This is initialized lazily in PyState_AddModule(), which is also + where modules get added. */ + PyObject *modules_by_index; + /* importlib module._bootstrap */ + PyObject *importlib; + /* override for config->use_frozen_modules (for tests) + (-1: "off", 1: "on", 0: no override) */ + int override_frozen_modules; +#ifdef HAVE_DLOPEN + int dlopenflags; +#endif + PyObject *import_func; +}; + +#ifdef HAVE_DLOPEN +# include +# if HAVE_DECL_RTLD_NOW +# define _Py_DLOPEN_FLAGS RTLD_NOW +# else +# define _Py_DLOPEN_FLAGS RTLD_LAZY +# endif +# define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS, +#else +# define _Py_DLOPEN_FLAGS 0 +# define DLOPENFLAGS_INIT +#endif + +#define IMPORTS_INIT \ + { \ + .override_frozen_modules = 0, \ + DLOPENFLAGS_INIT \ + } + +extern void _PyImport_ClearCore(PyInterpreterState *interp); + +extern Py_ssize_t _PyImport_GetNextModuleIndex(void); +extern const char * _PyImport_ResolveNameWithPackageContext(const char *name); +extern const char * _PyImport_SwapPackageContext(const char *newcontext); + +extern int _PyImport_GetDLOpenFlags(PyInterpreterState *interp); +extern void _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val); + +extern PyObject * _PyImport_InitModules(PyInterpreterState *interp); +extern PyObject * _PyImport_GetModules(PyInterpreterState *interp); +extern void _PyImport_ClearModules(PyInterpreterState *interp); + +extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp); + +extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp); +extern int _PyImport_IsDefaultImportFunc( + PyInterpreterState *interp, + PyObject *func); + +extern PyObject * _PyImport_GetImportlibLoader( + PyInterpreterState *interp, + const char *loader_name); +extern PyObject * _PyImport_GetImportlibExternalLoader( + PyInterpreterState *interp, + const char *loader_name); +extern PyObject * _PyImport_BlessMyLoader( + PyInterpreterState *interp, + PyObject *module_globals); +extern PyObject * _PyImport_ImportlibModuleRepr( + PyInterpreterState *interp, + PyObject *module); + + +extern PyStatus _PyImport_Init(void); +extern void _PyImport_Fini(void); +extern void _PyImport_Fini2(void); + +extern PyStatus _PyImport_InitCore( + PyThreadState *tstate, + PyObject *sysmod, + int importlib); +extern PyStatus _PyImport_InitExternal(PyThreadState *tstate); +extern void _PyImport_FiniCore(PyInterpreterState *interp); +extern void _PyImport_FiniExternal(PyInterpreterState *interp); + #ifdef HAVE_FORK extern PyStatus _PyImport_ReInitLock(void); #endif -extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate); + + +extern PyObject* _PyImport_GetBuiltinModuleNames(void); struct _module_alias { const char *name; /* ASCII encoded string */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 0e3d46852f2e6d..60de31b336f613 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -21,6 +21,7 @@ extern "C" { #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_import.h" // struct _import_state #include "pycore_list.h" // struct _Py_list_state #include "pycore_global_objects.h" // struct _Py_interp_static_objects #include "pycore_tuple.h" // struct _Py_tuple_state @@ -92,37 +93,12 @@ struct _is { struct _ceval_state ceval; struct _gc_runtime_state gc; - // sys.modules dictionary - PyObject *modules; - /* This is the list of module objects for all legacy (single-phase init) - extension modules ever loaded in this process (i.e. imported - in this interpreter or in any other). Py_None stands in for - modules that haven't actually been imported in this interpreter. - - A module's index (PyModuleDef.m_base.m_index) is used to look up - the corresponding module object for this interpreter, if any. - (See PyState_FindModule().) When any extension module - is initialized during import, its moduledef gets initialized by - PyModuleDef_Init(), and the first time that happens for each - PyModuleDef, its index gets set to the current value of - a global counter (see _PyRuntimeState.imports.last_module_index). - The entry for that index in this interpreter remains unset until - the module is actually imported here. (Py_None is used as - a placeholder.) Note that multi-phase init modules always get - an index for which there will never be a module set. - - This is initialized lazily in _PyState_AddModule(), which is also - where modules get added. */ - PyObject *modules_by_index; + struct _import_state imports; + // Dictionary of the sys module PyObject *sysdict; // Dictionary of the builtins module PyObject *builtins; - // importlib module - PyObject *importlib; - // override for config->use_frozen_modules (for tests) - // (-1: "off", 1: "on", 0: no override) - int override_frozen_modules; PyObject *codec_search_path; PyObject *codec_search_cache; @@ -130,15 +106,11 @@ struct _is { int codecs_initialized; PyConfig config; -#ifdef HAVE_DLOPEN - int dlopenflags; -#endif unsigned long feature_flags; PyObject *dict; /* Stores per-interpreter state */ PyObject *builtins_copy; - PyObject *import_func; // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; @@ -205,7 +177,6 @@ struct _is { /* other API */ -extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp); extern void _PyInterpreterState_Clear(PyThreadState *tstate); diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 2d431befd74f99..e7a31807205254 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -30,7 +30,6 @@ PyAPI_FUNC(int) _Py_IsLocaleCoercionTarget(const char *ctype_loc); /* Various one-time initializers */ extern void _Py_InitVersion(void); -extern PyStatus _PyImport_Init(void); extern PyStatus _PyFaulthandler_Init(int enable); extern int _PyTraceMalloc_Init(int enable); extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp); @@ -45,7 +44,6 @@ extern int _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); extern PyStatus _PyTime_Init(void); -extern PyStatus _PyImportZip_Init(PyThreadState *tstate); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); extern int _Py_Deepfreeze_Init(void); @@ -55,8 +53,6 @@ extern int _Py_Deepfreeze_Init(void); extern int _PySignal_Init(int install_signal_handlers); extern void _PySignal_Fini(void); -extern void _PyImport_Fini(void); -extern void _PyImport_Fini2(void); extern void _PyGC_Fini(PyInterpreterState *interp); extern void _Py_HashRandomization_Fini(void); extern void _PyFaulthandler_Fini(void); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 7046ec8d9adaaf..638b86253879ea 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -152,12 +152,6 @@ extern void _PySignal_AfterFork(void); #endif -PyAPI_FUNC(int) _PyState_AddModule( - PyThreadState *tstate, - PyObject* module, - PyModuleDef* def); - - PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate); #define HEAD_LOCK(runtime) \ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index c6a27d076eae2d..a8d5953ff98b0b 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -97,23 +97,10 @@ extern "C" { ._main_interpreter = _PyInterpreterState_INIT, \ } -#ifdef HAVE_DLOPEN -# include -# if HAVE_DECL_RTLD_NOW -# define _Py_DLOPEN_FLAGS RTLD_NOW -# else -# define _Py_DLOPEN_FLAGS RTLD_LAZY -# endif -# define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS, -#else -# define _Py_DLOPEN_FLAGS 0 -# define DLOPENFLAGS_INIT -#endif - #define _PyInterpreterState_INIT \ { \ .id_refcount = -1, \ - DLOPENFLAGS_INIT \ + .imports = IMPORTS_INIT, \ .ceval = { \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ }, \ diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index 10d092cdc30a2c..b4b1febafa4479 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -20,6 +20,9 @@ extern void _PySys_ClearAuditHooks(PyThreadState *tstate); PyAPI_FUNC(int) _PySys_SetAttr(PyObject *, PyObject *); +extern int _PySys_ClearAttrString(PyInterpreterState *interp, + const char *name, int verbose); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 31dce21587e2ca..c85ab92307de78 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -387,7 +387,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, check_basic_reloaded(mod, lookedup, initialized, init_count, before, reloaded) - # Currently _PyState_AddModule() always replaces the cached module. + # Currently PyState_AddModule() always replaces the cached module. self.assertIs(basic.look_up_self(), mod) self.assertEqual(basic.initialized_count(), expected_init_count) diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index e77c1c8409880d..7e50fbda2c07cb 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -864,7 +864,6 @@ def test_windows_feature_macros(self): "_PyObject_GC_Resize", "_PyObject_New", "_PyObject_NewVar", - "_PyState_AddModule", "_PyThreadState_Init", "_PyThreadState_Prealloc", "_PyWeakref_CallableProxyType", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 21ff9616133445..c04a3a228caf56 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1684,9 +1684,6 @@ [function._PyObject_NewVar] added = '3.2' abi_only = true -[function._PyState_AddModule] - added = '3.2' - abi_only = true [function._PyThreadState_Init] added = '3.2' abi_only = true diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 24190e320ee6d6..a0be19a3ca8ac8 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -42,10 +42,9 @@ PyModuleDef_Init(PyModuleDef* def) { assert(PyModuleDef_Type.tp_flags & Py_TPFLAGS_READY); if (def->m_base.m_index == 0) { - _PyRuntime.imports.last_module_index++; Py_SET_REFCNT(def, 1); Py_SET_TYPE(def, &PyModuleDef_Type); - def->m_base.m_index = _PyRuntime.imports.last_module_index; + def->m_base.m_index = _PyImport_GetNextModuleIndex(); } return (PyObject*)def; } @@ -209,24 +208,7 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) "module %s: PyModule_Create is incompatible with m_slots", name); return NULL; } - /* Make sure name is fully qualified. - - This is a bit of a hack: when the shared library is loaded, - the module name is "package.module", but the module calls - PyModule_Create*() with just "module" for the name. The shared - library loader squirrels away the true name of the module in - _Py_PackageContext, and PyModule_Create*() will substitute this - (if the name actually matches). - */ -#define _Py_PackageContext (_PyRuntime.imports.pkgcontext) - if (_Py_PackageContext != NULL) { - const char *p = strrchr(_Py_PackageContext, '.'); - if (p != NULL && strcmp(module->m_name, p+1) == 0) { - name = _Py_PackageContext; - _Py_PackageContext = NULL; - } - } -#undef _Py_PackageContext + name = _PyImport_ResolveNameWithPackageContext(name); if ((m = (PyModuleObject*)PyModule_New(name)) == NULL) return NULL; @@ -710,8 +692,7 @@ static PyObject * module_repr(PyModuleObject *m) { PyInterpreterState *interp = _PyInterpreterState_GET(); - - return PyObject_CallMethod(interp->importlib, "_module_repr", "O", m); + return _PyImport_ImportlibModuleRepr(interp, (PyObject *)m); } /* Check if the "_initializing" attribute of the module spec is set to true. diff --git a/PC/python3dll.c b/PC/python3dll.c index e300819365756e..79f09037282f54 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -34,7 +34,6 @@ EXPORT_FUNC(_PyObject_GC_NewVar) EXPORT_FUNC(_PyObject_GC_Resize) EXPORT_FUNC(_PyObject_New) EXPORT_FUNC(_PyObject_NewVar) -EXPORT_FUNC(_PyState_AddModule) EXPORT_FUNC(_PyThreadState_Init) EXPORT_FUNC(_PyThreadState_Prealloc) EXPORT_FUNC(Py_AddPendingCall) diff --git a/Python/_warnings.c b/Python/_warnings.c index e78f21644f372b..d510381c365b66 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -214,7 +214,7 @@ get_warnings_attr(PyInterpreterState *interp, PyObject *attr, int try_import) gone, then we can't even use PyImport_GetModule without triggering an interpreter abort. */ - if (!interp->modules) { + if (!_PyImport_GetModules(interp)) { return NULL; } warnings_module = PyImport_GetModule(&_Py_ID(warnings)); @@ -1050,7 +1050,6 @@ warnings_warn_impl(PyObject *module, PyObject *message, PyObject *category, static PyObject * get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno) { - PyObject *external; PyObject *loader; PyObject *module_name; PyObject *get_source; @@ -1059,13 +1058,7 @@ get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno PyObject *source_line; /* stolen from import.c */ - external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external"); - if (external == NULL) { - return NULL; - } - - loader = PyObject_CallMethod(external, "_bless_my_loader", "O", module_globals, NULL); - Py_DECREF(external); + loader = _PyImport_BlessMyLoader(interp, module_globals); if (loader == NULL) { return NULL; } diff --git a/Python/ceval.c b/Python/ceval.c index 611d62b0eba9af..09fd2f29266c87 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2688,7 +2688,7 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, } PyObject *locals = frame->f_locals; /* Fast path for not overloaded __import__. */ - if (import_func == tstate->interp->import_func) { + if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { int ilevel = _PyLong_AsInt(level); if (ilevel == -1 && _PyErr_Occurred(tstate)) { return NULL; diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 3c5fd83df584d5..6761bba457983b 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -75,7 +75,7 @@ _PyImport_FindSharedFuncptr(const char *prefix, return NULL; } - dlopenflags = _PyInterpreterState_GET()->dlopenflags; + dlopenflags = _PyImport_GetDLOpenFlags(_PyInterpreterState_GET()); handle = dlopen(pathname, dlopenflags); diff --git a/Python/import.c b/Python/import.c index 63ed2443657b29..ae27aaf56848d6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4,7 +4,7 @@ #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_interp.h" // _PyInterpreterState_ClearModules() +#include "pycore_interp.h" // struct _import_runtime_state #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pyhash.h" // _Py_KeyedHash() @@ -24,8 +24,18 @@ extern "C" { #endif -/* Forward references */ -static PyObject *import_add_module(PyThreadState *tstate, PyObject *name); + +/*[clinic input] +module _imp +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9c332475d8686284]*/ + +#include "clinic/import.c.h" + + +/*******************************/ +/* process-global import state */ +/*******************************/ /* This table is defined in config.c: */ extern struct _inittab _PyImport_Inittab[]; @@ -37,67 +47,51 @@ struct _inittab *PyImport_Inittab = _PyImport_Inittab; // we track the pointer here so we can deallocate it during finalization. static struct _inittab *inittab_copy = NULL; -/*[clinic input] -module _imp -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9c332475d8686284]*/ - -#include "clinic/import.c.h" - -/* Initialize things */ -PyStatus -_PyImportZip_Init(PyThreadState *tstate) -{ - PyObject *path_hooks; - int err = 0; +/*******************************/ +/* runtime-global import state */ +/*******************************/ - path_hooks = PySys_GetObject("path_hooks"); - if (path_hooks == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "unable to get sys.path_hooks"); - goto error; - } +#define INITTAB _PyRuntime.imports.inittab +#define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index +#define EXTENSIONS _PyRuntime.imports.extensions - int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; - if (verbose) { - PySys_WriteStderr("# installing zipimport hook\n"); - } +#define import_lock _PyRuntime.imports.lock.mutex +#define import_lock_thread _PyRuntime.imports.lock.thread +#define import_lock_level _PyRuntime.imports.lock.level - PyObject *zipimporter = _PyImport_GetModuleAttrString("zipimport", "zipimporter"); - if (zipimporter == NULL) { - _PyErr_Clear(tstate); /* No zipimporter object -- okay */ - if (verbose) { - PySys_WriteStderr("# can't import zipimport.zipimporter\n"); - } - } - else { - /* sys.path_hooks.insert(0, zipimporter) */ - err = PyList_Insert(path_hooks, 0, zipimporter); - Py_DECREF(zipimporter); - if (err < 0) { - goto error; - } - if (verbose) { - PySys_WriteStderr("# installed zipimport hook\n"); - } - } +#define FIND_AND_LOAD _PyRuntime.imports.find_and_load +#define PKGCONTEXT (_PyRuntime.imports.pkgcontext) + + +/*******************************/ +/* interpreter import state */ +/*******************************/ + +#define MODULES(interp) \ + (interp)->imports.modules +#define MODULES_BY_INDEX(interp) \ + (interp)->imports.modules_by_index +#define IMPORTLIB(interp) \ + (interp)->imports.importlib +#define OVERRIDE_FROZEN_MODULES(interp) \ + (interp)->imports.override_frozen_modules +#ifdef HAVE_DLOPEN +# define DLOPENFLAGS(interp) \ + (interp)->imports.dlopenflags +#endif +#define IMPORT_FUNC(interp) \ + (interp)->imports.import_func - return _PyStatus_OK(); - error: - PyErr_Print(); - return _PyStatus_ERR("initializing zipimport failed"); -} +/*******************/ +/* the import lock */ +/*******************/ /* Locking primitives to prevent parallel imports of the same module in different threads to return with a partially loaded module. These calls are serialized by the global interpreter lock. */ -#define import_lock _PyRuntime.imports.lock.mutex -#define import_lock_thread _PyRuntime.imports.lock.thread -#define import_lock_level _PyRuntime.imports.lock.level - void _PyImport_AcquireLock(void) { @@ -170,154 +164,45 @@ _PyImport_ReInitLock(void) } #endif -/*[clinic input] -_imp.lock_held - -Return True if the import lock is currently held, else False. - -On platforms without threads, return False. -[clinic start generated code]*/ - -static PyObject * -_imp_lock_held_impl(PyObject *module) -/*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ -{ - return PyBool_FromLong(import_lock_thread != PYTHREAD_INVALID_THREAD_ID); -} - -/*[clinic input] -_imp.acquire_lock - -Acquires the interpreter's import lock for the current thread. - -This lock should be used by import hooks to ensure thread-safety when importing -modules. On platforms without threads, this function does nothing. -[clinic start generated code]*/ - -static PyObject * -_imp_acquire_lock_impl(PyObject *module) -/*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ -{ - _PyImport_AcquireLock(); - Py_RETURN_NONE; -} - -/*[clinic input] -_imp.release_lock - -Release the interpreter's import lock. -On platforms without threads, this function does nothing. -[clinic start generated code]*/ +/***************/ +/* sys.modules */ +/***************/ -static PyObject * -_imp_release_lock_impl(PyObject *module) -/*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ +PyObject * +_PyImport_InitModules(PyInterpreterState *interp) { - if (_PyImport_ReleaseLock() < 0) { - PyErr_SetString(PyExc_RuntimeError, - "not holding the import lock"); + assert(MODULES(interp) == NULL); + MODULES(interp) = PyDict_New(); + if (MODULES(interp) == NULL) { return NULL; } - Py_RETURN_NONE; -} - -PyStatus -_PyImport_Init(void) -{ - if (_PyRuntime.imports.inittab != NULL) { - return _PyStatus_ERR("global import state already initialized"); - } - PyStatus status = _PyStatus_OK(); - - size_t size; - for (size = 0; PyImport_Inittab[size].name != NULL; size++) - ; - size++; - - /* Force default raw memory allocator to get a known allocator to be able - to release the memory in _PyImport_Fini() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Make the copy. */ - struct _inittab *copied = PyMem_RawMalloc(size * sizeof(struct _inittab)); - if (copied == NULL) { - status = PyStatus_NoMemory(); - goto done; - } - memcpy(copied, PyImport_Inittab, size * sizeof(struct _inittab)); - _PyRuntime.imports.inittab = copied; - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return status; + return MODULES(interp); } -static inline void _extensions_cache_clear(void); - -void -_PyImport_Fini(void) +PyObject * +_PyImport_GetModules(PyInterpreterState *interp) { - _extensions_cache_clear(); - if (import_lock != NULL) { - PyThread_free_lock(import_lock); - import_lock = NULL; - } - - /* Use the same memory allocator as _PyImport_Init(). */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Free memory allocated by _PyImport_Init() */ - struct _inittab *inittab = _PyRuntime.imports.inittab; - _PyRuntime.imports.inittab = NULL; - PyMem_RawFree(inittab); - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return MODULES(interp); } void -_PyImport_Fini2(void) +_PyImport_ClearModules(PyInterpreterState *interp) { - /* Use the same memory allocator than PyImport_ExtendInittab(). */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - // Reset PyImport_Inittab - PyImport_Inittab = _PyImport_Inittab; - - /* Free memory allocated by PyImport_ExtendInittab() */ - PyMem_RawFree(inittab_copy); - inittab_copy = NULL; - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + Py_SETREF(MODULES(interp), NULL); } -/* Helper for sys */ - PyObject * PyImport_GetModuleDict(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->modules == NULL) { + if (MODULES(interp) == NULL) { Py_FatalError("interpreter has no modules dictionary"); } - return interp->modules; -} - -/* In some corner cases it is important to be sure that the import - machinery has been initialized (or not cleaned up yet). For - example, see issue #4236 and PyModule_Create2(). */ - -int -_PyImport_IsInitialized(PyInterpreterState *interp) -{ - if (interp->modules == NULL) - return 0; - return 1; + return MODULES(interp); } +// This is only kept around for extensions that use _Py_IDENTIFIER. PyObject * _PyImport_GetModuleId(_Py_Identifier *nameid) { @@ -332,7 +217,7 @@ int _PyImport_SetModule(PyObject *name, PyObject *m) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *modules = interp->modules; + PyObject *modules = MODULES(interp); return PyObject_SetItem(modules, name, m); } @@ -340,14 +225,14 @@ int _PyImport_SetModuleString(const char *name, PyObject *m) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *modules = interp->modules; + PyObject *modules = MODULES(interp); return PyMapping_SetItemString(modules, name, m); } static PyObject * import_get_module(PyThreadState *tstate, PyObject *name) { - PyObject *modules = tstate->interp->modules; + PyObject *modules = MODULES(tstate->interp); if (modules == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "unable to get sys.modules"); @@ -370,7 +255,6 @@ import_get_module(PyThreadState *tstate, PyObject *name) return m; } - static int import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *name) { @@ -387,7 +271,7 @@ import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *n if (busy) { /* Wait until module is done importing. */ PyObject *value = _PyObject_CallMethodOneArg( - interp->importlib, &_Py_ID(_lock_unlock_module), name); + IMPORTLIB(interp), &_Py_ID(_lock_unlock_module), name); if (value == NULL) { return -1; } @@ -396,47 +280,381 @@ import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *n return 0; } +static void remove_importlib_frames(PyThreadState *tstate); -/* Helper for pythonrun.c -- return magic number and tag. */ - -long -PyImport_GetMagicNumber(void) +PyObject * +PyImport_GetModule(PyObject *name) { - long res; - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *external, *pyc_magic; + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *mod; - external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external"); - if (external == NULL) - return -1; - pyc_magic = PyObject_GetAttrString(external, "_RAW_MAGIC_NUMBER"); - Py_DECREF(external); - if (pyc_magic == NULL) - return -1; - res = PyLong_AsLong(pyc_magic); - Py_DECREF(pyc_magic); - return res; + mod = import_get_module(tstate, name); + if (mod != NULL && mod != Py_None) { + if (import_ensure_initialized(tstate->interp, mod, name) < 0) { + Py_DECREF(mod); + remove_importlib_frames(tstate); + return NULL; + } + } + return mod; } +/* Get the module object corresponding to a module name. + First check the modules dictionary if there's one there, + if not, create a new one and insert it in the modules dictionary. */ -extern const char * _PySys_ImplCacheTag; - -const char * -PyImport_GetMagicTag(void) +static PyObject * +import_add_module(PyThreadState *tstate, PyObject *name) { - return _PySys_ImplCacheTag; -} + PyObject *modules = MODULES(tstate->interp); + if (modules == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "no import module dictionary"); + return NULL; + } + PyObject *m; + if (PyDict_CheckExact(modules)) { + m = Py_XNewRef(PyDict_GetItemWithError(modules, name)); + } + else { + m = PyObject_GetItem(modules, name); + // For backward-compatibility we copy the behavior + // of PyDict_GetItemWithError(). + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyErr_Clear(tstate); + } + } + if (_PyErr_Occurred(tstate)) { + return NULL; + } + if (m != NULL && PyModule_Check(m)) { + return m; + } + Py_XDECREF(m); + m = PyModule_NewObject(name); + if (m == NULL) + return NULL; + if (PyObject_SetItem(modules, name, m) != 0) { + Py_DECREF(m); + return NULL; + } -/* -We support a number of kinds of single-phase init builtin/extension modules: + return m; +} -* "basic" - * no module state (PyModuleDef.m_size == -1) - * does not support repeated init (we use PyModuleDef.m_base.m_copy) - * may have process-global state - * the module's def is cached in _PyRuntime.imports.extensions, - by (name, filename) +PyObject * +PyImport_AddModuleObject(PyObject *name) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *mod = import_add_module(tstate, name); + if (mod) { + PyObject *ref = PyWeakref_NewRef(mod, NULL); + Py_DECREF(mod); + if (ref == NULL) { + return NULL; + } + mod = PyWeakref_GetObject(ref); + Py_DECREF(ref); + } + return mod; /* borrowed reference */ +} + + +PyObject * +PyImport_AddModule(const char *name) +{ + PyObject *nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) { + return NULL; + } + PyObject *module = PyImport_AddModuleObject(nameobj); + Py_DECREF(nameobj); + return module; +} + + +/* Remove name from sys.modules, if it's there. + * Can be called with an exception raised. + * If fail to remove name a new exception will be chained with the old + * exception, otherwise the old exception is preserved. + */ +static void +remove_module(PyThreadState *tstate, PyObject *name) +{ + PyObject *type, *value, *traceback; + _PyErr_Fetch(tstate, &type, &value, &traceback); + + PyObject *modules = MODULES(tstate->interp); + if (PyDict_CheckExact(modules)) { + PyObject *mod = _PyDict_Pop(modules, name, Py_None); + Py_XDECREF(mod); + } + else if (PyMapping_DelItem(modules, name) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyErr_Clear(tstate); + } + } + + _PyErr_ChainExceptions(type, value, traceback); +} + + +/************************************/ +/* per-interpreter modules-by-index */ +/************************************/ + +Py_ssize_t +_PyImport_GetNextModuleIndex(void) +{ + LAST_MODULE_INDEX++; + return LAST_MODULE_INDEX; +} + +static const char * +_modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) +{ + if (index == 0) { + return "invalid module index"; + } + if (MODULES_BY_INDEX(interp) == NULL) { + return "Interpreters module-list not accessible."; + } + if (index > PyList_GET_SIZE(MODULES_BY_INDEX(interp))) { + return "Module index out of bounds."; + } + return NULL; +} + +static PyObject * +_modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def) +{ + Py_ssize_t index = def->m_base.m_index; + if (_modules_by_index_check(interp, index) != NULL) { + return NULL; + } + PyObject *res = PyList_GET_ITEM(MODULES_BY_INDEX(interp), index); + return res==Py_None ? NULL : res; +} + +static int +_modules_by_index_set(PyInterpreterState *interp, + PyModuleDef *def, PyObject *module) +{ + assert(def != NULL); + assert(def->m_slots == NULL); + assert(def->m_base.m_index > 0); + + if (MODULES_BY_INDEX(interp) == NULL) { + MODULES_BY_INDEX(interp) = PyList_New(0); + if (MODULES_BY_INDEX(interp) == NULL) { + return -1; + } + } + + Py_ssize_t index = def->m_base.m_index; + while (PyList_GET_SIZE(MODULES_BY_INDEX(interp)) <= index) { + if (PyList_Append(MODULES_BY_INDEX(interp), Py_None) < 0) { + return -1; + } + } + + return PyList_SetItem(MODULES_BY_INDEX(interp), index, Py_NewRef(module)); +} + +static int +_modules_by_index_clear(PyInterpreterState *interp, PyModuleDef *def) +{ + Py_ssize_t index = def->m_base.m_index; + const char *err = _modules_by_index_check(interp, index); + if (err != NULL) { + Py_FatalError(err); + return -1; + } + return PyList_SetItem(MODULES_BY_INDEX(interp), index, Py_NewRef(Py_None)); +} + + +PyObject* +PyState_FindModule(PyModuleDef* module) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (module->m_slots) { + return NULL; + } + return _modules_by_index_get(interp, module); +} + +int +PyState_AddModule(PyObject* module, PyModuleDef* def) +{ + if (!def) { + Py_FatalError("module definition is NULL"); + return -1; + } + + PyThreadState *tstate = _PyThreadState_GET(); + if (def->m_slots) { + _PyErr_SetString(tstate, + PyExc_SystemError, + "PyState_AddModule called on module with slots"); + return -1; + } + + PyInterpreterState *interp = tstate->interp; + Py_ssize_t index = def->m_base.m_index; + if (MODULES_BY_INDEX(interp) && + index < PyList_GET_SIZE(MODULES_BY_INDEX(interp)) && + module == PyList_GET_ITEM(MODULES_BY_INDEX(interp), index)) + { + _Py_FatalErrorFormat(__func__, "module %p already added", module); + return -1; + } + + return _modules_by_index_set(interp, def, module); +} + +int +PyState_RemoveModule(PyModuleDef* def) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (def->m_slots) { + _PyErr_SetString(tstate, + PyExc_SystemError, + "PyState_RemoveModule called on module with slots"); + return -1; + } + return _modules_by_index_clear(tstate->interp, def); +} + + +// Used by finalize_modules() +void +_PyImport_ClearModulesByIndex(PyInterpreterState *interp) +{ + if (!MODULES_BY_INDEX(interp)) { + return; + } + + Py_ssize_t i; + for (i = 0; i < PyList_GET_SIZE(MODULES_BY_INDEX(interp)); i++) { + PyObject *m = PyList_GET_ITEM(MODULES_BY_INDEX(interp), i); + if (PyModule_Check(m)) { + /* cleanup the saved copy of module dicts */ + PyModuleDef *md = PyModule_GetDef(m); + if (md) { + Py_CLEAR(md->m_base.m_copy); + } + } + } + + /* Setting modules_by_index to NULL could be dangerous, so we + clear the list instead. */ + if (PyList_SetSlice(MODULES_BY_INDEX(interp), + 0, PyList_GET_SIZE(MODULES_BY_INDEX(interp)), + NULL)) { + PyErr_WriteUnraisable(MODULES_BY_INDEX(interp)); + } +} + + +/*********************/ +/* extension modules */ +/*********************/ + +/* Make sure name is fully qualified. + + This is a bit of a hack: when the shared library is loaded, + the module name is "package.module", but the module calls + PyModule_Create*() with just "module" for the name. The shared + library loader squirrels away the true name of the module in + _PyRuntime.imports.pkgcontext, and PyModule_Create*() will + substitute this (if the name actually matches). +*/ +const char * +_PyImport_ResolveNameWithPackageContext(const char *name) +{ + if (PKGCONTEXT != NULL) { + const char *p = strrchr(PKGCONTEXT, '.'); + if (p != NULL && strcmp(name, p+1) == 0) { + name = PKGCONTEXT; + PKGCONTEXT = NULL; + } + } + return name; +} + +const char * +_PyImport_SwapPackageContext(const char *newcontext) +{ + const char *oldcontext = PKGCONTEXT; + PKGCONTEXT = newcontext; + return oldcontext; +} + +#ifdef HAVE_DLOPEN +int +_PyImport_GetDLOpenFlags(PyInterpreterState *interp) +{ + return DLOPENFLAGS(interp); +} + +void +_PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val) +{ + DLOPENFLAGS(interp) = new_val; +} +#endif // HAVE_DLOPEN + + +/* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ +static int +exec_builtin_or_dynamic(PyObject *mod) { + PyModuleDef *def; + void *state; + + if (!PyModule_Check(mod)) { + return 0; + } + + def = PyModule_GetDef(mod); + if (def == NULL) { + return 0; + } + + state = PyModule_GetState(mod); + if (state) { + /* Already initialized; skip reload */ + return 0; + } + + return PyModule_ExecDef(mod, def); +} + + +/*******************/ + +#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) +#include +EM_JS(PyObject*, _PyImport_InitFunc_TrampolineCall, (PyModInitFunction func), { + return wasmTable.get(func)(); +}); +#endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE + + +/*****************************/ +/* single-phase init modules */ +/*****************************/ + +/* +We support a number of kinds of single-phase init builtin/extension modules: + +* "basic" + * no module state (PyModuleDef.m_size == -1) + * does not support repeated init (we use PyModuleDef.m_base.m_copy) + * may have process-global state + * the module's def is cached in _PyRuntime.imports.extensions, + by (name, filename) * "reinit" * no module state (PyModuleDef.m_size == 0) * supports repeated init (m_copy is never used) @@ -512,7 +730,7 @@ gets even messier. static PyModuleDef * _extensions_cache_get(PyObject *filename, PyObject *name) { - PyObject *extensions = _PyRuntime.imports.extensions; + PyObject *extensions = EXTENSIONS; if (extensions == NULL) { return NULL; } @@ -528,13 +746,13 @@ _extensions_cache_get(PyObject *filename, PyObject *name) static int _extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) { - PyObject *extensions = _PyRuntime.imports.extensions; + PyObject *extensions = EXTENSIONS; if (extensions == NULL) { extensions = PyDict_New(); if (extensions == NULL) { return -1; } - _PyRuntime.imports.extensions = extensions; + EXTENSIONS = extensions; } PyObject *key = PyTuple_Pack(2, filename, name); if (key == NULL) { @@ -551,7 +769,7 @@ _extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) static void _extensions_cache_clear(void) { - Py_CLEAR(_PyRuntime.imports.extensions); + Py_CLEAR(EXTENSIONS); } static int @@ -569,7 +787,7 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) } PyThreadState *tstate = _PyThreadState_GET(); - if (_PyState_AddModule(tstate, mod, def) < 0) { + if (_modules_by_index_set(tstate->interp, def, mod) < 0) { return -1; } @@ -616,40 +834,19 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, return 0; } -int -_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) -{ - int res = -1; - PyObject *nameobj; - nameobj = PyUnicode_InternFromString(name); - if (nameobj == NULL) { - return -1; - } - if (PyObject_SetItem(modules, nameobj, mod) < 0) { - goto finally; - } - if (fix_up_extension(mod, nameobj, nameobj) < 0) { - PyMapping_DelItem(modules, nameobj); - goto finally; - } - res = 0; - -finally: - Py_DECREF(nameobj); - return res; -} static PyObject * import_find_extension(PyThreadState *tstate, PyObject *name, PyObject *filename) { + /* Only single-phase init modules will be in the cache. */ PyModuleDef *def = _extensions_cache_get(filename, name); if (def == NULL) { return NULL; } PyObject *mod, *mdict; - PyObject *modules = tstate->interp->modules; + PyObject *modules = MODULES(tstate->interp); if (def->m_size == -1) { /* Module does not support repeated initialization */ @@ -679,7 +876,7 @@ import_find_extension(PyThreadState *tstate, PyObject *name, return NULL; } } - if (_PyState_AddModule(tstate, mod, def) < 0) { + if (_modules_by_index_set(tstate->interp, def, mod) < 0) { PyMapping_DelItem(modules, name); Py_DECREF(mod); return NULL; @@ -694,107 +891,268 @@ import_find_extension(PyThreadState *tstate, PyObject *name, } -/* Get the module object corresponding to a module name. - First check the modules dictionary if there's one there, - if not, create a new one and insert it in the modules dictionary. */ +/*******************/ +/* builtin modules */ +/*******************/ -static PyObject * -import_add_module(PyThreadState *tstate, PyObject *name) +int +_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) { - PyObject *modules = tstate->interp->modules; - if (modules == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "no import module dictionary"); - return NULL; + int res = -1; + PyObject *nameobj; + nameobj = PyUnicode_InternFromString(name); + if (nameobj == NULL) { + return -1; } - - PyObject *m; - if (PyDict_CheckExact(modules)) { - m = Py_XNewRef(PyDict_GetItemWithError(modules, name)); + if (PyObject_SetItem(modules, nameobj, mod) < 0) { + goto finally; } - else { - m = PyObject_GetItem(modules, name); - // For backward-compatibility we copy the behavior - // of PyDict_GetItemWithError(). - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyErr_Clear(tstate); - } + if (fix_up_extension(mod, nameobj, nameobj) < 0) { + PyMapping_DelItem(modules, nameobj); + goto finally; } - if (_PyErr_Occurred(tstate)) { - return NULL; + res = 0; + +finally: + Py_DECREF(nameobj); + return res; +} + +/* Helper to test for built-in module */ + +static int +is_builtin(PyObject *name) +{ + int i; + struct _inittab *inittab = INITTAB; + for (i = 0; inittab[i].name != NULL; i++) { + if (_PyUnicode_EqualToASCIIString(name, inittab[i].name)) { + if (inittab[i].initfunc == NULL) + return -1; + else + return 1; + } } - if (m != NULL && PyModule_Check(m)) { - return m; + return 0; +} + +static PyObject* +create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) +{ + PyObject *mod = import_find_extension(tstate, name, name); + if (mod || _PyErr_Occurred(tstate)) { + return mod; } - Py_XDECREF(m); - m = PyModule_NewObject(name); - if (m == NULL) - return NULL; - if (PyObject_SetItem(modules, name, m) != 0) { - Py_DECREF(m); - return NULL; + + PyObject *modules = MODULES(tstate->interp); + for (struct _inittab *p = INITTAB; p->name != NULL; p++) { + if (_PyUnicode_EqualToASCIIString(name, p->name)) { + if (p->initfunc == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + mod = PyImport_AddModuleObject(name); + return Py_XNewRef(mod); + } + mod = _PyImport_InitFunc_TrampolineCall(*p->initfunc); + if (mod == NULL) { + return NULL; + } + + if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { + return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); + } + else { + /* Remember pointer to module init function. */ + PyModuleDef *def = PyModule_GetDef(mod); + if (def == NULL) { + return NULL; + } + + def->m_base.m_init = p->initfunc; + if (_PyImport_FixupExtensionObject(mod, name, name, + modules) < 0) { + return NULL; + } + return mod; + } + } } - return m; + // not found + Py_RETURN_NONE; } -PyObject * -PyImport_AddModuleObject(PyObject *name) + +/*****************************/ +/* the builtin modules table */ +/*****************************/ + +/* API for embedding applications that want to add their own entries + to the table of built-in modules. This should normally be called + *before* Py_Initialize(). When the table resize fails, -1 is + returned and the existing table is unchanged. + + After a similar function by Just van Rossum. */ + +int +PyImport_ExtendInittab(struct _inittab *newtab) { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *mod = import_add_module(tstate, name); - if (mod) { - PyObject *ref = PyWeakref_NewRef(mod, NULL); - Py_DECREF(mod); - if (ref == NULL) { - return NULL; - } - mod = PyWeakref_GetObject(ref); - Py_DECREF(ref); + struct _inittab *p; + size_t i, n; + int res = 0; + + if (INITTAB != NULL) { + Py_FatalError("PyImport_ExtendInittab() may not be called after Py_Initialize()"); } - return mod; /* borrowed reference */ + + /* Count the number of entries in both tables */ + for (n = 0; newtab[n].name != NULL; n++) + ; + if (n == 0) + return 0; /* Nothing to do */ + for (i = 0; PyImport_Inittab[i].name != NULL; i++) + ; + + /* Force default raw memory allocator to get a known allocator to be able + to release the memory in _PyImport_Fini2() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* Allocate new memory for the combined table */ + p = NULL; + if (i + n <= SIZE_MAX / sizeof(struct _inittab) - 1) { + size_t size = sizeof(struct _inittab) * (i + n + 1); + p = PyMem_RawRealloc(inittab_copy, size); + } + if (p == NULL) { + res = -1; + goto done; + } + + /* Copy the tables into the new memory at the first call + to PyImport_ExtendInittab(). */ + if (inittab_copy != PyImport_Inittab) { + memcpy(p, PyImport_Inittab, (i+1) * sizeof(struct _inittab)); + } + memcpy(p + i, newtab, (n + 1) * sizeof(struct _inittab)); + PyImport_Inittab = inittab_copy = p; + +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return res; } +/* Shorthand to add a single entry given a name and a function */ -PyObject * -PyImport_AddModule(const char *name) +int +PyImport_AppendInittab(const char *name, PyObject* (*initfunc)(void)) { - PyObject *nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) { - return NULL; + struct _inittab newtab[2]; + + if (INITTAB != NULL) { + Py_FatalError("PyImport_AppendInittab() may not be called after Py_Initialize()"); } - PyObject *module = PyImport_AddModuleObject(nameobj); - Py_DECREF(nameobj); - return module; + + memset(newtab, '\0', sizeof newtab); + + newtab[0].name = name; + newtab[0].initfunc = initfunc; + + return PyImport_ExtendInittab(newtab); } -/* Remove name from sys.modules, if it's there. - * Can be called with an exception raised. - * If fail to remove name a new exception will be chained with the old - * exception, otherwise the old exception is preserved. - */ +/* the internal table */ + +static int +init_builtin_modules_table(void) +{ + size_t size; + for (size = 0; PyImport_Inittab[size].name != NULL; size++) + ; + size++; + + /* Make the copy. */ + struct _inittab *copied = PyMem_RawMalloc(size * sizeof(struct _inittab)); + if (copied == NULL) { + return -1; + } + memcpy(copied, PyImport_Inittab, size * sizeof(struct _inittab)); + INITTAB = copied; + return 0; +} + static void -remove_module(PyThreadState *tstate, PyObject *name) +fini_builtin_modules_table(void) { - PyObject *type, *value, *traceback; - _PyErr_Fetch(tstate, &type, &value, &traceback); + struct _inittab *inittab = INITTAB; + INITTAB = NULL; + PyMem_RawFree(inittab); +} - PyObject *modules = tstate->interp->modules; - if (PyDict_CheckExact(modules)) { - PyObject *mod = _PyDict_Pop(modules, name, Py_None); - Py_XDECREF(mod); +PyObject * +_PyImport_GetBuiltinModuleNames(void) +{ + PyObject *list = PyList_New(0); + if (list == NULL) { + return NULL; } - else if (PyMapping_DelItem(modules, name) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyErr_Clear(tstate); + struct _inittab *inittab = INITTAB; + for (Py_ssize_t i = 0; inittab[i].name != NULL; i++) { + PyObject *name = PyUnicode_FromString(inittab[i].name); + if (name == NULL) { + Py_DECREF(list); + return NULL; } + if (PyList_Append(list, name) < 0) { + Py_DECREF(name); + Py_DECREF(list); + return NULL; + } + Py_DECREF(name); } + return list; +} - _PyErr_ChainExceptions(type, value, traceback); + +/********************/ +/* the magic number */ +/********************/ + +/* Helper for pythonrun.c -- return magic number and tag. */ + +long +PyImport_GetMagicNumber(void) +{ + long res; + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *external, *pyc_magic; + + external = PyObject_GetAttrString(IMPORTLIB(interp), "_bootstrap_external"); + if (external == NULL) + return -1; + pyc_magic = PyObject_GetAttrString(external, "_RAW_MAGIC_NUMBER"); + Py_DECREF(external); + if (pyc_magic == NULL) + return -1; + res = PyLong_AsLong(pyc_magic); + Py_DECREF(pyc_magic); + return res; +} + + +extern const char * _PySys_ImplCacheTag; + +const char * +PyImport_GetMagicTag(void) +{ + return _PySys_ImplCacheTag; } +/*********************************/ +/* a Python module's code object */ +/*********************************/ + /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is * removed from sys.modules, to avoid leaving damaged module objects @@ -851,7 +1209,7 @@ PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, Py_FatalError("no current interpreter"); } - external= PyObject_GetAttrString(interp->importlib, + external= PyObject_GetAttrString(IMPORTLIB(interp), "_bootstrap_external"); if (external != NULL) { pathobj = _PyObject_CallMethodOneArg( @@ -936,7 +1294,7 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, if (pathname == NULL) { pathname = ((PyCodeObject *)co)->co_filename; } - external = PyObject_GetAttrString(tstate->interp->importlib, + external = PyObject_GetAttrString(IMPORTLIB(tstate->interp), "_bootstrap_external"); if (external == NULL) { Py_DECREF(d); @@ -989,204 +1347,10 @@ update_compiled_module(PyCodeObject *co, PyObject *newname) Py_DECREF(oldname); } -/*[clinic input] -_imp._fix_co_filename - - code: object(type="PyCodeObject *", subclass_of="&PyCode_Type") - Code object to change. - - path: unicode - File path to use. - / - -Changes code.co_filename to specify the passed-in file path. -[clinic start generated code]*/ - -static PyObject * -_imp__fix_co_filename_impl(PyObject *module, PyCodeObject *code, - PyObject *path) -/*[clinic end generated code: output=1d002f100235587d input=895ba50e78b82f05]*/ - -{ - update_compiled_module(code, path); - - Py_RETURN_NONE; -} - - -/* Helper to test for built-in module */ - -static int -is_builtin(PyObject *name) -{ - int i; - struct _inittab *inittab = _PyRuntime.imports.inittab; - for (i = 0; inittab[i].name != NULL; i++) { - if (_PyUnicode_EqualToASCIIString(name, inittab[i].name)) { - if (inittab[i].initfunc == NULL) - return -1; - else - return 1; - } - } - return 0; -} - - -/* Return a finder object for a sys.path/pkg.__path__ item 'p', - possibly by fetching it from the path_importer_cache dict. If it - wasn't yet cached, traverse path_hooks until a hook is found - that can handle the path item. Return None if no hook could; - this tells our caller that the path based finder could not find - a finder for this path item. Cache the result in - path_importer_cache. */ - -static PyObject * -get_path_importer(PyThreadState *tstate, PyObject *path_importer_cache, - PyObject *path_hooks, PyObject *p) -{ - PyObject *importer; - Py_ssize_t j, nhooks; - - /* These conditions are the caller's responsibility: */ - assert(PyList_Check(path_hooks)); - assert(PyDict_Check(path_importer_cache)); - - nhooks = PyList_Size(path_hooks); - if (nhooks < 0) - return NULL; /* Shouldn't happen */ - - importer = PyDict_GetItemWithError(path_importer_cache, p); - if (importer != NULL || _PyErr_Occurred(tstate)) { - return Py_XNewRef(importer); - } - - /* set path_importer_cache[p] to None to avoid recursion */ - if (PyDict_SetItem(path_importer_cache, p, Py_None) != 0) - return NULL; - - for (j = 0; j < nhooks; j++) { - PyObject *hook = PyList_GetItem(path_hooks, j); - if (hook == NULL) - return NULL; - importer = PyObject_CallOneArg(hook, p); - if (importer != NULL) - break; - - if (!_PyErr_ExceptionMatches(tstate, PyExc_ImportError)) { - return NULL; - } - _PyErr_Clear(tstate); - } - if (importer == NULL) { - Py_RETURN_NONE; - } - if (PyDict_SetItem(path_importer_cache, p, importer) < 0) { - Py_DECREF(importer); - return NULL; - } - return importer; -} - -PyObject * -PyImport_GetImporter(PyObject *path) -{ - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *path_importer_cache = PySys_GetObject("path_importer_cache"); - PyObject *path_hooks = PySys_GetObject("path_hooks"); - if (path_importer_cache == NULL || path_hooks == NULL) { - return NULL; - } - return get_path_importer(tstate, path_importer_cache, path_hooks, path); -} - -#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -#include -EM_JS(PyObject*, _PyImport_InitFunc_TrampolineCall, (PyModInitFunction func), { - return wasmTable.get(func)(); -}); -#endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE - -static PyObject* -create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) -{ - PyObject *mod = import_find_extension(tstate, name, name); - if (mod || _PyErr_Occurred(tstate)) { - return mod; - } - - PyObject *modules = tstate->interp->modules; - for (struct _inittab *p = _PyRuntime.imports.inittab; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(name, p->name)) { - if (p->initfunc == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - mod = PyImport_AddModuleObject(name); - return Py_XNewRef(mod); - } - mod = _PyImport_InitFunc_TrampolineCall(*p->initfunc); - if (mod == NULL) { - return NULL; - } - - if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { - return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); - } - else { - /* Remember pointer to module init function. */ - PyModuleDef *def = PyModule_GetDef(mod); - if (def == NULL) { - return NULL; - } - - def->m_base.m_init = p->initfunc; - if (_PyImport_FixupExtensionObject(mod, name, name, - modules) < 0) { - return NULL; - } - return mod; - } - } - } - - // not found - Py_RETURN_NONE; -} - - - -/*[clinic input] -_imp.create_builtin - - spec: object - / - -Create an extension module. -[clinic start generated code]*/ - -static PyObject * -_imp_create_builtin(PyObject *module, PyObject *spec) -/*[clinic end generated code: output=ace7ff22271e6f39 input=37f966f890384e47]*/ -{ - PyThreadState *tstate = _PyThreadState_GET(); - - PyObject *name = PyObject_GetAttrString(spec, "name"); - if (name == NULL) { - return NULL; - } - - if (!PyUnicode_Check(name)) { - PyErr_Format(PyExc_TypeError, - "name must be string, not %.200s", - Py_TYPE(name)->tp_name); - Py_DECREF(name); - return NULL; - } - - PyObject *mod = create_builtin(tstate, name, spec); - Py_DECREF(name); - return mod; -} +/******************/ +/* frozen modules */ +/******************/ /* Return true if the name is an alias. In that case, "alias" is set to the original module name. If it is an alias but the original @@ -1210,14 +1374,11 @@ resolve_module_alias(const char *name, const struct _module_alias *aliases, } } - -/* Frozen modules */ - static bool use_frozen(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int override = interp->override_frozen_modules; + int override = OVERRIDE_FROZEN_MODULES(interp); if (override > 0) { return true; } @@ -1587,54 +1748,315 @@ PyImport_ImportFrozenModule(const char *name) } -/* Import a module, either built-in, frozen, or external, and return - its module object WITH INCREMENTED REFERENCE COUNT */ +/*************/ +/* importlib */ +/*************/ -PyObject * -PyImport_ImportModule(const char *name) +/* Import the _imp extension by calling manually _imp.create_builtin() and + _imp.exec_builtin() since importlib is not initialized yet. Initializing + importlib requires the _imp module: this function fix the bootstrap issue. + */ +static PyObject* +bootstrap_imp(PyThreadState *tstate) { - PyObject *pname; - PyObject *result; - - pname = PyUnicode_FromString(name); - if (pname == NULL) + PyObject *name = PyUnicode_FromString("_imp"); + if (name == NULL) { return NULL; - result = PyImport_Import(pname); - Py_DECREF(pname); - return result; -} + } + // Mock a ModuleSpec object just good enough for PyModule_FromDefAndSpec(): + // an object with just a name attribute. + // + // _imp.__spec__ is overridden by importlib._bootstrap._instal() anyway. + PyObject *attrs = Py_BuildValue("{sO}", "name", name); + if (attrs == NULL) { + goto error; + } + PyObject *spec = _PyNamespace_New(attrs); + Py_DECREF(attrs); + if (spec == NULL) { + goto error; + } -/* Import a module without blocking - * - * At first it tries to fetch the module from sys.modules. If the module was - * never loaded before it loads it with PyImport_ImportModule() unless another - * thread holds the import lock. In the latter case the function raises an - * ImportError instead of blocking. - * - * Returns the module object with incremented ref count. - */ -PyObject * -PyImport_ImportModuleNoBlock(const char *name) -{ - return PyImport_ImportModule(name); + // Create the _imp module from its definition. + PyObject *mod = create_builtin(tstate, name, spec); + Py_CLEAR(name); + Py_DECREF(spec); + if (mod == NULL) { + goto error; + } + assert(mod != Py_None); // not found + + // Execute the _imp module: call imp_module_exec(). + if (exec_builtin_or_dynamic(mod) < 0) { + Py_DECREF(mod); + goto error; + } + return mod; + +error: + Py_XDECREF(name); + return NULL; } +/* Global initializations. Can be undone by Py_FinalizeEx(). Don't + call this twice without an intervening Py_FinalizeEx() call. When + initializations fail, a fatal error is issued and the function does + not return. On return, the first thread and interpreter state have + been created. -/* Remove importlib frames from the traceback, - * except in Verbose mode. */ -static void -remove_importlib_frames(PyThreadState *tstate) + Locking: you must hold the interpreter lock while calling this. + (If the lock has not yet been initialized, that's equivalent to + having the lock, but you cannot use multiple threads.) + +*/ +static int +init_importlib(PyThreadState *tstate, PyObject *sysmod) { - const char *importlib_filename = ""; - const char *external_filename = ""; - const char *remove_frames = "_call_with_frames_removed"; - int always_trim = 0; - int in_importlib = 0; - PyObject *exception, *value, *base_tb, *tb; - PyObject **prev_link, **outer_link = NULL; + assert(!_PyErr_Occurred(tstate)); - /* Synopsis: if it's an ImportError, we trim all importlib chunks + PyInterpreterState *interp = tstate->interp; + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; + + // Import _importlib through its frozen version, _frozen_importlib. + if (verbose) { + PySys_FormatStderr("import _frozen_importlib # frozen\n"); + } + if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) { + return -1; + } + PyObject *importlib = PyImport_AddModule("_frozen_importlib"); // borrowed + if (importlib == NULL) { + return -1; + } + IMPORTLIB(interp) = Py_NewRef(importlib); + + // Import the _imp module + if (verbose) { + PySys_FormatStderr("import _imp # builtin\n"); + } + PyObject *imp_mod = bootstrap_imp(tstate); + if (imp_mod == NULL) { + return -1; + } + if (_PyImport_SetModuleString("_imp", imp_mod) < 0) { + Py_DECREF(imp_mod); + return -1; + } + + // Install importlib as the implementation of import + PyObject *value = PyObject_CallMethod(importlib, "_install", + "OO", sysmod, imp_mod); + Py_DECREF(imp_mod); + if (value == NULL) { + return -1; + } + Py_DECREF(value); + + assert(!_PyErr_Occurred(tstate)); + return 0; +} + + +static int +init_importlib_external(PyInterpreterState *interp) +{ + PyObject *value; + value = PyObject_CallMethod(IMPORTLIB(interp), + "_install_external_importers", ""); + if (value == NULL) { + return -1; + } + Py_DECREF(value); + return 0; +} + +PyObject * +_PyImport_GetImportlibLoader(PyInterpreterState *interp, + const char *loader_name) +{ + return PyObject_GetAttrString(IMPORTLIB(interp), loader_name); +} + +PyObject * +_PyImport_GetImportlibExternalLoader(PyInterpreterState *interp, + const char *loader_name) +{ + PyObject *bootstrap = PyObject_GetAttrString(IMPORTLIB(interp), + "_bootstrap_external"); + if (bootstrap == NULL) { + return NULL; + } + + PyObject *loader_type = PyObject_GetAttrString(bootstrap, loader_name); + Py_DECREF(bootstrap); + return loader_type; +} + +PyObject * +_PyImport_BlessMyLoader(PyInterpreterState *interp, PyObject *module_globals) +{ + PyObject *external = PyObject_GetAttrString(IMPORTLIB(interp), + "_bootstrap_external"); + if (external == NULL) { + return NULL; + } + + PyObject *loader = PyObject_CallMethod(external, "_bless_my_loader", + "O", module_globals, NULL); + Py_DECREF(external); + return loader; +} + +PyObject * +_PyImport_ImportlibModuleRepr(PyInterpreterState *interp, PyObject *m) +{ + return PyObject_CallMethod(IMPORTLIB(interp), "_module_repr", "O", m); +} + + +/*******************/ + +/* Return a finder object for a sys.path/pkg.__path__ item 'p', + possibly by fetching it from the path_importer_cache dict. If it + wasn't yet cached, traverse path_hooks until a hook is found + that can handle the path item. Return None if no hook could; + this tells our caller that the path based finder could not find + a finder for this path item. Cache the result in + path_importer_cache. */ + +static PyObject * +get_path_importer(PyThreadState *tstate, PyObject *path_importer_cache, + PyObject *path_hooks, PyObject *p) +{ + PyObject *importer; + Py_ssize_t j, nhooks; + + /* These conditions are the caller's responsibility: */ + assert(PyList_Check(path_hooks)); + assert(PyDict_Check(path_importer_cache)); + + nhooks = PyList_Size(path_hooks); + if (nhooks < 0) + return NULL; /* Shouldn't happen */ + + importer = PyDict_GetItemWithError(path_importer_cache, p); + if (importer != NULL || _PyErr_Occurred(tstate)) { + return Py_XNewRef(importer); + } + + /* set path_importer_cache[p] to None to avoid recursion */ + if (PyDict_SetItem(path_importer_cache, p, Py_None) != 0) + return NULL; + + for (j = 0; j < nhooks; j++) { + PyObject *hook = PyList_GetItem(path_hooks, j); + if (hook == NULL) + return NULL; + importer = PyObject_CallOneArg(hook, p); + if (importer != NULL) + break; + + if (!_PyErr_ExceptionMatches(tstate, PyExc_ImportError)) { + return NULL; + } + _PyErr_Clear(tstate); + } + if (importer == NULL) { + Py_RETURN_NONE; + } + if (PyDict_SetItem(path_importer_cache, p, importer) < 0) { + Py_DECREF(importer); + return NULL; + } + return importer; +} + +PyObject * +PyImport_GetImporter(PyObject *path) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *path_importer_cache = PySys_GetObject("path_importer_cache"); + PyObject *path_hooks = PySys_GetObject("path_hooks"); + if (path_importer_cache == NULL || path_hooks == NULL) { + return NULL; + } + return get_path_importer(tstate, path_importer_cache, path_hooks, path); +} + + +/*********************/ +/* importing modules */ +/*********************/ + +int +_PyImport_InitDefaultImportFunc(PyInterpreterState *interp) +{ + // Get the __import__ function + PyObject *import_func = _PyDict_GetItemStringWithError(interp->builtins, + "__import__"); + if (import_func == NULL) { + return -1; + } + IMPORT_FUNC(interp) = Py_NewRef(import_func); + return 0; +} + +int +_PyImport_IsDefaultImportFunc(PyInterpreterState *interp, PyObject *func) +{ + return func == IMPORT_FUNC(interp); +} + + +/* Import a module, either built-in, frozen, or external, and return + its module object WITH INCREMENTED REFERENCE COUNT */ + +PyObject * +PyImport_ImportModule(const char *name) +{ + PyObject *pname; + PyObject *result; + + pname = PyUnicode_FromString(name); + if (pname == NULL) + return NULL; + result = PyImport_Import(pname); + Py_DECREF(pname); + return result; +} + + +/* Import a module without blocking + * + * At first it tries to fetch the module from sys.modules. If the module was + * never loaded before it loads it with PyImport_ImportModule() unless another + * thread holds the import lock. In the latter case the function raises an + * ImportError instead of blocking. + * + * Returns the module object with incremented ref count. + */ +PyObject * +PyImport_ImportModuleNoBlock(const char *name) +{ + return PyImport_ImportModule(name); +} + + +/* Remove importlib frames from the traceback, + * except in Verbose mode. */ +static void +remove_importlib_frames(PyThreadState *tstate) +{ + const char *importlib_filename = ""; + const char *external_filename = ""; + const char *remove_frames = "_call_with_frames_removed"; + int always_trim = 0; + int in_importlib = 0; + PyObject *exception, *value, *base_tb, *tb; + PyObject **prev_link, **outer_link = NULL; + + /* Synopsis: if it's an ImportError, we trim all importlib chunks from the traceback. We always trim chunks which end with a call to "_call_with_frames_removed". */ @@ -1851,8 +2273,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) PyObject *mod = NULL; PyInterpreterState *interp = tstate->interp; int import_time = _PyInterpreterState_GetConfig(interp)->import_time; -#define import_level _PyRuntime.imports.find_and_load.import_level -#define accumulated _PyRuntime.imports.find_and_load.accumulated +#define import_level FIND_AND_LOAD.import_level +#define accumulated FIND_AND_LOAD.accumulated _PyTime_t t1 = 0, accumulated_copy = accumulated; @@ -1873,7 +2295,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) * _PyDict_GetItemIdWithError(). */ if (import_time) { -#define header _PyRuntime.imports.find_and_load.header +#define header FIND_AND_LOAD.header if (header) { fputs("import time: self [us] | cumulative | imported package\n", stderr); @@ -1889,8 +2311,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) if (PyDTrace_IMPORT_FIND_LOAD_START_ENABLED()) PyDTrace_IMPORT_FIND_LOAD_START(PyUnicode_AsUTF8(abs_name)); - mod = PyObject_CallMethodObjArgs(interp->importlib, &_Py_ID(_find_and_load), - abs_name, interp->import_func, NULL); + mod = PyObject_CallMethodObjArgs(IMPORTLIB(interp), &_Py_ID(_find_and_load), + abs_name, IMPORT_FUNC(interp), NULL); if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) PyDTrace_IMPORT_FIND_LOAD_DONE(PyUnicode_AsUTF8(abs_name), @@ -1913,23 +2335,6 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef accumulated } -PyObject * -PyImport_GetModule(PyObject *name) -{ - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *mod; - - mod = import_get_module(tstate, name); - if (mod != NULL && mod != Py_None) { - if (import_ensure_initialized(tstate->interp, mod, name) < 0) { - Py_DECREF(mod); - remove_importlib_frames(tstate); - return NULL; - } - } - return mod; -} - PyObject * PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, @@ -2059,8 +2464,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, if (path) { Py_DECREF(path); final_mod = PyObject_CallMethodObjArgs( - interp->importlib, &_Py_ID(_handle_fromlist), - mod, fromlist, interp->import_func, NULL); + IMPORTLIB(interp), &_Py_ID(_handle_fromlist), + mod, fromlist, IMPORT_FUNC(interp), NULL); } else { final_mod = Py_NewRef(mod); @@ -2129,72 +2534,429 @@ PyImport_ReloadModule(PyObject *m) PyObject * PyImport_Import(PyObject *module_name) { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *globals = NULL; - PyObject *import = NULL; - PyObject *builtins = NULL; - PyObject *r = NULL; + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *globals = NULL; + PyObject *import = NULL; + PyObject *builtins = NULL; + PyObject *r = NULL; + + PyObject *from_list = PyList_New(0); + if (from_list == NULL) { + goto err; + } + + /* Get the builtins from current globals */ + globals = PyEval_GetGlobals(); + if (globals != NULL) { + Py_INCREF(globals); + builtins = PyObject_GetItem(globals, &_Py_ID(__builtins__)); + if (builtins == NULL) + goto err; + } + else { + /* No globals -- use standard builtins, and fake globals */ + builtins = PyImport_ImportModuleLevel("builtins", + NULL, NULL, NULL, 0); + if (builtins == NULL) { + goto err; + } + globals = Py_BuildValue("{OO}", &_Py_ID(__builtins__), builtins); + if (globals == NULL) + goto err; + } + + /* Get the __import__ function from the builtins */ + if (PyDict_Check(builtins)) { + import = PyObject_GetItem(builtins, &_Py_ID(__import__)); + if (import == NULL) { + _PyErr_SetObject(tstate, PyExc_KeyError, &_Py_ID(__import__)); + } + } + else + import = PyObject_GetAttr(builtins, &_Py_ID(__import__)); + if (import == NULL) + goto err; + + /* Call the __import__ function with the proper argument list + Always use absolute import here. + Calling for side-effect of import. */ + r = PyObject_CallFunction(import, "OOOOi", module_name, globals, + globals, from_list, 0, NULL); + if (r == NULL) + goto err; + Py_DECREF(r); + + r = import_get_module(tstate, module_name); + if (r == NULL && !_PyErr_Occurred(tstate)) { + _PyErr_SetObject(tstate, PyExc_KeyError, module_name); + } + + err: + Py_XDECREF(globals); + Py_XDECREF(builtins); + Py_XDECREF(import); + Py_XDECREF(from_list); + + return r; +} + + +/*********************/ +/* runtime lifecycle */ +/*********************/ + +PyStatus +_PyImport_Init(void) +{ + if (INITTAB != NULL) { + return _PyStatus_ERR("global import state already initialized"); + } + + PyStatus status = _PyStatus_OK(); + + /* Force default raw memory allocator to get a known allocator to be able + to release the memory in _PyImport_Fini() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (init_builtin_modules_table() != 0) { + status = PyStatus_NoMemory(); + goto done; + } + +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return status; +} + +void +_PyImport_Fini(void) +{ + /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ + _extensions_cache_clear(); + if (import_lock != NULL) { + PyThread_free_lock(import_lock); + import_lock = NULL; + } + + /* Use the same memory allocator as _PyImport_Init(). */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* Free memory allocated by _PyImport_Init() */ + fini_builtin_modules_table(); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + +void +_PyImport_Fini2(void) +{ + /* Use the same memory allocator than PyImport_ExtendInittab(). */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + // Reset PyImport_Inittab + PyImport_Inittab = _PyImport_Inittab; + + /* Free memory allocated by PyImport_ExtendInittab() */ + PyMem_RawFree(inittab_copy); + inittab_copy = NULL; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + + +/*************************/ +/* interpreter lifecycle */ +/*************************/ + +PyStatus +_PyImport_InitCore(PyThreadState *tstate, PyObject *sysmod, int importlib) +{ + // XXX Initialize here: interp->modules and interp->import_func. + // XXX Initialize here: sys.modules and sys.meta_path. + + if (importlib) { + /* This call sets up builtin and frozen import support */ + if (init_importlib(tstate, sysmod) < 0) { + return _PyStatus_ERR("failed to initialize importlib"); + } + } + + return _PyStatus_OK(); +} + +/* In some corner cases it is important to be sure that the import + machinery has been initialized (or not cleaned up yet). For + example, see issue #4236 and PyModule_Create2(). */ + +int +_PyImport_IsInitialized(PyInterpreterState *interp) +{ + if (MODULES(interp) == NULL) + return 0; + return 1; +} + +/* Clear the direct per-interpreter import state, if not cleared already. */ +void +_PyImport_ClearCore(PyInterpreterState *interp) +{ + /* interp->modules should have been cleaned up and cleared already + by _PyImport_FiniCore(). */ + Py_CLEAR(MODULES(interp)); + Py_CLEAR(MODULES_BY_INDEX(interp)); + Py_CLEAR(IMPORTLIB(interp)); + Py_CLEAR(IMPORT_FUNC(interp)); +} + +void +_PyImport_FiniCore(PyInterpreterState *interp) +{ + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; + + if (_PySys_ClearAttrString(interp, "meta_path", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } + + // XXX Pull in most of finalize_modules() in pylifecycle.c. + + if (_PySys_ClearAttrString(interp, "modules", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } + + _PyImport_ClearCore(interp); +} + +// XXX Add something like _PyImport_Disable() for use early in interp fini? + + +/* "external" imports */ + +static int +init_zipimport(PyThreadState *tstate, int verbose) +{ + PyObject *path_hooks = PySys_GetObject("path_hooks"); + if (path_hooks == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "unable to get sys.path_hooks"); + return -1; + } + + if (verbose) { + PySys_WriteStderr("# installing zipimport hook\n"); + } + + PyObject *zipimporter = _PyImport_GetModuleAttrString("zipimport", "zipimporter"); + if (zipimporter == NULL) { + _PyErr_Clear(tstate); /* No zipimporter object -- okay */ + if (verbose) { + PySys_WriteStderr("# can't import zipimport.zipimporter\n"); + } + } + else { + /* sys.path_hooks.insert(0, zipimporter) */ + int err = PyList_Insert(path_hooks, 0, zipimporter); + Py_DECREF(zipimporter); + if (err < 0) { + return -1; + } + if (verbose) { + PySys_WriteStderr("# installed zipimport hook\n"); + } + } + + return 0; +} + +PyStatus +_PyImport_InitExternal(PyThreadState *tstate) +{ + int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; + + // XXX Initialize here: sys.path_hooks and sys.path_importer_cache. + + if (init_importlib_external(tstate->interp) != 0) { + _PyErr_Print(tstate); + return _PyStatus_ERR("external importer setup failed"); + } + + if (init_zipimport(tstate, verbose) != 0) { + PyErr_Print(); + return _PyStatus_ERR("initializing zipimport failed"); + } + + return _PyStatus_OK(); +} + +void +_PyImport_FiniExternal(PyInterpreterState *interp) +{ + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; + + // XXX Uninstall importlib metapath importers here? + + if (_PySys_ClearAttrString(interp, "path_importer_cache", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } + if (_PySys_ClearAttrString(interp, "path_hooks", verbose) < 0) { + PyErr_WriteUnraisable(NULL); + } +} + + +/******************/ +/* module helpers */ +/******************/ + +PyObject * +_PyImport_GetModuleAttr(PyObject *modname, PyObject *attrname) +{ + PyObject *mod = PyImport_Import(modname); + if (mod == NULL) { + return NULL; + } + PyObject *result = PyObject_GetAttr(mod, attrname); + Py_DECREF(mod); + return result; +} + +PyObject * +_PyImport_GetModuleAttrString(const char *modname, const char *attrname) +{ + PyObject *pmodname = PyUnicode_FromString(modname); + if (pmodname == NULL) { + return NULL; + } + PyObject *pattrname = PyUnicode_FromString(attrname); + if (pattrname == NULL) { + Py_DECREF(pmodname); + return NULL; + } + PyObject *result = _PyImport_GetModuleAttr(pmodname, pattrname); + Py_DECREF(pattrname); + Py_DECREF(pmodname); + return result; +} + + +/**************/ +/* the module */ +/**************/ + +/*[clinic input] +_imp.lock_held + +Return True if the import lock is currently held, else False. + +On platforms without threads, return False. +[clinic start generated code]*/ + +static PyObject * +_imp_lock_held_impl(PyObject *module) +/*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ +{ + return PyBool_FromLong(import_lock_thread != PYTHREAD_INVALID_THREAD_ID); +} + +/*[clinic input] +_imp.acquire_lock + +Acquires the interpreter's import lock for the current thread. + +This lock should be used by import hooks to ensure thread-safety when importing +modules. On platforms without threads, this function does nothing. +[clinic start generated code]*/ + +static PyObject * +_imp_acquire_lock_impl(PyObject *module) +/*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ +{ + _PyImport_AcquireLock(); + Py_RETURN_NONE; +} + +/*[clinic input] +_imp.release_lock + +Release the interpreter's import lock. + +On platforms without threads, this function does nothing. +[clinic start generated code]*/ + +static PyObject * +_imp_release_lock_impl(PyObject *module) +/*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ +{ + if (_PyImport_ReleaseLock() < 0) { + PyErr_SetString(PyExc_RuntimeError, + "not holding the import lock"); + return NULL; + } + Py_RETURN_NONE; +} + + +/*[clinic input] +_imp._fix_co_filename + + code: object(type="PyCodeObject *", subclass_of="&PyCode_Type") + Code object to change. + + path: unicode + File path to use. + / + +Changes code.co_filename to specify the passed-in file path. +[clinic start generated code]*/ + +static PyObject * +_imp__fix_co_filename_impl(PyObject *module, PyCodeObject *code, + PyObject *path) +/*[clinic end generated code: output=1d002f100235587d input=895ba50e78b82f05]*/ + +{ + update_compiled_module(code, path); - PyObject *from_list = PyList_New(0); - if (from_list == NULL) { - goto err; - } + Py_RETURN_NONE; +} - /* Get the builtins from current globals */ - globals = PyEval_GetGlobals(); - if (globals != NULL) { - Py_INCREF(globals); - builtins = PyObject_GetItem(globals, &_Py_ID(__builtins__)); - if (builtins == NULL) - goto err; - } - else { - /* No globals -- use standard builtins, and fake globals */ - builtins = PyImport_ImportModuleLevel("builtins", - NULL, NULL, NULL, 0); - if (builtins == NULL) { - goto err; - } - globals = Py_BuildValue("{OO}", &_Py_ID(__builtins__), builtins); - if (globals == NULL) - goto err; - } - /* Get the __import__ function from the builtins */ - if (PyDict_Check(builtins)) { - import = PyObject_GetItem(builtins, &_Py_ID(__import__)); - if (import == NULL) { - _PyErr_SetObject(tstate, PyExc_KeyError, &_Py_ID(__import__)); - } - } - else - import = PyObject_GetAttr(builtins, &_Py_ID(__import__)); - if (import == NULL) - goto err; +/*[clinic input] +_imp.create_builtin - /* Call the __import__ function with the proper argument list - Always use absolute import here. - Calling for side-effect of import. */ - r = PyObject_CallFunction(import, "OOOOi", module_name, globals, - globals, from_list, 0, NULL); - if (r == NULL) - goto err; - Py_DECREF(r); + spec: object + / - r = import_get_module(tstate, module_name); - if (r == NULL && !_PyErr_Occurred(tstate)) { - _PyErr_SetObject(tstate, PyExc_KeyError, module_name); +Create an extension module. +[clinic start generated code]*/ + +static PyObject * +_imp_create_builtin(PyObject *module, PyObject *spec) +/*[clinic end generated code: output=ace7ff22271e6f39 input=37f966f890384e47]*/ +{ + PyThreadState *tstate = _PyThreadState_GET(); + + PyObject *name = PyObject_GetAttrString(spec, "name"); + if (name == NULL) { + return NULL; } - err: - Py_XDECREF(globals); - Py_XDECREF(builtins); - Py_XDECREF(import); - Py_XDECREF(from_list); + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "name must be string, not %.200s", + Py_TYPE(name)->tp_name); + Py_DECREF(name); + return NULL; + } - return r; + PyObject *mod = create_builtin(tstate, name, spec); + Py_DECREF(name); + return mod; } + /*[clinic input] _imp.extension_suffixes @@ -2459,34 +3221,10 @@ _imp__override_frozen_modules_for_tests_impl(PyObject *module, int override) /*[clinic end generated code: output=36d5cb1594160811 input=8f1f95a3ef21aec3]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - interp->override_frozen_modules = override; + OVERRIDE_FROZEN_MODULES(interp) = override; Py_RETURN_NONE; } -/* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ -static int -exec_builtin_or_dynamic(PyObject *mod) { - PyModuleDef *def; - void *state; - - if (!PyModule_Check(mod)) { - return 0; - } - - def = PyModule_GetDef(mod); - if (def == NULL) { - return 0; - } - - state = PyModule_GetState(mod); - if (state) { - /* Already initialized; skip reload */ - return 0; - } - - return PyModule_ExecDef(mod, def); -} - #ifdef HAVE_DYNAMIC_LOADING /*[clinic input] @@ -2674,158 +3412,6 @@ PyInit__imp(void) } -// Import the _imp extension by calling manually _imp.create_builtin() and -// _imp.exec_builtin() since importlib is not initialized yet. Initializing -// importlib requires the _imp module: this function fix the bootstrap issue. -PyObject* -_PyImport_BootstrapImp(PyThreadState *tstate) -{ - PyObject *name = PyUnicode_FromString("_imp"); - if (name == NULL) { - return NULL; - } - - // Mock a ModuleSpec object just good enough for PyModule_FromDefAndSpec(): - // an object with just a name attribute. - // - // _imp.__spec__ is overridden by importlib._bootstrap._instal() anyway. - PyObject *attrs = Py_BuildValue("{sO}", "name", name); - if (attrs == NULL) { - goto error; - } - PyObject *spec = _PyNamespace_New(attrs); - Py_DECREF(attrs); - if (spec == NULL) { - goto error; - } - - // Create the _imp module from its definition. - PyObject *mod = create_builtin(tstate, name, spec); - Py_CLEAR(name); - Py_DECREF(spec); - if (mod == NULL) { - goto error; - } - assert(mod != Py_None); // not found - - // Execute the _imp module: call imp_module_exec(). - if (exec_builtin_or_dynamic(mod) < 0) { - Py_DECREF(mod); - goto error; - } - return mod; - -error: - Py_XDECREF(name); - return NULL; -} - - -/* API for embedding applications that want to add their own entries - to the table of built-in modules. This should normally be called - *before* Py_Initialize(). When the table resize fails, -1 is - returned and the existing table is unchanged. - - After a similar function by Just van Rossum. */ - -int -PyImport_ExtendInittab(struct _inittab *newtab) -{ - struct _inittab *p; - size_t i, n; - int res = 0; - - if (_PyRuntime.imports.inittab != NULL) { - Py_FatalError("PyImport_ExtendInittab() may not be called after Py_Initialize()"); - } - - /* Count the number of entries in both tables */ - for (n = 0; newtab[n].name != NULL; n++) - ; - if (n == 0) - return 0; /* Nothing to do */ - for (i = 0; PyImport_Inittab[i].name != NULL; i++) - ; - - /* Force default raw memory allocator to get a known allocator to be able - to release the memory in _PyImport_Fini2() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Allocate new memory for the combined table */ - p = NULL; - if (i + n <= SIZE_MAX / sizeof(struct _inittab) - 1) { - size_t size = sizeof(struct _inittab) * (i + n + 1); - p = PyMem_RawRealloc(inittab_copy, size); - } - if (p == NULL) { - res = -1; - goto done; - } - - /* Copy the tables into the new memory at the first call - to PyImport_ExtendInittab(). */ - if (inittab_copy != PyImport_Inittab) { - memcpy(p, PyImport_Inittab, (i+1) * sizeof(struct _inittab)); - } - memcpy(p + i, newtab, (n + 1) * sizeof(struct _inittab)); - PyImport_Inittab = inittab_copy = p; - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return res; -} - -/* Shorthand to add a single entry given a name and a function */ - -int -PyImport_AppendInittab(const char *name, PyObject* (*initfunc)(void)) -{ - struct _inittab newtab[2]; - - if (_PyRuntime.imports.inittab != NULL) { - Py_FatalError("PyImport_AppendInittab() may not be called after Py_Initialize()"); - } - - memset(newtab, '\0', sizeof newtab); - - newtab[0].name = name; - newtab[0].initfunc = initfunc; - - return PyImport_ExtendInittab(newtab); -} - - -PyObject * -_PyImport_GetModuleAttr(PyObject *modname, PyObject *attrname) -{ - PyObject *mod = PyImport_Import(modname); - if (mod == NULL) { - return NULL; - } - PyObject *result = PyObject_GetAttr(mod, attrname); - Py_DECREF(mod); - return result; -} - -PyObject * -_PyImport_GetModuleAttrString(const char *modname, const char *attrname) -{ - PyObject *pmodname = PyUnicode_FromString(modname); - if (pmodname == NULL) { - return NULL; - } - PyObject *pattrname = PyUnicode_FromString(attrname); - if (pattrname == NULL) { - Py_DECREF(pmodname); - return NULL; - } - PyObject *result = _PyImport_GetModuleAttr(pmodname, pattrname); - Py_DECREF(pattrname); - Py_DECREF(pmodname); - return result; -} - #ifdef __cplusplus } #endif diff --git a/Python/importdl.c b/Python/importdl.c index 91fa06f49c2897..6dafb4541486e9 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -99,7 +99,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) #endif PyObject *name_unicode = NULL, *name = NULL, *path = NULL, *m = NULL; const char *name_buf, *hook_prefix; - const char *oldcontext; + const char *oldcontext, *newcontext; dl_funcptr exportfunc; PyModuleDef *def; PyModInitFunction p0; @@ -113,6 +113,10 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) "spec.name must be a string"); goto error; } + newcontext = PyUnicode_AsUTF8(name_unicode); + if (newcontext == NULL) { + goto error; + } name = get_encoded_name(name_unicode, &hook_prefix); if (name == NULL) { @@ -160,16 +164,9 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) p0 = (PyModInitFunction)exportfunc; /* Package context is needed for single-phase init */ -#define _Py_PackageContext (_PyRuntime.imports.pkgcontext) - oldcontext = _Py_PackageContext; - _Py_PackageContext = PyUnicode_AsUTF8(name_unicode); - if (_Py_PackageContext == NULL) { - _Py_PackageContext = oldcontext; - goto error; - } + oldcontext = _PyImport_SwapPackageContext(newcontext); m = _PyImport_InitFunc_TrampolineCall(p0); - _Py_PackageContext = oldcontext; -#undef _Py_PackageContext + _PyImport_SwapPackageContext(oldcontext); if (m == NULL) { if (!PyErr_Occurred()) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 045a2996e8988b..281035dafa9577 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -156,79 +156,6 @@ Py_IsInitialized(void) } -/* Global initializations. Can be undone by Py_FinalizeEx(). Don't - call this twice without an intervening Py_FinalizeEx() call. When - initializations fail, a fatal error is issued and the function does - not return. On return, the first thread and interpreter state have - been created. - - Locking: you must hold the interpreter lock while calling this. - (If the lock has not yet been initialized, that's equivalent to - having the lock, but you cannot use multiple threads.) - -*/ -static int -init_importlib(PyThreadState *tstate, PyObject *sysmod) -{ - assert(!_PyErr_Occurred(tstate)); - - PyInterpreterState *interp = tstate->interp; - int verbose = _PyInterpreterState_GetConfig(interp)->verbose; - - // Import _importlib through its frozen version, _frozen_importlib. - if (verbose) { - PySys_FormatStderr("import _frozen_importlib # frozen\n"); - } - if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) { - return -1; - } - PyObject *importlib = PyImport_AddModule("_frozen_importlib"); // borrowed - if (importlib == NULL) { - return -1; - } - interp->importlib = Py_NewRef(importlib); - - // Import the _imp module - if (verbose) { - PySys_FormatStderr("import _imp # builtin\n"); - } - PyObject *imp_mod = _PyImport_BootstrapImp(tstate); - if (imp_mod == NULL) { - return -1; - } - if (_PyImport_SetModuleString("_imp", imp_mod) < 0) { - Py_DECREF(imp_mod); - return -1; - } - - // Install importlib as the implementation of import - PyObject *value = PyObject_CallMethod(importlib, "_install", - "OO", sysmod, imp_mod); - Py_DECREF(imp_mod); - if (value == NULL) { - return -1; - } - Py_DECREF(value); - - assert(!_PyErr_Occurred(tstate)); - return 0; -} - - -static PyStatus -init_importlib_external(PyThreadState *tstate) -{ - PyObject *value; - value = PyObject_CallMethod(tstate->interp->importlib, - "_install_external_importers", ""); - if (value == NULL) { - _PyErr_Print(tstate); - return _PyStatus_ERR("external importer setup failed"); - } - Py_DECREF(value); - return _PyImportZip_Init(tstate); -} - /* Helper functions to better handle the legacy C locale * * The legacy C locale assumes ASCII as the default text encoding, which @@ -814,7 +741,8 @@ pycore_init_builtins(PyThreadState *tstate) goto error; } - if (_PyImport_FixupBuiltin(bimod, "builtins", interp->modules) < 0) { + PyObject *modules = _PyImport_GetModules(interp); + if (_PyImport_FixupBuiltin(bimod, "builtins", modules) < 0) { goto error; } @@ -850,13 +778,9 @@ pycore_init_builtins(PyThreadState *tstate) } Py_DECREF(bimod); - // Get the __import__ function - PyObject *import_func = _PyDict_GetItemStringWithError(interp->builtins, - "__import__"); - if (import_func == NULL) { + if (_PyImport_InitDefaultImportFunc(interp) < 0) { goto error; } - interp->import_func = Py_NewRef(import_func); assert(!_PyErr_Occurred(tstate)); return _PyStatus_OK(); @@ -918,11 +842,10 @@ pycore_interp_init(PyThreadState *tstate) } const PyConfig *config = _PyInterpreterState_GetConfig(interp); - if (config->_install_importlib) { - /* This call sets up builtin and frozen import support */ - if (init_importlib(tstate, sysmod) < 0) { - return _PyStatus_ERR("failed to initialize importlib"); - } + + status = _PyImport_InitCore(tstate, sysmod, config->_install_importlib); + if (_PyStatus_EXCEPTION(status)) { + goto done; } done: @@ -1172,7 +1095,7 @@ init_interp_main(PyThreadState *tstate) return _PyStatus_ERR("failed to update the Python config"); } - status = init_importlib_external(tstate); + status = _PyImport_InitExternal(tstate); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -1379,8 +1302,11 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose) static const char * const sys_deletes[] = { "path", "argv", "ps1", "ps2", "last_type", "last_value", "last_traceback", - "path_hooks", "path_importer_cache", "meta_path", "__interactivehook__", + // path_hooks and path_importer_cache are cleared + // by _PyImport_FiniExternal(). + // XXX Clear meta_path in _PyImport_FiniCore(). + "meta_path", NULL }; @@ -1401,10 +1327,7 @@ finalize_modules_delete_special(PyThreadState *tstate, int verbose) const char * const *p; for (p = sys_deletes; *p != NULL; p++) { - if (verbose) { - PySys_WriteStderr("# clear sys.%s\n", *p); - } - if (PyDict_SetItemString(interp->sysdict, *p, Py_None) < 0) { + if (_PySys_ClearAttrString(interp, *p, verbose) < 0) { PyErr_WriteUnraisable(NULL); } } @@ -1576,11 +1499,12 @@ finalize_clear_sys_builtins_dict(PyInterpreterState *interp, int verbose) /* Clear modules, as good as we can */ +// XXX Move most of this to import.c. static void finalize_modules(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; - PyObject *modules = interp->modules; + PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL) { // Already done return; @@ -1645,12 +1569,12 @@ finalize_modules(PyThreadState *tstate) // clear PyInterpreterState.modules_by_index and // clear PyModuleDef.m_base.m_copy (of extensions not using the multi-phase // initialization API) - _PyInterpreterState_ClearModules(interp); + _PyImport_ClearModulesByIndex(interp); // Clear and delete the modules directory. Actual modules will // still be there only if imported during the execution of some // destructor. - Py_SETREF(interp->modules, NULL); + _PyImport_ClearModules(interp); // Collect garbage once more _PyGC_CollectNoFail(tstate); @@ -1861,6 +1785,8 @@ Py_FinalizeEx(void) runtime->initialized = 0; runtime->core_initialized = 0; + // XXX Call something like _PyImport_Disable() here? + /* Destroy the state of all threads of the interpreter, except of the current thread. In practice, only daemon threads should still be alive, except if wait_for_thread_shutdown() has been cancelled by CTRL+C. @@ -1910,6 +1836,7 @@ Py_FinalizeEx(void) PyGC_Collect(); /* Destroy all modules */ + _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); /* Print debug stats if any */ @@ -1943,7 +1870,9 @@ Py_FinalizeEx(void) so it is possible to use tracemalloc in objects destructor. */ _PyTraceMalloc_Fini(); - /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ + /* Finalize any remaining import state */ + // XXX Move these up to where finalize_modules() is currently. + _PyImport_FiniCore(tstate->interp); _PyImport_Fini(); /* unload faulthandler module */ @@ -2183,7 +2112,11 @@ Py_EndInterpreter(PyThreadState *tstate) Py_FatalError("not the last thread"); } + // XXX Call something like _PyImport_Disable() here? + + _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); + _PyImport_FiniCore(tstate->interp); finalize_interp_clear(tstate); finalize_interp_delete(tstate->interp); @@ -2232,8 +2165,8 @@ add_main_module(PyInterpreterState *interp) if (PyErr_Occurred()) { return _PyStatus_ERR("Failed to test __main__.__loader__"); } - PyObject *loader = PyObject_GetAttrString(interp->importlib, - "BuiltinImporter"); + PyObject *loader = _PyImport_GetImportlibLoader(interp, + "BuiltinImporter"); if (loader == NULL) { return _PyStatus_ERR("Failed to retrieve BuiltinImporter"); } @@ -2739,7 +2672,7 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) if (interp == NULL) { return; } - PyObject *modules = interp->modules; + PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL || !PyDict_Check(modules)) { return; } diff --git a/Python/pystate.c b/Python/pystate.c index 1261092d1435fa..4770caaed0a363 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -772,11 +772,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->codec_search_path); Py_CLEAR(interp->codec_search_cache); Py_CLEAR(interp->codec_error_registry); - Py_CLEAR(interp->modules); - Py_CLEAR(interp->modules_by_index); + + assert(interp->imports.modules == NULL); + assert(interp->imports.modules_by_index == NULL); + assert(interp->imports.importlib == NULL); + assert(interp->imports.import_func == NULL); + Py_CLEAR(interp->builtins_copy); - Py_CLEAR(interp->importlib); - Py_CLEAR(interp->import_func); Py_CLEAR(interp->dict); #ifdef HAVE_FORK Py_CLEAR(interp->before_forkers); @@ -836,6 +838,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) // garbage. It can be different than the current Python thread state // of 'interp'. PyThreadState *current_tstate = current_fast_get(interp->runtime); + _PyImport_ClearCore(interp); interpreter_clear(interp, current_tstate); } @@ -843,6 +846,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) void _PyInterpreterState_Clear(PyThreadState *tstate) { + _PyImport_ClearCore(tstate->interp); interpreter_clear(tstate->interp, tstate); } @@ -945,36 +949,6 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) #endif -// Used by finalize_modules() -void -_PyInterpreterState_ClearModules(PyInterpreterState *interp) -{ - if (!interp->modules_by_index) { - return; - } - - Py_ssize_t i; - for (i = 0; i < PyList_GET_SIZE(interp->modules_by_index); i++) { - PyObject *m = PyList_GET_ITEM(interp->modules_by_index, i); - if (PyModule_Check(m)) { - /* cleanup the saved copy of module dicts */ - PyModuleDef *md = PyModule_GetDef(m); - if (md) { - Py_CLEAR(md->m_base.m_copy); - } - } - } - - /* Setting modules_by_index to NULL could be dangerous, so we - clear the list instead. */ - if (PyList_SetSlice(interp->modules_by_index, - 0, PyList_GET_SIZE(interp->modules_by_index), - NULL)) { - PyErr_WriteUnraisable(interp->modules_by_index); - } -} - - //---------- // accessors //---------- @@ -1058,11 +1032,12 @@ _PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required) PyObject * _PyInterpreterState_GetMainModule(PyInterpreterState *interp) { - if (interp->modules == NULL) { + PyObject *modules = _PyImport_GetModules(interp); + if (modules == NULL) { PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized"); return NULL; } - return PyMapping_GetItemString(interp->modules, "__main__"); + return PyMapping_GetItemString(modules, "__main__"); } PyObject * @@ -1922,110 +1897,6 @@ _PyThread_CurrentExceptions(void) } -/****************/ -/* module state */ -/****************/ - -PyObject* -PyState_FindModule(PyModuleDef* module) -{ - Py_ssize_t index = module->m_base.m_index; - PyInterpreterState *state = _PyInterpreterState_GET(); - PyObject *res; - if (module->m_slots) { - return NULL; - } - if (index == 0) - return NULL; - if (state->modules_by_index == NULL) - return NULL; - if (index >= PyList_GET_SIZE(state->modules_by_index)) - return NULL; - res = PyList_GET_ITEM(state->modules_by_index, index); - return res==Py_None ? NULL : res; -} - -int -_PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def) -{ - if (!def) { - assert(_PyErr_Occurred(tstate)); - return -1; - } - if (def->m_slots) { - _PyErr_SetString(tstate, - PyExc_SystemError, - "PyState_AddModule called on module with slots"); - return -1; - } - - PyInterpreterState *interp = tstate->interp; - if (!interp->modules_by_index) { - interp->modules_by_index = PyList_New(0); - if (!interp->modules_by_index) { - return -1; - } - } - - while (PyList_GET_SIZE(interp->modules_by_index) <= def->m_base.m_index) { - if (PyList_Append(interp->modules_by_index, Py_None) < 0) { - return -1; - } - } - - return PyList_SetItem(interp->modules_by_index, - def->m_base.m_index, Py_NewRef(module)); -} - -int -PyState_AddModule(PyObject* module, PyModuleDef* def) -{ - if (!def) { - Py_FatalError("module definition is NULL"); - return -1; - } - - PyThreadState *tstate = current_fast_get(&_PyRuntime); - PyInterpreterState *interp = tstate->interp; - Py_ssize_t index = def->m_base.m_index; - if (interp->modules_by_index && - index < PyList_GET_SIZE(interp->modules_by_index) && - module == PyList_GET_ITEM(interp->modules_by_index, index)) - { - _Py_FatalErrorFormat(__func__, "module %p already added", module); - return -1; - } - return _PyState_AddModule(tstate, module, def); -} - -int -PyState_RemoveModule(PyModuleDef* def) -{ - PyThreadState *tstate = current_fast_get(&_PyRuntime); - PyInterpreterState *interp = tstate->interp; - - if (def->m_slots) { - _PyErr_SetString(tstate, - PyExc_SystemError, - "PyState_RemoveModule called on module with slots"); - return -1; - } - - Py_ssize_t index = def->m_base.m_index; - if (index == 0) { - Py_FatalError("invalid module index"); - } - if (interp->modules_by_index == NULL) { - Py_FatalError("Interpreters module-list not accessible."); - } - if (index > PyList_GET_SIZE(interp->modules_by_index)) { - Py_FatalError("Module index out of bounds."); - } - - return PyList_SetItem(interp->modules_by_index, index, Py_NewRef(Py_None)); -} - - /***********************************/ /* Python "auto thread state" API. */ /***********************************/ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 6a4d593768690a..ce993ea8796cb7 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -350,14 +350,8 @@ static int set_main_loader(PyObject *d, PyObject *filename, const char *loader_name) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *bootstrap = PyObject_GetAttrString(interp->importlib, - "_bootstrap_external"); - if (bootstrap == NULL) { - return -1; - } - - PyObject *loader_type = PyObject_GetAttrString(bootstrap, loader_name); - Py_DECREF(bootstrap); + PyObject *loader_type = _PyImport_GetImportlibExternalLoader(interp, + loader_name); if (loader_type == NULL) { return -1; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 6e81ef92b67f70..b69b803560924c 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -142,6 +142,20 @@ PySys_SetObject(const char *name, PyObject *v) return sys_set_object_str(interp, name, v); } +int +_PySys_ClearAttrString(PyInterpreterState *interp, + const char *name, int verbose) +{ + if (verbose) { + PySys_WriteStderr("# clear sys.%s\n", name); + } + /* To play it safe, we set the attr to None instead of deleting it. */ + if (PyDict_SetItemString(interp->sysdict, name, Py_None) < 0) { + return -1; + } + return 0; +} + static int should_audit(PyInterpreterState *interp) @@ -1650,7 +1664,7 @@ sys_setdlopenflags_impl(PyObject *module, int new_val) /*[clinic end generated code: output=ec918b7fe0a37281 input=4c838211e857a77f]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - interp->dlopenflags = new_val; + _PyImport_SetDLOpenFlags(interp, new_val); Py_RETURN_NONE; } @@ -1668,7 +1682,8 @@ sys_getdlopenflags_impl(PyObject *module) /*[clinic end generated code: output=e92cd1bc5005da6e input=dc4ea0899c53b4b6]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyLong_FromLong(interp->dlopenflags); + return PyLong_FromLong( + _PyImport_GetDLOpenFlags(interp)); } #endif /* HAVE_DLOPEN */ @@ -2279,22 +2294,10 @@ static PyMethodDef sys_methods[] = { static PyObject * list_builtin_module_names(void) { - PyObject *list = PyList_New(0); + PyObject *list = _PyImport_GetBuiltinModuleNames(); if (list == NULL) { return NULL; } - struct _inittab *inittab = _PyRuntime.imports.inittab; - for (Py_ssize_t i = 0; inittab[i].name != NULL; i++) { - PyObject *name = PyUnicode_FromString(inittab[i].name); - if (name == NULL) { - goto error; - } - if (PyList_Append(list, name) < 0) { - Py_DECREF(name); - goto error; - } - Py_DECREF(name); - } if (PyList_Sort(list) != 0) { goto error; } @@ -3411,11 +3414,10 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) PyInterpreterState *interp = tstate->interp; - PyObject *modules = PyDict_New(); + PyObject *modules = _PyImport_InitModules(interp); if (modules == NULL) { goto error; } - interp->modules = modules; PyObject *sysmod = _PyModule_CreateInitialized(&sysmodule, PYTHON_API_VERSION); if (sysmod == NULL) { @@ -3428,7 +3430,7 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) } interp->sysdict = Py_NewRef(sysdict); - if (PyDict_SetItemString(sysdict, "modules", interp->modules) < 0) { + if (PyDict_SetItemString(sysdict, "modules", modules) < 0) { goto error; } @@ -3442,7 +3444,7 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) return status; } - if (_PyImport_FixupBuiltin(sysmod, "sys", interp->modules) < 0) { + if (_PyImport_FixupBuiltin(sysmod, "sys", modules) < 0) { goto error; } From b365d88465d9228ce4e9e0be20b88e9e4056ad88 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 15 Feb 2023 16:05:07 -0700 Subject: [PATCH 47/83] gh-101758: Add a Test For Single-Phase Init Modules in Multiple Interpreters (gh-101920) The test verifies the behavior of single-phase init modules when loaded in multiple interpreters. https://github.com/python/cpython/issues/101758 --- Include/internal/pycore_import.h | 3 ++ Lib/test/test_imp.py | 73 ++++++++++++++++++++++++++++++ Modules/_testinternalcapi.c | 15 +++++++ Modules/_testsinglephase.c | 49 +++++++++++++++++++- Python/import.c | 76 +++++++++++++++++++++++++++++++- 5 files changed, 212 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index da766253ef6b9c..6ee7356b41c021 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -153,6 +153,9 @@ PyAPI_DATA(const struct _frozen *) _PyImport_FrozenStdlib; PyAPI_DATA(const struct _frozen *) _PyImport_FrozenTest; extern const struct _module_alias * _PyImport_FrozenAliases; +// for testing +PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index c85ab92307de78..e81eb6f0a86fe8 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -10,10 +10,13 @@ from test.support import os_helper from test.support import script_helper from test.support import warnings_helper +import textwrap import unittest import warnings imp = warnings_helper.import_deprecated('imp') import _imp +import _testinternalcapi +import _xxsubinterpreters as _interpreters OS_PATH_NAME = os.path.__name__ @@ -251,6 +254,71 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) + @requires_load_dynamic + def test_singlephase_multiple_interpreters(self): + # Currently, for every single-phrase init module loaded + # in multiple interpreters, those interpreters share a + # PyModuleDef for that object, which can be a problem. + + # This single-phase module has global state, which is shared + # by the interpreters. + import _testsinglephase + name = _testsinglephase.__name__ + filename = _testsinglephase.__file__ + + del sys.modules[name] + _testsinglephase._clear_globals() + _testinternalcapi.clear_extension(name, filename) + init_count = _testsinglephase.initialized_count() + assert init_count == -1, (init_count,) + + def clean_up(): + _testsinglephase._clear_globals() + _testinternalcapi.clear_extension(name, filename) + self.addCleanup(clean_up) + + interp1 = _interpreters.create(isolated=False) + self.addCleanup(_interpreters.destroy, interp1) + interp2 = _interpreters.create(isolated=False) + self.addCleanup(_interpreters.destroy, interp2) + + script = textwrap.dedent(f''' + import _testsinglephase + + expected = %d + init_count = _testsinglephase.initialized_count() + if init_count != expected: + raise Exception(init_count) + + lookedup = _testsinglephase.look_up_self() + if lookedup is not _testsinglephase: + raise Exception((_testsinglephase, lookedup)) + + # Attrs set in the module init func are in m_copy. + _initialized = _testsinglephase._initialized + initialized = _testsinglephase.initialized() + if _initialized != initialized: + raise Exception((_initialized, initialized)) + + # Attrs set after loading are not in m_copy. + if hasattr(_testsinglephase, 'spam'): + raise Exception(_testsinglephase.spam) + _testsinglephase.spam = expected + ''') + + # Use an interpreter that gets destroyed right away. + ret = support.run_in_subinterp(script % 1) + self.assertEqual(ret, 0) + + # The module's init func gets run again. + # The module's globals did not get destroyed. + _interpreters.run_string(interp1, script % 2) + + # The module's init func is not run again. + # The second interpreter copies the module's m_copy. + # However, globals are still shared. + _interpreters.run_string(interp2, script % 2) + @requires_load_dynamic def test_singlephase_variants(self): '''Exercise the most meaningful variants described in Python/import.c.''' @@ -260,6 +328,11 @@ def test_singlephase_variants(self): fileobj, pathname, _ = imp.find_module(basename) fileobj.close() + def clean_up(): + import _testsinglephase + _testsinglephase._clear_globals() + self.addCleanup(clean_up) + modules = {} def load(name): assert name not in modules diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ba57719d92096b..632fac2de0c419 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -671,6 +671,20 @@ get_interp_settings(PyObject *self, PyObject *args) } +static PyObject * +clear_extension(PyObject *self, PyObject *args) +{ + PyObject *name = NULL, *filename = NULL; + if (!PyArg_ParseTuple(args, "OO:clear_extension", &name, &filename)) { + return NULL; + } + if (_PyImport_ClearExtension(name, filename) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -692,6 +706,7 @@ static PyMethodDef module_functions[] = { _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF {"get_interp_settings", get_interp_settings, METH_VARARGS, NULL}, + {"clear_extension", clear_extension, METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 9e8dd647ee761a..565221c887e5ae 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -17,12 +17,27 @@ typedef struct { PyObject *str_const; } module_state; + /* Process-global state is only used by _testsinglephase since it's the only one that does not support re-init. */ static struct { int initialized_count; module_state module; -} global_state = { .initialized_count = -1 }; +} global_state = { + +#define NOT_INITIALIZED -1 + .initialized_count = NOT_INITIALIZED, +}; + +static void clear_state(module_state *state); + +static void +clear_global_state(void) +{ + clear_state(&global_state.module); + global_state.initialized_count = NOT_INITIALIZED; +} + static inline module_state * get_module_state(PyObject *module) @@ -106,6 +121,7 @@ init_state(module_state *state) return -1; } + static int init_module(PyObject *module, module_state *state) { @@ -118,6 +134,16 @@ init_module(PyObject *module, module_state *state) if (PyModule_AddObjectRef(module, "str_const", state->str_const) != 0) { return -1; } + + double d = _PyTime_AsSecondsDouble(state->initialized); + PyObject *initialized = PyFloat_FromDouble(d); + if (initialized == NULL) { + return -1; + } + if (PyModule_AddObjectRef(module, "_initialized", initialized) != 0) { + return -1; + } + return 0; } @@ -198,10 +224,28 @@ basic_initialized_count(PyObject *self, PyObject *Py_UNUSED(ignored)) } #define INITIALIZED_COUNT_METHODDEF \ - {"initialized_count", basic_initialized_count, METH_VARARGS, \ + {"initialized_count", basic_initialized_count, METH_NOARGS, \ basic_initialized_count_doc} +PyDoc_STRVAR(basic__clear_globals_doc, +"_clear_globals()\n\ +\n\ +Free all global state and set it to uninitialized."); + +static PyObject * +basic__clear_globals(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + assert(PyModule_GetDef(self)->m_size == -1); + clear_global_state(); + Py_RETURN_NONE; +} + +#define _CLEAR_GLOBALS_METHODDEF \ + {"_clear_globals", basic__clear_globals, METH_NOARGS, \ + basic__clear_globals_doc} + + /*********************************************/ /* the _testsinglephase module (and aliases) */ /*********************************************/ @@ -223,6 +267,7 @@ static PyMethodDef TestMethods_Basic[] = { SUM_METHODDEF, INITIALIZED_METHODDEF, INITIALIZED_COUNT_METHODDEF, + _CLEAR_GLOBALS_METHODDEF, {NULL, NULL} /* sentinel */ }; diff --git a/Python/import.c b/Python/import.c index ae27aaf56848d6..87981668a30505 100644 --- a/Python/import.c +++ b/Python/import.c @@ -632,6 +632,28 @@ exec_builtin_or_dynamic(PyObject *mod) { } +static int clear_singlephase_extension(PyInterpreterState *interp, + PyObject *name, PyObject *filename); + +// Currently, this is only used for testing. +// (See _testinternalcapi.clear_extension().) +int +_PyImport_ClearExtension(PyObject *name, PyObject *filename) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + + /* Clearing a module's C globals is up to the module. */ + if (clear_singlephase_extension(interp, name, filename) < 0) { + return -1; + } + + // In the future we'll probably also make sure the extension's + // file handle (and DL handle) is closed (requires saving it). + + return 0; +} + + /*******************/ #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) @@ -766,8 +788,30 @@ _extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) return 0; } +static int +_extensions_cache_delete(PyObject *filename, PyObject *name) +{ + PyObject *extensions = EXTENSIONS; + if (extensions == NULL) { + return 0; + } + PyObject *key = PyTuple_Pack(2, filename, name); + if (key == NULL) { + return -1; + } + if (PyDict_DelItem(extensions, key) < 0) { + if (!PyErr_ExceptionMatches(PyExc_KeyError)) { + Py_DECREF(key); + return -1; + } + PyErr_Clear(); + } + Py_DECREF(key); + return 0; +} + static void -_extensions_cache_clear(void) +_extensions_cache_clear_all(void) { Py_CLEAR(EXTENSIONS); } @@ -890,6 +934,34 @@ import_find_extension(PyThreadState *tstate, PyObject *name, return mod; } +static int +clear_singlephase_extension(PyInterpreterState *interp, + PyObject *name, PyObject *filename) +{ + PyModuleDef *def = _extensions_cache_get(filename, name); + if (def == NULL) { + if (PyErr_Occurred()) { + return -1; + } + return 0; + } + + /* Clear data set when the module was initially loaded. */ + def->m_base.m_init = NULL; + Py_CLEAR(def->m_base.m_copy); + // We leave m_index alone since there's no reason to reset it. + + /* Clear the PyState_*Module() cache entry. */ + if (_modules_by_index_check(interp, def->m_base.m_index) == NULL) { + if (_modules_by_index_clear(interp, def) < 0) { + return -1; + } + } + + /* Clear the cached module def. */ + return _extensions_cache_delete(filename, name); +} + /*******************/ /* builtin modules */ @@ -2633,7 +2705,7 @@ void _PyImport_Fini(void) { /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ - _extensions_cache_clear(); + _extensions_cache_clear_all(); if (import_lock != NULL) { PyThread_free_lock(import_lock); import_lock = NULL; From 3dea4ba6c1b9237893d23574f931f33c940b74e8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 15 Feb 2023 17:54:05 -0700 Subject: [PATCH 48/83] gh-101758: Fix the wasm Buildbots (gh-101943) They were broken by gh-101920. https://github.com/python/cpython/issues/101758 --- Lib/test/test_imp.py | 12 +++++++++++- Python/pystate.c | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index e81eb6f0a86fe8..5997ffad8e1232 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -16,12 +16,21 @@ imp = warnings_helper.import_deprecated('imp') import _imp import _testinternalcapi -import _xxsubinterpreters as _interpreters +try: + import _xxsubinterpreters as _interpreters +except ModuleNotFoundError: + _interpreters = None OS_PATH_NAME = os.path.__name__ +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(_interpreters is None, + 'subinterpreters required')(meth) + + def requires_load_dynamic(meth): """Decorator to skip a test if not running under CPython or lacking imp.load_dynamic().""" @@ -254,6 +263,7 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) + @requires_subinterpreters @requires_load_dynamic def test_singlephase_multiple_interpreters(self): # Currently, for every single-phrase init module loaded diff --git a/Python/pystate.c b/Python/pystate.c index 4770caaed0a363..32b17fd19e348f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -197,6 +197,7 @@ gilstate_tss_clear(_PyRuntimeState *runtime) } +#ifndef NDEBUG static inline int tstate_is_alive(PyThreadState *tstate); static inline int @@ -204,6 +205,7 @@ tstate_is_bound(PyThreadState *tstate) { return tstate->_status.bound && !tstate->_status.unbound; } +#endif // !NDEBUG static void bind_gilstate_tstate(PyThreadState *); static void unbind_gilstate_tstate(PyThreadState *); @@ -1119,6 +1121,7 @@ _PyInterpreterState_LookUpID(int64_t requested_id) /* the per-thread runtime state */ /********************************/ +#ifndef NDEBUG static inline int tstate_is_alive(PyThreadState *tstate) { @@ -1127,6 +1130,7 @@ tstate_is_alive(PyThreadState *tstate) !tstate->_status.cleared && !tstate->_status.finalizing); } +#endif //---------- From 89ac665891dec1988bedec2ce9b2c4d016502a49 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 15 Feb 2023 18:16:00 -0700 Subject: [PATCH 49/83] gh-98627: Add an Optional Check for Extension Module Subinterpreter Compatibility (gh-99040) Enforcing (optionally) the restriction set by PEP 489 makes sense. Furthermore, this sets the stage for a potential restriction related to a per-interpreter GIL. This change includes the following: * add tests for extension module subinterpreter compatibility * add _PyInterpreterConfig.check_multi_interp_extensions * add Py_RTFLAGS_MULTI_INTERP_EXTENSIONS * add _PyImport_CheckSubinterpIncompatibleExtensionAllowed() * fail iff the module does not implement multi-phase init and the current interpreter is configured to check https://github.com/python/cpython/issues/98627 --- Include/cpython/initconfig.h | 3 + Include/cpython/pystate.h | 3 + Include/internal/pycore_import.h | 5 + Lib/test/support/import_helper.py | 18 ++ Lib/test/test_capi/check_config.py | 77 ++++++ Lib/test/test_capi/test_misc.py | 98 +++++++- Lib/test/test_embed.py | 4 +- Lib/test/test_import/__init__.py | 220 +++++++++++++++++- Lib/test/test_threading.py | 1 + ...2-11-02-20-23-47.gh-issue-98627.VJkdRM.rst | 5 + Modules/_testcapimodule.c | 12 +- Python/clinic/import.c.h | 33 ++- Python/import.c | 88 ++++++- Python/importdl.c | 5 + Python/pylifecycle.c | 4 + 15 files changed, 557 insertions(+), 19 deletions(-) create mode 100644 Lib/test/test_capi/check_config.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 6ce42b4c09502f..a070fa9ff3a038 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -248,6 +248,7 @@ typedef struct { int allow_exec; int allow_threads; int allow_daemon_threads; + int check_multi_interp_extensions; } _PyInterpreterConfig; #define _PyInterpreterConfig_INIT \ @@ -256,6 +257,7 @@ typedef struct { .allow_exec = 0, \ .allow_threads = 1, \ .allow_daemon_threads = 0, \ + .check_multi_interp_extensions = 1, \ } #define _PyInterpreterConfig_LEGACY_INIT \ @@ -264,6 +266,7 @@ typedef struct { .allow_exec = 1, \ .allow_threads = 1, \ .allow_daemon_threads = 1, \ + .check_multi_interp_extensions = 0, \ } /* --- Helper functions --------------------------------------- */ diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index be1fcb61fa2244..3efb241e8237e7 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -11,6 +11,9 @@ is available in a given context. For example, forking the process might not be allowed in the current interpreter (i.e. os.fork() would fail). */ +/* Set if import should check a module for subinterpreter support. */ +#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8) + /* Set if threads are allowed. */ #define Py_RTFLAGS_THREADS (1UL << 10) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 6ee7356b41c021..b7ffe01c0c0e20 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -64,6 +64,7 @@ struct _import_state { /* override for config->use_frozen_modules (for tests) (-1: "off", 1: "on", 0: no override) */ int override_frozen_modules; + int override_multi_interp_extensions_check; #ifdef HAVE_DLOPEN int dlopenflags; #endif @@ -153,6 +154,10 @@ PyAPI_DATA(const struct _frozen *) _PyImport_FrozenStdlib; PyAPI_DATA(const struct _frozen *) _PyImport_FrozenTest; extern const struct _module_alias * _PyImport_FrozenAliases; +PyAPI_FUNC(int) _PyImport_CheckSubinterpIncompatibleExtensionAllowed( + const char *name); + + // for testing PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 63a8a7952db7a6..772c0987c2ebef 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -105,6 +105,24 @@ def frozen_modules(enabled=True): _imp._override_frozen_modules_for_tests(0) +@contextlib.contextmanager +def multi_interp_extensions_check(enabled=True): + """Force legacy modules to be allowed in subinterpreters (or not). + + ("legacy" == single-phase init) + + This only applies to modules that haven't been imported yet. + It overrides the PyInterpreterConfig.check_multi_interp_extensions + setting (see support.run_in_subinterp_with_config() and + _xxsubinterpreters.create()). + """ + old = _imp._override_multi_interp_extensions_check(1 if enabled else -1) + try: + yield + finally: + _imp._override_multi_interp_extensions_check(old) + + def import_fresh_module(name, fresh=(), blocked=(), *, deprecated=False, usefrozen=False, diff --git a/Lib/test/test_capi/check_config.py b/Lib/test/test_capi/check_config.py new file mode 100644 index 00000000000000..aaedd82f39af50 --- /dev/null +++ b/Lib/test/test_capi/check_config.py @@ -0,0 +1,77 @@ +# This script is used by test_misc. + +import _imp +import _testinternalcapi +import json +import os +import sys + + +def import_singlephase(): + assert '_testsinglephase' not in sys.modules + try: + import _testsinglephase + except ImportError: + sys.modules.pop('_testsinglephase') + return False + else: + del sys.modules['_testsinglephase'] + return True + + +def check_singlephase(override): + # Check using the default setting. + settings_initial = _testinternalcapi.get_interp_settings() + allowed_initial = import_singlephase() + assert(_testinternalcapi.get_interp_settings() == settings_initial) + + # Apply the override and check. + override_initial = _imp._override_multi_interp_extensions_check(override) + settings_after = _testinternalcapi.get_interp_settings() + allowed_after = import_singlephase() + + # Apply the override again and check. + noop = {} + override_after = _imp._override_multi_interp_extensions_check(override) + settings_noop = _testinternalcapi.get_interp_settings() + if settings_noop != settings_after: + noop['settings_noop'] = settings_noop + allowed_noop = import_singlephase() + if allowed_noop != allowed_after: + noop['allowed_noop'] = allowed_noop + + # Restore the original setting and check. + override_noop = _imp._override_multi_interp_extensions_check(override_initial) + if override_noop != override_after: + noop['override_noop'] = override_noop + settings_restored = _testinternalcapi.get_interp_settings() + allowed_restored = import_singlephase() + + # Restore the original setting again. + override_restored = _imp._override_multi_interp_extensions_check(override_initial) + assert(_testinternalcapi.get_interp_settings() == settings_restored) + + return dict({ + 'requested': override, + 'override__initial': override_initial, + 'override_after': override_after, + 'override_restored': override_restored, + 'settings__initial': settings_initial, + 'settings_after': settings_after, + 'settings_restored': settings_restored, + 'allowed__initial': allowed_initial, + 'allowed_after': allowed_after, + 'allowed_restored': allowed_restored, + }, **noop) + + +def run_singlephase_check(override, outfd): + with os.fdopen(outfd, 'w') as outfile: + sys.stdout = outfile + sys.stderr = outfile + try: + results = check_singlephase(override) + json.dump(results, outfile) + finally: + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 7612cddb1f6576..f26b4723d1e68b 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -31,6 +31,10 @@ import _testmultiphase except ImportError: _testmultiphase = None +try: + import _testsinglephase +except ImportError: + _testsinglephase = None # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') @@ -1297,17 +1301,20 @@ def test_configured_settings(self): """ import json + EXTENSIONS = 1<<8 THREADS = 1<<10 DAEMON_THREADS = 1<<11 FORK = 1<<15 EXEC = 1<<16 - features = ['fork', 'exec', 'threads', 'daemon_threads'] + features = ['fork', 'exec', 'threads', 'daemon_threads', 'extensions'] kwlist = [f'allow_{n}' for n in features] + kwlist[-1] = 'check_multi_interp_extensions' for config, expected in { - (True, True, True, True): FORK | EXEC | THREADS | DAEMON_THREADS, - (False, False, False, False): 0, - (False, False, True, False): THREADS, + (True, True, True, True, True): + FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS, + (False, False, False, False, False): 0, + (False, False, True, False, True): THREADS | EXTENSIONS, }.items(): kwargs = dict(zip(kwlist, config)) expected = { @@ -1322,12 +1329,93 @@ def test_configured_settings(self): json.dump(settings, stdin) ''') with os.fdopen(r) as stdout: - support.run_in_subinterp_with_config(script, **kwargs) + ret = support.run_in_subinterp_with_config(script, **kwargs) + self.assertEqual(ret, 0) out = stdout.read() settings = json.loads(out) self.assertEqual(settings, expected) + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_overridden_setting_extensions_subinterp_check(self): + """ + PyInterpreterConfig.check_multi_interp_extensions can be overridden + with PyInterpreterState.override_multi_interp_extensions_check. + This verifies that the override works but does not modify + the underlying setting. + """ + import json + + EXTENSIONS = 1<<8 + THREADS = 1<<10 + DAEMON_THREADS = 1<<11 + FORK = 1<<15 + EXEC = 1<<16 + BASE_FLAGS = FORK | EXEC | THREADS | DAEMON_THREADS + base_kwargs = { + 'allow_fork': True, + 'allow_exec': True, + 'allow_threads': True, + 'allow_daemon_threads': True, + } + + def check(enabled, override): + kwargs = dict( + base_kwargs, + check_multi_interp_extensions=enabled, + ) + flags = BASE_FLAGS | EXTENSIONS if enabled else BASE_FLAGS + settings = { + 'feature_flags': flags, + } + + expected = { + 'requested': override, + 'override__initial': 0, + 'override_after': override, + 'override_restored': 0, + # The override should not affect the config or settings. + 'settings__initial': settings, + 'settings_after': settings, + 'settings_restored': settings, + # These are the most likely values to be wrong. + 'allowed__initial': not enabled, + 'allowed_after': not ((override > 0) if override else enabled), + 'allowed_restored': not enabled, + } + + r, w = os.pipe() + script = textwrap.dedent(f''' + from test.test_capi.check_config import run_singlephase_check + run_singlephase_check({override}, {w}) + ''') + with os.fdopen(r) as stdout: + ret = support.run_in_subinterp_with_config(script, **kwargs) + self.assertEqual(ret, 0) + out = stdout.read() + results = json.loads(out) + + self.assertEqual(results, expected) + + self.maxDiff = None + + # setting: check disabled + with self.subTest('config: check disabled; override: disabled'): + check(False, -1) + with self.subTest('config: check disabled; override: use config'): + check(False, 0) + with self.subTest('config: check disabled; override: enabled'): + check(False, 1) + + # setting: check enabled + with self.subTest('config: check enabled; override: disabled'): + check(True, -1) + with self.subTest('config: check enabled; override: use config'): + check(True, 0) + with self.subTest('config: check enabled; override: enabled'): + check(True, 1) + def test_mutate_exception(self): """ Exceptions saved in global module state get shared between diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 4d422da5b99f44..e56d0db8627e91 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1656,13 +1656,15 @@ def test_init_use_frozen_modules(self): api=API_PYTHON, env=env) def test_init_main_interpreter_settings(self): + EXTENSIONS = 1<<8 THREADS = 1<<10 DAEMON_THREADS = 1<<11 FORK = 1<<15 EXEC = 1<<16 expected = { # All optional features should be enabled. - 'feature_flags': FORK | EXEC | THREADS | DAEMON_THREADS, + 'feature_flags': + FORK | EXEC | THREADS | DAEMON_THREADS, } out, err = self.run_embedded_interpreter( 'test_init_main_interpreter_settings', diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 1e4429ed7efe13..96815b3f758a5b 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -21,7 +21,7 @@ from test.support import os_helper from test.support import ( STDLIB_DIR, swap_attr, swap_item, cpython_only, is_emscripten, - is_wasi) + is_wasi, run_in_subinterp_with_config) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport) from test.support.os_helper import ( @@ -30,6 +30,14 @@ from test.support import threading_helper from test.test_importlib.util import uncache from types import ModuleType +try: + import _testsinglephase +except ImportError: + _testsinglephase = None +try: + import _testmultiphase +except ImportError: + _testmultiphase = None skip_if_dont_write_bytecode = unittest.skipIf( @@ -1392,6 +1400,216 @@ def test_unwritable_module(self): unwritable.x = 42 +class SubinterpImportTests(unittest.TestCase): + + RUN_KWARGS = dict( + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + ) + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def pipe(self): + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + if hasattr(os, 'set_blocking'): + os.set_blocking(r, False) + return (r, w) + + def import_script(self, name, fd, check_override=None): + override_text = '' + if check_override is not None: + override_text = f''' + import _imp + _imp._override_multi_interp_extensions_check({check_override}) + ''' + return textwrap.dedent(f''' + import os, sys + {override_text} + try: + import {name} + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') + + def run_shared(self, name, *, + check_singlephase_setting=False, + check_singlephase_override=None, + ): + """ + Try importing the named module in a subinterpreter. + + The subinterpreter will be in the current process. + The module will have already been imported in the main interpreter. + Thus, for extension/builtin modules, the module definition will + have been loaded already and cached globally. + + "check_singlephase_setting" determines whether or not + the interpreter will be configured to check for modules + that are not compatible with use in multiple interpreters. + + This should always return "okay" for all modules if the + setting is False (with no override). + """ + __import__(name) + + kwargs = dict( + **self.RUN_KWARGS, + check_multi_interp_extensions=check_singlephase_setting, + ) + + r, w = self.pipe() + script = self.import_script(name, w, check_singlephase_override) + + ret = run_in_subinterp_with_config(script, **kwargs) + self.assertEqual(ret, 0) + return os.read(r, 100) + + def check_compatible_shared(self, name, *, strict=False): + # Verify that the named module may be imported in a subinterpreter. + # (See run_shared() for more info.) + out = self.run_shared(name, check_singlephase_setting=strict) + self.assertEqual(out, b'okay') + + def check_incompatible_shared(self, name): + # Differences from check_compatible_shared(): + # * verify that import fails + # * "strict" is always True + out = self.run_shared(name, check_singlephase_setting=True) + self.assertEqual( + out.decode('utf-8'), + f'ImportError: module {name} does not support loading in subinterpreters', + ) + + def check_compatible_isolated(self, name, *, strict=False): + # Differences from check_compatible_shared(): + # * subinterpreter in a new process + # * module has never been imported before in that process + # * this tests importing the module for the first time + _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' + import _testcapi, sys + assert ( + {name!r} in sys.builtin_module_names or + {name!r} not in sys.modules + ), repr({name!r}) + ret = _testcapi.run_in_subinterp_with_config( + {self.import_script(name, "sys.stdout.fileno()")!r}, + **{self.RUN_KWARGS}, + check_multi_interp_extensions={strict}, + ) + assert ret == 0, ret + ''')) + self.assertEqual(err, b'') + self.assertEqual(out, b'okay') + + def check_incompatible_isolated(self, name): + # Differences from check_compatible_isolated(): + # * verify that import fails + # * "strict" is always True + _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' + import _testcapi, sys + assert {name!r} not in sys.modules, {name!r} + ret = _testcapi.run_in_subinterp_with_config( + {self.import_script(name, "sys.stdout.fileno()")!r}, + **{self.RUN_KWARGS}, + check_multi_interp_extensions=True, + ) + assert ret == 0, ret + ''')) + self.assertEqual(err, b'') + self.assertEqual( + out.decode('utf-8'), + f'ImportError: module {name} does not support loading in subinterpreters', + ) + + def test_builtin_compat(self): + module = 'sys' + with self.subTest(f'{module}: not strict'): + self.check_compatible_shared(module, strict=False) + with self.subTest(f'{module}: strict, shared'): + self.check_compatible_shared(module, strict=True) + + @cpython_only + def test_frozen_compat(self): + module = '_frozen_importlib' + if __import__(module).__spec__.origin != 'frozen': + raise unittest.SkipTest(f'{module} is unexpectedly not frozen') + with self.subTest(f'{module}: not strict'): + self.check_compatible_shared(module, strict=False) + with self.subTest(f'{module}: strict, shared'): + self.check_compatible_shared(module, strict=True) + + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_single_init_extension_compat(self): + module = '_testsinglephase' + with self.subTest(f'{module}: not strict'): + self.check_compatible_shared(module, strict=False) + with self.subTest(f'{module}: strict, shared'): + self.check_incompatible_shared(module) + with self.subTest(f'{module}: strict, isolated'): + self.check_incompatible_isolated(module) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_compat(self): + module = '_testmultiphase' + with self.subTest(f'{module}: not strict'): + self.check_compatible_shared(module, strict=False) + with self.subTest(f'{module}: strict, shared'): + self.check_compatible_shared(module, strict=True) + with self.subTest(f'{module}: strict, isolated'): + self.check_compatible_isolated(module, strict=True) + + def test_python_compat(self): + module = 'threading' + if __import__(module).__spec__.origin == 'frozen': + raise unittest.SkipTest(f'{module} is unexpectedly frozen') + with self.subTest(f'{module}: not strict'): + self.check_compatible_shared(module, strict=False) + with self.subTest(f'{module}: strict, shared'): + self.check_compatible_shared(module, strict=True) + with self.subTest(f'{module}: strict, isolated'): + self.check_compatible_isolated(module, strict=True) + + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_singlephase_check_with_setting_and_override(self): + module = '_testsinglephase' + + def check_compatible(setting, override): + out = self.run_shared( + module, + check_singlephase_setting=setting, + check_singlephase_override=override, + ) + self.assertEqual(out, b'okay') + + def check_incompatible(setting, override): + out = self.run_shared( + module, + check_singlephase_setting=setting, + check_singlephase_override=override, + ) + self.assertNotEqual(out, b'okay') + + with self.subTest('config: check enabled; override: enabled'): + check_incompatible(True, 1) + with self.subTest('config: check enabled; override: use config'): + check_incompatible(True, 0) + with self.subTest('config: check enabled; override: disabled'): + check_compatible(True, -1) + + with self.subTest('config: check disabled; override: enabled'): + check_incompatible(False, 1) + with self.subTest('config: check disabled; override: use config'): + check_compatible(False, 0) + with self.subTest('config: check disabled; override: disabled'): + check_compatible(False, -1) + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 31bf46311a80dc..7fea2d38673eff 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1347,6 +1347,7 @@ def func(): allow_exec=True, allow_threads={allowed}, allow_daemon_threads={daemon_allowed}, + check_multi_interp_extensions=False, ) """) with test.support.SuppressCrashReport(): diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst new file mode 100644 index 00000000000000..3d2d6f6eb0c41f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst @@ -0,0 +1,5 @@ +When an interpreter is configured to check (and only then), importing an +extension module will now fail when the extension does not support multiple +interpreters (i.e. doesn't implement PEP 489 multi-phase init). This does +not apply to the main interpreter, nor to subinterpreters created with +``Py_NewInterpreter()``. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 5610a7689136f6..0d8d1d73fb2390 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1618,6 +1618,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) int allow_exec = -1; int allow_threads = -1; int allow_daemon_threads = -1; + int check_multi_interp_extensions = -1; int r; PyThreadState *substate, *mainstate; /* only initialise 'cflags.cf_flags' to test backwards compatibility */ @@ -1628,11 +1629,13 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) "allow_exec", "allow_threads", "allow_daemon_threads", + "check_multi_interp_extensions", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$pppp:run_in_subinterp_with_config", kwlist, + "s$ppppp:run_in_subinterp_with_config", kwlist, &code, &allow_fork, &allow_exec, - &allow_threads, &allow_daemon_threads)) { + &allow_threads, &allow_daemon_threads, + &check_multi_interp_extensions)) { return NULL; } if (allow_fork < 0) { @@ -1651,6 +1654,10 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); return NULL; } + if (check_multi_interp_extensions < 0) { + PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions"); + return NULL; + } mainstate = PyThreadState_Get(); @@ -1661,6 +1668,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) .allow_exec = allow_exec, .allow_threads = allow_threads, .allow_daemon_threads = allow_daemon_threads, + .check_multi_interp_extensions = check_multi_interp_extensions, }; substate = _Py_NewInterpreterFromConfig(&config); if (substate == NULL) { diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 819fb1c75c15c3..cb74be6a422124 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -442,6 +442,37 @@ _imp__override_frozen_modules_for_tests(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(_imp__override_multi_interp_extensions_check__doc__, +"_override_multi_interp_extensions_check($module, override, /)\n" +"--\n" +"\n" +"(internal-only) Override PyInterpreterConfig.check_multi_interp_extensions.\n" +"\n" +"(-1: \"never\", 1: \"always\", 0: no override)"); + +#define _IMP__OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK_METHODDEF \ + {"_override_multi_interp_extensions_check", (PyCFunction)_imp__override_multi_interp_extensions_check, METH_O, _imp__override_multi_interp_extensions_check__doc__}, + +static PyObject * +_imp__override_multi_interp_extensions_check_impl(PyObject *module, + int override); + +static PyObject * +_imp__override_multi_interp_extensions_check(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int override; + + override = _PyLong_AsInt(arg); + if (override == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _imp__override_multi_interp_extensions_check_impl(module, override); + +exit: + return return_value; +} + #if defined(HAVE_DYNAMIC_LOADING) PyDoc_STRVAR(_imp_create_dynamic__doc__, @@ -617,4 +648,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=806352838c3f7008 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b18d46e0036eff49 input=a9049054013a1b77]*/ diff --git a/Python/import.c b/Python/import.c index 87981668a30505..ec126f28b85816 100644 --- a/Python/import.c +++ b/Python/import.c @@ -74,6 +74,8 @@ static struct _inittab *inittab_copy = NULL; (interp)->imports.modules_by_index #define IMPORTLIB(interp) \ (interp)->imports.importlib +#define OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) \ + (interp)->imports.override_multi_interp_extensions_check #define OVERRIDE_FROZEN_MODULES(interp) \ (interp)->imports.override_frozen_modules #ifdef HAVE_DLOPEN @@ -816,6 +818,38 @@ _extensions_cache_clear_all(void) Py_CLEAR(EXTENSIONS); } + +static bool +check_multi_interp_extensions(PyInterpreterState *interp) +{ + int override = OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp); + if (override < 0) { + return false; + } + else if (override > 0) { + return true; + } + else if (_PyInterpreterState_HasFeature( + interp, Py_RTFLAGS_MULTI_INTERP_EXTENSIONS)) { + return true; + } + return false; +} + +int +_PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) +{ + PyInterpreterState *interp = _PyInterpreterState_Get(); + if (check_multi_interp_extensions(interp)) { + assert(!_Py_IsMainInterpreter(interp)); + PyErr_Format(PyExc_ImportError, + "module %s does not support loading in subinterpreters", + name); + return -1; + } + return 0; +} + static int fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) { @@ -3297,6 +3331,34 @@ _imp__override_frozen_modules_for_tests_impl(PyObject *module, int override) Py_RETURN_NONE; } +/*[clinic input] +_imp._override_multi_interp_extensions_check + + override: int + / + +(internal-only) Override PyInterpreterConfig.check_multi_interp_extensions. + +(-1: "never", 1: "always", 0: no override) +[clinic start generated code]*/ + +static PyObject * +_imp__override_multi_interp_extensions_check_impl(PyObject *module, + int override) +/*[clinic end generated code: output=3ff043af52bbf280 input=e086a2ea181f92ae]*/ +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_Py_IsMainInterpreter(interp)) { + PyErr_SetString(PyExc_RuntimeError, + "_imp._override_multi_interp_extensions_check() " + "cannot be used in the main interpreter"); + return NULL; + } + int oldvalue = OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp); + OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) = override; + return PyLong_FromLong(oldvalue); +} + #ifdef HAVE_DYNAMIC_LOADING /*[clinic input] @@ -3329,18 +3391,23 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) PyThreadState *tstate = _PyThreadState_GET(); mod = import_find_extension(tstate, name, path); - if (mod != NULL || PyErr_Occurred()) { - Py_DECREF(name); - Py_DECREF(path); - return mod; + if (mod != NULL) { + const char *name_buf = PyUnicode_AsUTF8(name); + assert(name_buf != NULL); + if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { + Py_DECREF(mod); + mod = NULL; + } + goto finally; + } + else if (PyErr_Occurred()) { + goto finally; } if (file != NULL) { fp = _Py_fopen_obj(path, "r"); if (fp == NULL) { - Py_DECREF(name); - Py_DECREF(path); - return NULL; + goto finally; } } else @@ -3348,10 +3415,12 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) mod = _PyImport_LoadDynamicModuleWithSpec(spec, fp); - Py_DECREF(name); - Py_DECREF(path); if (fp) fclose(fp); + +finally: + Py_DECREF(name); + Py_DECREF(path); return mod; } @@ -3436,6 +3505,7 @@ static PyMethodDef imp_methods[] = { _IMP_IS_FROZEN_METHODDEF _IMP__FROZEN_MODULE_NAMES_METHODDEF _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF + _IMP__OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK_METHODDEF _IMP_CREATE_DYNAMIC_METHODDEF _IMP_EXEC_DYNAMIC_METHODDEF _IMP_EXEC_BUILTIN_METHODDEF diff --git a/Python/importdl.c b/Python/importdl.c index 6dafb4541486e9..3a3a30ddbdcdb5 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_call.h" +#include "pycore_import.h" #include "pycore_pystate.h" #include "pycore_runtime.h" @@ -203,6 +204,10 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) /* Fall back to single-phase init mechanism */ + if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { + goto error; + } + if (hook_prefix == nonascii_prefix) { /* don't allow legacy init for non-ASCII module names */ PyErr_Format( diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 281035dafa9577..e80dd30c89dfd0 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -565,6 +565,10 @@ init_interp_settings(PyInterpreterState *interp, const _PyInterpreterConfig *con if (config->allow_daemon_threads) { interp->feature_flags |= Py_RTFLAGS_DAEMON_THREADS; } + + if (config->check_multi_interp_extensions) { + interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS; + } } From 0b13575e74ff3321364a3389eda6b4e92792afe1 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Wed, 15 Feb 2023 22:08:20 -0800 Subject: [PATCH 50/83] gh-99108: Refactor _sha256 & _sha512 into _sha2. (#101924) This merges their code. They're backed by the same single HACL* static library, having them be a single module simplifies maintenance. This should unbreak the wasm enscripten builds that currently fail due to linking in --whole-archive mode and the HACL* library appearing twice. Long unnoticed error fixed: _sha512.SHA384Type was doubly assigned and was actually SHA512Type. Nobody depends on those internal names. Also rename LIBHACL_ make vars to LIBHACL_SHA2_ in preperation for other future HACL things. --- Lib/hashlib.py | 12 +- Lib/test/test_hashlib.py | 22 +- Makefile.pre.in | 15 +- ...3-02-15-01-54-06.gh-issue-99108.rjTSic.rst | 3 + Modules/Setup | 3 +- Modules/Setup.stdlib.in | 3 +- Modules/clinic/sha256module.c.h | 225 ----- Modules/clinic/sha2module.c.h | 440 ++++++++++ Modules/clinic/sha512module.c.h | 225 ----- Modules/sha256module.c | 465 ---------- Modules/sha2module.c | 805 ++++++++++++++++++ Modules/sha512module.c | 456 ---------- PC/config.c | 6 +- PCbuild/pythoncore.vcxproj | 3 +- PCbuild/pythoncore.vcxproj.filters | 5 +- Python/stdlib_module_names.h | 3 +- configure | 96 +-- configure.ac | 16 +- 18 files changed, 1310 insertions(+), 1493 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst delete mode 100644 Modules/clinic/sha256module.c.h create mode 100644 Modules/clinic/sha2module.c.h delete mode 100644 Modules/clinic/sha512module.c.h delete mode 100644 Modules/sha256module.c create mode 100644 Modules/sha2module.c delete mode 100644 Modules/sha512module.c diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 21b5e912f3c771..1b16441cb60ba7 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -92,13 +92,13 @@ def __get_builtin_constructor(name): import _md5 cache['MD5'] = cache['md5'] = _md5.md5 elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}: - import _sha256 - cache['SHA224'] = cache['sha224'] = _sha256.sha224 - cache['SHA256'] = cache['sha256'] = _sha256.sha256 + import _sha2 + cache['SHA224'] = cache['sha224'] = _sha2.sha224 + cache['SHA256'] = cache['sha256'] = _sha2.sha256 elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}: - import _sha512 - cache['SHA384'] = cache['sha384'] = _sha512.sha384 - cache['SHA512'] = cache['sha512'] = _sha512.sha512 + import _sha2 + cache['SHA384'] = cache['sha384'] = _sha2.sha384 + cache['SHA512'] = cache['sha512'] = _sha2.sha512 elif name in {'blake2b', 'blake2s'}: import _blake2 cache['blake2b'] = _blake2.blake2b diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 9c92b4e9c280dc..5ead8857943592 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -1,6 +1,4 @@ -# Test hashlib module -# -# $Id$ +# Test the hashlib module. # # Copyright (C) 2005-2010 Gregory P. Smith (greg@krypto.org) # Licensed to PSF under a Contributor Agreement. @@ -28,7 +26,6 @@ from http.client import HTTPException -# default builtin hash module default_builtin_hashes = {'md5', 'sha1', 'sha256', 'sha512', 'sha3', 'blake2'} # --with-builtin-hashlib-hashes override builtin_hashes = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES") @@ -66,6 +63,7 @@ def get_fips_mode(): requires_blake2 = unittest.skipUnless(_blake2, 'requires _blake2') # bpo-46913: Don't test the _sha3 extension on a Python UBSAN build +# TODO(gh-99108): Revisit this after _sha3 uses HACL*. SKIP_SHA3 = support.check_sanitizer(ub=True) requires_sha3 = unittest.skipUnless(not SKIP_SHA3, 'requires _sha3') @@ -107,7 +105,7 @@ class HashLibTestCase(unittest.TestCase): shakes = {'shake_128', 'shake_256'} - # Issue #14693: fallback modules are always compiled under POSIX + # gh-58898: Fallback modules are always compiled under POSIX. _warn_on_extension_import = (os.name == 'posix' or support.Py_DEBUG) def _conditional_import_module(self, module_name): @@ -116,7 +114,7 @@ def _conditional_import_module(self, module_name): return importlib.import_module(module_name) except ModuleNotFoundError as error: if self._warn_on_extension_import and module_name in builtin_hashes: - warnings.warn('Did a C extension fail to compile? %s' % error) + warnings.warn(f'Did a C extension fail to compile? {error}') return None def __init__(self, *args, **kwargs): @@ -147,7 +145,7 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): _hashlib = self._conditional_import_module('_hashlib') self._hashlib = _hashlib if _hashlib: - # These two algorithms should always be present when this module + # These algorithms should always be present when this module # is compiled. If not, something was compiled wrong. self.assertTrue(hasattr(_hashlib, 'openssl_md5')) self.assertTrue(hasattr(_hashlib, 'openssl_sha1')) @@ -172,12 +170,10 @@ def add_builtin_constructor(name): _sha1 = self._conditional_import_module('_sha1') if _sha1: add_builtin_constructor('sha1') - _sha256 = self._conditional_import_module('_sha256') - if _sha256: + _sha2 = self._conditional_import_module('_sha2') + if _sha2: add_builtin_constructor('sha224') add_builtin_constructor('sha256') - _sha512 = self._conditional_import_module('_sha512') - if _sha512: add_builtin_constructor('sha384') add_builtin_constructor('sha512') if _blake2: @@ -460,9 +456,9 @@ def check_blocksize_name(self, name, block_size=0, digest_size=0, self.assertEqual(len(m.hexdigest()), 2*digest_size) self.assertEqual(m.name, name) # split for sha3_512 / _sha3.sha3 object - self.assertIn(name.split("_")[0], repr(m)) + self.assertIn(name.split("_")[0], repr(m).lower()) - def test_blocksize_name(self): + def test_blocksize_and_name(self): self.check_blocksize_name('md5', 64, 16) self.check_blocksize_name('sha1', 64, 20) self.check_blocksize_name('sha224', 64, 28) diff --git a/Makefile.pre.in b/Makefile.pre.in index ce3fed3d648536..490483a712014c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -207,7 +207,7 @@ ENSUREPIP= @ENSUREPIP@ # Internal static libraries LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a -LIBHACL_A= Modules/_hacl/libHacl_Streaming_SHA2.a +LIBHACL_SHA2_A= Modules/_hacl/libHacl_Streaming_SHA2.a # Module state, compiler flags and linker flags # Empty CFLAGS and LDFLAGS are omitted. @@ -575,10 +575,10 @@ LIBEXPAT_HEADERS= \ ########################################################################## # hashlib's HACL* library -LIBHACL_OBJS= \ +LIBHACL_SHA2_OBJS= \ Modules/_hacl/Hacl_Streaming_SHA2.o -LIBHACL_HEADERS= \ +LIBHACL_SHA2_HEADERS= \ Modules/_hacl/Hacl_Streaming_SHA2.h \ Modules/_hacl/include/krml/FStar_UInt128_Verified.h \ Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h \ @@ -912,12 +912,12 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) # Build HACL* static libraries for hashlib: libHacl_Streaming_SHA2.a LIBHACL_CFLAGS=-I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED) -Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_HEADERS) +Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_SHA2_HEADERS) $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c -$(LIBHACL_A): $(LIBHACL_OBJS) +$(LIBHACL_SHA2_A): $(LIBHACL_SHA2_OBJS) -rm -f $@ - $(AR) $(ARFLAGS) $@ $(LIBHACL_OBJS) + $(AR) $(ARFLAGS) $@ $(LIBHACL_SHA2_OBJS) # create relative links from build/lib.platform/egg.so to Modules/egg.so # pybuilddir.txt is created too late. We cannot use it in Makefile @@ -2635,9 +2635,8 @@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h -MODULE__SHA256_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A) +MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h -MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h diff --git a/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst b/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst new file mode 100644 index 00000000000000..1612c89c0ea6be --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst @@ -0,0 +1,3 @@ +The built-in extension modules for :mod:`hashlib` SHA2 algorithms, used when +OpenSSL does not provide them, now live in a single internal ``_sha2`` module +instead of separate ``_sha256`` and ``_sha512`` modules. diff --git a/Modules/Setup b/Modules/Setup index 428be0a1bf8fa1..1d5183bc2df118 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -165,8 +165,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c #_md5 md5module.c #_sha1 sha1module.c -#_sha256 sha256module.c -#_sha512 sha512module.c +#_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a #_sha3 _sha3/sha3module.c # text encodings and unicode diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 22bcb423db233f..8f5e14a4e80e22 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -79,8 +79,7 @@ # hashing builtins, can be disabled with --without-builtin-hashlib-hashes @MODULE__MD5_TRUE@_md5 md5module.c @MODULE__SHA1_TRUE@_sha1 sha1module.c -@MODULE__SHA256_TRUE@_sha256 sha256module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a -@MODULE__SHA512_TRUE@_sha512 sha512module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a +@MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a @MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c diff --git a/Modules/clinic/sha256module.c.h b/Modules/clinic/sha256module.c.h deleted file mode 100644 index 10d09fac695fc4..00000000000000 --- a/Modules/clinic/sha256module.c.h +++ /dev/null @@ -1,225 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - -PyDoc_STRVAR(SHA256Type_copy__doc__, -"copy($self, /)\n" -"--\n" -"\n" -"Return a copy of the hash object."); - -#define SHA256TYPE_COPY_METHODDEF \ - {"copy", _PyCFunction_CAST(SHA256Type_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, SHA256Type_copy__doc__}, - -static PyObject * -SHA256Type_copy_impl(SHAobject *self, PyTypeObject *cls); - -static PyObject * -SHA256Type_copy(SHAobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - if (nargs) { - PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); - return NULL; - } - return SHA256Type_copy_impl(self, cls); -} - -PyDoc_STRVAR(SHA256Type_digest__doc__, -"digest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a bytes object."); - -#define SHA256TYPE_DIGEST_METHODDEF \ - {"digest", (PyCFunction)SHA256Type_digest, METH_NOARGS, SHA256Type_digest__doc__}, - -static PyObject * -SHA256Type_digest_impl(SHAobject *self); - -static PyObject * -SHA256Type_digest(SHAobject *self, PyObject *Py_UNUSED(ignored)) -{ - return SHA256Type_digest_impl(self); -} - -PyDoc_STRVAR(SHA256Type_hexdigest__doc__, -"hexdigest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a string of hexadecimal digits."); - -#define SHA256TYPE_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)SHA256Type_hexdigest, METH_NOARGS, SHA256Type_hexdigest__doc__}, - -static PyObject * -SHA256Type_hexdigest_impl(SHAobject *self); - -static PyObject * -SHA256Type_hexdigest(SHAobject *self, PyObject *Py_UNUSED(ignored)) -{ - return SHA256Type_hexdigest_impl(self); -} - -PyDoc_STRVAR(SHA256Type_update__doc__, -"update($self, obj, /)\n" -"--\n" -"\n" -"Update this hash object\'s state with the provided string."); - -#define SHA256TYPE_UPDATE_METHODDEF \ - {"update", (PyCFunction)SHA256Type_update, METH_O, SHA256Type_update__doc__}, - -PyDoc_STRVAR(_sha256_sha256__doc__, -"sha256($module, /, string=b\'\', *, usedforsecurity=True)\n" -"--\n" -"\n" -"Return a new SHA-256 hash object; optionally initialized with a string."); - -#define _SHA256_SHA256_METHODDEF \ - {"sha256", _PyCFunction_CAST(_sha256_sha256), METH_FASTCALL|METH_KEYWORDS, _sha256_sha256__doc__}, - -static PyObject * -_sha256_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity); - -static PyObject * -_sha256_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "sha256", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; - int usedforsecurity = 1; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - string = args[0]; - if (!--noptargs) { - goto skip_optional_pos; - } - } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = _sha256_sha256_impl(module, string, usedforsecurity); - -exit: - return return_value; -} - -PyDoc_STRVAR(_sha256_sha224__doc__, -"sha224($module, /, string=b\'\', *, usedforsecurity=True)\n" -"--\n" -"\n" -"Return a new SHA-224 hash object; optionally initialized with a string."); - -#define _SHA256_SHA224_METHODDEF \ - {"sha224", _PyCFunction_CAST(_sha256_sha224), METH_FASTCALL|METH_KEYWORDS, _sha256_sha224__doc__}, - -static PyObject * -_sha256_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity); - -static PyObject * -_sha256_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "sha224", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; - int usedforsecurity = 1; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - string = args[0]; - if (!--noptargs) { - goto skip_optional_pos; - } - } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = _sha256_sha224_impl(module, string, usedforsecurity); - -exit: - return return_value; -} -/*[clinic end generated code: output=ae926f7ec85e7c97 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h new file mode 100644 index 00000000000000..8f855ca345e47a --- /dev/null +++ b/Modules/clinic/sha2module.c.h @@ -0,0 +1,440 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(SHA256Type_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a copy of the hash object."); + +#define SHA256TYPE_COPY_METHODDEF \ + {"copy", _PyCFunction_CAST(SHA256Type_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, SHA256Type_copy__doc__}, + +static PyObject * +SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls); + +static PyObject * +SHA256Type_copy(SHA256object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs) { + PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); + return NULL; + } + return SHA256Type_copy_impl(self, cls); +} + +PyDoc_STRVAR(SHA512Type_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a copy of the hash object."); + +#define SHA512TYPE_COPY_METHODDEF \ + {"copy", _PyCFunction_CAST(SHA512Type_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, SHA512Type_copy__doc__}, + +static PyObject * +SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls); + +static PyObject * +SHA512Type_copy(SHA512object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs) { + PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); + return NULL; + } + return SHA512Type_copy_impl(self, cls); +} + +PyDoc_STRVAR(SHA256Type_digest__doc__, +"digest($self, /)\n" +"--\n" +"\n" +"Return the digest value as a bytes object."); + +#define SHA256TYPE_DIGEST_METHODDEF \ + {"digest", (PyCFunction)SHA256Type_digest, METH_NOARGS, SHA256Type_digest__doc__}, + +static PyObject * +SHA256Type_digest_impl(SHA256object *self); + +static PyObject * +SHA256Type_digest(SHA256object *self, PyObject *Py_UNUSED(ignored)) +{ + return SHA256Type_digest_impl(self); +} + +PyDoc_STRVAR(SHA512Type_digest__doc__, +"digest($self, /)\n" +"--\n" +"\n" +"Return the digest value as a bytes object."); + +#define SHA512TYPE_DIGEST_METHODDEF \ + {"digest", (PyCFunction)SHA512Type_digest, METH_NOARGS, SHA512Type_digest__doc__}, + +static PyObject * +SHA512Type_digest_impl(SHA512object *self); + +static PyObject * +SHA512Type_digest(SHA512object *self, PyObject *Py_UNUSED(ignored)) +{ + return SHA512Type_digest_impl(self); +} + +PyDoc_STRVAR(SHA256Type_hexdigest__doc__, +"hexdigest($self, /)\n" +"--\n" +"\n" +"Return the digest value as a string of hexadecimal digits."); + +#define SHA256TYPE_HEXDIGEST_METHODDEF \ + {"hexdigest", (PyCFunction)SHA256Type_hexdigest, METH_NOARGS, SHA256Type_hexdigest__doc__}, + +static PyObject * +SHA256Type_hexdigest_impl(SHA256object *self); + +static PyObject * +SHA256Type_hexdigest(SHA256object *self, PyObject *Py_UNUSED(ignored)) +{ + return SHA256Type_hexdigest_impl(self); +} + +PyDoc_STRVAR(SHA512Type_hexdigest__doc__, +"hexdigest($self, /)\n" +"--\n" +"\n" +"Return the digest value as a string of hexadecimal digits."); + +#define SHA512TYPE_HEXDIGEST_METHODDEF \ + {"hexdigest", (PyCFunction)SHA512Type_hexdigest, METH_NOARGS, SHA512Type_hexdigest__doc__}, + +static PyObject * +SHA512Type_hexdigest_impl(SHA512object *self); + +static PyObject * +SHA512Type_hexdigest(SHA512object *self, PyObject *Py_UNUSED(ignored)) +{ + return SHA512Type_hexdigest_impl(self); +} + +PyDoc_STRVAR(SHA256Type_update__doc__, +"update($self, obj, /)\n" +"--\n" +"\n" +"Update this hash object\'s state with the provided string."); + +#define SHA256TYPE_UPDATE_METHODDEF \ + {"update", (PyCFunction)SHA256Type_update, METH_O, SHA256Type_update__doc__}, + +PyDoc_STRVAR(SHA512Type_update__doc__, +"update($self, obj, /)\n" +"--\n" +"\n" +"Update this hash object\'s state with the provided string."); + +#define SHA512TYPE_UPDATE_METHODDEF \ + {"update", (PyCFunction)SHA512Type_update, METH_O, SHA512Type_update__doc__}, + +PyDoc_STRVAR(_sha2_sha256__doc__, +"sha256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"--\n" +"\n" +"Return a new SHA-256 hash object; optionally initialized with a string."); + +#define _SHA2_SHA256_METHODDEF \ + {"sha256", _PyCFunction_CAST(_sha2_sha256), METH_FASTCALL|METH_KEYWORDS, _sha2_sha256__doc__}, + +static PyObject * +_sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity); + +static PyObject * +_sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sha256", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *string = NULL; + int usedforsecurity = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + string = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _sha2_sha256_impl(module, string, usedforsecurity); + +exit: + return return_value; +} + +PyDoc_STRVAR(_sha2_sha224__doc__, +"sha224($module, /, string=b\'\', *, usedforsecurity=True)\n" +"--\n" +"\n" +"Return a new SHA-224 hash object; optionally initialized with a string."); + +#define _SHA2_SHA224_METHODDEF \ + {"sha224", _PyCFunction_CAST(_sha2_sha224), METH_FASTCALL|METH_KEYWORDS, _sha2_sha224__doc__}, + +static PyObject * +_sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity); + +static PyObject * +_sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sha224", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *string = NULL; + int usedforsecurity = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + string = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _sha2_sha224_impl(module, string, usedforsecurity); + +exit: + return return_value; +} + +PyDoc_STRVAR(_sha2_sha512__doc__, +"sha512($module, /, string=b\'\', *, usedforsecurity=True)\n" +"--\n" +"\n" +"Return a new SHA-512 hash object; optionally initialized with a string."); + +#define _SHA2_SHA512_METHODDEF \ + {"sha512", _PyCFunction_CAST(_sha2_sha512), METH_FASTCALL|METH_KEYWORDS, _sha2_sha512__doc__}, + +static PyObject * +_sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity); + +static PyObject * +_sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sha512", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *string = NULL; + int usedforsecurity = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + string = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _sha2_sha512_impl(module, string, usedforsecurity); + +exit: + return return_value; +} + +PyDoc_STRVAR(_sha2_sha384__doc__, +"sha384($module, /, string=b\'\', *, usedforsecurity=True)\n" +"--\n" +"\n" +"Return a new SHA-384 hash object; optionally initialized with a string."); + +#define _SHA2_SHA384_METHODDEF \ + {"sha384", _PyCFunction_CAST(_sha2_sha384), METH_FASTCALL|METH_KEYWORDS, _sha2_sha384__doc__}, + +static PyObject * +_sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity); + +static PyObject * +_sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sha384", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *string = NULL; + int usedforsecurity = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + string = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _sha2_sha384_impl(module, string, usedforsecurity); + +exit: + return return_value; +} +/*[clinic end generated code: output=f81dacb48f3fee72 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha512module.c.h b/Modules/clinic/sha512module.c.h deleted file mode 100644 index f8d326363c398e..00000000000000 --- a/Modules/clinic/sha512module.c.h +++ /dev/null @@ -1,225 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - -PyDoc_STRVAR(SHA512Type_copy__doc__, -"copy($self, /)\n" -"--\n" -"\n" -"Return a copy of the hash object."); - -#define SHA512TYPE_COPY_METHODDEF \ - {"copy", _PyCFunction_CAST(SHA512Type_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, SHA512Type_copy__doc__}, - -static PyObject * -SHA512Type_copy_impl(SHAobject *self, PyTypeObject *cls); - -static PyObject * -SHA512Type_copy(SHAobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - if (nargs) { - PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); - return NULL; - } - return SHA512Type_copy_impl(self, cls); -} - -PyDoc_STRVAR(SHA512Type_digest__doc__, -"digest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a bytes object."); - -#define SHA512TYPE_DIGEST_METHODDEF \ - {"digest", (PyCFunction)SHA512Type_digest, METH_NOARGS, SHA512Type_digest__doc__}, - -static PyObject * -SHA512Type_digest_impl(SHAobject *self); - -static PyObject * -SHA512Type_digest(SHAobject *self, PyObject *Py_UNUSED(ignored)) -{ - return SHA512Type_digest_impl(self); -} - -PyDoc_STRVAR(SHA512Type_hexdigest__doc__, -"hexdigest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a string of hexadecimal digits."); - -#define SHA512TYPE_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)SHA512Type_hexdigest, METH_NOARGS, SHA512Type_hexdigest__doc__}, - -static PyObject * -SHA512Type_hexdigest_impl(SHAobject *self); - -static PyObject * -SHA512Type_hexdigest(SHAobject *self, PyObject *Py_UNUSED(ignored)) -{ - return SHA512Type_hexdigest_impl(self); -} - -PyDoc_STRVAR(SHA512Type_update__doc__, -"update($self, obj, /)\n" -"--\n" -"\n" -"Update this hash object\'s state with the provided string."); - -#define SHA512TYPE_UPDATE_METHODDEF \ - {"update", (PyCFunction)SHA512Type_update, METH_O, SHA512Type_update__doc__}, - -PyDoc_STRVAR(_sha512_sha512__doc__, -"sha512($module, /, string=b\'\', *, usedforsecurity=True)\n" -"--\n" -"\n" -"Return a new SHA-512 hash object; optionally initialized with a string."); - -#define _SHA512_SHA512_METHODDEF \ - {"sha512", _PyCFunction_CAST(_sha512_sha512), METH_FASTCALL|METH_KEYWORDS, _sha512_sha512__doc__}, - -static PyObject * -_sha512_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity); - -static PyObject * -_sha512_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "sha512", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; - int usedforsecurity = 1; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - string = args[0]; - if (!--noptargs) { - goto skip_optional_pos; - } - } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = _sha512_sha512_impl(module, string, usedforsecurity); - -exit: - return return_value; -} - -PyDoc_STRVAR(_sha512_sha384__doc__, -"sha384($module, /, string=b\'\', *, usedforsecurity=True)\n" -"--\n" -"\n" -"Return a new SHA-384 hash object; optionally initialized with a string."); - -#define _SHA512_SHA384_METHODDEF \ - {"sha384", _PyCFunction_CAST(_sha512_sha384), METH_FASTCALL|METH_KEYWORDS, _sha512_sha384__doc__}, - -static PyObject * -_sha512_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity); - -static PyObject * -_sha512_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "sha384", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; - int usedforsecurity = 1; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - string = args[0]; - if (!--noptargs) { - goto skip_optional_pos; - } - } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = _sha512_sha384_impl(module, string, usedforsecurity); - -exit: - return return_value; -} -/*[clinic end generated code: output=dd168f3f21097afe input=a9049054013a1b77]*/ diff --git a/Modules/sha256module.c b/Modules/sha256module.c deleted file mode 100644 index 301c9837bb6720..00000000000000 --- a/Modules/sha256module.c +++ /dev/null @@ -1,465 +0,0 @@ -/* SHA256 module */ - -/* This module provides an interface to NIST's SHA-256 and SHA-224 Algorithms */ - -/* See below for information about the original code this module was - based upon. Additional work performed by: - - Andrew Kuchling (amk@amk.ca) - Greg Stein (gstein@lyra.org) - Trevor Perrin (trevp@trevp.net) - Jonathan Protzenko (jonathan@protzenko.fr) - - Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) - Licensed to PSF under a Contributor Agreement. - -*/ - -/* SHA objects */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include "Python.h" -#include "pycore_bitutils.h" // _Py_bswap32() -#include "pycore_strhex.h" // _Py_strhex() -#include "structmember.h" // PyMemberDef -#include "hashlib.h" - -/*[clinic input] -module _sha256 -class SHA256Type "SHAobject *" "&PyType_Type" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=71a39174d4f0a744]*/ - - -/* The SHA block size and maximum message digest sizes, in bytes */ - -#define SHA_BLOCKSIZE 64 -#define SHA_DIGESTSIZE 32 - -/* The SHA2-224 and SHA2-256 implementations defer to the HACL* verified - * library. */ - -#include "_hacl/Hacl_Streaming_SHA2.h" - -typedef struct { - PyObject_HEAD - // Even though one could conceivably perform run-type checks to tell apart a - // sha224_type from a sha256_type (and thus deduce the digest size), we must - // keep this field because it's exposed as a member field on the underlying - // python object. - // TODO: could we transform this into a getter and get rid of the redundant - // field? - int digestsize; - Hacl_Streaming_SHA2_state_sha2_256 *state; -} SHAobject; - -#include "clinic/sha256module.c.h" - -/* We shall use run-time type information in the remainder of this module to - * tell apart SHA2-224 and SHA2-256 */ -typedef struct { - PyTypeObject* sha224_type; - PyTypeObject* sha256_type; -} _sha256_state; - -static inline _sha256_state* -_sha256_get_state(PyObject *module) -{ - void *state = PyModule_GetState(module); - assert(state != NULL); - return (_sha256_state *)state; -} - -static void SHAcopy(SHAobject *src, SHAobject *dest) -{ - dest->digestsize = src->digestsize; - dest->state = Hacl_Streaming_SHA2_copy_256(src->state); -} - -static SHAobject * -newSHA224object(_sha256_state *state) -{ - SHAobject *sha = (SHAobject *)PyObject_GC_New(SHAobject, - state->sha224_type); - PyObject_GC_Track(sha); - return sha; -} - -static SHAobject * -newSHA256object(_sha256_state *state) -{ - SHAobject *sha = (SHAobject *)PyObject_GC_New(SHAobject, - state->sha256_type); - PyObject_GC_Track(sha); - return sha; -} - -/* Internal methods for a hash object */ -static int -SHA_traverse(PyObject *ptr, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(ptr)); - return 0; -} - -static void -SHA_dealloc(SHAobject *ptr) -{ - Hacl_Streaming_SHA2_free_256(ptr->state); - PyTypeObject *tp = Py_TYPE(ptr); - PyObject_GC_UnTrack(ptr); - PyObject_GC_Del(ptr); - Py_DECREF(tp); -} - -/* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be - * 64 bits. */ -static void update_256(Hacl_Streaming_SHA2_state_sha2_256 *state, uint8_t *buf, Py_ssize_t len) { - /* Note: we explicitly ignore the error code on the basis that it would take > - * 1 billion years to overflow the maximum admissible length for SHA2-256 - * (namely, 2^61-1 bytes). */ - while (len > UINT32_MAX) { - Hacl_Streaming_SHA2_update_256(state, buf, UINT32_MAX); - len -= UINT32_MAX; - buf += UINT32_MAX; - } - /* Cast to uint32_t is safe: upon exiting the loop, len <= UINT32_MAX, and - * therefore fits in a uint32_t */ - Hacl_Streaming_SHA2_update_256(state, buf, (uint32_t) len); -} - - -/* External methods for a hash object */ - -/*[clinic input] -SHA256Type.copy - - cls:defining_class - -Return a copy of the hash object. -[clinic start generated code]*/ - -static PyObject * -SHA256Type_copy_impl(SHAobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=9273f92c382be12f input=3137146fcb88e212]*/ -{ - SHAobject *newobj; - _sha256_state *state = PyType_GetModuleState(cls); - if (Py_IS_TYPE(self, state->sha256_type)) { - if ( (newobj = newSHA256object(state)) == NULL) { - return NULL; - } - } else { - if ( (newobj = newSHA224object(state))==NULL) { - return NULL; - } - } - - SHAcopy(self, newobj); - return (PyObject *)newobj; -} - -/*[clinic input] -SHA256Type.digest - -Return the digest value as a bytes object. -[clinic start generated code]*/ - -static PyObject * -SHA256Type_digest_impl(SHAobject *self) -/*[clinic end generated code: output=46616a5e909fbc3d input=f1f4cfea5cbde35c]*/ -{ - uint8_t digest[SHA_DIGESTSIZE]; - // HACL performs copies under the hood so that self->state remains valid - // after this call. - Hacl_Streaming_SHA2_finish_256(self->state, digest); - return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); -} - -/*[clinic input] -SHA256Type.hexdigest - -Return the digest value as a string of hexadecimal digits. -[clinic start generated code]*/ - -static PyObject * -SHA256Type_hexdigest_impl(SHAobject *self) -/*[clinic end generated code: output=725f8a7041ae97f3 input=0cc4c714693010d1]*/ -{ - uint8_t digest[SHA_DIGESTSIZE]; - Hacl_Streaming_SHA2_finish_256(self->state, digest); - return _Py_strhex((const char *)digest, self->digestsize); -} - -/*[clinic input] -SHA256Type.update - - obj: object - / - -Update this hash object's state with the provided string. -[clinic start generated code]*/ - -static PyObject * -SHA256Type_update(SHAobject *self, PyObject *obj) -/*[clinic end generated code: output=0967fb2860c66af7 input=b2d449d5b30f0f5a]*/ -{ - Py_buffer buf; - - GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - - update_256(self->state, buf.buf, buf.len); - - PyBuffer_Release(&buf); - Py_RETURN_NONE; -} - -static PyMethodDef SHA_methods[] = { - SHA256TYPE_COPY_METHODDEF - SHA256TYPE_DIGEST_METHODDEF - SHA256TYPE_HEXDIGEST_METHODDEF - SHA256TYPE_UPDATE_METHODDEF - {NULL, NULL} /* sentinel */ -}; - -static PyObject * -SHA256_get_block_size(PyObject *self, void *closure) -{ - return PyLong_FromLong(SHA_BLOCKSIZE); -} - -static PyObject * -SHA256_get_name(SHAobject *self, void *closure) -{ - if (self->digestsize == 28) { - return PyUnicode_FromStringAndSize("sha224", 6); - } - return PyUnicode_FromStringAndSize("sha256", 6); -} - -static PyGetSetDef SHA_getseters[] = { - {"block_size", - (getter)SHA256_get_block_size, NULL, - NULL, - NULL}, - {"name", - (getter)SHA256_get_name, NULL, - NULL, - NULL}, - {NULL} /* Sentinel */ -}; - -static PyMemberDef SHA_members[] = { - {"digest_size", T_INT, offsetof(SHAobject, digestsize), READONLY, NULL}, - {NULL} /* Sentinel */ -}; - -static PyType_Slot sha256_types_slots[] = { - {Py_tp_dealloc, SHA_dealloc}, - {Py_tp_methods, SHA_methods}, - {Py_tp_members, SHA_members}, - {Py_tp_getset, SHA_getseters}, - {Py_tp_traverse, SHA_traverse}, - {0,0} -}; - -static PyType_Spec sha224_type_spec = { - .name = "_sha256.sha224", - .basicsize = sizeof(SHAobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), - .slots = sha256_types_slots -}; - -static PyType_Spec sha256_type_spec = { - .name = "_sha256.sha256", - .basicsize = sizeof(SHAobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), - .slots = sha256_types_slots -}; - -/* The single module-level function: new() */ - -/*[clinic input] -_sha256.sha256 - - string: object(c_default="NULL") = b'' - * - usedforsecurity: bool = True - -Return a new SHA-256 hash object; optionally initialized with a string. -[clinic start generated code]*/ - -static PyObject * -_sha256_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=a1de327e8e1185cf input=9be86301aeb14ea5]*/ -{ - Py_buffer buf; - - if (string) { - GET_BUFFER_VIEW_OR_ERROUT(string, &buf); - } - - _sha256_state *state = PyModule_GetState(module); - - SHAobject *new; - if ((new = newSHA256object(state)) == NULL) { - if (string) { - PyBuffer_Release(&buf); - } - return NULL; - } - - new->state = Hacl_Streaming_SHA2_create_in_256(); - new->digestsize = 32; - - if (PyErr_Occurred()) { - Py_DECREF(new); - if (string) { - PyBuffer_Release(&buf); - } - return NULL; - } - if (string) { - update_256(new->state, buf.buf, buf.len); - PyBuffer_Release(&buf); - } - - return (PyObject *)new; -} - -/*[clinic input] -_sha256.sha224 - - string: object(c_default="NULL") = b'' - * - usedforsecurity: bool = True - -Return a new SHA-224 hash object; optionally initialized with a string. -[clinic start generated code]*/ - -static PyObject * -_sha256_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=08be6b36569bc69c input=9fcfb46e460860ac]*/ -{ - Py_buffer buf; - if (string) { - GET_BUFFER_VIEW_OR_ERROUT(string, &buf); - } - - _sha256_state *state = PyModule_GetState(module); - SHAobject *new; - if ((new = newSHA224object(state)) == NULL) { - if (string) { - PyBuffer_Release(&buf); - } - return NULL; - } - - new->state = Hacl_Streaming_SHA2_create_in_224(); - new->digestsize = 28; - - if (PyErr_Occurred()) { - Py_DECREF(new); - if (string) { - PyBuffer_Release(&buf); - } - return NULL; - } - if (string) { - update_256(new->state, buf.buf, buf.len); - PyBuffer_Release(&buf); - } - - return (PyObject *)new; -} - - -/* List of functions exported by this module */ - -static struct PyMethodDef SHA_functions[] = { - _SHA256_SHA256_METHODDEF - _SHA256_SHA224_METHODDEF - {NULL, NULL} /* Sentinel */ -}; - -static int -_sha256_traverse(PyObject *module, visitproc visit, void *arg) -{ - _sha256_state *state = _sha256_get_state(module); - Py_VISIT(state->sha224_type); - Py_VISIT(state->sha256_type); - return 0; -} - -static int -_sha256_clear(PyObject *module) -{ - _sha256_state *state = _sha256_get_state(module); - Py_CLEAR(state->sha224_type); - Py_CLEAR(state->sha256_type); - return 0; -} - -static void -_sha256_free(void *module) -{ - _sha256_clear((PyObject *)module); -} - -static int sha256_exec(PyObject *module) -{ - _sha256_state *state = _sha256_get_state(module); - - state->sha224_type = (PyTypeObject *)PyType_FromModuleAndSpec( - module, &sha224_type_spec, NULL); - - if (state->sha224_type == NULL) { - return -1; - } - - state->sha256_type = (PyTypeObject *)PyType_FromModuleAndSpec( - module, &sha256_type_spec, NULL); - - if (state->sha256_type == NULL) { - return -1; - } - - Py_INCREF((PyObject *)state->sha224_type); - if (PyModule_AddObject(module, "SHA224Type", (PyObject *)state->sha224_type) < 0) { - Py_DECREF((PyObject *)state->sha224_type); - return -1; - } - Py_INCREF((PyObject *)state->sha256_type); - if (PyModule_AddObject(module, "SHA256Type", (PyObject *)state->sha256_type) < 0) { - Py_DECREF((PyObject *)state->sha256_type); - return -1; - } - return 0; -} - -static PyModuleDef_Slot _sha256_slots[] = { - {Py_mod_exec, sha256_exec}, - {0, NULL} -}; - -static struct PyModuleDef _sha256module = { - PyModuleDef_HEAD_INIT, - .m_name = "_sha256", - .m_size = sizeof(_sha256_state), - .m_methods = SHA_functions, - .m_slots = _sha256_slots, - .m_traverse = _sha256_traverse, - .m_clear = _sha256_clear, - .m_free = _sha256_free -}; - -/* Initialize this module. */ -PyMODINIT_FUNC -PyInit__sha256(void) -{ - return PyModuleDef_Init(&_sha256module); -} diff --git a/Modules/sha2module.c b/Modules/sha2module.c new file mode 100644 index 00000000000000..9999f255cd578a --- /dev/null +++ b/Modules/sha2module.c @@ -0,0 +1,805 @@ +/* SHA2 module */ + +/* This provides an interface to NIST's SHA2 224, 256, 384, & 512 Algorithms */ + +/* See below for information about the original code this module was + based upon. Additional work performed by: + + Andrew Kuchling (amk@amk.ca) + Greg Stein (gstein@lyra.org) + Trevor Perrin (trevp@trevp.net) + Jonathan Protzenko (jonathan@protzenko.fr) + + Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) + Licensed to PSF under a Contributor Agreement. + +*/ + +/* SHA objects */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" +#include "pycore_bitutils.h" // _Py_bswap32() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_strhex.h" // _Py_strhex() +#include "structmember.h" // PyMemberDef +#include "hashlib.h" + +/*[clinic input] +module _sha2 +class SHA256Type "SHA256object *" "&PyType_Type" +class SHA512Type "SHA512object *" "&PyType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b5315a7b611c9afc]*/ + + +/* The SHA block sizes and maximum message digest sizes, in bytes */ + +#define SHA256_BLOCKSIZE 64 +#define SHA256_DIGESTSIZE 32 +#define SHA512_BLOCKSIZE 128 +#define SHA512_DIGESTSIZE 64 + +/* Our SHA2 implementations defer to the HACL* verified library. */ + +#include "_hacl/Hacl_Streaming_SHA2.h" + +// TODO: Get rid of int digestsize in favor of Hacl state info? + +typedef struct { + PyObject_HEAD + int digestsize; + Hacl_Streaming_SHA2_state_sha2_256 *state; +} SHA256object; + +typedef struct { + PyObject_HEAD + int digestsize; + Hacl_Streaming_SHA2_state_sha2_512 *state; +} SHA512object; + +#include "clinic/sha2module.c.h" + +/* We shall use run-time type information in the remainder of this module to + * tell apart SHA2-224 and SHA2-256 */ +typedef struct { + PyTypeObject* sha224_type; + PyTypeObject* sha256_type; + PyTypeObject* sha384_type; + PyTypeObject* sha512_type; +} sha2_state; + +static inline sha2_state* +sha2_get_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (sha2_state *)state; +} + +static void SHA256copy(SHA256object *src, SHA256object *dest) +{ + dest->digestsize = src->digestsize; + dest->state = Hacl_Streaming_SHA2_copy_256(src->state); +} + +static void SHA512copy(SHA512object *src, SHA512object *dest) +{ + dest->digestsize = src->digestsize; + dest->state = Hacl_Streaming_SHA2_copy_512(src->state); +} + +static SHA256object * +newSHA224object(sha2_state *state) +{ + SHA256object *sha = (SHA256object *)PyObject_GC_New( + SHA256object, state->sha224_type); + if (!sha) { + return NULL; + } + PyObject_GC_Track(sha); + return sha; +} + +static SHA256object * +newSHA256object(sha2_state *state) +{ + SHA256object *sha = (SHA256object *)PyObject_GC_New( + SHA256object, state->sha256_type); + if (!sha) { + return NULL; + } + PyObject_GC_Track(sha); + return sha; +} + +static SHA512object * +newSHA384object(sha2_state *state) +{ + SHA512object *sha = (SHA512object *)PyObject_GC_New( + SHA512object, state->sha384_type); + if (!sha) { + return NULL; + } + PyObject_GC_Track(sha); + return sha; +} + +static SHA512object * +newSHA512object(sha2_state *state) +{ + SHA512object *sha = (SHA512object *)PyObject_GC_New( + SHA512object, state->sha512_type); + if (!sha) { + return NULL; + } + PyObject_GC_Track(sha); + return sha; +} + +/* Internal methods for our hash objects. */ + +static int +SHA2_traverse(PyObject *ptr, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(ptr)); + return 0; +} + +static void +SHA256_dealloc(SHA256object *ptr) +{ + Hacl_Streaming_SHA2_free_256(ptr->state); + PyTypeObject *tp = Py_TYPE(ptr); + PyObject_GC_UnTrack(ptr); + PyObject_GC_Del(ptr); + Py_DECREF(tp); +} + +static void +SHA512_dealloc(SHA512object *ptr) +{ + Hacl_Streaming_SHA2_free_512(ptr->state); + PyTypeObject *tp = Py_TYPE(ptr); + PyObject_GC_UnTrack(ptr); + PyObject_GC_Del(ptr); + Py_DECREF(tp); +} + +/* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be + * 64 bits so we loop in <4gig chunks when needed. */ + +static void update_256(Hacl_Streaming_SHA2_state_sha2_256 *state, uint8_t *buf, Py_ssize_t len) { + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to overflow the maximum admissible length for SHA2-256 + * (namely, 2^61-1 bytes). */ +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_SHA2_update_256(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ + Hacl_Streaming_SHA2_update_256(state, buf, (uint32_t) len); +} + +static void update_512(Hacl_Streaming_SHA2_state_sha2_512 *state, uint8_t *buf, Py_ssize_t len) { + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to overflow the maximum admissible length for this API + * (namely, 2^64-1 bytes). */ +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_SHA2_update_512(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ + Hacl_Streaming_SHA2_update_512(state, buf, (uint32_t) len); +} + + +/* External methods for our hash objects */ + +/*[clinic input] +SHA256Type.copy + + cls:defining_class + +Return a copy of the hash object. +[clinic start generated code]*/ + +static PyObject * +SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls) +/*[clinic end generated code: output=fabd515577805cd3 input=3137146fcb88e212]*/ +{ + SHA256object *newobj; + sha2_state *state = PyType_GetModuleState(cls); + if (Py_IS_TYPE(self, state->sha256_type)) { + if ((newobj = newSHA256object(state)) == NULL) { + return NULL; + } + } else { + if ((newobj = newSHA224object(state)) == NULL) { + return NULL; + } + } + + SHA256copy(self, newobj); + return (PyObject *)newobj; +} + +/*[clinic input] +SHA512Type.copy + + cls: defining_class + +Return a copy of the hash object. +[clinic start generated code]*/ + +static PyObject * +SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls) +/*[clinic end generated code: output=66d2a8ef20de8302 input=f673a18f66527c90]*/ +{ + SHA512object *newobj; + sha2_state *state = PyType_GetModuleState(cls); + + if (Py_IS_TYPE((PyObject*)self, state->sha512_type)) { + if ((newobj = newSHA512object(state)) == NULL) { + return NULL; + } + } + else { + if ((newobj = newSHA384object(state)) == NULL) { + return NULL; + } + } + + SHA512copy(self, newobj); + return (PyObject *)newobj; +} + +/*[clinic input] +SHA256Type.digest + +Return the digest value as a bytes object. +[clinic start generated code]*/ + +static PyObject * +SHA256Type_digest_impl(SHA256object *self) +/*[clinic end generated code: output=3a2e3997a98ee792 input=f1f4cfea5cbde35c]*/ +{ + uint8_t digest[SHA256_DIGESTSIZE]; + assert(self->digestsize <= SHA256_DIGESTSIZE); + // HACL* performs copies under the hood so that self->state remains valid + // after this call. + Hacl_Streaming_SHA2_finish_256(self->state, digest); + return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); +} + +/*[clinic input] +SHA512Type.digest + +Return the digest value as a bytes object. +[clinic start generated code]*/ + +static PyObject * +SHA512Type_digest_impl(SHA512object *self) +/*[clinic end generated code: output=dd8c6320070458e0 input=f6470dd359071f4b]*/ +{ + uint8_t digest[SHA512_DIGESTSIZE]; + assert(self->digestsize <= SHA512_DIGESTSIZE); + // HACL* performs copies under the hood so that self->state remains valid + // after this call. + Hacl_Streaming_SHA2_finish_512(self->state, digest); + return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); +} + +/*[clinic input] +SHA256Type.hexdigest + +Return the digest value as a string of hexadecimal digits. +[clinic start generated code]*/ + +static PyObject * +SHA256Type_hexdigest_impl(SHA256object *self) +/*[clinic end generated code: output=96cb68996a780ab3 input=0cc4c714693010d1]*/ +{ + uint8_t digest[SHA256_DIGESTSIZE]; + assert(self->digestsize <= SHA256_DIGESTSIZE); + Hacl_Streaming_SHA2_finish_256(self->state, digest); + return _Py_strhex((const char *)digest, self->digestsize); +} + +/*[clinic input] +SHA512Type.hexdigest + +Return the digest value as a string of hexadecimal digits. +[clinic start generated code]*/ + +static PyObject * +SHA512Type_hexdigest_impl(SHA512object *self) +/*[clinic end generated code: output=cbd6f844aba1fe7c input=498b877b25cbe0a2]*/ +{ + uint8_t digest[SHA512_DIGESTSIZE]; + assert(self->digestsize <= SHA512_DIGESTSIZE); + Hacl_Streaming_SHA2_finish_512(self->state, digest); + return _Py_strhex((const char *)digest, self->digestsize); +} + +/*[clinic input] +SHA256Type.update + + obj: object + / + +Update this hash object's state with the provided string. +[clinic start generated code]*/ + +static PyObject * +SHA256Type_update(SHA256object *self, PyObject *obj) +/*[clinic end generated code: output=1b240f965ddbd8c6 input=b2d449d5b30f0f5a]*/ +{ + Py_buffer buf; + + GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); + + update_256(self->state, buf.buf, buf.len); + + PyBuffer_Release(&buf); + Py_RETURN_NONE; +} + +/*[clinic input] +SHA512Type.update + + obj: object + / + +Update this hash object's state with the provided string. +[clinic start generated code]*/ + +static PyObject * +SHA512Type_update(SHA512object *self, PyObject *obj) +/*[clinic end generated code: output=745f51057a985884 input=ded2b46656566283]*/ +{ + Py_buffer buf; + + GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); + + update_512(self->state, buf.buf, buf.len); + + PyBuffer_Release(&buf); + Py_RETURN_NONE; +} + +static PyMethodDef SHA256_methods[] = { + SHA256TYPE_COPY_METHODDEF + SHA256TYPE_DIGEST_METHODDEF + SHA256TYPE_HEXDIGEST_METHODDEF + SHA256TYPE_UPDATE_METHODDEF + {NULL, NULL} /* sentinel */ +}; + +static PyMethodDef SHA512_methods[] = { + SHA512TYPE_COPY_METHODDEF + SHA512TYPE_DIGEST_METHODDEF + SHA512TYPE_HEXDIGEST_METHODDEF + SHA512TYPE_UPDATE_METHODDEF + {NULL, NULL} /* sentinel */ +}; + +static PyObject * +SHA256_get_block_size(PyObject *self, void *closure) +{ + return PyLong_FromLong(SHA256_BLOCKSIZE); +} + +static PyObject * +SHA512_get_block_size(PyObject *self, void *closure) +{ + return PyLong_FromLong(SHA512_BLOCKSIZE); +} + +static PyObject * +SHA256_get_digest_size(SHA256object *self, void *closure) +{ + return PyLong_FromLong(self->digestsize); +} + +static PyObject * +SHA512_get_digest_size(SHA512object *self, void *closure) +{ + return PyLong_FromLong(self->digestsize); +} + +static PyObject * +SHA256_get_name(SHA256object *self, void *closure) +{ + if (self->digestsize == 28) { + return PyUnicode_FromStringAndSize("sha224", 6); + } + return PyUnicode_FromStringAndSize("sha256", 6); +} + +static PyObject * +SHA512_get_name(SHA512object *self, void *closure) +{ + if (self->digestsize == 64) { + return PyUnicode_FromStringAndSize("sha512", 6); + } + return PyUnicode_FromStringAndSize("sha384", 6); +} + +static PyGetSetDef SHA256_getseters[] = { + {"block_size", + (getter)SHA256_get_block_size, NULL, + NULL, + NULL}, + {"name", + (getter)SHA256_get_name, NULL, + NULL, + NULL}, + {"digest_size", + (getter)SHA256_get_digest_size, NULL, + NULL, + NULL}, + {NULL} /* Sentinel */ +}; + +static PyGetSetDef SHA512_getseters[] = { + {"block_size", + (getter)SHA512_get_block_size, NULL, + NULL, + NULL}, + {"name", + (getter)SHA512_get_name, NULL, + NULL, + NULL}, + {"digest_size", + (getter)SHA512_get_digest_size, NULL, + NULL, + NULL}, + {NULL} /* Sentinel */ +}; + +static PyType_Slot sha256_types_slots[] = { + {Py_tp_dealloc, SHA256_dealloc}, + {Py_tp_methods, SHA256_methods}, + {Py_tp_getset, SHA256_getseters}, + {Py_tp_traverse, SHA2_traverse}, + {0,0} +}; + +static PyType_Slot sha512_type_slots[] = { + {Py_tp_dealloc, SHA512_dealloc}, + {Py_tp_methods, SHA512_methods}, + {Py_tp_getset, SHA512_getseters}, + {Py_tp_traverse, SHA2_traverse}, + {0,0} +}; + +// Using PyType_GetModuleState() on these types is safe since they +// cannot be subclassed: they don't have the Py_TPFLAGS_BASETYPE flag. +static PyType_Spec sha224_type_spec = { + .name = "_sha2.SHA224Type", + .basicsize = sizeof(SHA256object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), + .slots = sha256_types_slots +}; + +static PyType_Spec sha256_type_spec = { + .name = "_sha2.SHA256Type", + .basicsize = sizeof(SHA256object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), + .slots = sha256_types_slots +}; + +static PyType_Spec sha384_type_spec = { + .name = "_sha2.SHA384Type", + .basicsize = sizeof(SHA512object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), + .slots = sha512_type_slots +}; + +static PyType_Spec sha512_type_spec = { + .name = "_sha2.SHA512Type", + .basicsize = sizeof(SHA512object), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), + .slots = sha512_type_slots +}; + +/* The module-level constructors. */ + +/*[clinic input] +_sha2.sha256 + + string: object(c_default="NULL") = b'' + * + usedforsecurity: bool = True + +Return a new SHA-256 hash object; optionally initialized with a string. +[clinic start generated code]*/ + +static PyObject * +_sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) +/*[clinic end generated code: output=243c9dd289931f87 input=6249da1de607280a]*/ +{ + Py_buffer buf; + + if (string) { + GET_BUFFER_VIEW_OR_ERROUT(string, &buf); + } + + sha2_state *state = sha2_get_state(module); + + SHA256object *new; + if ((new = newSHA256object(state)) == NULL) { + if (string) { + PyBuffer_Release(&buf); + } + return NULL; + } + + new->state = Hacl_Streaming_SHA2_create_in_256(); + new->digestsize = 32; + + if (PyErr_Occurred()) { + Py_DECREF(new); + if (string) { + PyBuffer_Release(&buf); + } + return NULL; + } + if (string) { + update_256(new->state, buf.buf, buf.len); + PyBuffer_Release(&buf); + } + + return (PyObject *)new; +} + +/*[clinic input] +_sha2.sha224 + + string: object(c_default="NULL") = b'' + * + usedforsecurity: bool = True + +Return a new SHA-224 hash object; optionally initialized with a string. +[clinic start generated code]*/ + +static PyObject * +_sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) +/*[clinic end generated code: output=68191f232e4a3843 input=c42bcba47fd7d2b7]*/ +{ + Py_buffer buf; + if (string) { + GET_BUFFER_VIEW_OR_ERROUT(string, &buf); + } + + sha2_state *state = sha2_get_state(module); + SHA256object *new; + if ((new = newSHA224object(state)) == NULL) { + if (string) { + PyBuffer_Release(&buf); + } + return NULL; + } + + new->state = Hacl_Streaming_SHA2_create_in_224(); + new->digestsize = 28; + + if (PyErr_Occurred()) { + Py_DECREF(new); + if (string) { + PyBuffer_Release(&buf); + } + return NULL; + } + if (string) { + update_256(new->state, buf.buf, buf.len); + PyBuffer_Release(&buf); + } + + return (PyObject *)new; +} + +/*[clinic input] +_sha2.sha512 + + string: object(c_default="NULL") = b'' + * + usedforsecurity: bool = True + +Return a new SHA-512 hash object; optionally initialized with a string. +[clinic start generated code]*/ + +static PyObject * +_sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) +/*[clinic end generated code: output=d55c8996eca214d7 input=0576ae2a6ebfad25]*/ +{ + SHA512object *new; + Py_buffer buf; + + sha2_state *state = sha2_get_state(module); + + if (string) + GET_BUFFER_VIEW_OR_ERROUT(string, &buf); + + if ((new = newSHA512object(state)) == NULL) { + if (string) + PyBuffer_Release(&buf); + return NULL; + } + + new->state = Hacl_Streaming_SHA2_create_in_512(); + new->digestsize = 64; + + if (PyErr_Occurred()) { + Py_DECREF(new); + if (string) + PyBuffer_Release(&buf); + return NULL; + } + if (string) { + update_512(new->state, buf.buf, buf.len); + PyBuffer_Release(&buf); + } + + return (PyObject *)new; +} + +/*[clinic input] +_sha2.sha384 + + string: object(c_default="NULL") = b'' + * + usedforsecurity: bool = True + +Return a new SHA-384 hash object; optionally initialized with a string. +[clinic start generated code]*/ + +static PyObject * +_sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) +/*[clinic end generated code: output=b29a0d81d51d1368 input=4e9199d8de0d2f9b]*/ +{ + SHA512object *new; + Py_buffer buf; + + sha2_state *state = sha2_get_state(module); + + if (string) + GET_BUFFER_VIEW_OR_ERROUT(string, &buf); + + if ((new = newSHA384object(state)) == NULL) { + if (string) + PyBuffer_Release(&buf); + return NULL; + } + + new->state = Hacl_Streaming_SHA2_create_in_384(); + new->digestsize = 48; + + if (PyErr_Occurred()) { + Py_DECREF(new); + if (string) + PyBuffer_Release(&buf); + return NULL; + } + if (string) { + update_512(new->state, buf.buf, buf.len); + PyBuffer_Release(&buf); + } + + return (PyObject *)new; +} + +/* List of functions exported by this module */ + +static struct PyMethodDef SHA2_functions[] = { + _SHA2_SHA256_METHODDEF + _SHA2_SHA224_METHODDEF + _SHA2_SHA512_METHODDEF + _SHA2_SHA384_METHODDEF + {NULL, NULL} /* Sentinel */ +}; + +static int +_sha2_traverse(PyObject *module, visitproc visit, void *arg) +{ + sha2_state *state = sha2_get_state(module); + Py_VISIT(state->sha224_type); + Py_VISIT(state->sha256_type); + Py_VISIT(state->sha384_type); + Py_VISIT(state->sha512_type); + return 0; +} + +static int +_sha2_clear(PyObject *module) +{ + sha2_state *state = sha2_get_state(module); + Py_CLEAR(state->sha224_type); + Py_CLEAR(state->sha256_type); + Py_CLEAR(state->sha384_type); + Py_CLEAR(state->sha512_type); + return 0; +} + +static void +_sha2_free(void *module) +{ + _sha2_clear((PyObject *)module); +} + +/* Initialize this module. */ +static int sha2_exec(PyObject *module) +{ + sha2_state *state = sha2_get_state(module); + + state->sha224_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &sha224_type_spec, NULL); + if (state->sha224_type == NULL) { + return -1; + } + state->sha256_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &sha256_type_spec, NULL); + if (state->sha256_type == NULL) { + return -1; + } + state->sha384_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &sha384_type_spec, NULL); + if (state->sha384_type == NULL) { + return -1; + } + state->sha512_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &sha512_type_spec, NULL); + if (state->sha512_type == NULL) { + return -1; + } + + if (PyModule_AddType(module, state->sha224_type) < 0) { + return -1; + } + if (PyModule_AddType(module, state->sha256_type) < 0) { + return -1; + } + if (PyModule_AddType(module, state->sha384_type) < 0) { + return -1; + } + if (PyModule_AddType(module, state->sha512_type) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot _sha2_slots[] = { + {Py_mod_exec, sha2_exec}, + {0, NULL} +}; + +static struct PyModuleDef _sha2module = { + PyModuleDef_HEAD_INIT, + .m_name = "_sha2", + .m_size = sizeof(sha2_state), + .m_methods = SHA2_functions, + .m_slots = _sha2_slots, + .m_traverse = _sha2_traverse, + .m_clear = _sha2_clear, + .m_free = _sha2_free +}; + +PyMODINIT_FUNC +PyInit__sha2(void) +{ + return PyModuleDef_Init(&_sha2module); +} diff --git a/Modules/sha512module.c b/Modules/sha512module.c deleted file mode 100644 index d7dfed4e5db03a..00000000000000 --- a/Modules/sha512module.c +++ /dev/null @@ -1,456 +0,0 @@ -/* SHA512 module */ - -/* This module provides an interface to NIST's SHA-512 and SHA-384 Algorithms */ - -/* See below for information about the original code this module was - based upon. Additional work performed by: - - Andrew Kuchling (amk@amk.ca) - Greg Stein (gstein@lyra.org) - Trevor Perrin (trevp@trevp.net) - Jonathan Protzenko (jonathan@protzenko.fr) - - Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) - Licensed to PSF under a Contributor Agreement. - -*/ - -/* SHA objects */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include "Python.h" -#include "pycore_bitutils.h" // _Py_bswap64() -#include "pycore_strhex.h" // _Py_strhex() -#include "structmember.h" // PyMemberDef -#include "hashlib.h" - -/*[clinic input] -module _sha512 -class SHA512Type "SHAobject *" "&PyType_Type" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=81a3ccde92bcfe8d]*/ - - -/* The SHA block size and message digest sizes, in bytes */ - -#define SHA_BLOCKSIZE 128 -#define SHA_DIGESTSIZE 64 - -/* The SHA2-384 and SHA2-512 implementations defer to the HACL* verified - * library. */ - -#include "_hacl/Hacl_Streaming_SHA2.h" - -typedef struct { - PyObject_HEAD - int digestsize; - Hacl_Streaming_SHA2_state_sha2_512 *state; -} SHAobject; - -#include "clinic/sha512module.c.h" - - -static void SHAcopy(SHAobject *src, SHAobject *dest) -{ - dest->digestsize = src->digestsize; - dest->state = Hacl_Streaming_SHA2_copy_512(src->state); -} - -typedef struct { - PyTypeObject* sha384_type; - PyTypeObject* sha512_type; -} SHA512State; - -static inline SHA512State* -sha512_get_state(PyObject *module) -{ - void *state = PyModule_GetState(module); - assert(state != NULL); - return (SHA512State *)state; -} - -static SHAobject * -newSHA384object(SHA512State *st) -{ - SHAobject *sha = (SHAobject *)PyObject_GC_New(SHAobject, st->sha384_type); - PyObject_GC_Track(sha); - return sha; -} - -static SHAobject * -newSHA512object(SHA512State *st) -{ - SHAobject *sha = (SHAobject *)PyObject_GC_New(SHAobject, st->sha512_type); - PyObject_GC_Track(sha); - return sha; -} - -/* Internal methods for a hash object */ -static int -SHA_traverse(PyObject *ptr, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(ptr)); - return 0; -} - -static void -SHA512_dealloc(SHAobject *ptr) -{ - Hacl_Streaming_SHA2_free_512(ptr->state); - PyTypeObject *tp = Py_TYPE(ptr); - PyObject_GC_UnTrack(ptr); - PyObject_GC_Del(ptr); - Py_DECREF(tp); -} - -/* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be - * 64 bits. */ -static void update_512(Hacl_Streaming_SHA2_state_sha2_512 *state, uint8_t *buf, Py_ssize_t len) { - /* Note: we explicitly ignore the error code on the basis that it would take > - * 1 billion years to overflow the maximum admissible length for this API - * (namely, 2^64-1 bytes). */ - while (len > UINT32_MAX) { - Hacl_Streaming_SHA2_update_512(state, buf, UINT32_MAX); - len -= UINT32_MAX; - buf += UINT32_MAX; - } - /* Cast to uint32_t is safe: upon exiting the loop, len <= UINT32_MAX, and - * therefore fits in a uint32_t */ - Hacl_Streaming_SHA2_update_512(state, buf, (uint32_t) len); -} - - -/* External methods for a hash object */ - -/*[clinic input] -SHA512Type.copy - - cls: defining_class - -Return a copy of the hash object. -[clinic start generated code]*/ - -static PyObject * -SHA512Type_copy_impl(SHAobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=85ea5b47837a08e6 input=f673a18f66527c90]*/ -{ - SHAobject *newobj; - SHA512State *st = PyType_GetModuleState(cls); - - if (Py_IS_TYPE((PyObject*)self, st->sha512_type)) { - if ( (newobj = newSHA512object(st))==NULL) { - return NULL; - } - } - else { - if ( (newobj = newSHA384object(st))==NULL) { - return NULL; - } - } - - SHAcopy(self, newobj); - return (PyObject *)newobj; -} - -/*[clinic input] -SHA512Type.digest - -Return the digest value as a bytes object. -[clinic start generated code]*/ - -static PyObject * -SHA512Type_digest_impl(SHAobject *self) -/*[clinic end generated code: output=1080bbeeef7dde1b input=f6470dd359071f4b]*/ -{ - uint8_t digest[SHA_DIGESTSIZE]; - // HACL performs copies under the hood so that self->state remains valid - // after this call. - Hacl_Streaming_SHA2_finish_512(self->state, digest); - return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); -} - -/*[clinic input] -SHA512Type.hexdigest - -Return the digest value as a string of hexadecimal digits. -[clinic start generated code]*/ - -static PyObject * -SHA512Type_hexdigest_impl(SHAobject *self) -/*[clinic end generated code: output=7373305b8601e18b input=498b877b25cbe0a2]*/ -{ - uint8_t digest[SHA_DIGESTSIZE]; - Hacl_Streaming_SHA2_finish_512(self->state, digest); - return _Py_strhex((const char *)digest, self->digestsize); -} - -/*[clinic input] -SHA512Type.update - - obj: object - / - -Update this hash object's state with the provided string. -[clinic start generated code]*/ - -static PyObject * -SHA512Type_update(SHAobject *self, PyObject *obj) -/*[clinic end generated code: output=1cf333e73995a79e input=ded2b46656566283]*/ -{ - Py_buffer buf; - - GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - - update_512(self->state, buf.buf, buf.len); - - PyBuffer_Release(&buf); - Py_RETURN_NONE; -} - -static PyMethodDef SHA_methods[] = { - SHA512TYPE_COPY_METHODDEF - SHA512TYPE_DIGEST_METHODDEF - SHA512TYPE_HEXDIGEST_METHODDEF - SHA512TYPE_UPDATE_METHODDEF - {NULL, NULL} /* sentinel */ -}; - -static PyObject * -SHA512_get_block_size(PyObject *self, void *closure) -{ - return PyLong_FromLong(SHA_BLOCKSIZE); -} - -static PyObject * -SHA512_get_name(PyObject *self, void *closure) -{ - if (((SHAobject *)self)->digestsize == 64) - return PyUnicode_FromStringAndSize("sha512", 6); - else - return PyUnicode_FromStringAndSize("sha384", 6); -} - -static PyGetSetDef SHA_getseters[] = { - {"block_size", - (getter)SHA512_get_block_size, NULL, - NULL, - NULL}, - {"name", - (getter)SHA512_get_name, NULL, - NULL, - NULL}, - {NULL} /* Sentinel */ -}; - -static PyMemberDef SHA_members[] = { - {"digest_size", T_INT, offsetof(SHAobject, digestsize), READONLY, NULL}, - {NULL} /* Sentinel */ -}; - -static PyType_Slot sha512_sha384_type_slots[] = { - {Py_tp_dealloc, SHA512_dealloc}, - {Py_tp_methods, SHA_methods}, - {Py_tp_members, SHA_members}, - {Py_tp_getset, SHA_getseters}, - {Py_tp_traverse, SHA_traverse}, - {0,0} -}; - -static PyType_Spec sha512_sha384_type_spec = { - .name = "_sha512.sha384", - .basicsize = sizeof(SHAobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), - .slots = sha512_sha384_type_slots -}; - -// Using PyType_GetModuleState() on this type is safe since -// it cannot be subclassed: it does not have the Py_TPFLAGS_BASETYPE flag. -static PyType_Spec sha512_sha512_type_spec = { - .name = "_sha512.sha512", - .basicsize = sizeof(SHAobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), - .slots = sha512_sha384_type_slots -}; - -/* The single module-level function: new() */ - -/*[clinic input] -_sha512.sha512 - - string: object(c_default="NULL") = b'' - * - usedforsecurity: bool = True - -Return a new SHA-512 hash object; optionally initialized with a string. -[clinic start generated code]*/ - -static PyObject * -_sha512_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=a8d9e5f9e6a0831c input=23b4daebc2ebb9c9]*/ -{ - SHAobject *new; - Py_buffer buf; - - SHA512State *st = sha512_get_state(module); - - if (string) - GET_BUFFER_VIEW_OR_ERROUT(string, &buf); - - if ((new = newSHA512object(st)) == NULL) { - if (string) - PyBuffer_Release(&buf); - return NULL; - } - - new->state = Hacl_Streaming_SHA2_create_in_512(); - new->digestsize = 64; - - if (PyErr_Occurred()) { - Py_DECREF(new); - if (string) - PyBuffer_Release(&buf); - return NULL; - } - if (string) { - update_512(new->state, buf.buf, buf.len); - PyBuffer_Release(&buf); - } - - return (PyObject *)new; -} - -/*[clinic input] -_sha512.sha384 - - string: object(c_default="NULL") = b'' - * - usedforsecurity: bool = True - -Return a new SHA-384 hash object; optionally initialized with a string. -[clinic start generated code]*/ - -static PyObject * -_sha512_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=da7d594a08027ac3 input=59ef72f039a6b431]*/ -{ - SHAobject *new; - Py_buffer buf; - - SHA512State *st = sha512_get_state(module); - - if (string) - GET_BUFFER_VIEW_OR_ERROUT(string, &buf); - - if ((new = newSHA384object(st)) == NULL) { - if (string) - PyBuffer_Release(&buf); - return NULL; - } - - new->state = Hacl_Streaming_SHA2_create_in_384(); - new->digestsize = 48; - - if (PyErr_Occurred()) { - Py_DECREF(new); - if (string) - PyBuffer_Release(&buf); - return NULL; - } - if (string) { - update_512(new->state, buf.buf, buf.len); - PyBuffer_Release(&buf); - } - - return (PyObject *)new; -} - - -/* List of functions exported by this module */ - -static struct PyMethodDef SHA_functions[] = { - _SHA512_SHA512_METHODDEF - _SHA512_SHA384_METHODDEF - {NULL, NULL} /* Sentinel */ -}; - -static int -_sha512_traverse(PyObject *module, visitproc visit, void *arg) -{ - SHA512State *state = sha512_get_state(module); - Py_VISIT(state->sha384_type); - Py_VISIT(state->sha512_type); - return 0; -} - -static int -_sha512_clear(PyObject *module) -{ - SHA512State *state = sha512_get_state(module); - Py_CLEAR(state->sha384_type); - Py_CLEAR(state->sha512_type); - return 0; -} - -static void -_sha512_free(void *module) -{ - _sha512_clear((PyObject *)module); -} - - -/* Initialize this module. */ -static int -_sha512_exec(PyObject *m) -{ - SHA512State* st = sha512_get_state(m); - - st->sha384_type = (PyTypeObject *)PyType_FromModuleAndSpec( - m, &sha512_sha384_type_spec, NULL); - - st->sha512_type = (PyTypeObject *)PyType_FromModuleAndSpec( - m, &sha512_sha512_type_spec, NULL); - - if (st->sha384_type == NULL || st->sha512_type == NULL) { - return -1; - } - - Py_INCREF(st->sha384_type); - if (PyModule_AddObject(m, "SHA384Type", (PyObject *)st->sha384_type) < 0) { - Py_DECREF(st->sha384_type); - return -1; - } - - Py_INCREF(st->sha512_type); - if (PyModule_AddObject(m, "SHA384Type", (PyObject *)st->sha512_type) < 0) { - Py_DECREF(st->sha512_type); - return -1; - } - - return 0; -} - -static PyModuleDef_Slot _sha512_slots[] = { - {Py_mod_exec, _sha512_exec}, - {0, NULL} -}; - -static struct PyModuleDef _sha512module = { - PyModuleDef_HEAD_INIT, - .m_name = "_sha512", - .m_size = sizeof(SHA512State), - .m_methods = SHA_functions, - .m_slots = _sha512_slots, - .m_traverse = _sha512_traverse, - .m_clear = _sha512_clear, - .m_free = _sha512_free -}; - -PyMODINIT_FUNC -PyInit__sha512(void) -{ - return PyModuleDef_Init(&_sha512module); -} diff --git a/PC/config.c b/PC/config.c index cdb5db23c4ae49..b1481d79e6508d 100644 --- a/PC/config.c +++ b/PC/config.c @@ -20,8 +20,7 @@ extern PyObject* PyInit_nt(void); extern PyObject* PyInit__operator(void); extern PyObject* PyInit__signal(void); extern PyObject* PyInit__sha1(void); -extern PyObject* PyInit__sha256(void); -extern PyObject* PyInit__sha512(void); +extern PyObject* PyInit__sha2(void); extern PyObject* PyInit__sha3(void); extern PyObject* PyInit__statistics(void); extern PyObject* PyInit__typing(void); @@ -98,8 +97,7 @@ struct _inittab _PyImport_Inittab[] = { {"_signal", PyInit__signal}, {"_md5", PyInit__md5}, {"_sha1", PyInit__sha1}, - {"_sha256", PyInit__sha256}, - {"_sha512", PyInit__sha512}, + {"_sha2", PyInit__sha2}, {"_sha3", PyInit__sha3}, {"_blake2", PyInit__blake2}, {"time", PyInit_time}, diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index e8e9ff01e306bc..222963bc42d17c 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -408,8 +408,7 @@ - - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4820db6f2c32dc..efb96222043ac2 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -869,10 +869,7 @@ Modules - - Modules - - + Modules diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 4e7dfb14d19dec..e9f0061a59d3ba 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -63,9 +63,8 @@ static const char* _Py_stdlib_module_names[] = { "_random", "_scproxy", "_sha1", -"_sha256", +"_sha2", "_sha3", -"_sha512", "_signal", "_sitebuiltins", "_socket", diff --git a/configure b/configure index c00a1e1d2ec986..7c4254f3cb176f 100755 --- a/configure +++ b/configure @@ -686,10 +686,8 @@ MODULE__BLAKE2_FALSE MODULE__BLAKE2_TRUE MODULE__SHA3_FALSE MODULE__SHA3_TRUE -MODULE__SHA512_FALSE -MODULE__SHA512_TRUE -MODULE__SHA256_FALSE -MODULE__SHA256_TRUE +MODULE__SHA2_FALSE +MODULE__SHA2_TRUE MODULE__SHA1_FALSE MODULE__SHA1_TRUE MODULE__MD5_FALSE @@ -1891,9 +1889,9 @@ Optional Packages: leave OpenSSL's defaults untouched, STRING: use a custom string, python and STRING also set TLS 1.2 as minimum TLS version - --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2 - builtin hash modules, md5, sha1, sha256, sha512, - sha3 (with shake), blake2 + --with-builtin-hashlib-hashes=md5,sha1,sha2,sha3,blake2 + builtin hash modules, md5, sha1, sha2, sha3 (with + shake), blake2 Some influential environment variables: PKG_CONFIG path to pkg-config utility @@ -25346,7 +25344,7 @@ fi # builtin hash modules -default_hashlib_hashes="md5,sha1,sha256,sha512,sha3,blake2" +default_hashlib_hashes="md5,sha1,sha2,sha3,blake2" $as_echo "#define PY_BUILTIN_HASHLIB_HASHES /**/" >>confdefs.h @@ -25386,10 +25384,8 @@ for builtin_hash in $with_builtin_hashlib_hashes; do with_builtin_md5=yes ;; #( sha1) : with_builtin_sha1=yes ;; #( - sha256) : - with_builtin_sha256=yes ;; #( - sha512) : - with_builtin_sha512=yes ;; #( + sha2) : + with_builtin_sha2=yes ;; #( sha3) : with_builtin_sha3=yes ;; #( blake2) : @@ -26898,72 +26894,38 @@ fi $as_echo "$py_cv_module__sha1" >&6; } - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _sha256" >&5 -$as_echo_n "checking for stdlib extension module _sha256... " >&6; } - if test "$py_cv_module__sha256" != "n/a"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _sha2" >&5 +$as_echo_n "checking for stdlib extension module _sha2... " >&6; } + if test "$py_cv_module__sha2" != "n/a"; then : - if test "$with_builtin_sha256" = yes; then : + if test "$with_builtin_sha2" = yes; then : if true; then : - py_cv_module__sha256=yes + py_cv_module__sha2=yes else - py_cv_module__sha256=missing + py_cv_module__sha2=missing fi else - py_cv_module__sha256=disabled + py_cv_module__sha2=disabled fi fi - as_fn_append MODULE_BLOCK "MODULE__SHA256_STATE=$py_cv_module__sha256$as_nl" - if test "x$py_cv_module__sha256" = xyes; then : + as_fn_append MODULE_BLOCK "MODULE__SHA2_STATE=$py_cv_module__sha2$as_nl" + if test "x$py_cv_module__sha2" = xyes; then : - as_fn_append MODULE_BLOCK "MODULE__SHA256_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" fi - if test "$py_cv_module__sha256" = yes; then - MODULE__SHA256_TRUE= - MODULE__SHA256_FALSE='#' + if test "$py_cv_module__sha2" = yes; then + MODULE__SHA2_TRUE= + MODULE__SHA2_FALSE='#' else - MODULE__SHA256_TRUE='#' - MODULE__SHA256_FALSE= + MODULE__SHA2_TRUE='#' + MODULE__SHA2_FALSE= fi - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__sha256" >&5 -$as_echo "$py_cv_module__sha256" >&6; } - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _sha512" >&5 -$as_echo_n "checking for stdlib extension module _sha512... " >&6; } - if test "$py_cv_module__sha512" != "n/a"; then : - - if test "$with_builtin_sha512" = yes; then : - if true; then : - py_cv_module__sha512=yes -else - py_cv_module__sha512=missing -fi -else - py_cv_module__sha512=disabled -fi - -fi - as_fn_append MODULE_BLOCK "MODULE__SHA512_STATE=$py_cv_module__sha512$as_nl" - if test "x$py_cv_module__sha512" = xyes; then : - - as_fn_append MODULE_BLOCK "MODULE__SHA512_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" - - -fi - if test "$py_cv_module__sha512" = yes; then - MODULE__SHA512_TRUE= - MODULE__SHA512_FALSE='#' -else - MODULE__SHA512_TRUE='#' - MODULE__SHA512_FALSE= -fi - - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__sha512" >&5 -$as_echo "$py_cv_module__sha512" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__sha2" >&5 +$as_echo "$py_cv_module__sha2" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _sha3" >&5 @@ -28337,12 +28299,8 @@ if test -z "${MODULE__SHA1_TRUE}" && test -z "${MODULE__SHA1_FALSE}"; then as_fn_error $? "conditional \"MODULE__SHA1\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__SHA256_TRUE}" && test -z "${MODULE__SHA256_FALSE}"; then - as_fn_error $? "conditional \"MODULE__SHA256\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi -if test -z "${MODULE__SHA512_TRUE}" && test -z "${MODULE__SHA512_FALSE}"; then - as_fn_error $? "conditional \"MODULE__SHA512\" was never defined. +if test -z "${MODULE__SHA2_TRUE}" && test -z "${MODULE__SHA2_FALSE}"; then + as_fn_error $? "conditional \"MODULE__SHA2\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${MODULE__SHA3_TRUE}" && test -z "${MODULE__SHA3_FALSE}"; then diff --git a/configure.ac b/configure.ac index 92a05c011026f2..370bbe07c57634 100644 --- a/configure.ac +++ b/configure.ac @@ -6928,14 +6928,14 @@ AC_DEFINE(PY_SSL_DEFAULT_CIPHERS, 1) ]) # builtin hash modules -default_hashlib_hashes="md5,sha1,sha256,sha512,sha3,blake2" +default_hashlib_hashes="md5,sha1,sha2,sha3,blake2" AC_DEFINE([PY_BUILTIN_HASHLIB_HASHES], [], [enabled builtin hash modules] ) AC_MSG_CHECKING(for --with-builtin-hashlib-hashes) AC_ARG_WITH(builtin-hashlib-hashes, - AS_HELP_STRING([--with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2], + AS_HELP_STRING([--with-builtin-hashlib-hashes=md5,sha1,sha2,sha3,blake2], [builtin hash modules, - md5, sha1, sha256, sha512, sha3 (with shake), blake2]), + md5, sha1, sha2, sha3 (with shake), blake2]), [ AS_CASE([$with_builtin_hashlib_hashes], [yes], [with_builtin_hashlib_hashes=$default_hashlib_hashes], @@ -6952,8 +6952,7 @@ for builtin_hash in $with_builtin_hashlib_hashes; do AS_CASE($builtin_hash, [md5], [with_builtin_md5=yes], [sha1], [with_builtin_sha1=yes], - [sha256], [with_builtin_sha256=yes], - [sha512], [with_builtin_sha512=yes], + [sha2], [with_builtin_sha2=yes], [sha3], [with_builtin_sha3=yes], [blake2], [with_builtin_blake2=yes] ) @@ -7197,11 +7196,8 @@ dnl By default we always compile these even when OpenSSL is available dnl (issue #14693). The modules are small. PY_STDLIB_MOD([_md5], [test "$with_builtin_md5" = yes]) PY_STDLIB_MOD([_sha1], [test "$with_builtin_sha1" = yes]) -PY_STDLIB_MOD([_sha256], - [test "$with_builtin_sha256" = yes], [], - [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) -PY_STDLIB_MOD([_sha512], - [test "$with_builtin_sha512" = yes], [], +PY_STDLIB_MOD([_sha2], + [test "$with_builtin_sha2" = yes], [], [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) PY_STDLIB_MOD([_sha3], [test "$with_builtin_sha3" = yes]) PY_STDLIB_MOD([_blake2], From df7ccf6138b1a2ce0b82ff06aa3497ca4d38c90d Mon Sep 17 00:00:00 2001 From: penguin_wwy <940375606@qq.com> Date: Thu, 16 Feb 2023 19:31:41 +0800 Subject: [PATCH 51/83] gh-101928: fix crash in compiler on multi-line lambda in function call (#101933) --- Lib/test/test_compile.py | 11 +++++++++++ Python/compile.c | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 90b067bcf30912..a77742c0cfa6fc 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1155,6 +1155,17 @@ def test_if_expression_expression_empty_block(self): with self.subTest(expr=expr): compile(expr, "", "exec") + def test_multi_line_lambda_as_argument(self): + # See gh-101928 + compile(""" +def foo(param, lambda_exp): + pass + +foo(param=0, + lambda_exp=lambda: + 1) + """, "", "exec") + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): diff --git a/Python/compile.c b/Python/compile.c index 0534b536e3d12e..c3b344c7af2a7f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -9085,8 +9085,8 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) Py_DECREF(cnt); break; case RETURN_VALUE: - INSTR_SET_OP1(inst, RETURN_CONST, oparg); - INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP1(&bb->b_instr[++i], RETURN_CONST, oparg); break; } break; From 36b139af638cdeb671cb6b8b0315b254148688f7 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 16 Feb 2023 12:31:59 +0000 Subject: [PATCH 52/83] gh-101951: use textwrap.dedent in compiler tests to make them more readable (GH-101950) Fixes #101951. Automerge-Triggered-By: GH:iritkatriel --- Lib/test/test_compile.py | 288 ++++++++++++++++++++------------------- 1 file changed, 145 insertions(+), 143 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index a77742c0cfa6fc..fe775779c50f50 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -115,24 +115,24 @@ def test_extended_arg(self): repeat = 2000 longexpr = 'x = x or ' + '-x' * repeat g = {} - code = ''' -def f(x): - %s - %s - %s - %s - %s - %s - %s - %s - %s - %s - # the expressions above have no effect, x == argument - while x: - x -= 1 - # EXTENDED_ARG/JUMP_ABSOLUTE here - return x -''' % ((longexpr,)*10) + code = textwrap.dedent(''' + def f(x): + %s + %s + %s + %s + %s + %s + %s + %s + %s + %s + # the expressions above have no effect, x == argument + while x: + x -= 1 + # EXTENDED_ARG/JUMP_ABSOLUTE here + return x + ''' % ((longexpr,)*10)) exec(code, g) self.assertEqual(g['f'](5), 0) @@ -148,10 +148,11 @@ def test_float_literals(self): def test_indentation(self): # testing compile() of indented block w/o trailing newline" - s = """ -if 1: - if 2: - pass""" + s = textwrap.dedent(""" + if 1: + if 2: + pass + """) compile(s, "", "exec") # This test is probably specific to CPython and may not generalize @@ -1157,14 +1158,15 @@ def test_if_expression_expression_empty_block(self): def test_multi_line_lambda_as_argument(self): # See gh-101928 - compile(""" -def foo(param, lambda_exp): - pass + code = textwrap.dedent(""" + def foo(param, lambda_exp): + pass -foo(param=0, - lambda_exp=lambda: - 1) - """, "", "exec") + foo(param=0, + lambda_exp=lambda: + 1) + """) + compile(code, "", "exec") @requires_debug_ranges() @@ -1252,24 +1254,24 @@ def test_compiles_to_extended_op_arg(self): column=2, end_column=9, occurrence=2) def test_multiline_expression(self): - snippet = """\ -f( - 1, 2, 3, 4 -) -""" + snippet = textwrap.dedent("""\ + f( + 1, 2, 3, 4 + ) + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', line=1, end_line=3, column=0, end_column=1) @requires_specialization def test_multiline_boolean_expression(self): - snippet = """\ -if (a or - (b and not c) or - not ( - d > 0)): - x = 42 -""" + snippet = textwrap.dedent("""\ + if (a or + (b and not c) or + not ( + d > 0)): + x = 42 + """) compiled_code, _ = self.check_positions_against_ast(snippet) # jump if a is true: self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', @@ -1288,11 +1290,11 @@ def test_multiline_boolean_expression(self): line=4, end_line=4, column=8, end_column=13, occurrence=2) def test_multiline_assert(self): - snippet = """\ -assert (a > 0 and - bb > 0 and - ccc == 4), "error msg" -""" + snippet = textwrap.dedent("""\ + assert (a > 0 and + bb > 0 and + ccc == 4), "error msg" + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_ASSERTION_ERROR', line=1, end_line=3, column=0, end_column=30, occurrence=1) @@ -1305,14 +1307,14 @@ def test_multiline_assert(self): line=1, end_line=3, column=0, end_column=30, occurrence=1) def test_multiline_generator_expression(self): - snippet = """\ -((x, - 2*x) - for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)) -""" + snippet = textwrap.dedent("""\ + ((x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)) + """) compiled_code, _ = self.check_positions_against_ast(snippet) compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) @@ -1324,14 +1326,14 @@ def test_multiline_generator_expression(self): line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_generator_expression(self): - snippet = """\ -((x, - 2*x) - async for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)) -""" + snippet = textwrap.dedent("""\ + ((x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)) + """) compiled_code, _ = self.check_positions_against_ast(snippet) compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) @@ -1341,14 +1343,14 @@ def test_multiline_async_generator_expression(self): line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_list_comprehension(self): - snippet = """\ -[(x, - 2*x) - for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)] -""" + snippet = textwrap.dedent("""\ + [(x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)] + """) compiled_code, _ = self.check_positions_against_ast(snippet) compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) @@ -1360,15 +1362,15 @@ def test_multiline_list_comprehension(self): line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_list_comprehension(self): - snippet = """\ -async def f(): - [(x, - 2*x) - async for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)] -""" + snippet = textwrap.dedent("""\ + async def f(): + [(x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)] + """) compiled_code, _ = self.check_positions_against_ast(snippet) g = {} eval(compiled_code, g) @@ -1382,14 +1384,14 @@ async def f(): line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_multiline_set_comprehension(self): - snippet = """\ -{(x, - 2*x) - for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)} -""" + snippet = textwrap.dedent("""\ + {(x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) compiled_code, _ = self.check_positions_against_ast(snippet) compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) @@ -1401,15 +1403,15 @@ def test_multiline_set_comprehension(self): line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_set_comprehension(self): - snippet = """\ -async def f(): - {(x, - 2*x) - async for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)} -""" + snippet = textwrap.dedent("""\ + async def f(): + {(x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) compiled_code, _ = self.check_positions_against_ast(snippet) g = {} eval(compiled_code, g) @@ -1423,14 +1425,14 @@ async def f(): line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_multiline_dict_comprehension(self): - snippet = """\ -{x: - 2*x - for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)} -""" + snippet = textwrap.dedent("""\ + {x: + 2*x + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) compiled_code, _ = self.check_positions_against_ast(snippet) compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) @@ -1442,15 +1444,15 @@ def test_multiline_dict_comprehension(self): line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_dict_comprehension(self): - snippet = """\ -async def f(): - {x: - 2*x - async for x - in [1,2,3] if (x > 0 - and x < 100 - and x != 50)} -""" + snippet = textwrap.dedent("""\ + async def f(): + {x: + 2*x + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) compiled_code, _ = self.check_positions_against_ast(snippet) g = {} eval(compiled_code, g) @@ -1464,11 +1466,11 @@ async def f(): line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_matchcase_sequence(self): - snippet = """\ -match x: - case a, b: - pass -""" + snippet = textwrap.dedent("""\ + match x: + case a, b: + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', line=2, end_line=2, column=9, end_column=13, occurrence=1) @@ -1480,11 +1482,11 @@ def test_matchcase_sequence(self): line=2, end_line=2, column=9, end_column=13, occurrence=2) def test_matchcase_sequence_wildcard(self): - snippet = """\ -match x: - case a, *b, c: - pass -""" + snippet = textwrap.dedent("""\ + match x: + case a, *b, c: + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', line=2, end_line=2, column=9, end_column=17, occurrence=1) @@ -1498,11 +1500,11 @@ def test_matchcase_sequence_wildcard(self): line=2, end_line=2, column=9, end_column=17, occurrence=3) def test_matchcase_mapping(self): - snippet = """\ -match x: - case {"a" : a, "b": b}: - pass -""" + snippet = textwrap.dedent("""\ + match x: + case {"a" : a, "b": b}: + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', line=2, end_line=2, column=9, end_column=26, occurrence=1) @@ -1514,11 +1516,11 @@ def test_matchcase_mapping(self): line=2, end_line=2, column=9, end_column=26, occurrence=2) def test_matchcase_mapping_wildcard(self): - snippet = """\ -match x: - case {"a" : a, "b": b, **c}: - pass -""" + snippet = textwrap.dedent("""\ + match x: + case {"a" : a, "b": b, **c}: + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', line=2, end_line=2, column=9, end_column=31, occurrence=1) @@ -1530,11 +1532,11 @@ def test_matchcase_mapping_wildcard(self): line=2, end_line=2, column=9, end_column=31, occurrence=2) def test_matchcase_class(self): - snippet = """\ -match x: - case C(a, b): - pass -""" + snippet = textwrap.dedent("""\ + match x: + case C(a, b): + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', line=2, end_line=2, column=9, end_column=16, occurrence=1) @@ -1546,11 +1548,11 @@ def test_matchcase_class(self): line=2, end_line=2, column=9, end_column=16, occurrence=2) def test_matchcase_or(self): - snippet = """\ -match x: - case C(1) | C(2): - pass -""" + snippet = textwrap.dedent("""\ + match x: + case C(1) | C(2): + pass + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', line=2, end_line=2, column=9, end_column=13, occurrence=1) From 739c026f4488bd2e37d500a2c3d948aaf929b641 Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Thu, 16 Feb 2023 14:52:24 +0000 Subject: [PATCH 53/83] gh-101881: Support (non-)blocking read/write functions on Windows pipes (GH-101882) * fileutils: handle non-blocking pipe IO on Windows Handle erroring operations on non-blocking pipes by reading the _doserrno code. Limit writes on non-blocking pipes that are too large. * Support blocking functions on Windows Use the GetNamedPipeHandleState and SetNamedPipeHandleState Win32 API functions to add support for os.get_blocking and os.set_blocking. --- Doc/library/os.rst | 12 ++- Include/internal/pycore_fileutils.h | 4 +- Lib/test/test_os.py | 1 + ...-02-13-18-05-49.gh-issue-101881._TnHzN.rst | 1 + ...-02-15-11-08-10.gh-issue-101881.fScr3m.rst | 1 + Modules/clinic/posixmodule.c.h | 18 +--- Modules/posixmodule.c | 2 - Python/fileutils.c | 93 ++++++++++++++++++- 8 files changed, 107 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst create mode 100644 Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index fb091176767f7a..85924d0e48366b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1091,13 +1091,17 @@ as internal buffering of data. See also :func:`set_blocking` and :meth:`socket.socket.setblocking`. - .. availability:: Unix. + .. availability:: Unix, Windows. The function is limited on Emscripten and WASI, see :ref:`wasm-availability` for more information. + On Windows, this function is limited to pipes. + .. versionadded:: 3.5 + .. versionchanged:: 3.12 + Added support for pipes on Windows. .. function:: isatty(fd, /) @@ -1565,13 +1569,17 @@ or `the MSDN `_ on Windo See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. - .. availability:: Unix. + .. availability:: Unix, Windows. The function is limited on Emscripten and WASI, see :ref:`wasm-availability` for more information. + On Windows, this function is limited to pipes. + .. versionadded:: 3.5 + .. versionchanged:: 3.12 + Added support for pipes on Windows. .. data:: SF_NODISKIO SF_MNOWAIT diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index ac89c43d569c07..f8e2bf22590888 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -160,11 +160,11 @@ PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable, PyAPI_FUNC(int) _Py_dup(int fd); -#ifndef MS_WINDOWS PyAPI_FUNC(int) _Py_get_blocking(int fd); PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking); -#else /* MS_WINDOWS */ + +#ifdef MS_WINDOWS PyAPI_FUNC(void*) _Py_get_osfhandle_noraise(int fd); PyAPI_FUNC(void*) _Py_get_osfhandle(int fd); diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 387d2581c06fc6..deea207bfdadd9 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4099,6 +4099,7 @@ def test_path_t_converter_and_custom_class(self): @unittest.skipUnless(hasattr(os, 'get_blocking'), 'needs os.get_blocking() and os.set_blocking()') @unittest.skipIf(support.is_emscripten, "Cannot unset blocking flag") +@unittest.skipIf(sys.platform == 'win32', 'Windows only supports blocking on pipes') class BlockingTests(unittest.TestCase): def test_blocking(self): fd = os.open(__file__, os.O_RDONLY) diff --git a/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst b/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst new file mode 100644 index 00000000000000..ba58dd4f5cb450 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst @@ -0,0 +1 @@ +Add support for the os.get_blocking() and os.set_blocking() functions on Windows. diff --git a/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst b/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst new file mode 100644 index 00000000000000..099b2c1c07a665 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst @@ -0,0 +1 @@ +Handle read and write operations on non-blocking pipes properly on Windows. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 5e04507ddd6917..dcd25c28370c93 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10402,8 +10402,6 @@ os_set_handle_inheritable(PyObject *module, PyObject *const *args, Py_ssize_t na #endif /* defined(MS_WINDOWS) */ -#if !defined(MS_WINDOWS) - PyDoc_STRVAR(os_get_blocking__doc__, "get_blocking($module, fd, /)\n" "--\n" @@ -10439,10 +10437,6 @@ os_get_blocking(PyObject *module, PyObject *arg) return return_value; } -#endif /* !defined(MS_WINDOWS) */ - -#if !defined(MS_WINDOWS) - PyDoc_STRVAR(os_set_blocking__doc__, "set_blocking($module, fd, blocking, /)\n" "--\n" @@ -10482,8 +10476,6 @@ os_set_blocking(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* !defined(MS_WINDOWS) */ - PyDoc_STRVAR(os_DirEntry_is_symlink__doc__, "is_symlink($self, /)\n" "--\n" @@ -11789,14 +11781,6 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS_SET_HANDLE_INHERITABLE_METHODDEF #endif /* !defined(OS_SET_HANDLE_INHERITABLE_METHODDEF) */ -#ifndef OS_GET_BLOCKING_METHODDEF - #define OS_GET_BLOCKING_METHODDEF -#endif /* !defined(OS_GET_BLOCKING_METHODDEF) */ - -#ifndef OS_SET_BLOCKING_METHODDEF - #define OS_SET_BLOCKING_METHODDEF -#endif /* !defined(OS_SET_BLOCKING_METHODDEF) */ - #ifndef OS_GETRANDOM_METHODDEF #define OS_GETRANDOM_METHODDEF #endif /* !defined(OS_GETRANDOM_METHODDEF) */ @@ -11812,4 +11796,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=a3f76228b549e8ec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1b0eb6a76b1a0e28 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d9e93473aeadaa..524dc7eb1ccc97 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -13930,7 +13930,6 @@ os_set_handle_inheritable_impl(PyObject *module, intptr_t handle, } #endif /* MS_WINDOWS */ -#ifndef MS_WINDOWS /*[clinic input] os.get_blocking -> bool fd: int @@ -13978,7 +13977,6 @@ os_set_blocking_impl(PyObject *module, int fd, int blocking) return NULL; Py_RETURN_NONE; } -#endif /* !MS_WINDOWS */ /*[clinic input] diff --git a/Python/fileutils.c b/Python/fileutils.c index 22b2257a56d0ec..897c2f9f4ea160 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1750,7 +1750,15 @@ _Py_read(int fd, void *buf, size_t count) Py_BEGIN_ALLOW_THREADS errno = 0; #ifdef MS_WINDOWS + _doserrno = 0; n = read(fd, buf, (int)count); + // read() on a non-blocking empty pipe fails with EINVAL, which is + // mapped from the Windows error code ERROR_NO_DATA. + if (n < 0 && errno == EINVAL) { + if (_doserrno == ERROR_NO_DATA) { + errno = EAGAIN; + } + } #else n = read(fd, buf, count); #endif @@ -1804,6 +1812,7 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) } } } + #endif if (count > _PY_WRITE_MAX) { count = _PY_WRITE_MAX; @@ -1814,7 +1823,18 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) Py_BEGIN_ALLOW_THREADS errno = 0; #ifdef MS_WINDOWS - n = write(fd, buf, (int)count); + // write() on a non-blocking pipe fails with ENOSPC on Windows if + // the pipe lacks available space for the entire buffer. + int c = (int)count; + do { + _doserrno = 0; + n = write(fd, buf, c); + if (n >= 0 || errno != ENOSPC || _doserrno != 0) { + break; + } + errno = EAGAIN; + c /= 2; + } while (c > 0); #else n = write(fd, buf, count); #endif @@ -1829,7 +1849,18 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) do { errno = 0; #ifdef MS_WINDOWS - n = write(fd, buf, (int)count); + // write() on a non-blocking pipe fails with ENOSPC on Windows if + // the pipe lacks available space for the entire buffer. + int c = (int)count; + do { + _doserrno = 0; + n = write(fd, buf, c); + if (n >= 0 || errno != ENOSPC || _doserrno != 0) { + break; + } + errno = EAGAIN; + c /= 2; + } while (c > 0); #else n = write(fd, buf, count); #endif @@ -2450,6 +2481,64 @@ _Py_set_blocking(int fd, int blocking) return -1; } #else /* MS_WINDOWS */ +int +_Py_get_blocking(int fd) +{ + HANDLE handle; + DWORD mode; + BOOL success; + + handle = _Py_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + success = GetNamedPipeHandleStateW(handle, &mode, + NULL, NULL, NULL, NULL, 0); + Py_END_ALLOW_THREADS + + if (!success) { + PyErr_SetFromWindowsErr(0); + return -1; + } + + return !(mode & PIPE_NOWAIT); +} + +int +_Py_set_blocking(int fd, int blocking) +{ + HANDLE handle; + DWORD mode; + BOOL success; + + handle = _Py_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + success = GetNamedPipeHandleStateW(handle, &mode, + NULL, NULL, NULL, NULL, 0); + if (success) { + if (blocking) { + mode &= ~PIPE_NOWAIT; + } + else { + mode |= PIPE_NOWAIT; + } + success = SetNamedPipeHandleState(handle, &mode, NULL, NULL); + } + Py_END_ALLOW_THREADS + + if (!success) { + PyErr_SetFromWindowsErr(0); + return -1; + } + return 0; +} + void* _Py_get_osfhandle_noraise(int fd) { From 924a3bfa28578802eb9ca77a66fb5d4762a62f14 Mon Sep 17 00:00:00 2001 From: sblondon Date: Thu, 16 Feb 2023 16:13:21 +0100 Subject: [PATCH 54/83] gh-93573: Replace wrong example domains in configparser doc (GH-93574) * Replace bitbucket.org domain by forge.example * Update example to python.org * Use explicitly invalid domain topsecret.server.com domain is not controled by PSF. It's replaced by invalid topsecret.server.example domain. It follows RFC 2606, which advise .example as TLD for documentation. --- Doc/library/configparser.rst | 46 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index a925a3dd4fb9c2..a7f75fd6e84f4c 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -69,10 +69,10 @@ Let's take a very basic configuration file that looks like this: CompressionLevel = 9 ForwardX11 = yes - [bitbucket.org] + [forge.example] User = hg - [topsecret.server.com] + [topsecret.server.example] Port = 50022 ForwardX11 = no @@ -89,10 +89,10 @@ creating the above configuration file programmatically. >>> config['DEFAULT'] = {'ServerAliveInterval': '45', ... 'Compression': 'yes', ... 'CompressionLevel': '9'} - >>> config['bitbucket.org'] = {} - >>> config['bitbucket.org']['User'] = 'hg' - >>> config['topsecret.server.com'] = {} - >>> topsecret = config['topsecret.server.com'] + >>> config['forge.example'] = {} + >>> config['forge.example']['User'] = 'hg' + >>> config['topsecret.server.example'] = {} + >>> topsecret = config['topsecret.server.example'] >>> topsecret['Port'] = '50022' # mutates the parser >>> topsecret['ForwardX11'] = 'no' # same here >>> config['DEFAULT']['ForwardX11'] = 'yes' @@ -115,28 +115,28 @@ back and explore the data it holds. >>> config.read('example.ini') ['example.ini'] >>> config.sections() - ['bitbucket.org', 'topsecret.server.com'] - >>> 'bitbucket.org' in config + ['forge.example', 'topsecret.server.example'] + >>> 'forge.example' in config True - >>> 'bytebong.com' in config + >>> 'python.org' in config False - >>> config['bitbucket.org']['User'] + >>> config['forge.example']['User'] 'hg' >>> config['DEFAULT']['Compression'] 'yes' - >>> topsecret = config['topsecret.server.com'] + >>> topsecret = config['topsecret.server.example'] >>> topsecret['ForwardX11'] 'no' >>> topsecret['Port'] '50022' - >>> for key in config['bitbucket.org']: # doctest: +SKIP + >>> for key in config['forge.example']: # doctest: +SKIP ... print(key) user compressionlevel serveraliveinterval compression forwardx11 - >>> config['bitbucket.org']['ForwardX11'] + >>> config['forge.example']['ForwardX11'] 'yes' As we can see above, the API is pretty straightforward. The only bit of magic @@ -154,15 +154,15 @@ configuration while the previously existing keys are retained. >>> another_config = configparser.ConfigParser() >>> another_config.read('example.ini') ['example.ini'] - >>> another_config['topsecret.server.com']['Port'] + >>> another_config['topsecret.server.example']['Port'] '50022' - >>> another_config.read_string("[topsecret.server.com]\nPort=48484") - >>> another_config['topsecret.server.com']['Port'] + >>> another_config.read_string("[topsecret.server.example]\nPort=48484") + >>> another_config['topsecret.server.example']['Port'] '48484' - >>> another_config.read_dict({"topsecret.server.com": {"Port": 21212}}) - >>> another_config['topsecret.server.com']['Port'] + >>> another_config.read_dict({"topsecret.server.example": {"Port": 21212}}) + >>> another_config['topsecret.server.example']['Port'] '21212' - >>> another_config['topsecret.server.com']['ForwardX11'] + >>> another_config['topsecret.server.example']['ForwardX11'] 'no' This behaviour is equivalent to a :meth:`ConfigParser.read` call with several @@ -195,9 +195,9 @@ recognizes Boolean values from ``'yes'``/``'no'``, ``'on'``/``'off'``, >>> topsecret.getboolean('ForwardX11') False - >>> config['bitbucket.org'].getboolean('ForwardX11') + >>> config['forge.example'].getboolean('ForwardX11') True - >>> config.getboolean('bitbucket.org', 'Compression') + >>> config.getboolean('forge.example', 'Compression') True Apart from :meth:`~ConfigParser.getboolean`, config parsers also @@ -224,7 +224,7 @@ provide fallback values: Please note that default values have precedence over fallback values. For instance, in our example the ``'CompressionLevel'`` key was specified only in the ``'DEFAULT'`` section. If we try to get it from -the section ``'topsecret.server.com'``, we will always get the default, +the section ``'topsecret.server.example'``, we will always get the default, even if we specify a fallback: .. doctest:: @@ -239,7 +239,7 @@ the ``fallback`` keyword-only argument: .. doctest:: - >>> config.get('bitbucket.org', 'monster', + >>> config.get('forge.example', 'monster', ... fallback='No such things as monsters') 'No such things as monsters' From 68bd8c5e2efab64ff9d38a214775164182179431 Mon Sep 17 00:00:00 2001 From: Eclips4 <80244920+Eclips4@users.noreply.github.com> Date: Thu, 16 Feb 2023 20:46:43 +0300 Subject: [PATCH 55/83] gh-101952: Fix possible segfault in `BUILD_SET` opcode (#101958) --- .../2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst | 1 + Python/bytecodes.c | 2 ++ Python/generated_cases.c.h | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst new file mode 100644 index 00000000000000..3902c988c8bf9f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst @@ -0,0 +1 @@ +Fix possible segfault in ``BUILD_SET`` opcode, when new set created. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d5d5034cbfbf74..84747f1758e06c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1302,6 +1302,8 @@ dummy_func( inst(BUILD_SET, (values[oparg] -- set)) { set = PySet_New(NULL); + if (set == NULL) + goto error; int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8b8a7161ad898e..730dfb7426acbf 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1649,6 +1649,8 @@ PyObject **values = &PEEK(oparg); PyObject *set; set = PySet_New(NULL); + if (set == NULL) + goto error; int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; From 226484e47599a93f5bf033ac47198e68ff401432 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Thu, 16 Feb 2023 12:57:59 -0500 Subject: [PATCH 56/83] gh-99942: correct the pkg-config/python-config flags for cygwin/android --- .../next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst | 2 ++ configure | 2 +- configure.ac | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst diff --git a/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst b/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst new file mode 100644 index 00000000000000..5b692c3cc458c5 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst @@ -0,0 +1,2 @@ +On Android, in a static build, python-config in embed mode no longer +incorrectly reports a library to link to. diff --git a/configure b/configure index 7c4254f3cb176f..17dc62fb63de3b 100755 --- a/configure +++ b/configure @@ -21496,7 +21496,7 @@ $as_echo "$LDVERSION" >&6; } # On Android and Cygwin the shared libraries must be linked with libpython. -if test -n "$ANDROID_API_LEVEL" -o "$MACHDEP" = "cygwin"; then +if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" else LIBPYTHON='' diff --git a/configure.ac b/configure.ac index 370bbe07c57634..bc288b86cfa590 100644 --- a/configure.ac +++ b/configure.ac @@ -5759,7 +5759,7 @@ AC_MSG_RESULT($LDVERSION) # On Android and Cygwin the shared libraries must be linked with libpython. AC_SUBST(LIBPYTHON) -if test -n "$ANDROID_API_LEVEL" -o "$MACHDEP" = "cygwin"; then +if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" else LIBPYTHON='' From a5024a261a75dafa4fb6613298dcb64a9603d9c7 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 17 Feb 2023 00:18:21 +0530 Subject: [PATCH 57/83] GH-96764: rewrite `asyncio.wait_for` to use `asyncio.timeout` (#98518) Changes `asyncio.wait_for` to use `asyncio.timeout` as its underlying implementation. --- Lib/asyncio/tasks.py | 77 ++++------- Lib/test/test_asyncio/test_futures2.py | 7 +- Lib/test/test_asyncio/test_waitfor.py | 127 ++++++++++++++---- ...2-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst | 1 + 4 files changed, 133 insertions(+), 79 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index e78719de216fd0..a2e06d5ef72f42 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -24,6 +24,7 @@ from . import events from . import exceptions from . import futures +from . import timeouts from .coroutines import _is_coroutine # Helper to generate new task names @@ -437,65 +438,44 @@ async def wait_for(fut, timeout): If the wait is cancelled, the task is also cancelled. + If the task supresses the cancellation and returns a value instead, + that value is returned. + This function is a coroutine. """ - loop = events.get_running_loop() + # The special case for timeout <= 0 is for the following case: + # + # async def test_waitfor(): + # func_started = False + # + # async def func(): + # nonlocal func_started + # func_started = True + # + # try: + # await asyncio.wait_for(func(), 0) + # except asyncio.TimeoutError: + # assert not func_started + # else: + # assert False + # + # asyncio.run(test_waitfor()) - if timeout is None: - return await fut - if timeout <= 0: - fut = ensure_future(fut, loop=loop) + if timeout is not None and timeout <= 0: + fut = ensure_future(fut) if fut.done(): return fut.result() - await _cancel_and_wait(fut, loop=loop) + await _cancel_and_wait(fut) try: return fut.result() except exceptions.CancelledError as exc: - raise exceptions.TimeoutError() from exc - - waiter = loop.create_future() - timeout_handle = loop.call_later(timeout, _release_waiter, waiter) - cb = functools.partial(_release_waiter, waiter) - - fut = ensure_future(fut, loop=loop) - fut.add_done_callback(cb) - - try: - # wait until the future completes or the timeout - try: - await waiter - except exceptions.CancelledError: - if fut.done(): - return fut.result() - else: - fut.remove_done_callback(cb) - # We must ensure that the task is not running - # after wait_for() returns. - # See https://bugs.python.org/issue32751 - await _cancel_and_wait(fut, loop=loop) - raise - - if fut.done(): - return fut.result() - else: - fut.remove_done_callback(cb) - # We must ensure that the task is not running - # after wait_for() returns. - # See https://bugs.python.org/issue32751 - await _cancel_and_wait(fut, loop=loop) - # In case task cancellation failed with some - # exception, we should re-raise it - # See https://bugs.python.org/issue40607 - try: - return fut.result() - except exceptions.CancelledError as exc: - raise exceptions.TimeoutError() from exc - finally: - timeout_handle.cancel() + raise TimeoutError from exc + async with timeouts.timeout(timeout): + return await fut async def _wait(fs, timeout, return_when, loop): """Internal helper for wait(). @@ -541,9 +521,10 @@ def _on_completion(f): return done, pending -async def _cancel_and_wait(fut, loop): +async def _cancel_and_wait(fut): """Cancel the *fut* future or task and wait until it completes.""" + loop = events.get_running_loop() waiter = loop.create_future() cb = functools.partial(_release_waiter, waiter) fut.add_done_callback(cb) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index 9e7a5775a70383..b7cfffb76bd8f1 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -86,10 +86,9 @@ async def test_recursive_repr_for_pending_tasks(self): async def func(): return asyncio.all_tasks() - # The repr() call should not raise RecursiveError at first. - # The check for returned string is not very reliable but - # exact comparison for the whole string is even weaker. - self.assertIn('...', repr(await asyncio.wait_for(func(), timeout=10))) + # The repr() call should not raise RecursionError at first. + waiter = await asyncio.wait_for(asyncio.Task(func()),timeout=10) + self.assertIn('...', repr(waiter)) if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index 45498fa097f6bc..ed80540b2b3852 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -237,33 +237,6 @@ async def inner(): with self.assertRaises(FooException): await foo() - async def test_wait_for_self_cancellation(self): - async def inner(): - try: - await asyncio.sleep(0.3) - except asyncio.CancelledError: - try: - await asyncio.sleep(0.3) - except asyncio.CancelledError: - await asyncio.sleep(0.3) - - return 42 - - inner_task = asyncio.create_task(inner()) - - wait = asyncio.wait_for(inner_task, timeout=0.1) - - # Test that wait_for itself is properly cancellable - # even when the initial task holds up the initial cancellation. - task = asyncio.create_task(wait) - await asyncio.sleep(0.2) - task.cancel() - - with self.assertRaises(asyncio.CancelledError): - await task - - self.assertEqual(await inner_task, 42) - async def _test_cancel_wait_for(self, timeout): loop = asyncio.get_running_loop() @@ -289,6 +262,106 @@ async def test_cancel_blocking_wait_for(self): async def test_cancel_wait_for(self): await self._test_cancel_wait_for(60.0) + async def test_wait_for_cancel_suppressed(self): + # GH-86296: Supressing CancelledError is discouraged + # but if a task subpresses CancelledError and returns a value, + # `wait_for` should return the value instead of raising CancelledError. + # This is the same behavior as `asyncio.timeout`. + + async def return_42(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + return 42 + + res = await asyncio.wait_for(return_42(), timeout=0.1) + self.assertEqual(res, 42) + + + async def test_wait_for_issue86296(self): + # GH-86296: The task should get cancelled and not run to completion. + # inner completes in one cycle of the event loop so it + # completes before the task is cancelled. + + async def inner(): + return 'done' + + inner_task = asyncio.create_task(inner()) + reached_end = False + + async def wait_for_coro(): + await asyncio.wait_for(inner_task, timeout=100) + await asyncio.sleep(1) + nonlocal reached_end + reached_end = True + + task = asyncio.create_task(wait_for_coro()) + self.assertFalse(task.done()) + # Run the task + await asyncio.sleep(0) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + self.assertTrue(inner_task.done()) + self.assertEqual(await inner_task, 'done') + self.assertFalse(reached_end) + + +class WaitForShieldTests(unittest.IsolatedAsyncioTestCase): + + async def test_zero_timeout(self): + # `asyncio.shield` creates a new task which wraps the passed in + # awaitable and shields it from cancellation so with timeout=0 + # the task returned by `asyncio.shield` aka shielded_task gets + # cancelled immediately and the task wrapped by it is scheduled + # to run. + + async def coro(): + await asyncio.sleep(0.01) + return 'done' + + task = asyncio.create_task(coro()) + with self.assertRaises(asyncio.TimeoutError): + shielded_task = asyncio.shield(task) + await asyncio.wait_for(shielded_task, timeout=0) + + # Task is running in background + self.assertFalse(task.done()) + self.assertFalse(task.cancelled()) + self.assertTrue(shielded_task.cancelled()) + + # Wait for the task to complete + await asyncio.sleep(0.1) + self.assertTrue(task.done()) + + + async def test_none_timeout(self): + # With timeout=None the timeout is disabled so it + # runs till completion. + async def coro(): + await asyncio.sleep(0.1) + return 'done' + + task = asyncio.create_task(coro()) + await asyncio.wait_for(asyncio.shield(task), timeout=None) + + self.assertTrue(task.done()) + self.assertEqual(await task, "done") + + async def test_shielded_timeout(self): + # shield prevents the task from being cancelled. + async def coro(): + await asyncio.sleep(0.1) + return 'done' + + task = asyncio.create_task(coro()) + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(asyncio.shield(task), timeout=0.01) + + self.assertFalse(task.done()) + self.assertFalse(task.cancelled()) + self.assertEqual(await task, "done") + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst b/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst new file mode 100644 index 00000000000000..a0174291cbc311 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst @@ -0,0 +1 @@ +:func:`asyncio.wait_for` now uses :func:`asyncio.timeout` as its underlying implementation. Patch by Kumar Aditya. From 4d8959b73ac194ca9a2f623dcb5c23680f7d8536 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 16 Feb 2023 14:05:31 -0700 Subject: [PATCH 58/83] gh-101758: Add _PyState_AddModule() Back for the Stable ABI (gh-101956) We're adding the function back, only for the stable ABI symbol and not as any form of API. I had removed it yesterday. This undocumented "private" function was added with the implementation for PEP 3121 (3.0, 2007) for internal use and later moved out of the limited API (3.6, 2016) and then into the internal API (3.9, 2019). I removed it completely yesterday, including from the stable ABI manifest (where it was added because the symbol happened to be exported). It's unlikely that anyone is using _PyState_AddModule(), especially any stable ABI extensions built against 3.2-3.5, but we're playing it safe. https://github.com/python/cpython/issues/101758 --- Include/internal/pycore_pystate.h | 6 ++++++ Lib/test/test_stable_abi_ctypes.py | 1 + Misc/stable_abi.toml | 3 +++ PC/python3dll.c | 1 + Python/import.c | 20 ++++++++++++++++++++ 5 files changed, 31 insertions(+) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 638b86253879ea..7046ec8d9adaaf 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -152,6 +152,12 @@ extern void _PySignal_AfterFork(void); #endif +PyAPI_FUNC(int) _PyState_AddModule( + PyThreadState *tstate, + PyObject* module, + PyModuleDef* def); + + PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate); #define HEAD_LOCK(runtime) \ diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 7e50fbda2c07cb..e77c1c8409880d 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -864,6 +864,7 @@ def test_windows_feature_macros(self): "_PyObject_GC_Resize", "_PyObject_New", "_PyObject_NewVar", + "_PyState_AddModule", "_PyThreadState_Init", "_PyThreadState_Prealloc", "_PyWeakref_CallableProxyType", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index c04a3a228caf56..21ff9616133445 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1684,6 +1684,9 @@ [function._PyObject_NewVar] added = '3.2' abi_only = true +[function._PyState_AddModule] + added = '3.2' + abi_only = true [function._PyThreadState_Init] added = '3.2' abi_only = true diff --git a/PC/python3dll.c b/PC/python3dll.c index 79f09037282f54..e300819365756e 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -34,6 +34,7 @@ EXPORT_FUNC(_PyObject_GC_NewVar) EXPORT_FUNC(_PyObject_GC_Resize) EXPORT_FUNC(_PyObject_New) EXPORT_FUNC(_PyObject_NewVar) +EXPORT_FUNC(_PyState_AddModule) EXPORT_FUNC(_PyThreadState_Init) EXPORT_FUNC(_PyThreadState_Prealloc) EXPORT_FUNC(Py_AddPendingCall) diff --git a/Python/import.c b/Python/import.c index ec126f28b85816..fabf03b1c5d698 100644 --- a/Python/import.c +++ b/Python/import.c @@ -487,6 +487,26 @@ PyState_FindModule(PyModuleDef* module) return _modules_by_index_get(interp, module); } +/* _PyState_AddModule() has been completely removed from the C-API + (and was removed from the limited API in 3.6). However, we're + playing it safe and keeping it around for any stable ABI extensions + built against 3.2-3.5. */ +int +_PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def) +{ + if (!def) { + assert(_PyErr_Occurred(tstate)); + return -1; + } + if (def->m_slots) { + _PyErr_SetString(tstate, + PyExc_SystemError, + "PyState_AddModule called on module with slots"); + return -1; + } + return _modules_by_index_set(tstate->interp, def, module); +} + int PyState_AddModule(PyObject* module, PyModuleDef* def) { From 984f8ab018f847fe8d66768af962f69ec0e81849 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 16 Feb 2023 17:21:22 -0700 Subject: [PATCH 59/83] gh-101758: Fix Refleak-Related Failures in test_singlephase_variants (gh-101969) gh-101891 is causing failures under `$> ./python -m test test_imp -R 3:3`. Furthermore, with that fixed, "test_singlephase_variants" is leaking references. This change addresses the first part, but skips the leaking tests until we can follow up with a fix. https://github.com/python/cpython/issues/101758 --- Lib/test/test_imp.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 5997ffad8e1232..2292bb20939599 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -263,6 +263,7 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) + @unittest.skip('known refleak (temporarily skipping)') @requires_subinterpreters @requires_load_dynamic def test_singlephase_multiple_interpreters(self): @@ -329,9 +330,10 @@ def clean_up(): # However, globals are still shared. _interpreters.run_string(interp2, script % 2) + @unittest.skip('known refleak (temporarily skipping)') @requires_load_dynamic def test_singlephase_variants(self): - '''Exercise the most meaningful variants described in Python/import.c.''' + # Exercise the most meaningful variants described in Python/import.c. self.maxDiff = None basename = '_testsinglephase' @@ -343,6 +345,11 @@ def clean_up(): _testsinglephase._clear_globals() self.addCleanup(clean_up) + def add_ext_cleanup(name): + def clean_up(): + _testinternalcapi.clear_extension(name, pathname) + self.addCleanup(clean_up) + modules = {} def load(name): assert name not in modules @@ -440,6 +447,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, # Check the "basic" module. name = basename + add_ext_cleanup(name) expected_init_count = 1 with self.subTest(name): mod = load(name) @@ -457,6 +465,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, # Check its indirect variants. name = f'{basename}_basic_wrapper' + add_ext_cleanup(name) expected_init_count += 1 with self.subTest(name): mod = load(name) @@ -480,6 +489,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, # Check its direct variant. name = f'{basename}_basic_copy' + add_ext_cleanup(name) expected_init_count += 1 with self.subTest(name): mod = load(name) @@ -500,6 +510,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, # Check the non-basic variant that has no state. name = f'{basename}_with_reinit' + add_ext_cleanup(name) with self.subTest(name): mod = load(name) lookedup, initialized, cached = check_common(name, mod) @@ -518,6 +529,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, # Check the basic variant that has state. name = f'{basename}_with_state' + add_ext_cleanup(name) with self.subTest(name): mod = load(name) lookedup, initialized, cached = check_common(name, mod) From a3bb7fbe7eecfae6bf7b2f0912f9b2b12fac8db1 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Fri, 17 Feb 2023 12:43:07 +0400 Subject: [PATCH 60/83] gh-101973: Fix parameter reference for PyModule_FromDefAndSpec (#101976) --- Doc/c-api/module.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index e2ba157b32c7d9..c0351c8a6c72aa 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -388,7 +388,7 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and .. c:function:: PyObject * PyModule_FromDefAndSpec(PyModuleDef *def, PyObject *spec) - Create a new module object, given the definition in *module* and the + Create a new module object, given the definition in *def* and the ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2` with *module_api_version* set to :const:`PYTHON_API_VERSION`. @@ -396,7 +396,7 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and .. c:function:: PyObject * PyModule_FromDefAndSpec2(PyModuleDef *def, PyObject *spec, int module_api_version) - Create a new module object, given the definition in *module* and the + Create a new module object, given the definition in *def* and the ModuleSpec *spec*, assuming the API version *module_api_version*. If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. From 3c0a31cbfd1258bd96153a007dd44a96f2947dbf Mon Sep 17 00:00:00 2001 From: Yeojin Kim Date: Fri, 17 Feb 2023 17:47:02 +0900 Subject: [PATCH 61/83] Docs: fix typos in PyFunction_WatchCallback docs and in 3.12 NEWS (GH-101980) - possitibility => possibility - disaallowed => disallowed --- Doc/c-api/function.rst | 2 +- Misc/NEWS.d/3.12.0a2.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 3cce18bdde3057..bc7569d0add97d 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -169,7 +169,7 @@ There are a few functions specific to Python functions. before the modification to *func* takes place, so the prior state of *func* can be inspected. The runtime is permitted to optimize away the creation of function objects when possible. In such cases no event will be emitted. - Although this creates the possitibility of an observable difference of + Although this creates the possibility of an observable difference of runtime behavior depending on optimization decisions, it does not change the semantics of the Python code being executed. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 318f3f71f11546..41ad8cd22b5d89 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -1060,7 +1060,7 @@ Add ``getbufferproc`` and ``releasebufferproc`` to the stable API. Some configurable capabilities of sub-interpreters have changed. They always allow subprocesses (:mod:`subprocess`) now, whereas before subprocesses -could be optionally disaallowed for a sub-interpreter. Instead +could be optionally disallowed for a sub-interpreter. Instead :func:`os.exec` can now be disallowed. Disallowing daemon threads is now supported. Disallowing all threads is still allowed, but is never done by default. Note that the optional restrictions are only available through From 775f8819e319127f9bfb046773b74bcc62c68b6a Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 17 Feb 2023 19:14:07 +0900 Subject: [PATCH 62/83] gh-101766: Fix refleak for _BlockingOnManager resources (gh-101942) --- Lib/importlib/_bootstrap.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index bebe7e15cbce67..1ef7b6adb04434 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -85,6 +85,11 @@ def __enter__(self): def __exit__(self, *args, **kwargs): """Remove self.lock from this thread's _blocking_on list.""" self.blocked_on.remove(self.lock) + if len(self.blocked_on) == 0: + # gh-101766: glboal cache should be cleaned-up + # if there is no more _blocking_on for this thread. + del _blocking_on[self.thread_id] + del self.blocked_on class _DeadlockError(RuntimeError): From d401b20630965c0e1d2a5a0d60d5fc51aa5a8d80 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 17 Feb 2023 14:05:38 +0000 Subject: [PATCH 63/83] gh-101360: Fix anchor matching in pathlib.PureWindowsPath.match() (GH-101363) Use `fnmatch` to match path and pattern anchors, just as we do for other path parts. This allows patterns such as `'*:/Users/*'` to be matched. --- Lib/pathlib.py | 5 ----- Lib/test/test_ntpath.py | 4 ++++ Lib/test/test_pathlib.py | 9 ++++++--- .../2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst | 3 +++ 4 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 17659bcd3e2d7f..d7994a331d5eac 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -647,15 +647,10 @@ def match(self, path_pattern): drv, root, pat_parts = self._parse_parts((path_pattern,)) if not pat_parts: raise ValueError("empty pattern") - elif drv and drv != self._flavour.normcase(self._drv): - return False - elif root and root != self._root: - return False parts = self._parts_normcase if drv or root: if len(pat_parts) != len(parts): return False - pat_parts = pat_parts[1:] elif len(pat_parts) > len(parts): return False for part, pat in zip(reversed(parts), reversed(pat_parts)): diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index b32900697874b1..08c8a7a1f94b95 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -200,6 +200,10 @@ def test_splitroot(self): tester('ntpath.splitroot("//x")', ("//x", "", "")) # non-empty server & missing share tester('ntpath.splitroot("//x/")', ("//x/", "", "")) # non-empty server & empty share + # gh-101363: match GetFullPathNameW() drive letter parsing behaviour + tester('ntpath.splitroot(" :/foo")', (" :", "/", "foo")) + tester('ntpath.splitroot("/:/foo")', ("", "/", ":/foo")) + def test_split(self): tester('ntpath.split("c:\\foo\\bar")', ('c:\\foo', 'bar')) tester('ntpath.split("\\\\conky\\mountpoint\\foo\\bar")', diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index a596795b44f0fa..b8683796d9600d 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -852,8 +852,7 @@ def test_as_uri(self): def test_match_common(self): P = self.cls # Absolute patterns. - self.assertTrue(P('c:/b.py').match('/*.py')) - self.assertTrue(P('c:/b.py').match('c:*.py')) + self.assertTrue(P('c:/b.py').match('*:/*.py')) self.assertTrue(P('c:/b.py').match('c:/*.py')) self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive self.assertFalse(P('b.py').match('/*.py')) @@ -864,7 +863,7 @@ def test_match_common(self): self.assertFalse(P('/b.py').match('c:*.py')) self.assertFalse(P('/b.py').match('c:/*.py')) # UNC patterns. - self.assertTrue(P('//some/share/a.py').match('/*.py')) + self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) @@ -872,6 +871,10 @@ def test_match_common(self): self.assertTrue(P('B.py').match('b.PY')) self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) + # Path anchor doesn't match pattern anchor + self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' + self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' + self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' def test_ordering_common(self): # Case-insensitivity. diff --git a/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst b/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst new file mode 100644 index 00000000000000..4cfb136c5db853 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst @@ -0,0 +1,3 @@ +Fix anchor matching in :meth:`pathlib.PureWindowsPath.match`. Path and +pattern anchors are now matched with :mod:`fnmatch`, just like other path +parts. This allows patterns such as ``"*:/Users/*"`` to be matched. From 072011b3c38f871cdc3ab62630ea2234d09456d1 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 17 Feb 2023 14:08:14 +0000 Subject: [PATCH 64/83] gh-100809: Fix handling of drive-relative paths in pathlib.Path.absolute() (GH-100812) Resolving the drive independently uses the OS API, which ensures it starts from the current directory on that drive. --- Lib/pathlib.py | 7 +++- Lib/test/support/os_helper.py | 35 +++++++++++++++++++ Lib/test/test_pathlib.py | 20 +++++++++++ ...-01-06-21-14-41.gh-issue-100809.I697UT.rst | 3 ++ 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index d7994a331d5eac..dde573592fddce 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -816,7 +816,12 @@ def absolute(self): """ if self.is_absolute(): return self - return self._from_parts([os.getcwd()] + self._parts) + elif self._drv: + # There is a CWD on each drive-letter drive. + cwd = self._flavour.abspath(self._drv) + else: + cwd = os.getcwd() + return self._from_parts([cwd] + self._parts) def resolve(self, strict=False): """ diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 2d4356a1191b1e..821a4b1ffd5077 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -4,6 +4,7 @@ import os import re import stat +import string import sys import time import unittest @@ -716,3 +717,37 @@ def __exit__(self, *ignore_exc): else: self._environ[k] = v os.environ = self._environ + + +try: + import ctypes + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + + ERROR_FILE_NOT_FOUND = 2 + DDD_REMOVE_DEFINITION = 2 + DDD_EXACT_MATCH_ON_REMOVE = 4 + DDD_NO_BROADCAST_SYSTEM = 8 +except (ImportError, AttributeError): + def subst_drive(path): + raise unittest.SkipTest('ctypes or kernel32 is not available') +else: + @contextlib.contextmanager + def subst_drive(path): + """Temporarily yield a substitute drive for a given path.""" + for c in reversed(string.ascii_uppercase): + drive = f'{c}:' + if (not kernel32.QueryDosDeviceW(drive, None, 0) and + ctypes.get_last_error() == ERROR_FILE_NOT_FOUND): + break + else: + raise unittest.SkipTest('no available logical drive') + if not kernel32.DefineDosDeviceW( + DDD_NO_BROADCAST_SYSTEM, drive, path): + raise ctypes.WinError(ctypes.get_last_error()) + try: + yield drive + finally: + if not kernel32.DefineDosDeviceW( + DDD_REMOVE_DEFINITION | DDD_EXACT_MATCH_ON_REMOVE, + drive, path): + raise ctypes.WinError(ctypes.get_last_error()) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index b8683796d9600d..4de91d52c6d10c 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -2973,6 +2973,26 @@ def test_absolute(self): self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(share, 'a', 'b', 'c')) + drive = os.path.splitdrive(BASE)[0] + with os_helper.change_cwd(BASE): + # Relative path with root + self.assertEqual(str(P('\\').absolute()), drive + '\\') + self.assertEqual(str(P('\\foo').absolute()), drive + '\\foo') + + # Relative path on current drive + self.assertEqual(str(P(drive).absolute()), BASE) + self.assertEqual(str(P(drive + 'foo').absolute()), os.path.join(BASE, 'foo')) + + with os_helper.subst_drive(BASE) as other_drive: + # Set the working directory on the substitute drive + saved_cwd = os.getcwd() + other_cwd = f'{other_drive}\\dirA' + os.chdir(other_cwd) + os.chdir(saved_cwd) + + # Relative path on another drive + self.assertEqual(str(P(other_drive).absolute()), other_cwd) + self.assertEqual(str(P(other_drive + 'foo').absolute()), other_cwd + '\\foo') def test_glob(self): P = self.cls diff --git a/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst b/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst new file mode 100644 index 00000000000000..54082de88ccf4a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst @@ -0,0 +1,3 @@ +Fix handling of drive-relative paths (like 'C:' and 'C:foo') in +:meth:`pathlib.Path.absolute`. This method now uses the OS API +to retrieve the correct current working directory for the drive. From f482ade4c7887c49dfd8bba3be76f839e562608d Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sat, 18 Feb 2023 00:18:47 +0900 Subject: [PATCH 65/83] gh-101766: Fix refleak for _BlockingOnManager resources from test suite level (gh-101988) --- Lib/importlib/_bootstrap.py | 5 ----- Lib/test/test_importlib/test_locks.py | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 1ef7b6adb04434..bebe7e15cbce67 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -85,11 +85,6 @@ def __enter__(self): def __exit__(self, *args, **kwargs): """Remove self.lock from this thread's _blocking_on list.""" self.blocked_on.remove(self.lock) - if len(self.blocked_on) == 0: - # gh-101766: glboal cache should be cleaned-up - # if there is no more _blocking_on for this thread. - del _blocking_on[self.thread_id] - del self.blocked_on class _DeadlockError(RuntimeError): diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index 56d73c496e6bbb..ba9cf51c261d52 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -33,6 +33,11 @@ class ModuleLockAsRLockTests: test_repr = None test_locked_repr = None + def tearDown(self): + for splitinit in init.values(): + splitinit._bootstrap._blocking_on.clear() + + LOCK_TYPES = {kind: splitinit._bootstrap._ModuleLock for kind, splitinit in init.items()} From a1723caabfcdca5d675c4cb04554fb04c7edf601 Mon Sep 17 00:00:00 2001 From: Dustin Rodrigues Date: Fri, 17 Feb 2023 14:30:29 -0500 Subject: [PATCH 66/83] gh-101992: update plistlib examples to be runnable (#101994) * gh-101992: update plistlib examples to be runnable * Update Doc/library/plistlib.rst --------- Co-authored-by: Terry Jan Reedy --- Doc/library/plistlib.rst | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 5ded9661f08014..7aad15ec91a0ac 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -159,6 +159,9 @@ Examples Generating a plist:: + import datetime + import plistlib + pl = dict( aString = "Doodah", aList = ["A", "B", 12, 32.1, [1, 2, 3]], @@ -172,13 +175,19 @@ Generating a plist:: ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), + aDate = datetime.datetime.now() ) - with open(fileName, 'wb') as fp: - dump(pl, fp) + print(plistlib.dumps(pl).decode()) Parsing a plist:: - with open(fileName, 'rb') as fp: - pl = load(fp) - print(pl["aKey"]) + import plistlib + + plist = b""" + + foo + bar + + """ + pl = plistlib.loads(plist) + print(pl["foo"]) From 77d95c83733722ada35eb1ef89ae5b84a51ddd32 Mon Sep 17 00:00:00 2001 From: Jan Gosmann Date: Fri, 17 Feb 2023 22:01:26 +0100 Subject: [PATCH 67/83] gh-100226: Clarify StreamReader.read behavior (#101807) --- Doc/library/asyncio-stream.rst | 12 ++++++++++-- Lib/asyncio/streams.py | 17 +++++++++-------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 3b3c68ab6ef625..bbac1c32b5695f 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -206,12 +206,20 @@ StreamReader .. coroutinemethod:: read(n=-1) - Read up to *n* bytes. If *n* is not provided, or set to ``-1``, - read until EOF and return all read bytes. + Read up to *n* bytes from the stream. + If *n* is not provided or set to ``-1``, + read until EOF, then return all read :class:`bytes`. If EOF was received and the internal buffer is empty, return an empty ``bytes`` object. + If *n* is ``0``, return an empty ``bytes`` object immediately. + + If *n* is positive, return at most *n* available ``bytes`` + as soon as at least 1 byte is available in the internal buffer. + If EOF is received before any byte is read, return an empty + ``bytes`` object. + .. coroutinemethod:: readline() Read one line, where "line" is a sequence of bytes diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 7d13e961bd2de4..bf15f517e50dce 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -649,16 +649,17 @@ async def readuntil(self, separator=b'\n'): async def read(self, n=-1): """Read up to `n` bytes from the stream. - If n is not provided, or set to -1, read until EOF and return all read - bytes. If the EOF was received and the internal buffer is empty, return - an empty bytes object. + If `n` is not provided or set to -1, + read until EOF, then return all read bytes. + If EOF was received and the internal buffer is empty, + return an empty bytes object. - If n is zero, return empty bytes object immediately. + If `n` is 0, return an empty bytes object immediately. - If n is positive, this function try to read `n` bytes, and may return - less or equal bytes than requested, but at least one byte. If EOF was - received before any byte is read, this function returns empty byte - object. + If `n` is positive, return at most `n` available bytes + as soon as at least 1 byte is available in the internal buffer. + If EOF is received before any byte is read, return an empty + bytes object. Returned value is not limited with limit, configured at stream creation. From 7f1c72175600b21c1c840e8988cc6e6b4b244582 Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Sat, 18 Feb 2023 04:36:47 +0700 Subject: [PATCH 68/83] gh-101739: [Enum] update docs - default boundary for Flag is CONFORM (GH-101746) --- Doc/library/enum.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 13591a1bdc7347..24b6dbfe37cd38 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -696,10 +696,9 @@ Data Types .. attribute:: STRICT - Out-of-range values cause a :exc:`ValueError` to be raised. This is the - default for :class:`Flag`:: + Out-of-range values cause a :exc:`ValueError` to be raised:: - >>> from enum import Flag, STRICT + >>> from enum import Flag, STRICT, auto >>> class StrictFlag(Flag, boundary=STRICT): ... RED = auto() ... GREEN = auto() @@ -715,9 +714,9 @@ Data Types .. attribute:: CONFORM Out-of-range values have invalid values removed, leaving a valid *Flag* - value:: + value. This is the default for :class:`Flag`:: - >>> from enum import Flag, CONFORM + >>> from enum import Flag, CONFORM, auto >>> class ConformFlag(Flag, boundary=CONFORM): ... RED = auto() ... GREEN = auto() @@ -731,7 +730,7 @@ Data Types Out-of-range values lose their *Flag* membership and revert to :class:`int`. This is the default for :class:`IntFlag`:: - >>> from enum import Flag, EJECT + >>> from enum import Flag, EJECT, auto >>> class EjectFlag(Flag, boundary=EJECT): ... RED = auto() ... GREEN = auto() @@ -742,10 +741,10 @@ Data Types .. attribute:: KEEP - Out-of-range values are kept, and the *Flag* membership is kept. This is - used for some stdlib flags: + Out-of-range values are kept, and the *Flag* membership is kept. This is + used for some stdlib flags:: - >>> from enum import Flag, KEEP + >>> from enum import Flag, KEEP, auto >>> class KeepFlag(Flag, boundary=KEEP): ... RED = auto() ... GREEN = auto() From 89413bbccb9261b72190e275eefe4b0d49671477 Mon Sep 17 00:00:00 2001 From: Eclips4 <80244920+Eclips4@users.noreply.github.com> Date: Sat, 18 Feb 2023 03:52:23 +0300 Subject: [PATCH 69/83] gh-101967: add a missing error check (#101968) --- .../2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst | 1 + Python/ceval.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst new file mode 100644 index 00000000000000..6e681f910f5359 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst @@ -0,0 +1 @@ +Fix possible segfault in ``positional_only_passed_as_keyword`` function, when new list created. diff --git a/Python/ceval.c b/Python/ceval.c index 09fd2f29266c87..308ef52259df3d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1255,7 +1255,9 @@ positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, { int posonly_conflicts = 0; PyObject* posonly_names = PyList_New(0); - + if (posonly_names == NULL) { + goto fail; + } for(int k=0; k < co->co_posonlyargcount; k++){ PyObject* posonly_name = PyTuple_GET_ITEM(co->co_localsplusnames, k); From af446bbb76f64e67831444a0ceee6863a1527088 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 18 Feb 2023 18:46:33 +0300 Subject: [PATCH 70/83] gh-101536: [docs] Improve attributes of `urllib.error.HTTPError` (#101612) * gh-101536: [docs] Improve attributes of `urllib.error.HTTPError` * Address review --- Doc/library/urllib.error.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst index f7d47ed76aca18..3adbdd26132273 100644 --- a/Doc/library/urllib.error.rst +++ b/Doc/library/urllib.error.rst @@ -31,7 +31,7 @@ The following exceptions are raised by :mod:`urllib.error` as appropriate: of :exc:`IOError`. -.. exception:: HTTPError +.. exception:: HTTPError(url, code, msg, hdrs, fp) Though being an exception (a subclass of :exc:`URLError`), an :exc:`HTTPError` can also function as a non-exceptional file-like return @@ -39,6 +39,11 @@ The following exceptions are raised by :mod:`urllib.error` as appropriate: is useful when handling exotic HTTP errors, such as requests for authentication. + .. attribute:: url + + Contains the request URL. + An alias for *filename* attribute. + .. attribute:: code An HTTP status code as defined in :rfc:`2616`. This numeric value corresponds @@ -48,14 +53,20 @@ The following exceptions are raised by :mod:`urllib.error` as appropriate: .. attribute:: reason This is usually a string explaining the reason for this error. + An alias for *msg* attribute. .. attribute:: headers The HTTP response headers for the HTTP request that caused the :exc:`HTTPError`. + An alias for *hdrs* attribute. .. versionadded:: 3.4 + .. attribute:: fp + + A file-like object where the HTTP error body can be read from. + .. exception:: ContentTooShortError(msg, content) This exception is raised when the :func:`~urllib.request.urlretrieve` From 128379b8cdb88a6d3d7fed24df082c9a654b3fb8 Mon Sep 17 00:00:00 2001 From: Nicko van Someren Date: Sat, 18 Feb 2023 11:44:41 -0700 Subject: [PATCH 71/83] bpo-46978: Correct docstrings for in-place builtin operators (#31802) --- .../Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst | 1 + Objects/typeobject.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst b/Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst new file mode 100644 index 00000000000000..72291d042a0394 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst @@ -0,0 +1 @@ +Fixed docstrings for in-place operators of built-in types. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index bf6ccdb77a90f0..f2e8092aa37eec 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8564,7 +8564,7 @@ an all-zero entry. #NAME "($self, /)\n--\n\n" DOC) #define IBSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ ETSLOT(NAME, as_number.SLOT, FUNCTION, WRAPPER, \ - #NAME "($self, value, /)\n--\n\nReturn self" DOC "value.") + #NAME "($self, value, /)\n--\n\nCompute self " DOC " value.") #define BINSLOT(NAME, SLOT, FUNCTION, DOC) \ ETSLOT(NAME, as_number.SLOT, FUNCTION, wrap_binaryfunc_l, \ #NAME "($self, value, /)\n--\n\nReturn self" DOC "value.") From 5170caf3059fdacc92d7370eecb9fe4f0c5a1c76 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 18 Feb 2023 16:29:22 -0500 Subject: [PATCH 72/83] gh-97930: Apply changes from importlib_resources 5.12. (GH-102010) --- Lib/importlib/resources/_adapters.py | 4 +- Lib/importlib/resources/_itertools.py | 69 ++++++------ Lib/importlib/resources/readers.py | 36 +++++-- Lib/test/test_importlib/resources/_path.py | 18 ++-- .../subdirectory/subsubdir/resource.txt | 1 + .../resources/test_compatibilty_files.py | 6 +- .../test_importlib/resources/test_custom.py | 46 ++++++++ .../test_importlib/resources/test_files.py | 4 +- .../test_importlib/resources/test_open.py | 14 ++- .../test_importlib/resources/test_path.py | 15 ++- .../test_importlib/resources/test_read.py | 12 ++- .../test_importlib/resources/test_reader.py | 11 ++ .../test_importlib/resources/test_resource.py | 100 ++++++++---------- Lib/test/test_importlib/resources/util.py | 28 +++-- ...3-02-17-19-00-58.gh-issue-97930.C_nQjb.rst | 4 + 15 files changed, 241 insertions(+), 127 deletions(-) create mode 100644 Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt create mode 100644 Lib/test/test_importlib/resources/test_custom.py create mode 100644 Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst diff --git a/Lib/importlib/resources/_adapters.py b/Lib/importlib/resources/_adapters.py index f22f6bc509a3d7..50688fbb666658 100644 --- a/Lib/importlib/resources/_adapters.py +++ b/Lib/importlib/resources/_adapters.py @@ -34,9 +34,7 @@ def _io_wrapper(file, mode='r', *args, **kwargs): return TextIOWrapper(file, *args, **kwargs) elif mode == 'rb': return file - raise ValueError( - f"Invalid mode value '{mode}', only 'r' and 'rb' are supported" - ) + raise ValueError(f"Invalid mode value '{mode}', only 'r' and 'rb' are supported") class CompatibilityFiles: diff --git a/Lib/importlib/resources/_itertools.py b/Lib/importlib/resources/_itertools.py index cce05582ffc6fe..7b775ef5ae893f 100644 --- a/Lib/importlib/resources/_itertools.py +++ b/Lib/importlib/resources/_itertools.py @@ -1,35 +1,38 @@ -from itertools import filterfalse +# from more_itertools 9.0 +def only(iterable, default=None, too_long=None): + """If *iterable* has only one item, return it. + If it has zero items, return *default*. + If it has more than one item, raise the exception given by *too_long*, + which is ``ValueError`` by default. + >>> only([], default='missing') + 'missing' + >>> only([1]) + 1 + >>> only([1, 2]) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Expected exactly one item in iterable, but got 1, 2, + and perhaps more.' + >>> only([1, 2], too_long=TypeError) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + TypeError + Note that :func:`only` attempts to advance *iterable* twice to ensure there + is only one item. See :func:`spy` or :func:`peekable` to check + iterable contents less destructively. + """ + it = iter(iterable) + first_value = next(it, default) -from typing import ( - Callable, - Iterable, - Iterator, - Optional, - Set, - TypeVar, - Union, -) - -# Type and type variable definitions -_T = TypeVar('_T') -_U = TypeVar('_U') - - -def unique_everseen( - iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = None -) -> Iterator[_T]: - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen: Set[Union[_T, _U]] = set() - seen_add = seen.add - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element + try: + second_value = next(it) + except StopIteration: + pass else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element + msg = ( + 'Expected exactly one item in iterable, but got {!r}, {!r}, ' + 'and perhaps more.'.format(first_value, second_value) + ) + raise too_long or ValueError(msg) + + return first_value diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index 80cb320dd8bda0..c3cdf769cbecb0 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -1,11 +1,12 @@ import collections -import operator +import itertools import pathlib +import operator import zipfile from . import abc -from ._itertools import unique_everseen +from ._itertools import only def remove_duplicates(items): @@ -41,8 +42,10 @@ def open_resource(self, resource): raise FileNotFoundError(exc.args[0]) def is_resource(self, path): - # workaround for `zipfile.Path.is_file` returning true - # for non-existent paths. + """ + Workaround for `zipfile.Path.is_file` returning true + for non-existent paths. + """ target = self.files().joinpath(path) return target.is_file() and target.exists() @@ -67,8 +70,10 @@ def __init__(self, *paths): raise NotADirectoryError('MultiplexedPath only supports directories') def iterdir(self): - files = (file for path in self._paths for file in path.iterdir()) - return unique_everseen(files, key=operator.attrgetter('name')) + children = (child for path in self._paths for child in path.iterdir()) + by_name = operator.attrgetter('name') + groups = itertools.groupby(sorted(children, key=by_name), key=by_name) + return map(self._follow, (locs for name, locs in groups)) def read_bytes(self): raise FileNotFoundError(f'{self} is not a file') @@ -90,6 +95,25 @@ def joinpath(self, *descendants): # Just return something that will not exist. return self._paths[0].joinpath(*descendants) + @classmethod + def _follow(cls, children): + """ + Construct a MultiplexedPath if needed. + + If children contains a sole element, return it. + Otherwise, return a MultiplexedPath of the items. + Unless one of the items is not a Directory, then return the first. + """ + subdirs, one_dir, one_file = itertools.tee(children, 3) + + try: + return only(one_dir) + except ValueError: + try: + return cls(*subdirs) + except NotADirectoryError: + return next(one_file) + def open(self, *args, **kwargs): raise FileNotFoundError(f'{self} is not a file') diff --git a/Lib/test/test_importlib/resources/_path.py b/Lib/test/test_importlib/resources/_path.py index c630e4d3d3f352..1f97c96146960d 100644 --- a/Lib/test/test_importlib/resources/_path.py +++ b/Lib/test/test_importlib/resources/_path.py @@ -1,12 +1,16 @@ import pathlib import functools +from typing import Dict, Union + #### -# from jaraco.path 3.4 +# from jaraco.path 3.4.1 + +FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore -def build(spec, prefix=pathlib.Path()): +def build(spec: FilesSpec, prefix=pathlib.Path()): """ Build a set of files/directories, as described by the spec. @@ -23,15 +27,17 @@ def build(spec, prefix=pathlib.Path()): ... "baz.py": "# Some code", ... } ... } - >>> tmpdir = getfixture('tmpdir') - >>> build(spec, tmpdir) + >>> target = getfixture('tmp_path') + >>> build(spec, target) + >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') + '# Some code' """ for name, contents in spec.items(): create(contents, pathlib.Path(prefix) / name) @functools.singledispatch -def create(content, path): +def create(content: Union[str, bytes, FilesSpec], path): path.mkdir(exist_ok=True) build(content, prefix=path) # type: ignore @@ -43,7 +49,7 @@ def _(content: bytes, path): @create.register def _(content: str, path): - path.write_text(content) + path.write_text(content, encoding='utf-8') # end from jaraco.path diff --git a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt b/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt new file mode 100644 index 00000000000000..48f587a2d0ac53 --- /dev/null +++ b/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt @@ -0,0 +1 @@ +a resource \ No newline at end of file diff --git a/Lib/test/test_importlib/resources/test_compatibilty_files.py b/Lib/test/test_importlib/resources/test_compatibilty_files.py index 6fa18a24973f64..bcf608d9e2cbdf 100644 --- a/Lib/test/test_importlib/resources/test_compatibilty_files.py +++ b/Lib/test/test_importlib/resources/test_compatibilty_files.py @@ -64,11 +64,13 @@ def test_orphan_path_name(self): def test_spec_path_open(self): self.assertEqual(self.files.read_bytes(), b'Hello, world!') - self.assertEqual(self.files.read_text(), 'Hello, world!') + self.assertEqual(self.files.read_text(encoding='utf-8'), 'Hello, world!') def test_child_path_open(self): self.assertEqual((self.files / 'a').read_bytes(), b'Hello, world!') - self.assertEqual((self.files / 'a').read_text(), 'Hello, world!') + self.assertEqual( + (self.files / 'a').read_text(encoding='utf-8'), 'Hello, world!' + ) def test_orphan_path_open(self): with self.assertRaises(FileNotFoundError): diff --git a/Lib/test/test_importlib/resources/test_custom.py b/Lib/test/test_importlib/resources/test_custom.py new file mode 100644 index 00000000000000..73127209a2761b --- /dev/null +++ b/Lib/test/test_importlib/resources/test_custom.py @@ -0,0 +1,46 @@ +import unittest +import contextlib +import pathlib + +from test.support import os_helper + +from importlib import resources +from importlib.resources.abc import TraversableResources, ResourceReader +from . import util + + +class SimpleLoader: + """ + A simple loader that only implements a resource reader. + """ + + def __init__(self, reader: ResourceReader): + self.reader = reader + + def get_resource_reader(self, package): + return self.reader + + +class MagicResources(TraversableResources): + """ + Magically returns the resources at path. + """ + + def __init__(self, path: pathlib.Path): + self.path = path + + def files(self): + return self.path + + +class CustomTraversableResourcesTests(unittest.TestCase): + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + def test_custom_loader(self): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + loader = SimpleLoader(MagicResources(temp_dir)) + pkg = util.create_package_from_loader(loader) + files = resources.files(pkg) + assert files is temp_dir diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index fe813ae7d08881..1450cfb310926a 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -85,7 +85,7 @@ def test_module_resources(self): _path.build(spec, self.site_dir) import mod - actual = resources.files(mod).joinpath('res.txt').read_text() + actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8') assert actual == spec['res.txt'] @@ -99,7 +99,7 @@ def test_implicit_files(self): '__init__.py': textwrap.dedent( """ import importlib.resources as res - val = res.files().joinpath('res.txt').read_text() + val = res.files().joinpath('res.txt').read_text(encoding='utf-8') """ ), 'res.txt': 'resources are the best', diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 0554c41ba67d0e..86becb4bfaad37 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -15,7 +15,7 @@ def execute(self, package, path): class CommonTextTests(util.CommonTests, unittest.TestCase): def execute(self, package, path): target = resources.files(package).joinpath(path) - with target.open(): + with target.open(encoding='utf-8'): pass @@ -28,7 +28,7 @@ def test_open_binary(self): def test_open_text_default_encoding(self): target = resources.files(self.data) / 'utf-8.file' - with target.open() as fp: + with target.open(encoding='utf-8') as fp: result = fp.read() self.assertEqual(result, 'Hello, UTF-8 world!\n') @@ -39,7 +39,9 @@ def test_open_text_given_encoding(self): self.assertEqual(result, 'Hello, UTF-16 world!\n') def test_open_text_with_errors(self): - # Raises UnicodeError without the 'errors' argument. + """ + Raises UnicodeError without the 'errors' argument. + """ target = resources.files(self.data) / 'utf-16.file' with target.open(encoding='utf-8', errors='strict') as fp: self.assertRaises(UnicodeError, fp.read) @@ -54,11 +56,13 @@ def test_open_text_with_errors(self): def test_open_binary_FileNotFoundError(self): target = resources.files(self.data) / 'does-not-exist' - self.assertRaises(FileNotFoundError, target.open, 'rb') + with self.assertRaises(FileNotFoundError): + target.open('rb') def test_open_text_FileNotFoundError(self): target = resources.files(self.data) / 'does-not-exist' - self.assertRaises(FileNotFoundError, target.open) + with self.assertRaises(FileNotFoundError): + target.open(encoding='utf-8') class OpenDiskTests(OpenTests, unittest.TestCase): diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index adcf75feea78ec..34a6bdd2d58b91 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -14,9 +14,12 @@ def execute(self, package, path): class PathTests: def test_reading(self): - # Path should be readable. - # Test also implicitly verifies the returned object is a pathlib.Path - # instance. + """ + Path should be readable. + + Test also implicitly verifies the returned object is a pathlib.Path + instance. + """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) @@ -51,8 +54,10 @@ def setUp(self): class PathZipTests(PathTests, util.ZipSetup, unittest.TestCase): def test_remove_in_context_manager(self): - # It is not an error if the file that was temporarily stashed on the - # file system is removed inside the `with` stanza. + """ + It is not an error if the file that was temporarily stashed on the + file system is removed inside the `with` stanza. + """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: path.unlink() diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 0ca8ee9d02856b..088982681e8b0c 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -12,7 +12,7 @@ def execute(self, package, path): class CommonTextTests(util.CommonTests, unittest.TestCase): def execute(self, package, path): - resources.files(package).joinpath(path).read_text() + resources.files(package).joinpath(path).read_text(encoding='utf-8') class ReadTests: @@ -21,7 +21,11 @@ def test_read_bytes(self): self.assertEqual(result, b'\0\1\2\3') def test_read_text_default_encoding(self): - result = resources.files(self.data).joinpath('utf-8.file').read_text() + result = ( + resources.files(self.data) + .joinpath('utf-8.file') + .read_text(encoding='utf-8') + ) self.assertEqual(result, 'Hello, UTF-8 world!\n') def test_read_text_given_encoding(self): @@ -33,7 +37,9 @@ def test_read_text_given_encoding(self): self.assertEqual(result, 'Hello, UTF-16 world!\n') def test_read_text_with_errors(self): - # Raises UnicodeError without the 'errors' argument. + """ + Raises UnicodeError without the 'errors' argument. + """ target = resources.files(self.data) / 'utf-16.file' self.assertRaises(UnicodeError, target.read_text, encoding='utf-8') result = target.read_text(encoding='utf-8', errors='ignore') diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index 4fd9e6bbe4281c..8670f72a334585 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -81,6 +81,17 @@ def test_join_path_compound(self): path = MultiplexedPath(self.folder) assert not path.joinpath('imaginary/foo.py').exists() + def test_join_path_common_subdir(self): + prefix = os.path.abspath(os.path.join(__file__, '..')) + data01 = os.path.join(prefix, 'data01') + data02 = os.path.join(prefix, 'data02') + path = MultiplexedPath(data01, data02) + self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) + self.assertEqual( + str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], + os.path.join('data02', 'subdirectory', 'subsubdir'), + ) + def test_repr(self): self.assertEqual( repr(MultiplexedPath(self.folder)), diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index f7e3abbdc805a7..6f75cf57f03d02 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,3 +1,4 @@ +import contextlib import sys import unittest import uuid @@ -7,7 +8,7 @@ from . import zipdata01, zipdata02 from . import util from importlib import resources, import_module -from test.support import import_helper +from test.support import import_helper, os_helper from test.support.os_helper import unlink @@ -69,10 +70,12 @@ def test_resource_missing(self): class ResourceCornerCaseTests(unittest.TestCase): def test_package_has_no_reader_fallback(self): - # Test odd ball packages which: + """ + Test odd ball packages which: # 1. Do not have a ResourceReader as a loader # 2. Are not on the file system # 3. Are not in a zip file + """ module = util.create_package( file=data01, path=data01.__file__, contents=['A', 'B', 'C'] ) @@ -138,82 +141,71 @@ def test_unrelated_contents(self): ) +@contextlib.contextmanager +def zip_on_path(dir): + data_path = pathlib.Path(zipdata01.__file__) + source_zip_path = data_path.parent.joinpath('ziptestdata.zip') + zip_path = pathlib.Path(dir) / f'{uuid.uuid4()}.zip' + zip_path.write_bytes(source_zip_path.read_bytes()) + sys.path.append(str(zip_path)) + import_module('ziptestdata') + + try: + yield + finally: + with contextlib.suppress(ValueError): + sys.path.remove(str(zip_path)) + + with contextlib.suppress(KeyError): + del sys.path_importer_cache[str(zip_path)] + del sys.modules['ziptestdata'] + + with contextlib.suppress(OSError): + unlink(zip_path) + + class DeletingZipsTest(unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - ZIP_MODULE = zipdata01 - def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + modules = import_helper.modules_setup() self.addCleanup(import_helper.modules_cleanup, *modules) - data_path = pathlib.Path(self.ZIP_MODULE.__file__) - data_dir = data_path.parent - self.source_zip_path = data_dir / 'ziptestdata.zip' - self.zip_path = pathlib.Path(f'{uuid.uuid4()}.zip').absolute() - self.zip_path.write_bytes(self.source_zip_path.read_bytes()) - sys.path.append(str(self.zip_path)) - self.data = import_module('ziptestdata') - - def tearDown(self): - try: - sys.path.remove(str(self.zip_path)) - except ValueError: - pass - - try: - del sys.path_importer_cache[str(self.zip_path)] - del sys.modules[self.data.__name__] - except KeyError: - pass - - try: - unlink(self.zip_path) - except OSError: - # If the test fails, this will probably fail too - pass + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + self.fixtures.enter_context(zip_on_path(temp_dir)) def test_iterdir_does_not_keep_open(self): - c = [item.name for item in resources.files('ziptestdata').iterdir()] - self.zip_path.unlink() - del c + [item.name for item in resources.files('ziptestdata').iterdir()] def test_is_file_does_not_keep_open(self): - c = resources.files('ziptestdata').joinpath('binary.file').is_file() - self.zip_path.unlink() - del c + resources.files('ziptestdata').joinpath('binary.file').is_file() def test_is_file_failure_does_not_keep_open(self): - c = resources.files('ziptestdata').joinpath('not-present').is_file() - self.zip_path.unlink() - del c + resources.files('ziptestdata').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") def test_as_file_does_not_keep_open(self): # pragma: no cover - c = resources.as_file(resources.files('ziptestdata') / 'binary.file') - self.zip_path.unlink() - del c + resources.as_file(resources.files('ziptestdata') / 'binary.file') def test_entered_path_does_not_keep_open(self): - # This is what certifi does on import to make its bundle - # available for the process duration. - c = resources.as_file( - resources.files('ziptestdata') / 'binary.file' - ).__enter__() - self.zip_path.unlink() - del c + """ + Mimic what certifi does on import to make its bundle + available for the process duration. + """ + resources.as_file(resources.files('ziptestdata') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - c = resources.files('ziptestdata').joinpath('binary.file').read_bytes() - self.zip_path.unlink() - del c + resources.files('ziptestdata').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - c = resources.files('ziptestdata').joinpath('utf-8.file').read_text() - self.zip_path.unlink() - del c + resources.files('ziptestdata').joinpath('utf-8.file').read_text( + encoding='utf-8' + ) class ResourceFromNamespaceTest01(unittest.TestCase): diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index 1e72b91ff6b31f..dbe6ee81476699 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -80,32 +80,44 @@ def execute(self, package, path): """ def test_package_name(self): - # Passing in the package name should succeed. + """ + Passing in the package name should succeed. + """ self.execute(data01.__name__, 'utf-8.file') def test_package_object(self): - # Passing in the package itself should succeed. + """ + Passing in the package itself should succeed. + """ self.execute(data01, 'utf-8.file') def test_string_path(self): - # Passing in a string for the path should succeed. + """ + Passing in a string for the path should succeed. + """ path = 'utf-8.file' self.execute(data01, path) def test_pathlib_path(self): - # Passing in a pathlib.PurePath object for the path should succeed. + """ + Passing in a pathlib.PurePath object for the path should succeed. + """ path = pathlib.PurePath('utf-8.file') self.execute(data01, path) def test_importing_module_as_side_effect(self): - # The anchor package can already be imported. + """ + The anchor package can already be imported. + """ del sys.modules[data01.__name__] self.execute(data01.__name__, 'utf-8.file') def test_missing_path(self): - # Attempting to open or read or request the path for a - # non-existent path should succeed if open_resource - # can return a viable data stream. + """ + Attempting to open or read or request the path for a + non-existent path should succeed if open_resource + can return a viable data stream. + """ bytes_data = io.BytesIO(b'Hello, world!') package = create_package(file=bytes_data, path=FileNotFoundError()) self.execute(package, 'utf-8.file') diff --git a/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst b/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst new file mode 100644 index 00000000000000..967e13f752bcd1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst @@ -0,0 +1,4 @@ +Apply changes from `importlib_resources 5.12 +`_, +including fix for ``MultiplexedPath`` to support directories in multiple +namespaces (python/importlib_resources#265). From 61f1e67c6fcbf80eb9be2b75f7d62954e28c89e6 Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 19 Feb 2023 00:22:02 +0000 Subject: [PATCH 73/83] GH-84783: Make the slice object hashable (GH-101264) --- Doc/library/functions.rst | 3 ++ Lib/test/test_capi/test_misc.py | 7 +--- Lib/test/test_doctest.py | 2 +- Lib/test/test_slice.py | 12 ++++-- ...3-02-11-23-14-06.gh-issue-84783._P5sMa.rst | 1 + Objects/sliceobject.c | 38 ++++++++++++++++++- Objects/tupleobject.c | 2 +- 7 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 658d6768457d16..3ff28849025153 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1635,6 +1635,9 @@ are always available. They are listed here in alphabetical order. example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See :func:`itertools.islice` for an alternate version that returns an iterator. + .. versionchanged:: 3.12 + Slice objects are now :term:`hashable` (provided :attr:`~slice.start`, + :attr:`~slice.stop`, and :attr:`~slice.step` are hashable). .. function:: sorted(iterable, /, *, key=None, reverse=False) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index f26b4723d1e68b..ad099c61463b66 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -419,11 +419,6 @@ def __setitem__(self, index, value): with self.assertRaises(TypeError): _testcapi.sequence_set_slice(None, 1, 3, 'xy') - mapping = {1: 'a', 2: 'b', 3: 'c'} - with self.assertRaises(TypeError): - _testcapi.sequence_set_slice(mapping, 1, 3, 'xy') - self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) - def test_sequence_del_slice(self): # Correct case: data = [1, 2, 3, 4, 5] @@ -459,7 +454,7 @@ def __delitem__(self, index): _testcapi.sequence_del_slice(None, 1, 3) mapping = {1: 'a', 2: 'b', 3: 'c'} - with self.assertRaises(TypeError): + with self.assertRaises(KeyError): _testcapi.sequence_del_slice(mapping, 1, 3) self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 65e215f1cdda4a..3491d4cdb1c18b 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -707,7 +707,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 825 < len(tests) < 845 # approximate number of objects with docstrings + >>> 830 < len(tests) < 850 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests diff --git a/Lib/test/test_slice.py b/Lib/test/test_slice.py index 03fde3275e1475..c35a2293f790a2 100644 --- a/Lib/test/test_slice.py +++ b/Lib/test/test_slice.py @@ -80,10 +80,16 @@ def test_repr(self): self.assertEqual(repr(slice(1, 2, 3)), "slice(1, 2, 3)") def test_hash(self): - # Verify clearing of SF bug #800796 - self.assertRaises(TypeError, hash, slice(5)) + self.assertEqual(hash(slice(5)), slice(5).__hash__()) + self.assertEqual(hash(slice(1, 2)), slice(1, 2).__hash__()) + self.assertEqual(hash(slice(1, 2, 3)), slice(1, 2, 3).__hash__()) + self.assertNotEqual(slice(5), slice(6)) + + with self.assertRaises(TypeError): + hash(slice(1, 2, [])) + with self.assertRaises(TypeError): - slice(5).__hash__() + hash(slice(4, {})) def test_cmp(self): s1 = slice(1, 2, 3) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst new file mode 100644 index 00000000000000..e1c851a0825a7f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst @@ -0,0 +1 @@ +Make the slice object hashable. diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 5694bd9c661fa5..5d2e6ad522bcf2 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -628,6 +628,42 @@ slice_traverse(PySliceObject *v, visitproc visit, void *arg) return 0; } +/* code based on tuplehash() of Objects/tupleobject.c */ +#if SIZEOF_PY_UHASH_T > 4 +#define _PyHASH_XXPRIME_1 ((Py_uhash_t)11400714785074694791ULL) +#define _PyHASH_XXPRIME_2 ((Py_uhash_t)14029467366897019727ULL) +#define _PyHASH_XXPRIME_5 ((Py_uhash_t)2870177450012600261ULL) +#define _PyHASH_XXROTATE(x) ((x << 31) | (x >> 33)) /* Rotate left 31 bits */ +#else +#define _PyHASH_XXPRIME_1 ((Py_uhash_t)2654435761UL) +#define _PyHASH_XXPRIME_2 ((Py_uhash_t)2246822519UL) +#define _PyHASH_XXPRIME_5 ((Py_uhash_t)374761393UL) +#define _PyHASH_XXROTATE(x) ((x << 13) | (x >> 19)) /* Rotate left 13 bits */ +#endif + +static Py_hash_t +slicehash(PySliceObject *v) +{ + Py_uhash_t acc = _PyHASH_XXPRIME_5; +#define _PyHASH_SLICE_PART(com) { \ + Py_uhash_t lane = PyObject_Hash(v->com); \ + if(lane == (Py_uhash_t)-1) { \ + return -1; \ + } \ + acc += lane * _PyHASH_XXPRIME_2; \ + acc = _PyHASH_XXROTATE(acc); \ + acc *= _PyHASH_XXPRIME_1; \ +} + _PyHASH_SLICE_PART(start); + _PyHASH_SLICE_PART(stop); + _PyHASH_SLICE_PART(step); +#undef _PyHASH_SLICE_PART + if(acc == (Py_uhash_t)-1) { + return 1546275796; + } + return acc; +} + PyTypeObject PySlice_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "slice", /* Name of this type */ @@ -642,7 +678,7 @@ PyTypeObject PySlice_Type = { 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - PyObject_HashNotImplemented, /* tp_hash */ + (hashfunc)slicehash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index e1b9953226c0d7..7d6d0e1bea249e 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -288,7 +288,7 @@ tuplerepr(PyTupleObject *v) /* Hash for tuples. This is a slightly simplified version of the xxHash non-cryptographic hash: - - we do not use any parallellism, there is only 1 accumulator. + - we do not use any parallelism, there is only 1 accumulator. - we drop the final mixing since this is just a permutation of the output space: it does not help against collisions. - at the end, we mangle the length with a single constant. From 36b670908b3546f46283aae4dbf311e53289f3d1 Mon Sep 17 00:00:00 2001 From: Reza Rastak Date: Sat, 18 Feb 2023 19:55:43 -0500 Subject: [PATCH 74/83] Fix incorrectly documented attribute in csv docs (#101250) --- Doc/library/csv.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 41f11505aa1030..f1776554d8b9f2 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -458,7 +458,7 @@ Reader objects have the following public attributes: DictReader objects have the following public attribute: -.. attribute:: csvreader.fieldnames +.. attribute:: DictReader.fieldnames If not passed as a parameter when creating the object, this attribute is initialized upon first access or when the first record is read from the From 6aab56f3c2ee331116eba242d2fcdca592577328 Mon Sep 17 00:00:00 2001 From: Patricio Paez Date: Sat, 18 Feb 2023 19:06:03 -0600 Subject: [PATCH 75/83] gh-99735: Use required=True in argparse subparsers example (#100927) --- Doc/library/argparse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index dbaa5d0d9b995b..34b4c61649b99f 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1867,7 +1867,7 @@ Sub-commands ... >>> # create the top-level parser >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers() + >>> subparsers = parser.add_subparsers(required=True) >>> >>> # create the parser for the "foo" command >>> parser_foo = subparsers.add_parser('foo') From 072935951f7cd44b40ee37fe561478b2e431c2fb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 18 Feb 2023 21:32:50 -0500 Subject: [PATCH 76/83] gh-97930: Also include subdirectory in makefile. (#102030) --- Makefile.pre.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 490483a712014c..b28c6067a535b8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2075,6 +2075,8 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/resources/data01/subdirectory \ test/test_importlib/resources/data02 \ test/test_importlib/resources/data02/one \ + test/test_importlib/resources/data02/subdirectory \ + test/test_importlib/resources/data02/subdirectory/subsubdir \ test/test_importlib/resources/data02/two \ test/test_importlib/resources/data03 \ test/test_importlib/resources/data03/namespace \ From 9a07eff628c1cd88b7cdda88a8fd0db3fe7ea552 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Amiel Date: Sun, 19 Feb 2023 11:18:12 +0100 Subject: [PATCH 77/83] gh-100210: Correct the comment link for unescaping HTML (#100212) gh-100210: correct the comment link for unescaping HTML --- Lib/html/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/html/__init__.py b/Lib/html/__init__.py index da0a0a3ce70eed..1543460ca33b0a 100644 --- a/Lib/html/__init__.py +++ b/Lib/html/__init__.py @@ -25,7 +25,7 @@ def escape(s, quote=True): return s -# see http://www.w3.org/TR/html5/syntax.html#tokenizing-character-references +# see https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state _invalid_charrefs = { 0x00: '\ufffd', # REPLACEMENT CHARACTER From 71f614ef2a3d66213b9cae807cbbc1ed03741221 Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Sun, 19 Feb 2023 22:00:59 +0700 Subject: [PATCH 78/83] Add missing 'is' to `cmath.log()` docstring (#102049) Fix missing 'is' in cmath.log() docstring --- Modules/clinic/cmathmodule.c.h | 4 ++-- Modules/cmathmodule.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index b1da9452c61db8..941448e76e80de 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -644,7 +644,7 @@ PyDoc_STRVAR(cmath_log__doc__, "\n" "log(z[, base]) -> the logarithm of z to the given base.\n" "\n" -"If the base not specified, returns the natural logarithm (base e) of z."); +"If the base is not specified, returns the natural logarithm (base e) of z."); #define CMATH_LOG_METHODDEF \ {"log", _PyCFunction_CAST(cmath_log), METH_FASTCALL, cmath_log__doc__}, @@ -982,4 +982,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=0146c656e67f5d5f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=87f609786ef270cd input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 2038ac26e65857..53e34061d53773 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -957,12 +957,12 @@ cmath.log log(z[, base]) -> the logarithm of z to the given base. -If the base not specified, returns the natural logarithm (base e) of z. +If the base is not specified, returns the natural logarithm (base e) of z. [clinic start generated code]*/ static PyObject * cmath_log_impl(PyObject *module, Py_complex x, PyObject *y_obj) -/*[clinic end generated code: output=4effdb7d258e0d94 input=230ed3a71ecd000a]*/ +/*[clinic end generated code: output=4effdb7d258e0d94 input=e1f81d4fcfd26497]*/ { Py_complex y; From 32df540635cacce1053ee0ef98ee23f3f6a43c02 Mon Sep 17 00:00:00 2001 From: neuralstring <107343209+neuralstring@users.noreply.github.com> Date: Sun, 19 Feb 2023 16:39:03 +0000 Subject: [PATCH 79/83] gh-100425: Update tutorial docs related to sum() accuracy (FH-101854) --- Doc/tutorial/stdlib2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 0c101c1f207235..33f311db3a24d2 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -394,7 +394,7 @@ point:: >>> sum([Decimal('0.1')]*10) == Decimal('1.0') True - >>> sum([0.1]*10) == 1.0 + >>> 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 == 1.0 False The :mod:`decimal` module provides arithmetic with as much precision as needed:: From b513c46d998344dc07eb6d510782c2e23d2b859e Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Sun, 19 Feb 2023 19:15:44 +0000 Subject: [PATCH 80/83] gh-85417: Clarify behaviour on branch cuts in cmath module (#102046) This PR updates the cmath module documentation to reflect the reality that Python is almost always (and as far as I can tell, that "almost" can be omitted) running on a machine whose C double supports signed zeros. * Removes misleading references to functions being continuous from above / below / the left / the right at branch cuts * Expands the note on branch cuts at the top of the module documentation to explain the double-sided sign-of-zero-based behaviour --- Doc/library/cmath.rst | 66 +++++++++++-------- ...3-02-19-10-33-01.gh-issue-85417.kYO8u3.rst | 1 + 2 files changed, 39 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 28cd96b0e12da9..5ed7a09b3e9db2 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -15,11 +15,27 @@ the function is then applied to the result of the conversion. .. note:: - On platforms with hardware and system-level support for signed - zeros, functions involving branch cuts are continuous on *both* - sides of the branch cut: the sign of the zero distinguishes one - side of the branch cut from the other. On platforms that do not - support signed zeros the continuity is as specified below. + For functions involving branch cuts, we have the problem of deciding how to + define those functions on the cut itself. Following Kahan's "Branch cuts for + complex elementary functions" paper, as well as Annex G of C99 and later C + standards, we use the sign of zero to distinguish one side of the branch cut + from the other: for a branch cut along (a portion of) the real axis we look + at the sign of the imaginary part, while for a branch cut along the + imaginary axis we look at the sign of the real part. + + For example, the :func:`cmath.sqrt` function has a branch cut along the + negative real axis. An argument of ``complex(-2.0, -0.0)`` is treated as + though it lies *below* the branch cut, and so gives a result on the negative + imaginary axis:: + + >>> cmath.sqrt(complex(-2.0, -0.0)) + -1.4142135623730951j + + But an argument of ``complex(-2.0, 0.0)`` is treated as though it lies above + the branch cut:: + + >>> cmath.sqrt(complex(-2.0, 0.0)) + 1.4142135623730951j Conversions to and from polar coordinates @@ -44,14 +60,11 @@ rectangular coordinates to polar coordinates and back. .. function:: phase(x) - Return the phase of *x* (also known as the *argument* of *x*), as a - float. ``phase(x)`` is equivalent to ``math.atan2(x.imag, - x.real)``. The result lies in the range [-\ *π*, *π*], and the branch - cut for this operation lies along the negative real axis, - continuous from above. On systems with support for signed zeros - (which includes most systems in current use), this means that the - sign of the result is the same as the sign of ``x.imag``, even when - ``x.imag`` is zero:: + Return the phase of *x* (also known as the *argument* of *x*), as a float. + ``phase(x)`` is equivalent to ``math.atan2(x.imag, x.real)``. The result + lies in the range [-\ *π*, *π*], and the branch cut for this operation lies + along the negative real axis. The sign of the result is the same as the + sign of ``x.imag``, even when ``x.imag`` is zero:: >>> phase(complex(-1.0, 0.0)) 3.141592653589793 @@ -92,8 +105,8 @@ Power and logarithmic functions .. function:: log(x[, base]) Returns the logarithm of *x* to the given *base*. If the *base* is not - specified, returns the natural logarithm of *x*. There is one branch cut, from 0 - along the negative real axis to -∞, continuous from above. + specified, returns the natural logarithm of *x*. There is one branch cut, + from 0 along the negative real axis to -∞. .. function:: log10(x) @@ -112,9 +125,9 @@ Trigonometric functions .. function:: acos(x) - Return the arc cosine of *x*. There are two branch cuts: One extends right from - 1 along the real axis to ∞, continuous from below. The other extends left from - -1 along the real axis to -∞, continuous from above. + Return the arc cosine of *x*. There are two branch cuts: One extends right + from 1 along the real axis to ∞. The other extends left from -1 along the + real axis to -∞. .. function:: asin(x) @@ -125,9 +138,8 @@ Trigonometric functions .. function:: atan(x) Return the arc tangent of *x*. There are two branch cuts: One extends from - ``1j`` along the imaginary axis to ``∞j``, continuous from the right. The - other extends from ``-1j`` along the imaginary axis to ``-∞j``, continuous - from the left. + ``1j`` along the imaginary axis to ``∞j``. The other extends from ``-1j`` + along the imaginary axis to ``-∞j``. .. function:: cos(x) @@ -151,23 +163,21 @@ Hyperbolic functions .. function:: acosh(x) Return the inverse hyperbolic cosine of *x*. There is one branch cut, - extending left from 1 along the real axis to -∞, continuous from above. + extending left from 1 along the real axis to -∞. .. function:: asinh(x) Return the inverse hyperbolic sine of *x*. There are two branch cuts: - One extends from ``1j`` along the imaginary axis to ``∞j``, - continuous from the right. The other extends from ``-1j`` along - the imaginary axis to ``-∞j``, continuous from the left. + One extends from ``1j`` along the imaginary axis to ``∞j``. The other + extends from ``-1j`` along the imaginary axis to ``-∞j``. .. function:: atanh(x) Return the inverse hyperbolic tangent of *x*. There are two branch cuts: One - extends from ``1`` along the real axis to ``∞``, continuous from below. The - other extends from ``-1`` along the real axis to ``-∞``, continuous from - above. + extends from ``1`` along the real axis to ``∞``. The other extends from + ``-1`` along the real axis to ``-∞``. .. function:: cosh(x) diff --git a/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst b/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst new file mode 100644 index 00000000000000..a5532df14795d2 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst @@ -0,0 +1 @@ +Update :mod:`cmath` documentation to clarify behaviour on branch cuts. From 3b264df470c82d77f5b01c6f9d1d7173d1cb9597 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 19 Feb 2023 13:21:37 -0600 Subject: [PATCH 81/83] Misc improvements to the float tutorial (GH-102052) --- Doc/tutorial/floatingpoint.rst | 139 +++++++++++++++++++++++++-------- 1 file changed, 106 insertions(+), 33 deletions(-) diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index cedade6e336608..306b1eba3c45b8 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -1,6 +1,7 @@ .. testsetup:: import math + from fractions import Fraction .. _tut-fp-issues: @@ -9,12 +10,13 @@ Floating Point Arithmetic: Issues and Limitations ************************************************** .. sectionauthor:: Tim Peters +.. sectionauthor:: Raymond Hettinger Floating-point numbers are represented in computer hardware as base 2 (binary) -fractions. For example, the **decimal** fraction ``0.125`` -has value 1/10 + 2/100 + 5/1000, and in the same way the **binary** fraction ``0.001`` -has value 0/2 + 0/4 + 1/8. These two fractions have identical values, the only +fractions. For example, the **decimal** fraction ``0.625`` +has value 6/10 + 2/100 + 5/1000, and in the same way the **binary** fraction ``0.101`` +has value 1/2 + 0/4 + 1/8. These two fractions have identical values, the only real difference being that the first is written in base 10 fractional notation, and the second in base 2. @@ -57,13 +59,15 @@ Many users are not aware of the approximation because of the way values are displayed. Python only prints a decimal approximation to the true decimal value of the binary approximation stored by the machine. On most machines, if Python were to print the true decimal value of the binary approximation stored -for 0.1, it would have to display :: +for 0.1, it would have to display:: >>> 0.1 0.1000000000000000055511151231257827021181583404541015625 That is more digits than most people find useful, so Python keeps the number -of digits manageable by displaying a rounded value instead :: +of digits manageable by displaying a rounded value instead: + +.. doctest:: >>> 1 / 10 0.1 @@ -90,7 +94,10 @@ thing in all languages that support your hardware's floating-point arithmetic (although some languages may not *display* the difference by default, or in all output modes). -For more pleasant output, you may wish to use string formatting to produce a limited number of significant digits:: +For more pleasant output, you may wish to use string formatting to produce a +limited number of significant digits: + +.. doctest:: >>> format(math.pi, '.12g') # give 12 significant digits '3.14159265359' @@ -101,33 +108,49 @@ For more pleasant output, you may wish to use string formatting to produce a lim >>> repr(math.pi) '3.141592653589793' - It's important to realize that this is, in a real sense, an illusion: you're simply rounding the *display* of the true machine value. One illusion may beget another. For example, since 0.1 is not exactly 1/10, -summing three values of 0.1 may not yield exactly 0.3, either:: +summing three values of 0.1 may not yield exactly 0.3, either: + +.. doctest:: - >>> .1 + .1 + .1 == .3 + >>> 0.1 + 0.1 + 0.1 == 0.3 False Also, since the 0.1 cannot get any closer to the exact value of 1/10 and 0.3 cannot get any closer to the exact value of 3/10, then pre-rounding with -:func:`round` function cannot help:: +:func:`round` function cannot help: - >>> round(.1, 1) + round(.1, 1) + round(.1, 1) == round(.3, 1) +.. doctest:: + + >>> round(0.1, 1) + round(0.1, 1) + round(0.1, 1) == round(0.3, 1) False Though the numbers cannot be made closer to their intended exact values, -the :func:`round` function can be useful for post-rounding so that results -with inexact values become comparable to one another:: +the :func:`math.isclose` function can be useful for comparing inexact values: - >>> round(.1 + .1 + .1, 10) == round(.3, 10) - True +.. doctest:: + + >>> math.isclose(0.1 + 0.1 + 0.1, 0.3) + True + +Alternatively, the :func:`round` function can be used to compare rough +approximations:: + +.. doctest:: + + >>> round(math.pi, ndigits=2) == round(22 / 7, ndigits=2) + True Binary floating-point arithmetic holds many surprises like this. The problem with "0.1" is explained in precise detail below, in the "Representation Error" -section. See `The Perils of Floating Point `_ +section. See `Examples of Floating Point Problems +`_ for +a pleasant summary of how binary floating point works and the kinds of +problems commonly encountered in practice. Also see +`The Perils of Floating Point `_ for a more complete account of other common surprises. As that says near the end, "there are no easy answers." Still, don't be unduly @@ -158,26 +181,34 @@ statistical operations supplied by the SciPy project. See . Python provides tools that may help on those rare occasions when you really *do* want to know the exact value of a float. The :meth:`float.as_integer_ratio` method expresses the value of a float as a -fraction:: +fraction: + +.. doctest:: >>> x = 3.14159 >>> x.as_integer_ratio() (3537115888337719, 1125899906842624) Since the ratio is exact, it can be used to losslessly recreate the -original value:: +original value: + +.. doctest:: >>> x == 3537115888337719 / 1125899906842624 True The :meth:`float.hex` method expresses a float in hexadecimal (base -16), again giving the exact value stored by your computer:: +16), again giving the exact value stored by your computer: + +.. doctest:: >>> x.hex() '0x1.921f9f01b866ep+1' This precise hexadecimal representation can be used to reconstruct -the float value exactly:: +the float value exactly: + +.. doctest:: >>> x == float.fromhex('0x1.921f9f01b866ep+1') True @@ -186,17 +217,43 @@ Since the representation is exact, it is useful for reliably porting values across different versions of Python (platform independence) and exchanging data with other languages that support the same format (such as Java and C99). -Another helpful tool is the :func:`math.fsum` function which helps mitigate -loss-of-precision during summation. It tracks "lost digits" as values are -added onto a running total. That can make a difference in overall accuracy -so that the errors do not accumulate to the point where they affect the -final total: +Another helpful tool is the :func:`sum` function which helps mitigate +loss-of-precision during summation. It uses extended precision for +intermediate rounding steps as values are added onto a running total. +That can make a difference in overall accuracy so that the errors do not +accumulate to the point where they affect the final total: + +.. doctest:: >>> 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 + 0.1 == 1.0 False - >>> math.fsum([0.1] * 10) == 1.0 + >>> sum([0.1] * 10) == 1.0 True +The :func:`math.fsum()` goes further and tracks all of the "lost digits" +as values are added onto a running total so that the result has only a +single rounding. This is slower than :func:`sum` but will be more +accurate in uncommon cases where large magnitude inputs mostly cancel +each other out leaving a final sum near zero: + +.. doctest:: + + >>> arr = [-0.10430216751806065, -266310978.67179024, 143401161448607.16, + ... -143401161400469.7, 266262841.31058735, -0.003244936839808227] + >>> float(sum(map(Fraction, arr))) # Exact summation with single rounding + 8.042173697819788e-13 + >>> math.fsum(arr) # Single rounding + 8.042173697819788e-13 + >>> sum(arr) # Multiple roundings in extended precision + 8.042178034628478e-13 + >>> total = 0.0 + >>> for x in arr: + ... total += x # Multiple roundings in standard precision + ... + >>> total # Straight addition has no correct digits! + -0.0051575902860057365 + + .. _tut-fp-error: Representation Error @@ -225,20 +282,28 @@ as :: J ~= 2**N / 10 and recalling that *J* has exactly 53 bits (is ``>= 2**52`` but ``< 2**53``), -the best value for *N* is 56:: +the best value for *N* is 56: + +.. doctest:: >>> 2**52 <= 2**56 // 10 < 2**53 True That is, 56 is the only value for *N* that leaves *J* with exactly 53 bits. The -best possible value for *J* is then that quotient rounded:: +best possible value for *J* is then that quotient rounded: + +.. doctest:: >>> q, r = divmod(2**56, 10) >>> r 6 Since the remainder is more than half of 10, the best approximation is obtained -by rounding up:: +by rounding up: + +.. doctest:: + + >>> q+1 7205759403792794 @@ -256,13 +321,17 @@ if we had not rounded up, the quotient would have been a little bit smaller than 1/10. But in no case can it be *exactly* 1/10! So the computer never "sees" 1/10: what it sees is the exact fraction given -above, the best 754 double approximation it can get:: +above, the best 754 double approximation it can get: + +.. doctest:: >>> 0.1 * 2 ** 55 3602879701896397.0 If we multiply that fraction by 10\*\*55, we can see the value out to -55 decimal digits:: +55 decimal digits: + +.. doctest:: >>> 3602879701896397 * 10 ** 55 // 2 ** 55 1000000000000000055511151231257827021181583404541015625 @@ -270,13 +339,17 @@ If we multiply that fraction by 10\*\*55, we can see the value out to meaning that the exact number stored in the computer is equal to the decimal value 0.1000000000000000055511151231257827021181583404541015625. Instead of displaying the full decimal value, many languages (including -older versions of Python), round the result to 17 significant digits:: +older versions of Python), round the result to 17 significant digits: + +.. doctest:: >>> format(0.1, '.17f') '0.10000000000000001' The :mod:`fractions` and :mod:`decimal` modules make these calculations -easy:: +easy: + +.. doctest:: >>> from decimal import Decimal >>> from fractions import Fraction From 60bbed7f174e481d3fc69984430a4667951678b3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 19 Feb 2023 21:22:29 +0100 Subject: [PATCH 82/83] gh-101578: Amend PyErr_{Set,Get}RaisedException docs (#101962) Co-authored-by: C.A.M. Gerlach --- Doc/c-api/exceptions.rst | 49 ++++++++++++++-------------------------- Doc/data/refcounts.dat | 3 +++ 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index de9b15edd6859a..e735f8db402363 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -402,51 +402,36 @@ Querying the error indicator .. c:function:: PyObject *PyErr_GetRaisedException(void) - Returns the exception currently being raised, clearing the exception at - the same time. Do not confuse this with the exception currently being - handled which can be accessed with :c:func:`PyErr_GetHandledException`. + Return the exception currently being raised, clearing the error indicator at + the same time. - .. note:: + This function is used by code that needs to catch exceptions, + or code that needs to save and restore the error indicator temporarily. - This function is normally only used by code that needs to catch exceptions or - by code that needs to save and restore the error indicator temporarily, e.g.:: + For example:: - { - PyObject *exc = PyErr_GetRaisedException(); + { + PyObject *exc = PyErr_GetRaisedException(); - /* ... code that might produce other errors ... */ + /* ... code that might produce other errors ... */ - PyErr_SetRaisedException(exc); - } + PyErr_SetRaisedException(exc); + } + + .. seealso:: :c:func:`PyErr_GetHandledException`, + to save the exception currently being handled. .. versionadded:: 3.12 .. c:function:: void PyErr_SetRaisedException(PyObject *exc) - Sets the exception currently being raised ``exc``. - If the exception is already set, it is cleared first. - - ``exc`` must be a valid exception. - (Violating this rules will cause subtle problems later.) - This call consumes a reference to the ``exc`` object: you must own a - reference to that object before the call and after the call you no longer own - that reference. - (If you don't understand this, don't use this function. I warned you.) + Set *exc* as the exception currently being raised, + clearing the existing exception if one is set. - .. note:: - - This function is normally only used by code that needs to save and restore the - error indicator temporarily. Use :c:func:`PyErr_GetRaisedException` to save - the current exception, e.g.:: - - { - PyObject *exc = PyErr_GetRaisedException(); - - /* ... code that might produce other errors ... */ + .. warning:: - PyErr_SetRaisedException(exc); - } + This call steals a reference to *exc*, which must be a valid exception. .. versionadded:: 3.12 diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 349c4dd5be3d81..ed2958f8cd2205 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -606,6 +606,9 @@ PyErr_GetExcInfo:PyObject**:ptype:+1: PyErr_GetExcInfo:PyObject**:pvalue:+1: PyErr_GetExcInfo:PyObject**:ptraceback:+1: +PyErr_GetRaisedException:PyObject*::+1: +PyErr_SetRaisedException:::: + PyErr_GivenExceptionMatches:int::: PyErr_GivenExceptionMatches:PyObject*:given:0: PyErr_GivenExceptionMatches:PyObject*:exc:0: From b1b375e2670a58fc37cb4c2629ed73b045159918 Mon Sep 17 00:00:00 2001 From: Mark Dickinson Date: Mon, 20 Feb 2023 01:16:11 +0000 Subject: [PATCH 83/83] gh-97786: Fix compiler warnings in pytime.c (#101826) Fixes compiler warnings in pytime.c. --- ...3-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst | 2 ++ Python/pytime.c | 35 +++++++++++++++---- 2 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst diff --git a/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst b/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst new file mode 100644 index 00000000000000..df194b67590d67 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst @@ -0,0 +1,2 @@ +Fix potential undefined behaviour in corner cases of floating-point-to-time +conversions. diff --git a/Python/pytime.c b/Python/pytime.c index 01c07da074757e..acd1842056af43 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1,5 +1,4 @@ #include "Python.h" -#include "pycore_pymath.h" // _Py_InIntegralTypeRange() #ifdef MS_WINDOWS # include // struct timeval #endif @@ -41,6 +40,14 @@ # error "unsupported time_t size" #endif +#if PY_TIME_T_MAX + PY_TIME_T_MIN != -1 +# error "time_t is not a two's complement integer type" +#endif + +#if _PyTime_MIN + _PyTime_MAX != -1 +# error "_PyTime_t is not a two's complement integer type" +#endif + static void pytime_time_t_overflow(void) @@ -294,7 +301,21 @@ pytime_double_to_denominator(double d, time_t *sec, long *numerator, } assert(0.0 <= floatpart && floatpart < denominator); - if (!_Py_InIntegralTypeRange(time_t, intpart)) { + /* + Conversion of an out-of-range value to time_t gives undefined behaviour + (C99 §6.3.1.4p1), so we must guard against it. However, checking that + `intpart` is in range is delicate: the obvious expression `intpart <= + PY_TIME_T_MAX` will first convert the value `PY_TIME_T_MAX` to a double, + potentially changing its value and leading to us failing to catch some + UB-inducing values. The code below works correctly under the mild + assumption that time_t is a two's complement integer type with no trap + representation, and that `PY_TIME_T_MIN` is within the representable + range of a C double. + + Note: we want the `if` condition below to be true for NaNs; therefore, + resist any temptation to simplify by applying De Morgan's laws. + */ + if (!((double)PY_TIME_T_MIN <= intpart && intpart < -(double)PY_TIME_T_MIN)) { pytime_time_t_overflow(); return -1; } @@ -349,7 +370,8 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) d = pytime_round(d, round); (void)modf(d, &intpart); - if (!_Py_InIntegralTypeRange(time_t, intpart)) { + /* See comments in pytime_double_to_denominator */ + if (!((double)PY_TIME_T_MIN <= intpart && intpart < -(double)PY_TIME_T_MIN)) { pytime_time_t_overflow(); return -1; } @@ -515,8 +537,9 @@ pytime_from_double(_PyTime_t *tp, double value, _PyTime_round_t round, d *= (double)unit_to_ns; d = pytime_round(d, round); - if (!_Py_InIntegralTypeRange(_PyTime_t, d)) { - pytime_overflow(); + /* See comments in pytime_double_to_denominator */ + if (!((double)_PyTime_MIN <= d && d < -(double)_PyTime_MIN)) { + pytime_time_t_overflow(); return -1; } _PyTime_t ns = (_PyTime_t)d; @@ -910,7 +933,7 @@ py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) info->monotonic = 0; info->adjustable = 1; if (clock_getres(CLOCK_REALTIME, &res) == 0) { - info->resolution = res.tv_sec + res.tv_nsec * 1e-9; + info->resolution = (double)res.tv_sec + (double)res.tv_nsec * 1e-9; } else { info->resolution = 1e-9;