From eaede0ded72e67cee4a91c086847d54cb64ca74c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 17 May 2021 23:48:35 +0200 Subject: [PATCH 001/160] bpo-44131: Test Py_FrozenMain() (GH-26126) * Add test_frozenmain to test_embed * Add Programs/test_frozenmain.py * Add Programs/freeze_test_frozenmain.py * Add Programs/test_frozenmain.h * Add make regen-test-frozenmain * Add test_frozenmain command to Programs/_testembed * _testembed.c: add error(msg) function --- .gitattributes | 1 + Lib/test/test_embed.py | 15 ++++ Makefile.pre.in | 12 +++- .../2021-05-14-14-13-44.bpo-44131.YPizoC.rst | 2 + Programs/_testembed.c | 72 ++++++++++++++++--- Programs/freeze_test_frozenmain.py | 48 +++++++++++++ Programs/test_frozenmain.h | 30 ++++++++ Programs/test_frozenmain.py | 9 +++ Python/frozenmain.c | 26 ++++--- Tools/freeze/makefreeze.py | 20 +++--- 10 files changed, 203 insertions(+), 32 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2021-05-14-14-13-44.bpo-44131.YPizoC.rst create mode 100644 Programs/freeze_test_frozenmain.py create mode 100644 Programs/test_frozenmain.h create mode 100644 Programs/test_frozenmain.py diff --git a/.gitattributes b/.gitattributes index c66e765266382f..fd303806dac21a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -57,6 +57,7 @@ Doc/library/token-list.inc linguist-generated=true Include/token.h linguist-generated=true Lib/token.py linguist-generated=true Parser/token.c linguist-generated=true +Programs/test_frozenmain.h linguist-generated=true # Language aware diff headers # https://tekin.co.uk/2020/10/better-git-diff-output-for-ruby-python-elixir-and-more diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 23cf297d4ab624..c68a66270cfbd5 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1480,6 +1480,21 @@ def test_unicode_id_init(self): # when Python is initialized multiples times. self.run_embedded_interpreter("test_unicode_id_init") + # See bpo-44133 + @unittest.skipIf(os.name == 'nt', + 'Py_FrozenMain is not exported on Windows') + def test_frozenmain(self): + out, err = self.run_embedded_interpreter("test_frozenmain") + exe = os.path.realpath('./argv0') + expected = textwrap.dedent(f""" + Frozen Hello World + sys.argv ['./argv0', '-E', 'arg1', 'arg2'] + config program_name: ./argv0 + config executable: {exe} + config use_environment: 1 + """).lstrip() + self.assertEqual(out, expected) + class StdPrinterTests(EmbeddingTestsMixin, unittest.TestCase): # Test PyStdPrinter_Type which is used by _PySys_SetPreliminaryStderr(): diff --git a/Makefile.pre.in b/Makefile.pre.in index 080318bf454f6c..83788a640389e0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -720,6 +720,14 @@ Makefile Modules/config.c: Makefile.pre \ @mv config.c Modules @echo "The Makefile was updated, you may need to re-run make." +regen-test-frozenmain: $(BUILDPYTHON) + # Regenerate Programs/test_frozenmain.h + # from Programs/test_frozenmain.py + # using Programs/freeze_test_frozenmain.py + $(RUNSHARED) ./$(BUILDPYTHON) Programs/freeze_test_frozenmain.py Programs/test_frozenmain.h + +Programs/test_frozenmain.h: Programs/freeze_test_frozenmain.py Programs/test_frozenmain.py + $(MAKE) regen-test-frozenmain Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) @@ -763,7 +771,7 @@ regen-limited-abi: all regen-all: regen-opcode regen-opcode-targets regen-typeslots \ regen-token regen-ast regen-keyword regen-importlib clinic \ - regen-pegen-metaparser regen-pegen regen-frozen + regen-pegen-metaparser regen-pegen regen-frozen regen-test-frozenmain @echo @echo "Note: make regen-stdlib-module-names and autoconf should be run manually" @@ -794,7 +802,7 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile Programs/python.o: $(srcdir)/Programs/python.c $(MAINCC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/python.c -Programs/_testembed.o: $(srcdir)/Programs/_testembed.c +Programs/_testembed.o: $(srcdir)/Programs/_testembed.c Programs/test_frozenmain.h $(MAINCC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/_testembed.c Modules/_sre.o: $(srcdir)/Modules/_sre.c $(srcdir)/Modules/sre.h $(srcdir)/Modules/sre_constants.h $(srcdir)/Modules/sre_lib.h diff --git a/Misc/NEWS.d/next/Tests/2021-05-14-14-13-44.bpo-44131.YPizoC.rst b/Misc/NEWS.d/next/Tests/2021-05-14-14-13-44.bpo-44131.YPizoC.rst new file mode 100644 index 00000000000000..a646acf8e44431 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2021-05-14-14-13-44.bpo-44131.YPizoC.rst @@ -0,0 +1,2 @@ +Add test_frozenmain to test_embed to test the :c:func:`Py_FrozenMain` C +function. Patch by Victor Stinner. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 21b24f7d51e402..a5ae7c1d863357 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -27,6 +27,14 @@ _Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS + +static void error(const char *msg) +{ + fprintf(stderr, "ERROR: %s\n", msg); + fflush(stderr); +} + + static void _testembed_Py_Initialize(void) { Py_SetProgramName(PROGRAM_NAME); @@ -239,7 +247,7 @@ static void bpo20891_thread(void *lockp) PyGILState_STATE state = PyGILState_Ensure(); if (!PyGILState_Check()) { - fprintf(stderr, "PyGILState_Check failed!"); + error("PyGILState_Check failed!"); abort(); } @@ -259,7 +267,7 @@ static int test_bpo20891(void) crash. */ PyThread_type_lock lock = PyThread_allocate_lock(); if (!lock) { - fprintf(stderr, "PyThread_allocate_lock failed!"); + error("PyThread_allocate_lock failed!"); return 1; } @@ -267,7 +275,7 @@ static int test_bpo20891(void) unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &lock); if (thrd == PYTHREAD_INVALID_THREAD_ID) { - fprintf(stderr, "PyThread_start_new_thread failed!"); + error("PyThread_start_new_thread failed!"); return 1; } PyThread_acquire_lock(lock, WAIT_LOCK); @@ -1397,12 +1405,12 @@ static int test_init_setpath(void) { char *env = getenv("TESTPATH"); if (!env) { - fprintf(stderr, "missing TESTPATH env var\n"); + error("missing TESTPATH env var"); return 1; } wchar_t *path = Py_DecodeLocale(env, NULL); if (path == NULL) { - fprintf(stderr, "failed to decode TESTPATH\n"); + error("failed to decode TESTPATH"); return 1; } Py_SetPath(path); @@ -1430,12 +1438,12 @@ static int test_init_setpath_config(void) char *env = getenv("TESTPATH"); if (!env) { - fprintf(stderr, "missing TESTPATH env var\n"); + error("missing TESTPATH env var"); return 1; } wchar_t *path = Py_DecodeLocale(env, NULL); if (path == NULL) { - fprintf(stderr, "failed to decode TESTPATH\n"); + error("failed to decode TESTPATH"); return 1; } Py_SetPath(path); @@ -1459,12 +1467,12 @@ static int test_init_setpythonhome(void) { char *env = getenv("TESTHOME"); if (!env) { - fprintf(stderr, "missing TESTHOME env var\n"); + error("missing TESTHOME env var"); return 1; } wchar_t *home = Py_DecodeLocale(env, NULL); if (home == NULL) { - fprintf(stderr, "failed to decode TESTHOME\n"); + error("failed to decode TESTHOME"); return 1; } Py_SetPythonHome(home); @@ -1726,6 +1734,48 @@ static int test_unicode_id_init(void) } +#ifndef MS_WINDOWS +#include "test_frozenmain.h" // M_test_frozenmain + +static int test_frozenmain(void) +{ + // Get "_frozen_importlib" and "_frozen_importlib_external" + // from PyImport_FrozenModules + const struct _frozen *importlib = NULL, *importlib_external = NULL; + for (const struct _frozen *mod = PyImport_FrozenModules; mod->name != NULL; mod++) { + if (strcmp(mod->name, "_frozen_importlib") == 0) { + importlib = mod; + } + else if (strcmp(mod->name, "_frozen_importlib_external") == 0) { + importlib_external = mod; + } + } + if (importlib == NULL || importlib_external == NULL) { + error("cannot find frozen importlib and importlib_external"); + return 1; + } + + static struct _frozen frozen_modules[4] = { + {0, 0, 0}, // importlib + {0, 0, 0}, // importlib_external + {"__main__", M_test_frozenmain, sizeof(M_test_frozenmain)}, + {0, 0, 0} // sentinel + }; + frozen_modules[0] = *importlib; + frozen_modules[1] = *importlib_external; + + char* argv[] = { + "./argv0", + "-E", + "arg1", + "arg2", + }; + PyImport_FrozenModules = frozen_modules; + return Py_FrozenMain(Py_ARRAY_LENGTH(argv), argv); +} +#endif // !MS_WINDOWS + + // List frozen modules. // Command used by Tools/scripts/generate_stdlib_module_names.py script. static int list_frozen(void) @@ -1811,11 +1861,15 @@ static struct TestCase TestCases[] = { {"test_audit_run_stdin", test_audit_run_stdin}, {"test_unicode_id_init", test_unicode_id_init}, +#ifndef MS_WINDOWS + {"test_frozenmain", test_frozenmain}, +#endif {"list_frozen", list_frozen}, {NULL, NULL} }; + int main(int argc, char *argv[]) { if (argc > 1) { diff --git a/Programs/freeze_test_frozenmain.py b/Programs/freeze_test_frozenmain.py new file mode 100644 index 00000000000000..848fc31b3d6f44 --- /dev/null +++ b/Programs/freeze_test_frozenmain.py @@ -0,0 +1,48 @@ +import marshal +import tokenize +import os.path +import sys + +PROGRAM_DIR = os.path.dirname(__file__) +SRC_DIR = os.path.dirname(PROGRAM_DIR) + + +def writecode(fp, mod, data): + print('unsigned char M_%s[] = {' % mod, file=fp) + indent = ' ' * 4 + for i in range(0, len(data), 16): + print(indent, file=fp, end='') + for c in bytes(data[i:i+16]): + print('%d,' % c, file=fp, end='') + print('', file=fp) + print('};', file=fp) + + +def dump(fp, filename, name): + # Strip the directory to get reproducible marshal dump + code_filename = os.path.basename(filename) + + with tokenize.open(filename) as source_fp: + source = source_fp.read() + code = compile(source, code_filename, 'exec') + + data = marshal.dumps(code) + writecode(fp, name, data) + + +def main(): + if len(sys.argv) < 2: + print(f"usage: {sys.argv[0]} filename") + sys.exit(1) + filename = sys.argv[1] + + with open(filename, "w") as fp: + print("// Auto-generated by Programs/freeze_test_frozenmain.py", file=fp) + frozenmain = os.path.join(PROGRAM_DIR, 'test_frozenmain.py') + dump(fp, frozenmain, 'test_frozenmain') + + print(f"{filename} written") + + +if __name__ == "__main__": + main() diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h new file mode 100644 index 00000000000000..ac3dfd32c262ec --- /dev/null +++ b/Programs/test_frozenmain.h @@ -0,0 +1,30 @@ +// Auto-generated by Programs/freeze_test_frozenmain.py +unsigned char M_test_frozenmain[] = { + 227,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,4,0,0,0,64,0,0,0,115,106,0,0,0,100,0, + 100,1,108,0,90,0,100,0,100,1,108,1,90,1,101,2, + 100,2,131,1,1,0,101,2,100,3,101,0,106,3,131,2, + 1,0,101,1,160,4,161,0,100,4,25,0,90,5,101,2, + 100,5,101,5,100,6,25,0,155,0,157,2,131,1,1,0, + 101,2,100,7,101,5,100,8,25,0,155,0,157,2,131,1, + 1,0,101,2,100,9,101,5,100,10,25,0,155,0,157,2, + 131,1,1,0,100,1,83,0,41,11,233,0,0,0,0,78, + 122,18,70,114,111,122,101,110,32,72,101,108,108,111,32,87, + 111,114,108,100,122,8,115,121,115,46,97,114,103,118,218,6, + 99,111,110,102,105,103,122,21,99,111,110,102,105,103,32,112, + 114,111,103,114,97,109,95,110,97,109,101,58,32,90,12,112, + 114,111,103,114,97,109,95,110,97,109,101,122,19,99,111,110, + 102,105,103,32,101,120,101,99,117,116,97,98,108,101,58,32, + 218,10,101,120,101,99,117,116,97,98,108,101,122,24,99,111, + 110,102,105,103,32,117,115,101,95,101,110,118,105,114,111,110, + 109,101,110,116,58,32,90,15,117,115,101,95,101,110,118,105, + 114,111,110,109,101,110,116,41,6,218,3,115,121,115,90,17, + 95,116,101,115,116,105,110,116,101,114,110,97,108,99,97,112, + 105,218,5,112,114,105,110,116,218,4,97,114,103,118,90,11, + 103,101,116,95,99,111,110,102,105,103,115,114,2,0,0,0, + 169,0,114,7,0,0,0,114,7,0,0,0,250,18,116,101, + 115,116,95,102,114,111,122,101,110,109,97,105,110,46,112,121, + 218,8,60,109,111,100,117,108,101,62,1,0,0,0,115,16, + 0,0,0,8,0,8,1,8,2,12,1,12,1,18,1,18, + 1,22,1,243,0,0,0,0, +}; diff --git a/Programs/test_frozenmain.py b/Programs/test_frozenmain.py new file mode 100644 index 00000000000000..aa79106df63966 --- /dev/null +++ b/Programs/test_frozenmain.py @@ -0,0 +1,9 @@ +import sys +import _testinternalcapi + +print("Frozen Hello World") +print("sys.argv", sys.argv) +config = _testinternalcapi.get_configs()['config'] +print(f"config program_name: {config['program_name']}") +print(f"config executable: {config['executable']}") +print(f"config use_environment: {config['use_environment']}") diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 5eb9e31112484a..c3104dae449771 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -1,4 +1,3 @@ - /* Python interpreter main program for frozen scripts */ #include "Python.h" @@ -43,10 +42,12 @@ Py_FrozenMain(int argc, char **argv) PyConfig_InitPythonConfig(&config); config.pathconfig_warnings = 0; /* Suppress errors from getpath.c */ - if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') + if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') { inspect = 1; - if ((p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0') + } + if ((p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0') { unbuffered = 1; + } if (unbuffered) { setbuf(stdin, (char *)NULL); @@ -65,8 +66,9 @@ Py_FrozenMain(int argc, char **argv) argv_copy[i] = Py_DecodeLocale(argv[i], NULL); argv_copy2[i] = argv_copy[i]; if (!argv_copy[i]) { - fprintf(stderr, "Unable to decode the command line argument #%i\n", - i + 1); + fprintf(stderr, + "Unable to decode the command line argument #%i\n", + i + 1); argc = i; goto error; } @@ -97,24 +99,28 @@ Py_FrozenMain(int argc, char **argv) PyWinFreeze_ExeInit(); #endif - if (Py_VerboseFlag) + if (Py_VerboseFlag) { fprintf(stderr, "Python %s\n%s\n", - Py_GetVersion(), Py_GetCopyright()); + Py_GetVersion(), Py_GetCopyright()); + } PySys_SetArgv(argc, argv_copy); n = PyImport_ImportFrozenModule("__main__"); - if (n == 0) + if (n == 0) { Py_FatalError("the __main__ module is not frozen"); + } if (n < 0) { PyErr_Print(); sts = 1; } - else + else { sts = 0; + } - if (inspect && isatty((int)fileno(stdin))) + if (inspect && isatty((int)fileno(stdin))) { sts = PyRun_AnyFile(stdin, "") != 0; + } #ifdef MS_WINDOWS PyWinFreeze_ExeTerm(); diff --git a/Tools/freeze/makefreeze.py b/Tools/freeze/makefreeze.py index 64e3e6bf71e774..d7d05db88a96c9 100644 --- a/Tools/freeze/makefreeze.py +++ b/Tools/freeze/makefreeze.py @@ -74,14 +74,12 @@ def makefreeze(base, dict, debug=0, entry_point=None, fail_import=()): # Write a C initializer for a module containing the frozen python code. # The array is called M_. -def writecode(outfp, mod, str): - outfp.write('unsigned char M_%s[] = {' % mod) - for i in range(0, len(str), 16): - outfp.write('\n\t') - for c in bytes(str[i:i+16]): - outfp.write('%d,' % c) - outfp.write('\n};\n') - -## def writecode(outfp, mod, str): -## outfp.write('unsigned char M_%s[%d] = "%s";\n' % (mod, len(str), -## '\\"'.join(map(lambda s: repr(s)[1:-1], str.split('"'))))) +def writecode(fp, mod, data): + print('unsigned char M_%s[] = {' % mod, file=fp) + indent = ' ' * 4 + for i in range(0, len(data), 16): + print(indent, file=fp, end='') + for c in bytes(data[i:i+16]): + print('%d,' % c, file=fp, end='') + print('', file=fp) + print('};', file=fp) From 834498e178684a7e2da49b4efe1adea33e0026b0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 18 May 2021 08:40:37 +0200 Subject: [PATCH 002/160] bpo-44131: Fix Makefile for test_frozenmain (GH-26203) Remove Programs/test_frozenmain.h Makefile target: it ran make in parallel which caused build errors on LTO+PGO builds. --- Makefile.pre.in | 4 +--- Programs/test_frozenmain.h | 2 +- Programs/test_frozenmain.py | 3 +++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 83788a640389e0..798c53fa7c695f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -720,15 +720,13 @@ Makefile Modules/config.c: Makefile.pre \ @mv config.c Modules @echo "The Makefile was updated, you may need to re-run make." +.PHONY: regen-test-frozenmain regen-test-frozenmain: $(BUILDPYTHON) # Regenerate Programs/test_frozenmain.h # from Programs/test_frozenmain.py # using Programs/freeze_test_frozenmain.py $(RUNSHARED) ./$(BUILDPYTHON) Programs/freeze_test_frozenmain.py Programs/test_frozenmain.h -Programs/test_frozenmain.h: Programs/freeze_test_frozenmain.py Programs/test_frozenmain.py - $(MAKE) regen-test-frozenmain - Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index ac3dfd32c262ec..e680b80e7ae8ac 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -25,6 +25,6 @@ unsigned char M_test_frozenmain[] = { 169,0,114,7,0,0,0,114,7,0,0,0,250,18,116,101, 115,116,95,102,114,111,122,101,110,109,97,105,110,46,112,121, 218,8,60,109,111,100,117,108,101,62,1,0,0,0,115,16, - 0,0,0,8,0,8,1,8,2,12,1,12,1,18,1,18, + 0,0,0,8,3,8,1,8,2,12,1,12,1,18,1,18, 1,22,1,243,0,0,0,0, }; diff --git a/Programs/test_frozenmain.py b/Programs/test_frozenmain.py index aa79106df63966..928b9ea816accb 100644 --- a/Programs/test_frozenmain.py +++ b/Programs/test_frozenmain.py @@ -1,3 +1,6 @@ +# Script used to test Py_FrozenMain(): see test_embed.test_frozenmain(). +# Run "make regen-test-frozenmain" if you modify this test. + import sys import _testinternalcapi From 115dea9e2602b96b63390f00cc880e90c433efa2 Mon Sep 17 00:00:00 2001 From: uniocto Date: Tue, 18 May 2021 17:56:52 +0900 Subject: [PATCH 003/160] bpo-25872: Add unit tests for linecache and threading (GH-25913) --- Lib/test/test_linecache.py | 42 ++++++++++++++++++++++++++++++++++++++ Lib/test/test_threading.py | 19 ++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 59e00da242ad63..c6e2dadbb25e1f 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -239,5 +239,47 @@ def raise_memoryerror(*args, **kwargs): self.assertEqual(linecache.getlines(FILENAME), lines) +class LineCacheInvalidationTests(unittest.TestCase): + def setUp(self): + super().setUp() + linecache.clearcache() + self.deleted_file = os_helper.TESTFN + '.1' + self.modified_file = os_helper.TESTFN + '.2' + self.unchanged_file = os_helper.TESTFN + '.3' + + for fname in (self.deleted_file, + self.modified_file, + self.unchanged_file): + self.addCleanup(os_helper.unlink, fname) + with open(fname, 'w', encoding='utf-8') as source: + source.write(f'print("I am {fname}")') + + self.assertNotIn(fname, linecache.cache) + linecache.getlines(fname) + self.assertIn(fname, linecache.cache) + + os.remove(self.deleted_file) + with open(self.modified_file, 'w', encoding='utf-8') as source: + source.write('print("was modified")') + + def test_checkcache_for_deleted_file(self): + linecache.checkcache(self.deleted_file) + self.assertNotIn(self.deleted_file, linecache.cache) + self.assertIn(self.modified_file, linecache.cache) + self.assertIn(self.unchanged_file, linecache.cache) + + def test_checkcache_for_modified_file(self): + linecache.checkcache(self.modified_file) + self.assertIn(self.deleted_file, linecache.cache) + self.assertNotIn(self.modified_file, linecache.cache) + self.assertIn(self.unchanged_file, linecache.cache) + + def test_checkcache_with_no_parameter(self): + linecache.checkcache() + self.assertNotIn(self.deleted_file, linecache.cache) + self.assertNotIn(self.modified_file, linecache.cache) + self.assertIn(self.unchanged_file, linecache.cache) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 08c0ccd9a79b2a..b563797cbd0d39 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -4,7 +4,7 @@ import test.support from test.support import threading_helper -from test.support import verbose, cpython_only +from test.support import verbose, cpython_only, os_helper from test.support.import_helper import import_module from test.support.script_helper import assert_python_ok, assert_python_failure @@ -19,6 +19,7 @@ import subprocess import signal import textwrap +import traceback from unittest import mock from test import lock_tests @@ -1345,6 +1346,22 @@ def run(self): # explicitly break the reference cycle to not leak a dangling thread thread.exc = None + def test_multithread_modify_file_noerror(self): + # See issue25872 + def modify_file(): + with open(os_helper.TESTFN, 'w', encoding='utf-8') as fp: + fp.write(' ') + traceback.format_stack() + + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + threads = [ + threading.Thread(target=modify_file) + for i in range(100) + ] + for t in threads: + t.start() + t.join() + class ThreadRunFail(threading.Thread): def run(self): From 02ee8191263848f8c8999f72286148946b83e5c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Gmach?= Date: Tue, 18 May 2021 18:11:23 +0200 Subject: [PATCH 004/160] bpo-41963: document that ConfigParser strips off comments (GH-26197) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Laura Gutierrez Funderburk <58710704+lgfunderburk@users.noreply.github.com> --- Doc/library/configparser.rst | 7 +++++++ Lib/configparser.py | 5 ++++- .../Documentation/2021-05-17-20-03-47.bpo-41963.eUz9_o.rst | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Documentation/2021-05-17-20-03-47.bpo-41963.eUz9_o.rst diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 646e8a317f52c3..b0c2a2cfb06a47 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -1153,6 +1153,13 @@ ConfigParser Objects *space_around_delimiters* is true, delimiters between keys and values are surrounded by spaces. + .. note:: + + Comments in the original configuration file are not preserved when + writing the configuration back. + What is considered a comment, depends on the given values for + *comment_prefix* and *inline_comment_prefix*. + .. method:: remove_option(section, option) diff --git a/Lib/configparser.py b/Lib/configparser.py index 3b4cb5e6b2407f..2f45e242b49426 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -908,6 +908,9 @@ def write(self, fp, space_around_delimiters=True): If `space_around_delimiters' is True (the default), delimiters between keys and values are surrounded by spaces. + + Please note that comments in the original configuration file are not + preserved when writing the configuration back. """ if space_around_delimiters: d = " {} ".format(self._delimiters[0]) @@ -1006,7 +1009,7 @@ def _read(self, fp, fpname): Configuration files may include comments, prefixed by specific characters (`#' and `;' by default). Comments may appear on their own in an otherwise empty line or may be entered in lines holding values or - section names. + section names. Please note that comments get stripped off when reading configuration files. """ elements_added = set() cursect = None # None, or a dictionary diff --git a/Misc/NEWS.d/next/Documentation/2021-05-17-20-03-47.bpo-41963.eUz9_o.rst b/Misc/NEWS.d/next/Documentation/2021-05-17-20-03-47.bpo-41963.eUz9_o.rst new file mode 100644 index 00000000000000..b9fe722fa3a795 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2021-05-17-20-03-47.bpo-41963.eUz9_o.rst @@ -0,0 +1 @@ +Document that ``ConfigParser`` strips off comments when reading configuration files. \ No newline at end of file From 4fdcc39f711e1b586a94e2c5088fdd7e78fd9f58 Mon Sep 17 00:00:00 2001 From: Zackery Spytz Date: Tue, 18 May 2021 14:56:01 -0700 Subject: [PATCH 005/160] bpo-35765: Clarify references to "object x" in the JSON tutorial (GH-22411) --- Doc/tutorial/inputoutput.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 4e27cff83ce59f..7f83c4d4612eb3 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -480,7 +480,8 @@ If you have an object ``x``, you can view its JSON string representation with a simple line of code:: >>> import json - >>> json.dumps([1, 'simple', 'list']) + >>> x = [1, 'simple', 'list'] + >>> json.dumps(x) '[1, "simple", "list"]' Another variant of the :func:`~json.dumps` function, called :func:`~json.dump`, From 901443757333a66ff2b5c85eba30dc1c48eac321 Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Wed, 19 May 2021 09:05:48 +0200 Subject: [PATCH 006/160] bpo-30593: Doc'ed that executescript() disregards isolation level (GH-26220) --- Doc/library/sqlite3.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index d0f28db12fda16..f9e4c8a269027b 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -648,7 +648,8 @@ Cursor Objects This is a nonstandard convenience method for executing multiple SQL statements at once. It issues a ``COMMIT`` statement first, then executes the SQL script it - gets as a parameter. + gets as a parameter. This method disregards :attr:`isolation_level`; any + transation control must be added to *sql_script*. *sql_script* can be an instance of :class:`str`. @@ -1048,6 +1049,9 @@ setting :attr:`isolation_level` to ``None``. This will leave the underlying control the transaction state by explicitly issuing ``BEGIN``, ``ROLLBACK``, ``SAVEPOINT``, and ``RELEASE`` statements in your code. +Note that :meth:`~Cursor.executescript` disregards +:attr:`isolation_level`; any transaction control must be added explicitly. + .. versionchanged:: 3.6 :mod:`sqlite3` used to implicitly commit an open transaction before DDL statements. This is no longer the case. From 92d1064727d6b7f4136f9c09ab52ae15e3676afe Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Wed, 19 May 2021 09:41:19 +0200 Subject: [PATCH 007/160] bpo-44106: Improve sqlite3 example database contents (GH-26027) --- Doc/includes/sqlite3/createdb.py | 10 +++++----- Doc/includes/sqlite3/ctx_manager.py | 8 ++++---- Doc/includes/sqlite3/execsql_fetchonerow.py | 10 +++++----- Doc/includes/sqlite3/execsql_printall_1.py | 2 +- Doc/includes/sqlite3/execute_1.py | 13 ++++++------- ...nsert_more_people.py => insert_more_langs.py} | 12 ++++++------ Doc/includes/sqlite3/shortcut_methods.py | 16 ++++++++-------- Doc/includes/sqlite3/simple_tableprinter.py | 5 +---- Doc/tools/susp-ignored.csv | 3 +-- 9 files changed, 37 insertions(+), 42 deletions(-) rename Doc/includes/sqlite3/{insert_more_people.py => insert_more_langs.py} (50%) diff --git a/Doc/includes/sqlite3/createdb.py b/Doc/includes/sqlite3/createdb.py index ee2950bdf81646..49702121f72534 100644 --- a/Doc/includes/sqlite3/createdb.py +++ b/Doc/includes/sqlite3/createdb.py @@ -12,15 +12,15 @@ con = sqlite3.connect(DB_FILE) cur = con.cursor() cur.execute(""" - create table people + create table lang ( - name_last varchar(20), - age integer + name varchar(20), + first_appeared integer ) """) -cur.execute("insert into people (name_last, age) values ('Yeltsin', 72)") -cur.execute("insert into people (name_last, age) values ('Putin', 51)") +cur.execute("insert into lang (name, first_appeared) values ('Forth', 1970)") +cur.execute("insert into lang (name, first_appeared) values ('Ada', 1980)") con.commit() diff --git a/Doc/includes/sqlite3/ctx_manager.py b/Doc/includes/sqlite3/ctx_manager.py index 6db77d45046e1f..2e1175ef44c641 100644 --- a/Doc/includes/sqlite3/ctx_manager.py +++ b/Doc/includes/sqlite3/ctx_manager.py @@ -1,19 +1,19 @@ import sqlite3 con = sqlite3.connect(":memory:") -con.execute("create table person (id integer primary key, firstname varchar unique)") +con.execute("create table lang (id integer primary key, name varchar unique)") # Successful, con.commit() is called automatically afterwards with con: - con.execute("insert into person(firstname) values (?)", ("Joe",)) + con.execute("insert into lang(name) values (?)", ("Python",)) # con.rollback() is called after the with block finishes with an exception, the # exception is still raised and must be caught try: with con: - con.execute("insert into person(firstname) values (?)", ("Joe",)) + con.execute("insert into lang(name) values (?)", ("Python",)) except sqlite3.IntegrityError: - print("couldn't add Joe twice") + print("couldn't add Python twice") # Connection object used as context manager only commits or rollbacks transactions, # so the connection object should be closed manually diff --git a/Doc/includes/sqlite3/execsql_fetchonerow.py b/Doc/includes/sqlite3/execsql_fetchonerow.py index 115bcb50c7c754..0ca7e14469760b 100644 --- a/Doc/includes/sqlite3/execsql_fetchonerow.py +++ b/Doc/includes/sqlite3/execsql_fetchonerow.py @@ -3,17 +3,17 @@ con = sqlite3.connect("mydb") cur = con.cursor() -SELECT = "select name_last, age from people order by age, name_last" +SELECT = "select name, first_appeared from people order by first_appeared, name" # 1. Iterate over the rows available from the cursor, unpacking the -# resulting sequences to yield their elements (name_last, age): +# resulting sequences to yield their elements (name, first_appeared): cur.execute(SELECT) -for (name_last, age) in cur: - print('%s is %d years old.' % (name_last, age)) +for name, first_appeared in cur: + print(f"The {name} programming language appeared in {first_appeared}.") # 2. Equivalently: cur.execute(SELECT) for row in cur: - print('%s is %d years old.' % (row[0], row[1])) + print(f"The {row[0]} programming language appeared in {row[1]}.") con.close() diff --git a/Doc/includes/sqlite3/execsql_printall_1.py b/Doc/includes/sqlite3/execsql_printall_1.py index 19306e6e3ca7d1..b3b42b5567df3b 100644 --- a/Doc/includes/sqlite3/execsql_printall_1.py +++ b/Doc/includes/sqlite3/execsql_printall_1.py @@ -7,7 +7,7 @@ cur = con.cursor() # Execute the SELECT statement: -cur.execute("select * from people order by age") +cur.execute("select * from lang order by first_appeared") # Retrieve all rows as a sequence and print that sequence: print(cur.fetchall()) diff --git a/Doc/includes/sqlite3/execute_1.py b/Doc/includes/sqlite3/execute_1.py index 42aad4d5839f06..ee0000e2b94a32 100644 --- a/Doc/includes/sqlite3/execute_1.py +++ b/Doc/includes/sqlite3/execute_1.py @@ -2,22 +2,21 @@ con = sqlite3.connect(":memory:") cur = con.cursor() -cur.execute("create table lang (lang_name, lang_age)") +cur.execute("create table lang (name, first_appeared)") # This is the qmark style: -cur.execute("insert into lang values (?, ?)", ("C", 49)) +cur.execute("insert into lang values (?, ?)", ("C", 1972)) # The qmark style used with executemany(): lang_list = [ - ("Fortran", 64), - ("Python", 30), - ("Go", 11), + ("Fortran", 1957), + ("Python", 1991), + ("Go", 2009), ] cur.executemany("insert into lang values (?, ?)", lang_list) # And this is the named style: -cur.execute("select * from lang where lang_name=:name and lang_age=:age", - {"name": "C", "age": 49}) +cur.execute("select * from lang where first_appeared=:year", {"year": 1972}) print(cur.fetchall()) con.close() diff --git a/Doc/includes/sqlite3/insert_more_people.py b/Doc/includes/sqlite3/insert_more_langs.py similarity index 50% rename from Doc/includes/sqlite3/insert_more_people.py rename to Doc/includes/sqlite3/insert_more_langs.py index 10cf937243f6da..ceef949608449e 100644 --- a/Doc/includes/sqlite3/insert_more_people.py +++ b/Doc/includes/sqlite3/insert_more_langs.py @@ -4,13 +4,13 @@ cur = con.cursor() -newPeople = ( - ('Lebed' , 53), - ('Zhirinovsky' , 57), - ) +languages = ( + ("Smalltalk", 1972), + ("Swift", 2014), +) -for person in newPeople: - cur.execute("insert into people (name_last, age) values (?, ?)", person) +for lang in languages: + cur.execute("insert into lang (name, first_appeared) values (?, ?)", lang) # The changes will not be saved unless the transaction is committed explicitly: con.commit() diff --git a/Doc/includes/sqlite3/shortcut_methods.py b/Doc/includes/sqlite3/shortcut_methods.py index 98a39411495cba..48ea6fad15a898 100644 --- a/Doc/includes/sqlite3/shortcut_methods.py +++ b/Doc/includes/sqlite3/shortcut_methods.py @@ -1,23 +1,23 @@ import sqlite3 -persons = [ - ("Hugo", "Boss"), - ("Calvin", "Klein") - ] +langs = [ + ("C++", 1985), + ("Objective-C", 1984), +] con = sqlite3.connect(":memory:") # Create the table -con.execute("create table person(firstname, lastname)") +con.execute("create table lang(name, first_appeared)") # Fill the table -con.executemany("insert into person(firstname, lastname) values (?, ?)", persons) +con.executemany("insert into lang(name, first_appeared) values (?, ?)", langs) # Print the table contents -for row in con.execute("select firstname, lastname from person"): +for row in con.execute("select name, first_appeared from lang"): print(row) -print("I just deleted", con.execute("delete from person").rowcount, "rows") +print("I just deleted", con.execute("delete from lang").rowcount, "rows") # close is not a shortcut method and it's not called automatically, # so the connection object should be closed manually diff --git a/Doc/includes/sqlite3/simple_tableprinter.py b/Doc/includes/sqlite3/simple_tableprinter.py index 148a1707f948bc..9be6e4f414acd8 100644 --- a/Doc/includes/sqlite3/simple_tableprinter.py +++ b/Doc/includes/sqlite3/simple_tableprinter.py @@ -1,13 +1,10 @@ import sqlite3 FIELD_MAX_WIDTH = 20 -TABLE_NAME = 'people' -SELECT = 'select * from %s order by age, name_last' % TABLE_NAME con = sqlite3.connect("mydb") - cur = con.cursor() -cur.execute(SELECT) +cur.execute("select * from lang order by name, first_appeared") # Print a header. for fieldDesc in cur.description: diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index d56a2b9fd0bfb9..1fde253feac2fa 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -209,8 +209,7 @@ library/smtplib,,:port,method must support that as well as a regular host:port library/socket,,::,'5aef:2b::8' library/socket,,:can,"return (can_id, can_dlc, data[:can_dlc])" library/socket,,:len,fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) -library/sqlite3,,:name,"cur.execute(""select * from lang where lang_name=:name and lang_age=:age""," -library/sqlite3,,:age,"cur.execute(""select * from lang where lang_name=:name and lang_age=:age""," +library/sqlite3,,:year,"cur.execute(""select * from lang where first_appeared=:year"", {""year"": 1972})" library/sqlite3,,:memory, library/sqlite3,,:path,"db = sqlite3.connect('file:path/to/database?mode=ro', uri=True)" library/ssl,,:My,"Organizational Unit Name (eg, section) []:My Group" From d798acc8733b605f7fc9c3c1a85cd14ee2a56add Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Wed, 19 May 2021 10:08:12 +0200 Subject: [PATCH 008/160] bpo-44106: Purge unused sqlite3 doc includes (GH-26234) --- Doc/includes/sqlite3/countcursors.py | 17 ------------- Doc/includes/sqlite3/createdb.py | 28 --------------------- Doc/includes/sqlite3/execsql_fetchonerow.py | 19 -------------- Doc/includes/sqlite3/execsql_printall_1.py | 15 ----------- Doc/includes/sqlite3/insert_more_langs.py | 18 ------------- Doc/includes/sqlite3/parse_colnames.py | 10 -------- Doc/includes/sqlite3/shared_cache.py | 6 ----- Doc/includes/sqlite3/simple_tableprinter.py | 25 ------------------ 8 files changed, 138 deletions(-) delete mode 100644 Doc/includes/sqlite3/countcursors.py delete mode 100644 Doc/includes/sqlite3/createdb.py delete mode 100644 Doc/includes/sqlite3/execsql_fetchonerow.py delete mode 100644 Doc/includes/sqlite3/execsql_printall_1.py delete mode 100644 Doc/includes/sqlite3/insert_more_langs.py delete mode 100644 Doc/includes/sqlite3/parse_colnames.py delete mode 100644 Doc/includes/sqlite3/shared_cache.py delete mode 100644 Doc/includes/sqlite3/simple_tableprinter.py diff --git a/Doc/includes/sqlite3/countcursors.py b/Doc/includes/sqlite3/countcursors.py deleted file mode 100644 index 112f47703a2ff4..00000000000000 --- a/Doc/includes/sqlite3/countcursors.py +++ /dev/null @@ -1,17 +0,0 @@ -import sqlite3 - -class CountCursorsConnection(sqlite3.Connection): - def __init__(self, *args, **kwargs): - sqlite3.Connection.__init__(self, *args, **kwargs) - self.numcursors = 0 - - def cursor(self, *args, **kwargs): - self.numcursors += 1 - return sqlite3.Connection.cursor(self, *args, **kwargs) - -con = sqlite3.connect(":memory:", factory=CountCursorsConnection) -cur1 = con.cursor() -cur2 = con.cursor() -print(con.numcursors) - -con.close() diff --git a/Doc/includes/sqlite3/createdb.py b/Doc/includes/sqlite3/createdb.py deleted file mode 100644 index 49702121f72534..00000000000000 --- a/Doc/includes/sqlite3/createdb.py +++ /dev/null @@ -1,28 +0,0 @@ -# Not referenced from the documentation, but builds the database file the other -# code snippets expect. - -import sqlite3 -import os - -DB_FILE = "mydb" - -if os.path.exists(DB_FILE): - os.remove(DB_FILE) - -con = sqlite3.connect(DB_FILE) -cur = con.cursor() -cur.execute(""" - create table lang - ( - name varchar(20), - first_appeared integer - ) - """) - -cur.execute("insert into lang (name, first_appeared) values ('Forth', 1970)") -cur.execute("insert into lang (name, first_appeared) values ('Ada', 1980)") - -con.commit() - -cur.close() -con.close() diff --git a/Doc/includes/sqlite3/execsql_fetchonerow.py b/Doc/includes/sqlite3/execsql_fetchonerow.py deleted file mode 100644 index 0ca7e14469760b..00000000000000 --- a/Doc/includes/sqlite3/execsql_fetchonerow.py +++ /dev/null @@ -1,19 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") - -cur = con.cursor() -SELECT = "select name, first_appeared from people order by first_appeared, name" - -# 1. Iterate over the rows available from the cursor, unpacking the -# resulting sequences to yield their elements (name, first_appeared): -cur.execute(SELECT) -for name, first_appeared in cur: - print(f"The {name} programming language appeared in {first_appeared}.") - -# 2. Equivalently: -cur.execute(SELECT) -for row in cur: - print(f"The {row[0]} programming language appeared in {row[1]}.") - -con.close() diff --git a/Doc/includes/sqlite3/execsql_printall_1.py b/Doc/includes/sqlite3/execsql_printall_1.py deleted file mode 100644 index b3b42b5567df3b..00000000000000 --- a/Doc/includes/sqlite3/execsql_printall_1.py +++ /dev/null @@ -1,15 +0,0 @@ -import sqlite3 - -# Create a connection to the database file "mydb": -con = sqlite3.connect("mydb") - -# Get a Cursor object that operates in the context of Connection con: -cur = con.cursor() - -# Execute the SELECT statement: -cur.execute("select * from lang order by first_appeared") - -# Retrieve all rows as a sequence and print that sequence: -print(cur.fetchall()) - -con.close() diff --git a/Doc/includes/sqlite3/insert_more_langs.py b/Doc/includes/sqlite3/insert_more_langs.py deleted file mode 100644 index ceef949608449e..00000000000000 --- a/Doc/includes/sqlite3/insert_more_langs.py +++ /dev/null @@ -1,18 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") - -cur = con.cursor() - -languages = ( - ("Smalltalk", 1972), - ("Swift", 2014), -) - -for lang in languages: - cur.execute("insert into lang (name, first_appeared) values (?, ?)", lang) - -# The changes will not be saved unless the transaction is committed explicitly: -con.commit() - -con.close() diff --git a/Doc/includes/sqlite3/parse_colnames.py b/Doc/includes/sqlite3/parse_colnames.py deleted file mode 100644 index 5f01dbfe1cb524..00000000000000 --- a/Doc/includes/sqlite3/parse_colnames.py +++ /dev/null @@ -1,10 +0,0 @@ -import sqlite3 -import datetime - -con = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_COLNAMES) -cur = con.cursor() -cur.execute('select ? as "x [timestamp]"', (datetime.datetime.now(),)) -dt = cur.fetchone()[0] -print(dt, type(dt)) - -con.close() diff --git a/Doc/includes/sqlite3/shared_cache.py b/Doc/includes/sqlite3/shared_cache.py deleted file mode 100644 index 30e71c935ff62e..00000000000000 --- a/Doc/includes/sqlite3/shared_cache.py +++ /dev/null @@ -1,6 +0,0 @@ -import sqlite3 - -# The shared cache is only available in SQLite versions 3.3.3 or later -# See the SQLite documentation for details. - -sqlite3.enable_shared_cache(True) diff --git a/Doc/includes/sqlite3/simple_tableprinter.py b/Doc/includes/sqlite3/simple_tableprinter.py deleted file mode 100644 index 9be6e4f414acd8..00000000000000 --- a/Doc/includes/sqlite3/simple_tableprinter.py +++ /dev/null @@ -1,25 +0,0 @@ -import sqlite3 - -FIELD_MAX_WIDTH = 20 - -con = sqlite3.connect("mydb") -cur = con.cursor() -cur.execute("select * from lang order by name, first_appeared") - -# Print a header. -for fieldDesc in cur.description: - print(fieldDesc[0].ljust(FIELD_MAX_WIDTH), end=' ') -print() # Finish the header with a newline. -print('-' * 78) - -# For each row, print the value of each field left-justified within -# the maximum possible width of that field. -fieldIndices = range(len(cur.description)) -for row in cur: - for fieldIndex in fieldIndices: - fieldValue = str(row[fieldIndex]) - print(fieldValue.ljust(FIELD_MAX_WIDTH), end=' ') - - print() # Finish the row with a newline. - -con.close() From 60d343a81679ea90ae0e08fadcd132c16906a51a Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Wed, 19 May 2021 12:18:10 +0300 Subject: [PATCH 009/160] bpo-44010: IDLE: colorize pattern-matching soft keywords (GH-25851) --- Doc/library/idle.rst | 6 + Doc/whatsnew/3.10.rst | 6 + Lib/idlelib/colorizer.py | 135 ++++++---- Lib/idlelib/help.html | 27 +- Lib/idlelib/idle_test/test_colorizer.py | 239 ++++++++++++++++-- .../2021-05-09-09-02-09.bpo-44010.TaLe9x.rst | 5 + 6 files changed, 345 insertions(+), 73 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 3c302115b5f408..faa34e69ff15d7 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -613,6 +613,12 @@ keywords, builtin class and function names, names following ``class`` and ``def``, strings, and comments. For any text window, these are the cursor (when present), found text (when possible), and selected text. +IDLE also highlights the :ref:`soft keywords ` :keyword:`match`, +:keyword:`case `, and :keyword:`_ ` in +pattern-matching statements. However, this highlighting is not perfect and +will be incorrect in some rare cases, including some ``_``-s in ``case`` +patterns. + Text coloring is done in the background, so uncolorized text is occasionally visible. To change the color scheme, use the Configure IDLE dialog Highlighting tab. The marking of debugger breakpoint lines in the editor and diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 926679e6f32dc5..570af7f3b61814 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1030,6 +1030,12 @@ Terry Jan Reedy in :issue:`37892`.) We expect to backport these shell changes to a future 3.9 maintenance release. +Highlight the new :ref:`soft keywords ` :keyword:`match`, +:keyword:`case `, and :keyword:`_ ` in +pattern-matching statements. However, this highlighting is not perfect +and will be incorrect in some rare cases, including some ``_``-s in +``case`` patterns. (Contributed by Tal Einat in bpo-44010.) + importlib.metadata ------------------ diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index 3c527409731afa..e9f19c145c8673 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -16,6 +16,32 @@ def any(name, alternates): def make_pat(): kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b" + match_softkw = ( + r"^[ \t]*" + # at beginning of line + possible indentation + r"(?Pmatch)\b" + + r"(?![ \t]*(?:" + "|".join([ # not followed by ... + r"[:,;=^&|@~)\]}]", # a character which means it can't be a + # pattern-matching statement + r"\b(?:" + r"|".join(keyword.kwlist) + r")\b", # a keyword + ]) + + r"))" + ) + case_default = ( + r"^[ \t]*" + # at beginning of line + possible indentation + r"(?Pcase)" + + r"[ \t]+(?P_\b)" + ) + case_softkw_and_pattern = ( + r"^[ \t]*" + # at beginning of line + possible indentation + r"(?Pcase)\b" + + r"(?![ \t]*(?:" + "|".join([ # not followed by ... + r"_\b", # a lone underscore + r"[:,;=^&|@~)\]}]", # a character which means it can't be a + # pattern-matching case + r"\b(?:" + r"|".join(keyword.kwlist) + r")\b", # a keyword + ]) + + r"))" + ) builtinlist = [str(name) for name in dir(builtins) if not name.startswith('_') and name not in keyword.kwlist] @@ -27,12 +53,29 @@ def make_pat(): sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) - return (kw + "|" + builtin + "|" + comment + "|" + string + - "|" + any("SYNC", [r"\n"])) + prog = re.compile("|".join([ + builtin, comment, string, kw, + match_softkw, case_default, + case_softkw_and_pattern, + any("SYNC", [r"\n"]), + ]), + re.DOTALL | re.MULTILINE) + return prog -prog = re.compile(make_pat(), re.S) -idprog = re.compile(r"\s+(\w+)", re.S) +prog = make_pat() +idprog = re.compile(r"\s+(\w+)") +prog_group_name_to_tag = { + "MATCH_SOFTKW": "KEYWORD", + "CASE_SOFTKW": "KEYWORD", + "CASE_DEFAULT_UNDERSCORE": "KEYWORD", + "CASE_SOFTKW2": "KEYWORD", +} + + +def matched_named_groups(re_match): + "Get only the non-empty named groups from an re.Match object." + return ((k, v) for (k, v) in re_match.groupdict().items() if v) def color_config(text): @@ -231,14 +274,10 @@ def recolorize(self): def recolorize_main(self): "Evaluate text and apply colorizing tags." next = "1.0" - while True: - item = self.tag_nextrange("TODO", next) - if not item: - break - head, tail = item - self.tag_remove("SYNC", head, tail) - item = self.tag_prevrange("SYNC", head) - head = item[1] if item else "1.0" + while todo_tag_range := self.tag_nextrange("TODO", next): + self.tag_remove("SYNC", todo_tag_range[0], todo_tag_range[1]) + sync_tag_range = self.tag_prevrange("SYNC", todo_tag_range[0]) + head = sync_tag_range[1] if sync_tag_range else "1.0" chars = "" next = head @@ -256,23 +295,8 @@ def recolorize_main(self): return for tag in self.tagdefs: self.tag_remove(tag, mark, next) - chars = chars + line - m = self.prog.search(chars) - while m: - for key, value in m.groupdict().items(): - if value: - a, b = m.span(key) - self.tag_add(key, - head + "+%dc" % a, - head + "+%dc" % b) - if value in ("def", "class"): - m1 = self.idprog.match(chars, b) - if m1: - a, b = m1.span(1) - self.tag_add("DEFINITION", - head + "+%dc" % a, - head + "+%dc" % b) - m = self.prog.search(chars, m.end()) + chars += line + self._add_tags_in_section(chars, head) if "SYNC" in self.tag_names(next + "-1c"): head = next chars = "" @@ -291,6 +315,40 @@ def recolorize_main(self): if DEBUG: print("colorizing stopped") return + def _add_tag(self, start, end, head, matched_group_name): + """Add a tag to a given range in the text widget. + + This is a utility function, receiving the range as `start` and + `end` positions, each of which is a number of characters + relative to the given `head` index in the text widget. + + The tag to add is determined by `matched_group_name`, which is + the name of a regular expression "named group" as matched by + by the relevant highlighting regexps. + """ + tag = prog_group_name_to_tag.get(matched_group_name, + matched_group_name) + self.tag_add(tag, + f"{head}+{start:d}c", + f"{head}+{end:d}c") + + def _add_tags_in_section(self, chars, head): + """Parse and add highlighting tags to a given part of the text. + + `chars` is a string with the text to parse and to which + highlighting is to be applied. + + `head` is the index in the text widget where the text is found. + """ + for m in self.prog.finditer(chars): + for name, matched_text in matched_named_groups(m): + a, b = m.span(name) + self._add_tag(a, b, head, name) + if matched_text in ("def", "class"): + if m1 := self.idprog.match(chars, b): + a, b = m1.span(1) + self._add_tag(a, b, head, "DEFINITION") + def removecolors(self): "Remove all colorizing tags." for tag in self.tagdefs: @@ -299,27 +357,14 @@ def removecolors(self): def _color_delegator(parent): # htest # from tkinter import Toplevel, Text + from idlelib.idle_test.test_colorizer import source from idlelib.percolator import Percolator top = Toplevel(parent) top.title("Test ColorDelegator") x, y = map(int, parent.geometry().split('+')[1:]) - top.geometry("700x250+%d+%d" % (x + 20, y + 175)) - source = ( - "if True: int ('1') # keyword, builtin, string, comment\n" - "elif False: print(0)\n" - "else: float(None)\n" - "if iF + If + IF: 'keyword matching must respect case'\n" - "if'': x or'' # valid keyword-string no-space combinations\n" - "async def f(): await g()\n" - "# All valid prefixes for unicode and byte strings should be colored.\n" - "'x', '''x''', \"x\", \"\"\"x\"\"\"\n" - "r'x', u'x', R'x', U'x', f'x', F'x'\n" - "fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x'\n" - "b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x'\n" - "# Invalid combinations of legal characters should be half colored.\n" - "ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x'\n" - ) + top.geometry("700x550+%d+%d" % (x + 20, y + 175)) + text = Text(top, background="white") text.pack(expand=1, fill="both") text.insert("insert", source) diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index e80384b7775222..19041c6054e4cc 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -5,7 +5,7 @@ - IDLE — Python 3.10.0a6 documentation + IDLE — Python 3.11.0a0 documentation @@ -18,7 +18,7 @@ @@ -71,7 +71,7 @@

Navigation

  • - 3.10.0a6 Documentation » + 3.11.0a0 Documentation »
  • @@ -102,7 +102,7 @@

    Navigation

    IDLE

    -

    Source code: Lib/idlelib/

    +

    Source code: Lib/idlelib/


    IDLE is Python’s Integrated Development and Learning Environment.

    IDLE has the following features:

    @@ -581,6 +581,11 @@

    Text colorsclass and def, strings, and comments. For any text window, these are the cursor (when present), found text (when possible), and selected text.

    +

    IDLE also highlights the soft keywords match, +case, and _ in +pattern-matching statements. However, this highlighting is not perfect and +will be incorrect in some rare cases, including some _-s in case +patterns.

    Text coloring is done in the background, so uncolorized text is occasionally visible. To change the color scheme, use the Configure IDLE dialog Highlighting tab. The marking of debugger breakpoint lines in the editor and @@ -685,7 +690,7 @@

    Running user codesys.modules starts with more entries, -and threading.activeCount() returns 2 instead of 1.

    +and threading.active_count() returns 2 instead of 1.

    By default, IDLE runs user code in a separate OS process rather than in the user interface process that runs the shell and editor. In the execution process, it replaces sys.stdin, sys.stdout, and sys.stderr @@ -939,7 +944,7 @@

    This Page

    • Report a Bug
    • - Show Source
    • @@ -971,7 +976,7 @@

      Navigation

    • - 3.10.0a6 Documentation » + 3.11.0a0 Documentation »
    • @@ -997,13 +1002,19 @@

      Navigation