From 035681bb902da90a5916e24f085aa8f882d6d1d5 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 26 Oct 2023 15:17:28 +0900 Subject: [PATCH] gh-67224: Show source lines in tracebacks when using the -c option when running Python (#111200) --- Include/internal/pycore_pylifecycle.h | 3 + Lib/linecache.py | 8 +- Lib/test/test_cmd_line_script.py | 10 +++ Lib/test/test_io.py | 4 +- Lib/test/test_repl.py | 2 +- Lib/test/test_subprocess.py | 4 +- Lib/test/test_sys.py | 8 +- Lib/test/test_traceback.py | 2 + Lib/test/test_warnings/__init__.py | 4 + Lib/traceback.py | 11 ++- ...3-10-23-15-44-47.gh-issue-67224.S4D6CR.rst | 2 + Modules/main.c | 2 +- Python/pythonrun.c | 80 ++++++++++++++----- 13 files changed, 104 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-23-15-44-47.gh-issue-67224.S4D6CR.rst diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index ec003a1dad25953..61e0150e89009ce 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -114,6 +114,9 @@ extern int _Py_LegacyLocaleDetected(int warn); // Export for 'readline' shared extension PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); +// Export for special main.c string compiling with source tracebacks +int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); + #ifdef __cplusplus } #endif diff --git a/Lib/linecache.py b/Lib/linecache.py index c1c988d9df436c4..329a14053458b7b 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -5,10 +5,8 @@ that name. """ -import functools import sys import os -import tokenize __all__ = ["getline", "clearcache", "checkcache", "lazycache"] @@ -82,6 +80,8 @@ def updatecache(filename, module_globals=None): If something's wrong, print a message, discard the cache entry, and return an empty list.""" + import tokenize + if filename in cache: if len(cache[filename]) != 1: cache.pop(filename, None) @@ -176,11 +176,13 @@ def lazycache(filename, module_globals): get_source = getattr(loader, 'get_source', None) if name and get_source: - get_lines = functools.partial(get_source, name) + def get_lines(name=name, *args, **kwargs): + return get_source(name, *args, **kwargs) cache[filename] = (get_lines,) return True return False + def _register_code(code, string, name): cache[code] = ( len(string), diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e9405ff8b214145..48754d5a63da3b1 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -684,6 +684,16 @@ def test_syntaxerror_null_bytes_in_multiline_string(self): ] ) + def test_source_lines_are_shown_when_running_source(self): + _, _, stderr = assert_python_failure("-c", "1/0") + expected_lines = [ + b'Traceback (most recent call last):', + b' File "", line 1, in ', + b' 1/0', + b' ~^~', + b'ZeroDivisionError: division by zero'] + self.assertEqual(stderr.splitlines(), expected_lines) + def test_syntaxerror_does_not_crash(self): script = "nonlocal x\n" with os_helper.temp_dir() as script_dir: diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 022cf21a4709a27..4c80429684e525d 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4396,11 +4396,11 @@ def test_check_encoding_warning(self): ''') proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code) warnings = proc.err.splitlines() - self.assertEqual(len(warnings), 2) + self.assertEqual(len(warnings), 4) self.assertTrue( warnings[0].startswith(b":5: EncodingWarning: ")) self.assertTrue( - warnings[1].startswith(b":8: EncodingWarning: ")) + warnings[2].startswith(b":8: EncodingWarning: ")) def test_text_encoding(self): # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 2ee5117bcd7bd40..7533376e015e731 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -184,7 +184,7 @@ def bar(x): p.stdin.write(user_input) user_input2 = dedent(""" import linecache - print(linecache.cache['']) + print(linecache.cache['-1']) """) p.stdin.write(user_input2) output = kill_python(p) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index a865df1082fba36..fe1a3675fced653 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1769,9 +1769,9 @@ def test_encoding_warning(self): cp = subprocess.run([sys.executable, "-Xwarn_default_encoding", "-c", code], capture_output=True) lines = cp.stderr.splitlines() - self.assertEqual(len(lines), 2, lines) + self.assertEqual(len(lines), 4, lines) self.assertTrue(lines[0].startswith(b":2: EncodingWarning: ")) - self.assertTrue(lines[1].startswith(b":3: EncodingWarning: ")) + self.assertTrue(lines[2].startswith(b":3: EncodingWarning: ")) def _get_test_grp_name(): diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index da2135061515943..b16e1e7f70a24ac 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1114,14 +1114,18 @@ def check(tracebacklimit, expected): traceback = [ b'Traceback (most recent call last):', b' File "", line 8, in ', + b' f2()', b' File "", line 6, in f2', + b' f1()', b' File "", line 4, in f1', + b' 1 / 0', + b' ~~^~~', b'ZeroDivisionError: division by zero' ] check(10, traceback) check(3, traceback) - check(2, traceback[:1] + traceback[2:]) - check(1, traceback[:1] + traceback[3:]) + check(2, traceback[:1] + traceback[3:]) + check(1, traceback[:1] + traceback[5:]) check(0, [traceback[-1]]) check(-1, [traceback[-1]]) check(1<<1000, traceback) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index a0f7ad81bd540d7..43f15ab6179bebc 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -313,6 +313,8 @@ def __del__(self): rc, stdout, stderr = assert_python_ok('-c', code) expected = [b'Traceback (most recent call last):', b' File "", line 8, in __init__', + b' x = 1 / 0', + b' ^^^^^', b'ZeroDivisionError: division by zero'] self.assertEqual(stderr.splitlines(), expected) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 83237f5fe0d1b3b..2c523230e7e97ff 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1233,6 +1233,10 @@ def test_conflicting_envvar_and_command_line(self): self.assertEqual(stderr.splitlines(), [b"Traceback (most recent call last):", b" File \"\", line 1, in ", + b' import sys, warnings; sys.stdout.write(str(sys.warnoptions)); warnings.w' + b"arn('Message', DeprecationWarning)", + b' ^^^^^^^^^^' + b'^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', b"DeprecationWarning: Message"]) def test_default_filter_configuration(self): diff --git a/Lib/traceback.py b/Lib/traceback.py index d3c581f92ba1b87..4f0dff9bed08666 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -476,12 +476,11 @@ def format_frame_summary(self, frame_summary): gets called for every frame to be printed in the stack summary. """ row = [] - if frame_summary.filename.startswith("", line {}, in {}\n'.format( - frame_summary.lineno, frame_summary.name)) - else: - row.append(' File "{}", line {}, in {}\n'.format( - frame_summary.filename, frame_summary.lineno, frame_summary.name)) + filename = frame_summary.filename + if frame_summary.filename.startswith("-"): + filename = "" + row.append(' File "{}", line {}, in {}\n'.format( + filename, frame_summary.lineno, frame_summary.name)) if frame_summary.line: stripped_line = frame_summary.line.strip() row.append(' {}\n'.format(stripped_line)) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-23-15-44-47.gh-issue-67224.S4D6CR.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-23-15-44-47.gh-issue-67224.S4D6CR.rst new file mode 100644 index 000000000000000..b0474f38cff6f6b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-23-15-44-47.gh-issue-67224.S4D6CR.rst @@ -0,0 +1,2 @@ +Show source lines in tracebacks when using the ``-c`` option when running +Python. Patch by Pablo Galindo diff --git a/Modules/main.c b/Modules/main.c index b5ee34d0141daf8..df2ce5502450889 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -249,7 +249,7 @@ pymain_run_command(wchar_t *command) PyCompilerFlags cf = _PyCompilerFlags_INIT; cf.cf_flags |= PyCF_IGNORE_COOKIE; - ret = PyRun_SimpleStringFlags(PyBytes_AsString(bytes), &cf); + ret = _PyRun_SimpleStringFlagsWithName(PyBytes_AsString(bytes), "", &cf); Py_DECREF(bytes); return (ret != 0); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index db4991662b8bb1c..79aeee1eb6f6b72 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -40,14 +40,17 @@ /* Forward */ static void flush_io(void); static PyObject *run_mod(mod_ty, PyObject *, PyObject *, PyObject *, - PyCompilerFlags *, PyArena *, PyObject*); + PyCompilerFlags *, PyArena *, PyObject*, int); static PyObject *run_pyc_file(FILE *, PyObject *, PyObject *, PyCompilerFlags *); static int PyRun_InteractiveOneObjectEx(FILE *, PyObject *, PyCompilerFlags *); static PyObject* pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags); - +static PyObject * +_PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, + PyObject *globals, PyObject *locals, PyCompilerFlags *flags, + int generate_new_source); int _PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit, @@ -281,7 +284,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, } PyObject *main_dict = PyModule_GetDict(main_module); // borrowed ref - PyObject *res = run_mod(mod, filename, main_dict, main_dict, flags, arena, interactive_src); + PyObject *res = run_mod(mod, filename, main_dict, main_dict, flags, arena, interactive_src, 1); _PyArena_Free(arena); Py_DECREF(main_module); if (res == NULL) { @@ -499,16 +502,25 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, int -PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) -{ +_PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags) { PyObject *main_module = PyImport_AddModuleRef("__main__"); if (main_module == NULL) { return -1; } PyObject *dict = PyModule_GetDict(main_module); // borrowed ref - PyObject *res = PyRun_StringFlags(command, Py_file_input, - dict, dict, flags); + PyObject *res = NULL; + if (name == NULL) { + res = PyRun_StringFlags(command, Py_file_input, dict, dict, flags); + } else { + PyObject* the_name = PyUnicode_FromString(name); + if (!the_name) { + PyErr_Print(); + return -1; + } + res = _PyRun_StringFlagsWithName(command, the_name, Py_file_input, dict, dict, flags, 0); + Py_DECREF(the_name); + } Py_DECREF(main_module); if (res == NULL) { PyErr_Print(); @@ -519,6 +531,12 @@ PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) return 0; } +int +PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) +{ + return _PyRun_SimpleStringFlagsWithName(command, NULL, flags); +} + int _Py_HandleSystemExit(int *exitcode_p) { @@ -1131,9 +1149,10 @@ void PyErr_DisplayException(PyObject *exc) PyErr_Display(NULL, exc, NULL); } -PyObject * -PyRun_StringFlags(const char *str, int start, PyObject *globals, - PyObject *locals, PyCompilerFlags *flags) +static PyObject * +_PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, + PyObject *globals, PyObject *locals, PyCompilerFlags *flags, + int generate_new_source) { PyObject *ret = NULL; mod_ty mod; @@ -1143,17 +1162,36 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals, if (arena == NULL) return NULL; + PyObject* source = NULL; _Py_DECLARE_STR(anon_string, ""); - mod = _PyParser_ASTFromString( - str, &_Py_STR(anon_string), start, flags, arena); - if (mod != NULL) - ret = run_mod(mod, &_Py_STR(anon_string), globals, locals, flags, arena, NULL); + if (name) { + source = PyUnicode_FromString(str); + if (!source) { + PyErr_Clear(); + } + } else { + name = &_Py_STR(anon_string); + } + + mod = _PyParser_ASTFromString(str, name, start, flags, arena); + + if (mod != NULL) { + ret = run_mod(mod, name, globals, locals, flags, arena, source, generate_new_source); + } + Py_XDECREF(source); _PyArena_Free(arena); return ret; } +PyObject * +PyRun_StringFlags(const char *str, int start, PyObject *globals, + PyObject *locals, PyCompilerFlags *flags) { + + return _PyRun_StringFlagsWithName(str, NULL, start, globals, locals, flags, 0); +} + static PyObject * pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags) @@ -1173,7 +1211,7 @@ pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *ret; if (mod != NULL) { - ret = run_mod(mod, filename, globals, locals, flags, arena, NULL); + ret = run_mod(mod, filename, globals, locals, flags, arena, NULL, 0); } else { ret = NULL; @@ -1261,15 +1299,19 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py static PyObject * run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, - PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src) + PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src, + int generate_new_source) { PyThreadState *tstate = _PyThreadState_GET(); PyObject* interactive_filename = filename; if (interactive_src) { PyInterpreterState *interp = tstate->interp; - interactive_filename = PyUnicode_FromFormat( - "", interp->_interactive_src_count++ - ); + if (generate_new_source) { + interactive_filename = PyUnicode_FromFormat( + "%U-%d", filename, interp->_interactive_src_count++); + } else { + Py_INCREF(interactive_filename); + } if (interactive_filename == NULL) { return NULL; }