From 039e45f1a0c44a6b4fffd66a64bceb02c4f5653d Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Fri, 5 Nov 2021 08:05:32 +0100 Subject: [PATCH 01/10] git subrepo clone (merge) --branch=2.x --force https://github.com/spyder-ide/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "efe0a3f22" upstream: origin: "https://github.com/spyder-ide/spyder-kernels.git" branch: "2.x" commit: "efe0a3f22" git-subrepo: version: "0.4.1" origin: "https://github.com/ingydotnet/git-subrepo" commit: "a04d8c2" --- external-deps/spyder-kernels/.gitrepo | 4 +- .../spyder_kernels/console/kernel.py | 46 +++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 9f379dcf08e..f37455dce4e 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = 2.x - commit = ac40350c9260e2e5d73f0fc3f8a9cc15da8d8a03 - parent = 01421c2ae26c7a3861fc4c912f295f9f2b2c1c1f + commit = efe0a3f2294d022b848a4929a7a1966abd131718 + parent = 575e34d5a84de29eca0cd1fcf3db9ded4030416b method = merge cmdver = 0.4.1 diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index 0ceecaa0ddf..ae19295912b 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -31,6 +31,8 @@ from spyder_kernels.utils.nsview import get_remote_data, make_remote_view from spyder_kernels.console.shell import SpyderShell +if PY3: + import faulthandler # Excluded variables from the Variable Explorer (i.e. they are not # shown at all there) @@ -78,6 +80,8 @@ def __init__(self, *args, **kwargs): 'get_matplotlib_backend': self.get_matplotlib_backend, 'pdb_input_reply': self.pdb_input_reply, '_interrupt_eventloop': self._interrupt_eventloop, + 'enable_faulthandler': self.enable_faulthandler, + "flush_std": self.flush_std, } for call_id in handlers: self.frontend_comm.register_call_handler( @@ -88,8 +92,14 @@ def __init__(self, *args, **kwargs): self._mpl_backend_error = None self._running_namespace = None self._pdb_input_line = None + self.faulthandler_handle = None # -- Public API ----------------------------------------------------------- + def do_shutdown(self, restart): + """Disable faulthandler if enabled before proceeding.""" + self.disable_faulthandler() + super(SpyderKernel, self).do_shutdown(restart) + def frontend_call(self, blocking=False, broadcast=True, timeout=None, callback=None): """Call the frontend.""" @@ -105,6 +115,42 @@ def frontend_call(self, blocking=False, broadcast=True, callback=callback, timeout=timeout) + def flush_std(self): + """Flush C standard streams.""" + sys.__stderr__.flush() + sys.__stdout__.flush() + + def enable_faulthandler(self, fn): + """ + Open a file to save the faulthandling and identifiers for + internal threads. + """ + if not PY3: + # Not implemented + return + self.disable_faulthandler() + f = open(fn, 'w') + self.faulthandler_handle = f + f.write("Main thread id:\n") + f.write(hex(threading.main_thread().ident)) + f.write('\nSystem threads ids:\n') + f.write(" ".join([hex(thread.ident) for thread in threading.enumerate() + if thread is not threading.main_thread()])) + f.write('\n') + faulthandler.enable(f) + + def disable_faulthandler(self): + """ + Cancel the faulthandling, close the file handle and remove the file. + """ + if not PY3: + # Not implemented + return + if self.faulthandler_handle: + faulthandler.disable() + self.faulthandler_handle.close() + self.faulthandler_handle = None + # --- For the Variable Explorer def set_namespace_view_settings(self, settings): """Set namespace_view_settings.""" From 2895ea942e606b0072228c9fb8f4070d4bbcf800 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 10 Nov 2021 13:38:23 +0100 Subject: [PATCH 02/10] tests --- spyder/app/tests/test_mainwindow.py | 69 +++++++++++++++++++ .../tests/test_ipythonconsole.py | 61 +++++++++++----- 2 files changed, 114 insertions(+), 16 deletions(-) diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index d6f52e207fa..09ba3fe5ee7 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -4229,5 +4229,74 @@ def test_add_external_plugins_to_dependencies(main_window): assert 'spyder-boilerplate' in external_names +@pytest.mark.slow +@flaky(max_runs=3) +def test_print_multiprocessing(main_window, qtbot, tmpdir): + """Test the runcell command.""" + # Write code with a cell to a file + code = """ +import multiprocessing +import sys +def test_func(): + print("Test stdout") + print("Test stderr", file=sys.stderr) + +if __name__ == "__main__": + p = multiprocessing.Process(target=test_func) + p.start() + p.join() +""" + + p = tmpdir.join("print-test.py") + p.write(code) + main_window.editor.load(to_text_string(p)) + shell = main_window.ipyconsole.get_current_shellwidget() + qtbot.waitUntil(lambda: shell._prompt_html is not None, + timeout=SHELL_TIMEOUT) + control = main_window.ipyconsole.get_widget().get_focus_widget() + + # Click the run button + run_action = main_window.run_toolbar_actions[0] + run_button = main_window.run_toolbar.widgetForAction(run_action) + with qtbot.waitSignal(shell.executed): + qtbot.mouseClick(run_button, Qt.LeftButton) + qtbot.wait(1000) + + assert 'Test stdout' in control.toPlainText() + assert 'Test stderr' in control.toPlainText() + + +@pytest.mark.slow +@flaky(max_runs=3) +@pytest.mark.skipif( + os.name == 'nt', + reason="ctypes.string_at(0) doesn't segfaults on windows") +def test_print_faulthandler(main_window, qtbot, tmpdir): + """Test the runcell command.""" + # Write code with a cell to a file + code = """ +def crash_func(): + import ctypes; ctypes.string_at(0) +crash_func() +""" + + p = tmpdir.join("print-test.py") + p.write(code) + main_window.editor.load(to_text_string(p)) + shell = main_window.ipyconsole.get_current_shellwidget() + qtbot.waitUntil(lambda: shell._prompt_html is not None, + timeout=SHELL_TIMEOUT) + control = main_window.ipyconsole.get_widget().get_focus_widget() + + # Click the run button + run_action = main_window.run_toolbar_actions[0] + run_button = main_window.run_toolbar.widgetForAction(run_action) + qtbot.mouseClick(run_button, Qt.LeftButton) + qtbot.wait(5000) + + assert 'Segmentation fault' in control.toPlainText() + assert 'in crash_func' in control.toPlainText() + + if __name__ == "__main__": pytest.main() diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py index 3fb17bc8431..e7b4a74e71c 100644 --- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py +++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py @@ -878,10 +878,10 @@ def test_read_stderr(ipyconsole, qtbot): # Set contents of the stderr file of the kernel content = 'Test text' - stderr_file = client.stderr_file + stderr_file = client.stderr_obj.filename codecs.open(stderr_file, 'w', 'cp437').write(content) # Assert that content is correct - assert content == client._read_stderr() + assert content == client.stderr_obj.get_contents() @flaky(max_runs=10) @@ -1267,9 +1267,9 @@ def test_stderr_file_is_removed_one_kernel(ipyconsole, qtbot, monkeypatch): # In a normal situation file should exist monkeypatch.setattr(QMessageBox, 'question', classmethod(lambda *args: QMessageBox.Yes)) - assert osp.exists(client.stderr_file) + assert osp.exists(client.stderr_obj.filename) ipyconsole.close_client(client=client) - assert not osp.exists(client.stderr_file) + assert not osp.exists(client.stderr_obj.filename) @flaky(max_runs=3) @@ -1288,14 +1288,14 @@ def test_stderr_file_is_removed_two_kernels(ipyconsole, qtbot, monkeypatch): client.connection_file, None, None, None) assert len(ipyconsole.get_widget().get_related_clients(client)) == 1 other_client = ipyconsole.get_widget().get_related_clients(client)[0] - assert client.stderr_file == other_client.stderr_file + assert client.stderr_obj.filename == other_client.stderr_obj.filename # In a normal situation file should exist monkeypatch.setattr(QMessageBox, 'question', classmethod(lambda *args: QMessageBox.Yes)) - assert osp.exists(client.stderr_file) + assert osp.exists(client.stderr_obj.filename) ipyconsole.close_client(client=client) - assert not osp.exists(client.stderr_file) + assert not osp.exists(client.stderr_obj.filename) @flaky(max_runs=3) @@ -1315,14 +1315,14 @@ def test_stderr_file_remains_two_kernels(ipyconsole, qtbot, monkeypatch): assert len(ipyconsole.get_widget().get_related_clients(client)) == 1 other_client = ipyconsole.get_widget().get_related_clients(client)[0] - assert client.stderr_file == other_client.stderr_file + assert client.stderr_obj.filename == other_client.stderr_obj.filename # In a normal situation file should exist monkeypatch.setattr(QMessageBox, "question", classmethod(lambda *args: QMessageBox.No)) - assert osp.exists(client.stderr_file) + assert osp.exists(client.stderr_obj.filename) ipyconsole.close_client(client=client) - assert osp.exists(client.stderr_file) + assert osp.exists(client.stderr_obj.filename) @flaky(max_runs=3) @@ -1361,15 +1361,15 @@ def test_kernel_crash(ipyconsole, qtbot): @flaky(max_runs=3) @pytest.mark.skipif(not os.name == 'nt', reason="Only works on Windows") -def test_remove_old_stderr_files(ipyconsole, qtbot): - """Test that we are removing old stderr files.""" +def test_remove_old_std_files(ipyconsole, qtbot): + """Test that we are removing old std files.""" # Create empty stderr file in our temp dir to see # if it's removed correctly. tmpdir = get_temp_dir() open(osp.join(tmpdir, 'foo.stderr'), 'a').close() # Assert that only that file is removed - ipyconsole._remove_old_stderr_files() + ipyconsole._remove_old_std_files() assert not osp.isfile(osp.join(tmpdir, 'foo.stderr')) @@ -1753,11 +1753,40 @@ def test_stderr_poll(ipyconsole, qtbot): qtbot.waitUntil(lambda: shell._prompt_html is not None, timeout=SHELL_TIMEOUT) client = ipyconsole.get_current_client() - with open(client.stderr_file, 'w') as f: + client.stderr_obj.handle.flush() + with open(client.stderr_obj.filename, 'a') as f: f.write("test_test") # Wait for the poll - qtbot.wait(2000) - assert "test_test" in ipyconsole.get_widget().get_focus_widget().toPlainText() + qtbot.waitUntil(lambda: "test_test" in ipyconsole.get_widget( + ).get_focus_widget().toPlainText()) + assert "test_test" in ipyconsole.get_widget( + ).get_focus_widget().toPlainText() + # Write a second time, makes sure it is not duplicated + client.stderr_obj.handle.flush() + with open(client.stderr_obj.filename, 'a') as f: + f.write("\ntest_test") + # Wait for the poll + qtbot.waitUntil(lambda: ipyconsole.get_widget().get_focus_widget( + ).toPlainText().count("test_test") == 2) + assert ipyconsole.get_widget().get_focus_widget().toPlainText( + ).count("test_test") == 2 + + +@flaky(max_runs=3) +def test_stdout_poll(ipyconsole, qtbot): + """Test if the content of stdout is printed to the console.""" + shell = ipyconsole.get_current_shellwidget() + qtbot.waitUntil(lambda: shell._prompt_html is not None, + timeout=SHELL_TIMEOUT) + client = ipyconsole.get_current_client() + client.stdout_obj.handle.flush() + with open(client.stdout_obj.filename, 'a') as f: + f.write("test_test") + # Wait for the poll + qtbot.waitUntil(lambda: "test_test" in ipyconsole.get_widget( + ).get_focus_widget().toPlainText(), timeout=5000) + assert "test_test" in ipyconsole.get_widget().get_focus_widget( + ).toPlainText() @flaky(max_runs=10) From 11d4357073fc9b3d533ba746a527c0fa0a284ea5 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 10 Nov 2021 13:51:37 +0100 Subject: [PATCH 03/10] Fix std show --- spyder/plugins/ipythonconsole/plugin.py | 10 +- .../plugins/ipythonconsole/widgets/client.py | 291 +++++++++++++----- .../ipythonconsole/widgets/main_widget.py | 22 +- 3 files changed, 229 insertions(+), 94 deletions(-) diff --git a/spyder/plugins/ipythonconsole/plugin.py b/spyder/plugins/ipythonconsole/plugin.py index 09b5b913e95..584d07fdaf4 100644 --- a/spyder/plugins/ipythonconsole/plugin.py +++ b/spyder/plugins/ipythonconsole/plugin.py @@ -244,7 +244,7 @@ def on_initialize(self): self.main.sig_pythonpath_changed.connect(self.update_path) self.sig_focus_changed.connect(self.main.plugin_focus_changed) - self._remove_old_stderr_files() + self._remove_old_std_files() @on_plugin_available(plugin=Plugins.Preferences) def on_preferences_available(self): @@ -366,17 +366,17 @@ def _on_project_loaded(self): def _on_project_closed(self): self.get_widget().update_active_project_path(None) - def _remove_old_stderr_files(self): + def _remove_old_std_files(self): """ - Remove stderr files left by previous Spyder instances. + Remove std files left by previous Spyder instances. This is only required on Windows because we can't - clean up stderr files while Spyder is running on it. + clean up std files while Spyder is running on it. """ if os.name == 'nt': tmpdir = get_temp_dir() for fname in os.listdir(tmpdir): - if osp.splitext(fname)[1] == '.stderr': + if osp.splitext(fname)[1] in ('.stderr', '.stdout', '.fault'): try: os.remove(osp.join(tmpdir, fname)) except Exception: diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 69ba75d7c3d..482e7af98ff 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -21,6 +21,7 @@ import codecs import os import os.path as osp +import re from string import Template import time @@ -71,6 +72,81 @@ time.monotonic = time.time +class Std_File(): + def __init__(self, filename): + self.filename = filename + self._mtime = 0 + self._cursor = 0 + self._handle = None + + @property + def handle(self): + """Get handle to file.""" + if self._handle is None and self.filename is not None: + # Needed to prevent any error that could appear. + # See spyder-ide/spyder#6267. + try: + self._handle = codecs.open( + self.filename, 'w', encoding='utf-8') + except Exception: + pass + return self._handle + + def remove(self): + """Remove file associated with the client.""" + try: + # Defer closing the handle until the client + # is closed because jupyter_client needs it open + # while it tries to restart the kernel + if self._handle is not None: + self._handle.close() + os.remove(self.filename) + self._handle = None + except Exception: + pass + + def get_contents(self): + """Get the contents of the std kernel file.""" + try: + with open(self.filename, 'rb') as f: + # We need to read the file as bytes to be able to + # detect its encoding with chardet + text = f.read() + + # This is needed to avoid showing an empty error message + # when the kernel takes too much time to start. + # See spyder-ide/spyder#8581. + if not text: + return '' + + # This is needed since the file could be encoded + # in something different to utf-8. + # See spyder-ide/spyder#4191. + encoding = get_coding(text) + text = to_text_string(text, encoding) + return text + except Exception: + return None + + def poll_file_change(self): + """Check if the stdout file just changed""" + if self._handle is not None and not self._handle.closed: + self._handle.flush() + try: + mtime = os.stat(self.filename).st_mtime + except Exception: + return + + if mtime == self._mtime: + return + self._mtime = mtime + text = self.get_contents() + if text: + ret_text = text[self._cursor:] + self._cursor = len(text) + return ret_text + + # ---------------------------------------------------------------------------- # Client widget # ---------------------------------------------------------------------------- @@ -108,7 +184,8 @@ def __init__(self, parent, id_, ask_before_closing=False, css_path=None, configuration=None, - handlers={}): + handlers={}, + std_dir=None): super(ClientWidget, self).__init__(parent) SaveHistoryMixin.__init__(self, history_filename) @@ -131,8 +208,9 @@ def __init__(self, parent, id_, self.options_button = options_button self.history = [] self.allow_rename = True - self.stderr_dir = None + self.std_dir = std_dir self.is_error_shown = False + self.error_text = None self.restart_thread = None self.give_focus = give_focus @@ -177,12 +255,22 @@ def __init__(self, parent, id_, # --- Dialog manager self.dialog_manager = DialogManager() - # Poll for stderr changes - self.stderr_mtime = 0 - self.stderr_timer = QTimer(self) - self.stderr_timer.timeout.connect(self.poll_stderr_file_change) - self.stderr_timer.setInterval(1000) - self.stderr_timer.start() + self.stderr_obj = None + self.stdout_obj = None + self.fault_obj = None + self.std_poll_timer = None + if not self.is_external_kernel: + # Can not connect for external kernels + self.stderr_obj = Std_File(self.std_filename('.stderr')) + self.stdout_obj = Std_File(self.std_filename('.stdout')) + self.std_poll_timer = QTimer(self) + self.std_poll_timer.timeout.connect(self.poll_std_file_change) + self.std_poll_timer.setInterval(1000) + self.std_poll_timer.start() + self.shellwidget.executed.connect(self.poll_std_file_change) + if self.hostname is None: + # Can not read file that is not on this computer + self.fault_obj = Std_File(self.std_filename('.fault')) def __del__(self): """Close threads to avoid segfault.""" @@ -310,11 +398,14 @@ def _abort_kernel_restart(self): We also ignore errors about comms, which are irrelevant. """ - stderr = self.get_stderr_contents() - if stderr and 'No such comm' not in stderr: - return True - else: + stderr = self.stderr_obj.get_contents() + if not stderr: return False + # There is an error. If it is only about comms, ignore. + for line in stderr.splitlines(): + if line and 'No such comm' not in line: + return True + return False def _connect_control_signals(self): """Connect signals of control widgets.""" @@ -332,78 +423,62 @@ def _connect_control_signals(self): page_control.sig_show_find_widget_requested.connect( self.container.find_widget.show) - # ---- Public methods ----------------------------------------------------- - @property - def kernel_id(self): - """Get kernel id""" - if self.connection_file is not None: - json_file = osp.basename(self.connection_file) - return json_file.split('.json')[0] - - @property - def stderr_file(self): - """Filename to save kernel stderr output.""" - stderr_file = None + # ----- Public API -------------------------------------------------------- + def std_filename(self, extension): + """Filename to save kernel output.""" + file = None if self.connection_file is not None: - stderr_file = self.kernel_id + '.stderr' - if self.stderr_dir is not None: - stderr_file = osp.join(self.stderr_dir, stderr_file) + file = self.kernel_id + extension + if self.std_dir is not None: + file = osp.join(self.std_dir, file) else: try: - stderr_file = osp.join(get_temp_dir(), stderr_file) + file = osp.join(get_temp_dir(), file) except (IOError, OSError): - stderr_file = None - return stderr_file + file = None + return file @property - def stderr_handle(self): - """Get handle to stderr_file.""" - if self.stderr_file is not None: - # Needed to prevent any error that could appear. - # See spyder-ide/spyder#6267. - try: - handle = codecs.open(self.stderr_file, 'w', encoding='utf-8') - except Exception: - handle = None - else: - handle = None - - return handle + def kernel_id(self): + """Get kernel id.""" + if self.connection_file is not None: + json_file = osp.basename(self.connection_file) + return json_file.split('.json')[0] - def remove_stderr_file(self): + def remove_std_files(self, is_last_client=True): """Remove stderr_file associated with the client.""" try: - # Defer closing the stderr_handle until the client - # is closed because jupyter_client needs it open - # while it tries to restart the kernel - self.stderr_handle.close() - os.remove(self.stderr_file) - except Exception: + self.shellwidget.executed.disconnect(self.poll_std_file_change) + except TypeError: pass - - def get_stderr_contents(self): - """Get the contents of the stderr kernel file.""" - try: - stderr = self._read_stderr() - except Exception: - stderr = None - return stderr + if self.std_poll_timer is not None: + self.std_poll_timer.stop() + if is_last_client: + if self.stderr_obj is not None: + self.stderr_obj.remove() + if self.stdout_obj is not None: + self.stdout_obj.remove() + if self.fault_obj is not None: + self.fault_obj.remove() @Slot() - def poll_stderr_file_change(self): - """Check if the stderr file just changed""" - try: - mtime = os.stat(self.stderr_file).st_mtime - except Exception: - return - - if mtime == self.stderr_mtime: - return - self.stderr_mtime = mtime - stderr = self.get_stderr_contents() + def poll_std_file_change(self): + """Check if the stderr or stdout file just changed.""" + self.shellwidget.call_kernel().flush_std() + stderr = self.stderr_obj.poll_file_change() if stderr: + if self.shellwidget.isHidden(): + # Avoid printing the same thing again + if self.error_text != '%s' % stderr: + full_stderr = self.stderr_obj.get_contents() + self.show_kernel_error('%s' % full_stderr) + else: + self.shellwidget._append_plain_text( + '\n' + stderr, before_prompt=True) + stdout = self.stdout_obj.poll_file_change() + if stdout: self.shellwidget._append_plain_text( - '\n' + stderr, before_prompt=True) + '\n' + stdout, before_prompt=True) def configure_shellwidget(self, give_focus=True): """Configure shellwidget after kernel is connected.""" @@ -457,6 +532,11 @@ def configure_shellwidget(self, give_focus=True): # To apply style self.set_color_scheme(self.shellwidget.syntax_style, reset=False) + if self.fault_obj is not None: + # To display faulthandler + self.shellwidget.call_kernel().enable_faulthandler( + self.fault_obj.filename) + def add_to_history(self, command): """Add command to history""" if self.shellwidget.is_debugging(): @@ -478,7 +558,10 @@ def stop_button_click_handler(self): def show_kernel_error(self, error): """Show kernel initialization errors in infowidget.""" - InstallerIPythonKernelError(error) + self.error_text = error + if "issue1666807" not in error: + # The installer has some strange relative python + InstallerIPythonKernelError(error) # Replace end of line chars with
eol = sourcecode.get_eol_chars(error) @@ -559,8 +642,7 @@ def set_color_scheme(self, color_scheme, reset=True): def shutdown(self, is_last_client): """Shutdown connection and kernel if needed.""" self.dialog_manager.close_all() - if is_last_client: - self.remove_stderr_file() + self.remove_std_files(is_last_client) shutdown_kernel = is_last_client and not self.is_external_kernel self.shellwidget.shutdown(shutdown_kernel) @@ -635,7 +717,8 @@ def _restart_thread_main(self): """Restart the kernel in a thread.""" try: self.shellwidget.kernel_manager.restart_kernel( - stderr=self.stderr_handle) + stderr=self.stderr_obj.handle, + stdout=self.stdout_obj.handle) except RuntimeError as e: self.restart_thread.error = e @@ -653,6 +736,13 @@ def _finalise_restart(self, reset=False): before_prompt=True ) else: + if self.fault_obj is not None: + fault = self.fault_obj.get_contents() + if fault: + fault = self.filter_fault(fault) + self.shellwidget._append_plain_text( + '\n' + fault, before_prompt=True) + # Reset Pdb state and reopen comm sw._pdb_in_loop = False sw.spyder_kernel_comm.remove() @@ -669,24 +759,69 @@ def _finalise_restart(self, reset=False): sw._append_html(_("
Restarting kernel...
"), before_prompt=True) sw.insert_horizontal_ruler() + if self.fault_obj is not None: + self.shellwidget.call_kernel().enable_faulthandler( + self.fault_obj.filename) self._hide_loading_page() self.restart_thread = None self.sig_execution_state_changed.emit() + def filter_fault(self, fault): + """Get a fault from a previous session.""" + thread_regex = ( + r"(Current thread|Thread) " + r"(0x[\da-f]+) \(most recent call first\):" + r"(?:.|\r\n|\r|\n)+?(?=Current thread|Thread|\Z)") + # Keep line for future improvments + # files_regex = r"File \"([^\"]+)\", line (\d+) in (\S+)" + + main_re = "Main thread id:(?:\r\n|\r|\n)(0x[0-9a-f]+)" + main_id = 0 + for match in re.finditer(main_re, fault): + main_id = int(match.group(1), base=16) + + system_re = ("System threads ids:" + "(?:\r\n|\r|\n)(0x[0-9a-f]+(?: 0x[0-9a-f]+)+)") + ignore_ids = [] + start_idx = 0 + for match in re.finditer(system_re, fault): + ignore_ids = [int(i, base=16) for i in match.group(1).split()] + start_idx = match.span()[1] + text = "" + for idx, match in enumerate(re.finditer(thread_regex, fault)): + if idx == 0: + text += fault[start_idx:match.span()[0]] + thread_id = int(match.group(2), base=16) + if thread_id != main_id: + if thread_id in ignore_ids: + continue + if "wurlitzer.py" in match.group(0): + # Wurlitzer threads are launched later + continue + text += "\n" + match.group(0) + "\n" + else: + try: + pattern = (r".*(?:/IPython/core/interactiveshell\.py|" + r"\\IPython\\core\\interactiveshell\.py).*") + match_internal = next(re.finditer(pattern, match.group(0))) + end_idx = match_internal.span()[0] + except StopIteration: + end_idx = None + text += "\nMain thread:\n" + match.group(0)[:end_idx] + "\n" + return text + @Slot(str) def kernel_restarted_message(self, msg): """Show kernel restarted/died messages.""" - if not self.is_error_shown: + if self.stderr_obj is not None: # If there are kernel creation errors, jupyter_client will # try to restart the kernel and qtconsole prints a # message about it. # So we read the kernel's stderr_file and display its # contents in the client instead of the usual message shown # by qtconsole. - stderr = self.get_stderr_contents() - if stderr: - self.show_kernel_error('%s' % stderr) + self.poll_std_file_change() else: self.shellwidget._append_html("
%s

" % msg, before_prompt=False) diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 483ca4c1d1c..e03a973112d 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -1081,11 +1081,8 @@ def _create_client_for_kernel(self, connection_file, hostname, sshkey, ask_before_restart=ask_before_restart, css_path=self.css_path, configuration=self.CONFIGURATION, - handlers=self.registered_spyder_kernel_handlers) - - # Change stderr_dir if requested - if self._test_dir: - client.stderr_dir = self._test_dir + handlers=self.registered_spyder_kernel_handlers, + std_dir=self._test_dir) # Create kernel client kernel_client = QtKernelClient(connection_file=connection_file) @@ -1488,11 +1485,8 @@ def create_new_client(self, give_focus=True, filename='', is_cython=False, ask_before_closing=ask_before_closing, css_path=self.css_path, configuration=self.CONFIGURATION, - handlers=self.registered_spyder_kernel_handlers) - - # Change stderr_dir if requested - if self._test_dir: - client.stderr_dir = self._test_dir + handlers=self.registered_spyder_kernel_handlers, + std_dir=self._test_dir) self.add_tab( client, name=client.get_name(), filename=filename, @@ -1559,10 +1553,14 @@ def connect_client_to_kernel(self, client, is_cython=False, is_pylab=False, is_sympy=False): """Connect a client to its kernel.""" connection_file = client.connection_file - stderr_handle = None if self._test_no_stderr else client.stderr_handle + stderr_handle = ( + None if self._test_no_stderr else client.stderr_obj.handle) + stdout_handle = ( + None if self._test_no_stderr else client.stdout_obj.handle) km, kc = self.create_kernel_manager_and_kernel_client( connection_file, stderr_handle, + stdout_handle, is_cython=is_cython, is_pylab=is_pylab, is_sympy=is_sympy, @@ -1925,6 +1923,7 @@ def create_kernel_spec(self, is_cython=False, def create_kernel_manager_and_kernel_client(self, connection_file, stderr_handle, + stdout_handle, is_cython=False, is_pylab=False, is_sympy=False): @@ -1951,6 +1950,7 @@ def create_kernel_manager_and_kernel_client(self, connection_file, # See spyder-ide/spyder#7302. try: kernel_manager.start_kernel(stderr=stderr_handle, + stdout=stdout_handle, env=kernel_spec.env) except Exception: error_msg = _("The error is:

" From f1734c77f4e8306e10c5969795c6520746a66080 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 13 Nov 2021 19:09:29 +0000 Subject: [PATCH 04/10] Update .gitrepo --- external-deps/spyder-kernels/.gitrepo | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 408b2d5e0a3..0a98e3f5c45 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = 2.x - commit = efe0a3f2294d022b848a4929a7a1966abd131718 - parent = 575e34d5a84de29eca0cd1fcf3db9ded4030416b + commit = 23f584b0e291b99064518ee987b26cfed8553987 + parent = 33dce2a0ec6f2d8f9f861992ff646936810b8afb method = merge cmdver = 0.4.3 From 6427fe53027b150f5d0e7d7fe9fadc76b28d16c8 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 14 Nov 2021 22:28:39 +0000 Subject: [PATCH 05/10] Apply suggestions from code review Co-authored-by: Carlos Cordoba --- spyder/app/tests/test_mainwindow.py | 6 +++--- spyder/plugins/ipythonconsole/widgets/client.py | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index 09ba3fe5ee7..f034fdd322f 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -4232,7 +4232,7 @@ def test_add_external_plugins_to_dependencies(main_window): @pytest.mark.slow @flaky(max_runs=3) def test_print_multiprocessing(main_window, qtbot, tmpdir): - """Test the runcell command.""" + """Test print commands from multiprocessing.""" # Write code with a cell to a file code = """ import multiprocessing @@ -4270,9 +4270,9 @@ def test_func(): @flaky(max_runs=3) @pytest.mark.skipif( os.name == 'nt', - reason="ctypes.string_at(0) doesn't segfaults on windows") + reason="ctypes.string_at(0) doesn't segfaults on Windows") def test_print_faulthandler(main_window, qtbot, tmpdir): - """Test the runcell command.""" + """Test printing segfault info from kernel crashes.""" # Write code with a cell to a file code = """ def crash_func(): diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 482e7af98ff..9e869a469e9 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -129,7 +129,7 @@ def get_contents(self): return None def poll_file_change(self): - """Check if the stdout file just changed""" + """Check if the std kernel file just changed.""" if self._handle is not None and not self._handle.closed: self._handle.flush() try: @@ -260,7 +260,7 @@ def __init__(self, parent, id_, self.fault_obj = None self.std_poll_timer = None if not self.is_external_kernel: - # Can not connect for external kernels + # Cannot create std files for external kernels self.stderr_obj = Std_File(self.std_filename('.stderr')) self.stdout_obj = Std_File(self.std_filename('.stdout')) self.std_poll_timer = QTimer(self) @@ -269,7 +269,7 @@ def __init__(self, parent, id_, self.std_poll_timer.start() self.shellwidget.executed.connect(self.poll_std_file_change) if self.hostname is None: - # Can not read file that is not on this computer + # Cannot read file that is not on this computer self.fault_obj = Std_File(self.std_filename('.fault')) def __del__(self): @@ -475,6 +475,7 @@ def poll_std_file_change(self): else: self.shellwidget._append_plain_text( '\n' + stderr, before_prompt=True) + stdout = self.stdout_obj.poll_file_change() if stdout: self.shellwidget._append_plain_text( From ccc66e4d2a638dcd6d85558bae197c4e9d5f8832 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 14 Nov 2021 23:35:05 +0100 Subject: [PATCH 06/10] Move StdFile --- .../plugins/ipythonconsole/utils/stdfile.py | 92 +++++++++++++++++++ .../plugins/ipythonconsole/widgets/client.py | 84 +---------------- 2 files changed, 96 insertions(+), 80 deletions(-) create mode 100644 spyder/plugins/ipythonconsole/utils/stdfile.py diff --git a/spyder/plugins/ipythonconsole/utils/stdfile.py b/spyder/plugins/ipythonconsole/utils/stdfile.py new file mode 100644 index 00000000000..035ded6278f --- /dev/null +++ b/spyder/plugins/ipythonconsole/utils/stdfile.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# Copyright (c) 2009- Spyder Project Contributors +# +# Distributed under the terms of the MIT License +# (see spyder/__init__.py for details) +# ----------------------------------------------------------------------------- +""" +Class to control a file where stanbdard output can be written. +""" +# Standard library imports. +import codecs +import os + +# Local imports +from spyder.py3compat import to_text_string +from spyder.utils.encoding import get_coding + + +class StdFile(): + def __init__(self, filename): + self.filename = filename + self._mtime = 0 + self._cursor = 0 + self._handle = None + + @property + def handle(self): + """Get handle to file.""" + if self._handle is None and self.filename is not None: + # Needed to prevent any error that could appear. + # See spyder-ide/spyder#6267. + try: + self._handle = codecs.open( + self.filename, 'w', encoding='utf-8') + except Exception: + pass + return self._handle + + def remove(self): + """Remove file associated with the client.""" + try: + # Defer closing the handle until the client + # is closed because jupyter_client needs it open + # while it tries to restart the kernel + if self._handle is not None: + self._handle.close() + os.remove(self.filename) + self._handle = None + except Exception: + pass + + def get_contents(self): + """Get the contents of the std kernel file.""" + try: + with open(self.filename, 'rb') as f: + # We need to read the file as bytes to be able to + # detect its encoding with chardet + text = f.read() + + # This is needed to avoid showing an empty error message + # when the kernel takes too much time to start. + # See spyder-ide/spyder#8581. + if not text: + return '' + + # This is needed since the file could be encoded + # in something different to utf-8. + # See spyder-ide/spyder#4191. + encoding = get_coding(text) + text = to_text_string(text, encoding) + return text + except Exception: + return None + + def poll_file_change(self): + """Check if the std kernel file just changed.""" + if self._handle is not None and not self._handle.closed: + self._handle.flush() + try: + mtime = os.stat(self.filename).st_mtime + except Exception: + return + + if mtime == self._mtime: + return + self._mtime = mtime + text = self.get_contents() + if text: + ret_text = text[self._cursor:] + self._cursor = len(text) + return ret_text diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 9e869a469e9..f3c66cb7bcc 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -18,7 +18,6 @@ # Fix for spyder-ide/spyder#1356. from __future__ import absolute_import -import codecs import os import os.path as osp import re @@ -42,6 +41,7 @@ from spyder.utils.palette import QStylePalette from spyder.utils.programs import get_temp_dir from spyder.utils.qthelpers import add_actions, DialogManager +from spyder.utils.stdfile import StdFile from spyder.py3compat import to_text_string from spyder.plugins.ipythonconsole.widgets import ShellWidget from spyder.widgets.collectionseditor import CollectionsEditor @@ -71,82 +71,6 @@ except AttributeError: time.monotonic = time.time - -class Std_File(): - def __init__(self, filename): - self.filename = filename - self._mtime = 0 - self._cursor = 0 - self._handle = None - - @property - def handle(self): - """Get handle to file.""" - if self._handle is None and self.filename is not None: - # Needed to prevent any error that could appear. - # See spyder-ide/spyder#6267. - try: - self._handle = codecs.open( - self.filename, 'w', encoding='utf-8') - except Exception: - pass - return self._handle - - def remove(self): - """Remove file associated with the client.""" - try: - # Defer closing the handle until the client - # is closed because jupyter_client needs it open - # while it tries to restart the kernel - if self._handle is not None: - self._handle.close() - os.remove(self.filename) - self._handle = None - except Exception: - pass - - def get_contents(self): - """Get the contents of the std kernel file.""" - try: - with open(self.filename, 'rb') as f: - # We need to read the file as bytes to be able to - # detect its encoding with chardet - text = f.read() - - # This is needed to avoid showing an empty error message - # when the kernel takes too much time to start. - # See spyder-ide/spyder#8581. - if not text: - return '' - - # This is needed since the file could be encoded - # in something different to utf-8. - # See spyder-ide/spyder#4191. - encoding = get_coding(text) - text = to_text_string(text, encoding) - return text - except Exception: - return None - - def poll_file_change(self): - """Check if the std kernel file just changed.""" - if self._handle is not None and not self._handle.closed: - self._handle.flush() - try: - mtime = os.stat(self.filename).st_mtime - except Exception: - return - - if mtime == self._mtime: - return - self._mtime = mtime - text = self.get_contents() - if text: - ret_text = text[self._cursor:] - self._cursor = len(text) - return ret_text - - # ---------------------------------------------------------------------------- # Client widget # ---------------------------------------------------------------------------- @@ -261,8 +185,8 @@ def __init__(self, parent, id_, self.std_poll_timer = None if not self.is_external_kernel: # Cannot create std files for external kernels - self.stderr_obj = Std_File(self.std_filename('.stderr')) - self.stdout_obj = Std_File(self.std_filename('.stdout')) + self.stderr_obj = StdFile(self.std_filename('.stderr')) + self.stdout_obj = StdFile(self.std_filename('.stdout')) self.std_poll_timer = QTimer(self) self.std_poll_timer.timeout.connect(self.poll_std_file_change) self.std_poll_timer.setInterval(1000) @@ -270,7 +194,7 @@ def __init__(self, parent, id_, self.shellwidget.executed.connect(self.poll_std_file_change) if self.hostname is None: # Cannot read file that is not on this computer - self.fault_obj = Std_File(self.std_filename('.fault')) + self.fault_obj = StdFile(self.std_filename('.fault')) def __del__(self): """Close threads to avoid segfault.""" From db8f7f6d814dd109ab143a09e9a984355ac69427 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 14 Nov 2021 23:35:26 +0100 Subject: [PATCH 07/10] revert fix for issue1666807 --- spyder/plugins/ipythonconsole/widgets/client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index f3c66cb7bcc..e2ab18b08c7 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -484,9 +484,7 @@ def stop_button_click_handler(self): def show_kernel_error(self, error): """Show kernel initialization errors in infowidget.""" self.error_text = error - if "issue1666807" not in error: - # The installer has some strange relative python - InstallerIPythonKernelError(error) + InstallerIPythonKernelError(error) # Replace end of line chars with
eol = sourcecode.get_eol_chars(error) From 5bfe10a4734680000e3ba04013a756fd4fac74ef Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 14 Nov 2021 23:50:06 +0100 Subject: [PATCH 08/10] Fix path --- spyder/plugins/ipythonconsole/widgets/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index e2ab18b08c7..05ce6a22477 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -41,8 +41,8 @@ from spyder.utils.palette import QStylePalette from spyder.utils.programs import get_temp_dir from spyder.utils.qthelpers import add_actions, DialogManager -from spyder.utils.stdfile import StdFile from spyder.py3compat import to_text_string +from spyder.plugins.ipythonconsole.utils.stdfile import StdFile from spyder.plugins.ipythonconsole.widgets import ShellWidget from spyder.widgets.collectionseditor import CollectionsEditor from spyder.widgets.mixins import SaveHistoryMixin From d364920b247352f0cf0ef801a2990d0e655f5e4e Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 17 Nov 2021 10:57:48 +0100 Subject: [PATCH 09/10] skip error for macos installer --- spyder/plugins/ipythonconsole/widgets/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 05ce6a22477..4e70ed1fab6 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -484,7 +484,9 @@ def stop_button_click_handler(self): def show_kernel_error(self, error): """Show kernel initialization errors in infowidget.""" self.error_text = error - InstallerIPythonKernelError(error) + if "issue1666807" not in error: + # TODO: remove the above if, see spyder-ide/spyder#16828 + InstallerIPythonKernelError(error) # Replace end of line chars with
eol = sourcecode.get_eol_chars(error) From 5fc1599f1bf26ae6e99493ddca2b2fd1ee2869d5 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 17 Nov 2021 19:49:28 +0100 Subject: [PATCH 10/10] Update spyder/plugins/ipythonconsole/widgets/client.py Co-authored-by: Carlos Cordoba --- spyder/plugins/ipythonconsole/widgets/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 4e70ed1fab6..5c9b85ca7b2 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -179,6 +179,7 @@ def __init__(self, parent, id_, # --- Dialog manager self.dialog_manager = DialogManager() + # --- Standard files handling self.stderr_obj = None self.stdout_obj = None self.fault_obj = None