PR: Try to decrease timeouts when running tests and check general tes…

…ts robustness and speed (CI/Testing) (#22077)

conftest.py

@@ -84,7 +84,7 @@ def pytest_collection_modifyitems(config, items):
         ]
 
         if os.name == 'nt':
-            percentage = 0.5
+            percentage = 0.4
         elif sys.platform == 'darwin':
             percentage = 0.5
         else:

spyder/app/tests/conftest.py

@@ -482,10 +482,14 @@ def main_window(request, tmpdir, qtbot):
             # after running test that uses the fixture
             # Currently 'test_out_runfile_runcell' is the last tests so
             # in order to prevent errors finalizing the test suit such test has
-            # this marker
+            # this marker.
+            # Also, try to decrease chances of freezes/timeouts from tests that
+            # are known to have leaks by also closing the main window for them.
+            known_leak = request.node.get_closest_marker(
+                'known_leak')
             close_main_window = request.node.get_closest_marker(
                 'close_main_window')
-            if close_main_window:
+            if close_main_window or known_leak:
                 main_window.window = None
                 window.closing(close_immediately=True)
                 window.close()
@@ -545,12 +549,6 @@ def main_window(request, tmpdir, qtbot):
                     # Do not test leaks on windows
                     return
 
-                known_leak = request.node.get_closest_marker(
-                    'known_leak')
-                if known_leak:
-                    # This test has a known leak
-                    return
-
                 def show_diff(init_list, now_list, name):
                     sys.stderr.write(f"Extra {name} before test:\n")
                     for item in init_list:

spyder/app/tests/spyder-boilerplate/spyder_boilerplate/spyder/plugin.py

@@ -64,7 +64,7 @@ def get_title(self):
         return "Spyder boilerplate plugin"
 
     def get_focus_widget(self):
-        pass
+        return self
 
     def setup(self):
         # Create an example action

spyder/app/tests/test_mainwindow.py

@@ -3986,6 +3986,9 @@ def processEvents():
 
         # Make sure the events are not processed.
         assert not processEvents.called
+
+        with qtbot.waitSignal(shell.executed):
+            shell.execute("q")
     finally:
         QApplication.processEvents = super_processEvents
 
@@ -4087,6 +4090,9 @@ def test_pdb_step(main_window, qtbot, tmpdir, where):
             main_window.editor.get_current_editor().filename,
             str(test_file))
 
+    with qtbot.waitSignal(shell.executed):
+        shell.execute("q")
+
 
 @flaky(max_runs=3)
 @pytest.mark.skipif(sys.platform == 'darwin' or os.name == 'nt',
@@ -4391,6 +4397,9 @@ def test_post_mortem(main_window, qtbot, tmpdir):
 
     assert "IPdb [" in control.toPlainText()
 
+    with qtbot.waitSignal(shell.executed):
+        shell.execute("q")
+
 
 @flaky(max_runs=3)
 @pytest.mark.order(after="test_debug_unsaved_function")
@@ -4701,6 +4710,7 @@ def test_ordering_lsp_requests_at_startup(main_window, qtbot):
 
 
 @flaky(max_runs=3)
+@pytest.mark.close_main_window
 @pytest.mark.parametrize(
     'main_window',
     [{'spy_config': ('tours', 'show_tour_message', True)}],
@@ -4751,6 +4761,7 @@ def test_tour_message(main_window, qtbot):
     # Close the tour
     animated_tour.close_tour()
     qtbot.waitUntil(lambda: not animated_tour.is_running, timeout=9000)
+    qtbot.wait(2000)
 
 
 @flaky(max_runs=3)
@@ -6815,6 +6826,12 @@ def test_undock_plugin_and_close(main_window, qtbot):
 
     This checks the functionality added in PR spyder-ide/spyder#19784.
     """
+    # Wait until the window is fully up
+    shell = main_window.ipyconsole.get_current_shellwidget()
+    qtbot.waitUntil(
+        lambda: shell.spyder_kernel_ready and shell._prompt_html is not None,
+        timeout=SHELL_TIMEOUT)
+
     # Select a random plugin and undock it
     plugin = get_random_dockable_plugin(main_window)
     plugin.get_widget().undock_action.trigger()
@@ -6865,6 +6882,12 @@ def test_outline_in_maximized_editor(main_window, qtbot):
 
     This is a regression test for issue spyder-ide/spyder#16265.
     """
+    # Wait until the window is fully up
+    shell = main_window.ipyconsole.get_current_shellwidget()
+    qtbot.waitUntil(
+        lambda: shell.spyder_kernel_ready and shell._prompt_html is not None,
+        timeout=SHELL_TIMEOUT)
+
     editor = main_window.get_plugin(Plugins.Editor)
     outline = main_window.get_plugin(Plugins.OutlineExplorer)
 

spyder/plugins/editor/tests/test_plugin.py

@@ -407,6 +407,7 @@ def test_toggle_eol_chars(editor_plugin, python_files, qtbot, os_name):
     assert get_eol_chars(text) == get_eol_chars_from_os_name(os_name)
 
 
+@pytest.mark.order(1)
 @pytest.mark.parametrize('os_name', ['nt', 'mac', 'posix'])
 def test_save_with_preferred_eol_chars(editor_plugin, python_files, qtbot,
                                        os_name):
@@ -415,16 +416,29 @@ def test_save_with_preferred_eol_chars(editor_plugin, python_files, qtbot,
     editorstack = editor_plugin.get_current_editorstack()
     eol_lookup = {'posix': 'LF', 'nt': 'CRLF', 'mac': 'CR'}
 
-    # Set options
-    editor_plugin.set_conf('convert_eol_on_save', True)
-    editor_plugin.set_conf('convert_eol_on_save_to', eol_lookup[os_name])
+    # Check default options value are set
+    qtbot.waitUntil(
+        lambda:
+            not editorstack.convert_eol_on_save
+            and editorstack.convert_eol_on_save_to == 'LF'
+    )
+
 
     # Load a test file
     fname = filenames[0]
     editor_plugin.load(fname)
     qtbot.wait(500)
     codeeditor = editor_plugin.get_current_editor()
 
+    # Set options
+    editor_plugin.set_conf('convert_eol_on_save', True)
+    editor_plugin.set_conf('convert_eol_on_save_to', eol_lookup[os_name])
+    qtbot.waitUntil(
+        lambda:
+            editorstack.convert_eol_on_save
+            and editorstack.convert_eol_on_save_to == eol_lookup[os_name]
+    )
+
     # Set file as dirty, save it and check that it has the right eol.
     codeeditor.document().setModified(True)
     editorstack.save()

spyder/plugins/editor/widgets/editorstack/editorstack.py

@@ -832,10 +832,9 @@ def display_help(self, help_text, clicked):
         self.send_to_help(name, help_text, force=True)
 
     # ---- Editor Widget Settings
-    @on_conf_change(option='connect_to_oi')
+    @on_conf_change(section='help', option='connect/editor')
     def on_help_connection_change(self, value):
-        help_option_value = self.get_conf('connect/editor', section='help')
-        self.set_help_enabled(help_option_value)
+        self.set_help_enabled(value)
 
     @on_conf_change(section='appearance', option=['selected', 'ui_theme'])
     def on_color_scheme_change(self, option, value):

spyder/plugins/editor/widgets/main_widget.py

@@ -1030,6 +1030,8 @@ def on_close(self):
             'windows_layout_settings',
             [win.get_layout_settings() for win in self.editorwindows]
         )
+        for window in self.editorwindows:
+            window.close()
         self.set_conf('recent_files', self.recent_files)
         self.autosave.stop_autosave_timer()
 

spyder/plugins/help/plugin.py

@@ -194,13 +194,6 @@ def on_close(self, cancelable=False):
     def apply_conf(self, options_set, notify=False):
         super().apply_conf(options_set)
 
-        # To make auto-connection changes take place instantly
-        try:
-            editor = self.get_plugin(Plugins.Editor)
-            editor.apply_plugin_settings({'connect_to_oi'})
-        except SpyderAPIError:
-            pass
-
     # --- Private API
     # ------------------------------------------------------------------------
     def _setup_menus(self):

spyder/plugins/history/tests/test_plugin.py

@@ -63,6 +63,7 @@ def test_init(historylog):
     assert len(hl.get_actions()) == 7
 
 
+@pytest.mark.order(1)
 @pytest.mark.skipif(
     sys.platform == "darwin" and running_in_ci(),
     reason="Fails on Mac when running on CI "

spyder/plugins/ipythonconsole/widgets/status.py

@@ -127,8 +127,10 @@ def on_kernel_start(self, shellwidget):
         # Reset value of interactive backend
         self._interactive_gui = None
 
-        # Avoid errors when running our test suite on Mac.
-        if running_in_ci() and sys.platform == "darwin":
+        # Avoid errors when running our test suite on Mac and Windows.
+        # On Windows the following error appears:
+        # `spyder_kernels.comms.commbase.CommError: The comm is not connected.`
+        if running_in_ci() and not sys.platform.startswith("linux"):
             mpl_backend = "inline"
         else:
             mpl_backend = shellwidget.get_matplotlib_backend()

spyder/plugins/plots/tests/test_plugin.py

@@ -12,6 +12,7 @@
 
 import pytest
 
+from spyder.config.base import running_in_ci
 from spyder.config.manager import CONF
 from spyder.plugins.plots.plugin import Plots
 from spyder.plugins.plots.widgets.main_widget import PlotsWidgetActions
@@ -27,8 +28,11 @@ def plots_plugin(qapp, qtbot):
     plots.get_widget().setMinimumSize(700, 500)
     plots.get_widget().add_shellwidget(Mock())
     qtbot.addWidget(plots.get_widget())
-    qapp.setStyleSheet(str(APP_STYLESHEET))
-    plots.get_widget().show()
+    if not running_in_ci():
+        qapp.setStyleSheet(str(APP_STYLESHEET))
+    with qtbot.waitExposed(plots.get_widget()):
+        plots.get_widget().show()
+
     return plots
 
 

spyder/plugins/plots/widgets/figurebrowser.py

@@ -481,15 +481,15 @@ def load_figure(self, fig, fmt):
         # immediately after the figure is loaded doesn't work.
         QTimer.singleShot(
             20,
-            lambda: self.verticalScrollBar().setValue(
-                self.current_thumbnail.vscrollbar_value
-            ),
+            self.update_scrollbars_values,
         )
-        QTimer.singleShot(
-            20,
-            lambda: self.horizontalScrollBar().setValue(
-                self.current_thumbnail.hscrollbar_value
-            ),
+
+    def update_scrollbars_values(self):
+        self.verticalScrollBar().setValue(
+            self.current_thumbnail.vscrollbar_value
+        )
+        self.horizontalScrollBar().setValue(
+            self.current_thumbnail.hscrollbar_value
         )
 
     def eventFilter(self, widget, event):

spyder/plugins/updatemanager/tests/test_update_manager.py

@@ -11,7 +11,7 @@
 
 from spyder.config.base import running_in_ci
 from spyder.plugins.updatemanager import workers
-from spyder.plugins.updatemanager.workers import WorkerUpdate
+from spyder.plugins.updatemanager.workers import WorkerUpdate, HTTP_ERROR_MSG
 from spyder.plugins.updatemanager.widgets import update
 from spyder.plugins.updatemanager.widgets.update import UpdateManagerWidget
 
@@ -59,6 +59,16 @@ def test_updates_appenv(qtbot, mocker, version, caplog):
         um.start_check_update()
         qtbot.waitUntil(um.update_thread.isFinished)
 
+    if um.update_worker.error:
+        # Possible 403 error - rate limit error, was encountered while doing
+        # the tests
+        # Check error message corresponds to the status code and exit early to
+        # prevent failing the test
+        assert um.update_worker.error == HTTP_ERROR_MSG.format(status_code="403")
+        return
+
+    assert not um.update_worker.error
+
     update_available = um.update_worker.update_available
     if version.split('.')[0] == '1':
         assert update_available

spyder/widgets/tests/test_sidebardialog.py

@@ -11,6 +11,7 @@
 import pytest
 
 # Local imports
+from spyder.config.base import running_in_ci
 from spyder.utils.stylesheet import APP_STYLESHEET
 from spyder.widgets.sidebardialog import SidebarDialog, SidebarPage
 
@@ -55,10 +56,15 @@ def setup_page(self):
     class TestDialog(SidebarDialog):
         PAGE_CLASSES = [Page1, Page2]
 
-    qapp.setStyleSheet(str(APP_STYLESHEET))
+    if not running_in_ci():
+        qapp.setStyleSheet(str(APP_STYLESHEET))
     dialog = TestDialog()
     qtbot.addWidget(dialog)
-    dialog.show()
+
+    # To check the dialog visually
+    with qtbot.waitExposed(dialog):
+        dialog.show()
+
     return dialog
 
 
@@ -68,9 +74,6 @@ def test_sidebardialog(sidebar_dialog, qtbot):
     dialog = sidebar_dialog
     assert dialog is not None
 
-    # To check the dialog visually
-    qtbot.wait(1000)
-
     # Check label displayed in the initial page
     assert "one" in dialog.get_page().label.text()