Partially fix non-deterministic coverage (#2318)

* Partially fix non-deterministic coverage

This is a band-aid fix that mostly resolves the issue with
non-deterministic coverage. Without the wait, tests abruptly end and
coverage can fluctuate. With the wait, Qt has time to handle its stuff
for a while longer, which stabilizes the coverage. The proper fix would
be to clean up everything Qt-related that is created in a test after
each test, but I couldn't get that to work yet.

There are also at least two more cases of non-deterministic coverage
that seem to be unrelated to Qt and which aren't fixed with this.

* Add docstring to qtbot overwrite

tests/_test_msui/test_editor.py

@@ -41,7 +41,7 @@ class Test_Editor:
     save_file_name = fs.path.join(ROOT_DIR, "testeditor_save.json")
 
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         with mock.patch("PyQt5.QtWidgets.QMessageBox.warning", return_value=QtWidgets.QMessageBox.Yes):
             self.window = editor.EditorMainWindow()
             self.save_file_name = self.save_file_name

tests/_test_msui/test_kmloverlay_dockwidget.py

@@ -42,7 +42,7 @@
 class Test_KmlOverlayDockWidget:
 
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.view = mock.Mock()
         self.view.map = mock.Mock(side_effect=lambda x, y: (x, y))
         self.view.map.plot = mock.Mock(return_value=[mock.Mock()])

tests/_test_msui/test_linearview.py

@@ -38,7 +38,7 @@
 
 class Test_MSS_LV_Options_Dialog:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.window = tv.MSUI_LV_Options_Dialog(settings=_DEFAULT_SETTINGS_LINEARVIEW)
         self.window.show()
         QtTest.QTest.qWaitForWindowExposed(self.window)
@@ -54,7 +54,7 @@ def test_get(self):
 
 class Test_MSSLinearViewWindow:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         initial_waypoints = [ft.Waypoint(40., 25., 300), ft.Waypoint(60., -10., 400), ft.Waypoint(40., 10, 300)]
 
         waypoints_model = ft.WaypointsTableModel("")
@@ -86,7 +86,7 @@ def test_options(self, mockdlg):
 
 class Test_LinearViewWMS:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp, mswms_server):
+    def setup(self, qtbot, mswms_server):
         self.url = mswms_server
         self.tempdir = tempfile.mkdtemp()
         if not os.path.exists(self.tempdir):

tests/_test_msui/test_mpl_map.py

@@ -32,7 +32,7 @@
 
 class Test_MapCanvas:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         kwargs = {'resolution': 'l', 'area_thresh': 1000.0, 'ax': plt.gca(), 'llcrnrlon': -15.0, 'llcrnrlat': 35.0,
                   'urcrnrlon': 30.0, 'urcrnrlat': 65.0, 'epsg': '4326'}
         self.map = MapCanvas(**kwargs)

tests/_test_msui/test_mscolab.py

@@ -47,7 +47,7 @@
 
 class Test_Mscolab_connect_window:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp, mscolab_server):
+    def setup(self, qtbot, mscolab_server):
         self.url = mscolab_server
         self.userdata = 'UV10@uv10', 'UV10', 'uv10'
         self.operation_name = "europe"
@@ -258,7 +258,7 @@ class Test_Mscolab:
     }
 
     @pytest.fixture(autouse=True)
-    def setup(self, qapp, mscolab_app, mscolab_server):
+    def setup(self, qtbot, mscolab_app, mscolab_server):
         self.app = mscolab_app
         self.url = mscolab_server
         self.userdata = 'UV10@uv10', 'UV10', 'uv10'

tests/_test_msui/test_mscolab_merge_waypoints.py

@@ -41,7 +41,7 @@
 
 class Test_Mscolab_Merge_Waypoints:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp, mscolab_app, mscolab_server):
+    def setup(self, qtbot, mscolab_app, mscolab_server):
         self.app = mscolab_app
         self.url = mscolab_server
         self.window = msui.MSUIMainWindow(mscolab_data_dir=mscolab_settings.MSCOLAB_DATA_DIR)

tests/_test_msui/test_mss.py

@@ -28,7 +28,7 @@
 from mslib.msui import mss
 
 
-def test_mss_rename_message(qapp):
+def test_mss_rename_message(qtbot):
     main_window = mss.MSSMainWindow()
     main_window.show()
     QtTest.QTest.mouseClick(main_window.pushButton, QtCore.Qt.LeftButton)

tests/_test_msui/test_msui.py

@@ -67,7 +67,7 @@ def test_main():
 
 class Test_MSS_TutorialMode:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot, qapp):
         qapp.setApplicationDisplayName("MSUI")
         self.main_window = msui_mw.MSUIMainWindow(tutorial_mode=True)
         self.main_window.create_new_flight_track()
@@ -95,7 +95,7 @@ def test_tutorial_dir(self):
 
 class Test_MSS_AboutDialog:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.window = msui_mw.MSUI_AboutDialog()
         yield
         self.window.hide()
@@ -109,7 +109,7 @@ def test_milestone_url(self):
 
 class Test_MSS_ShortcutDialog:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.main_window = msui_mw.MSUIMainWindow()
         self.main_window.show()
         self.shortcuts = msui_mw.MSUI_ShortcutsDialog()
@@ -169,7 +169,7 @@ class Test_MSSSideViewWindow:
     }
 
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.sample_path = os.path.join(
             os.path.dirname(os.path.abspath(__file__)),
             '../',

tests/_test_msui/test_multiple_flightpath_dockwidget.py

@@ -34,7 +34,7 @@
 
 class Test_MultipleFlightpathControlWidget:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.window = msui.MSUIMainWindow()
         self.window.create_new_flight_track()
 

tests/_test_msui/test_remotesensing.py

@@ -46,7 +46,7 @@ class Test_RemoteSensingControlWidget:
     Tests about RemoteSensingControlWidget
     """
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.view = Mock()
         self.map = qt.TopViewPlotter()
         self.map.init_map()

tests/_test_msui/test_satellite_dockwidget.py

@@ -34,7 +34,7 @@
 
 class Test_SatelliteDockWidget:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.view = mock.Mock()
         self.window = sd.SatelliteControlWidget(view=self.view)
         self.window.show()

tests/_test_msui/test_sideview.py

@@ -39,7 +39,7 @@
 
 class Test_MSS_SV_OptionsDialog:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.window = tv.MSUI_SV_OptionsDialog(settings=_DEFAULT_SETTINGS_SIDEVIEW)
         self.window.show()
         QtTest.QTest.qWaitForWindowExposed(self.window)
@@ -73,7 +73,7 @@ def test_setColour(self, mockdlg):
 
 class Test_MSSSideViewWindow:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         initial_waypoints = [ft.Waypoint(40., 25., 300), ft.Waypoint(60., -10., 400), ft.Waypoint(40., 10, 300)]
 
         waypoints_model = ft.WaypointsTableModel("")
@@ -126,7 +126,7 @@ def test_y_axes(self):
 
 class Test_SideViewWMS:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp, mswms_server):
+    def setup(self, qtbot, mswms_server):
         self.url = mswms_server
         self.tempdir = tempfile.mkdtemp()
         if not os.path.exists(self.tempdir):

tests/_test_msui/test_suffix.py

@@ -35,7 +35,7 @@
 
 class Test_SuffixChange:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.window = tv.MSUI_SV_OptionsDialog(settings=_DEFAULT_SETTINGS_SIDEVIEW)
         self.window.show()
         QtTest.QTest.qWaitForWindowExposed(self.window)

tests/_test_msui/test_tableview.py

@@ -37,7 +37,7 @@
 
 class Test_TableView:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         # Create an initital flight track.
         initial_waypoints = [ft.Waypoint(flightlevel=0, location="EDMO", comments="take off OP"),
                              ft.Waypoint(48.10, 10.27, 200),

tests/_test_msui/test_topview.py

@@ -39,7 +39,7 @@
 
 class Test_MSS_TV_MapAppearanceDialog:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.window = tv.MSUI_TV_MapAppearanceDialog(settings=_DEFAULT_SETTINGS_TOPVIEW)
         self.window.show()
         QtTest.QTest.qWaitForWindowExposed(self.window)
@@ -55,7 +55,7 @@ def test_get(self):
 
 class Test_MSSTopViewWindow:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         mainwindow = MSUIMainWindow()
         initial_waypoints = [ft.Waypoint(40., 25., 0), ft.Waypoint(60., -10., 0), ft.Waypoint(40., 10, 0)]
         waypoints_model = ft.WaypointsTableModel("")
@@ -199,7 +199,7 @@ def test_map_options(self):
 
 class Test_TopViewWMS:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp, mswms_server):
+    def setup(self, qtbot, mswms_server):
         self.url = mswms_server
         self.tempdir = tempfile.mkdtemp()
         if not os.path.exists(self.tempdir):
@@ -242,7 +242,7 @@ def test_server_getmap(self, qtbot):
 
 class Test_MSUITopViewWindow:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         pass
 
     def test_kwargs_update_does_not_harm(self):

tests/_test_msui/test_updater.py

@@ -56,7 +56,7 @@ def __init__(self, args=None, **named_args):
 @mock.patch("mslib.utils.qt.Worker.start", Worker.run)
 class Test_MSS_ShortcutDialog:
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.updater = Updater()
         self.status = ""
         self.update_available = False

tests/_test_msui/test_wms_capabilities.py

@@ -35,7 +35,7 @@
 class Test_WMSCapabilities:
 
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.capabilities = mock.Mock()
         self.capabilities.capabilities_document = u"Hölla die Waldfee".encode("utf-8")
         self.capabilities.provider = mock.Mock()

tests/_test_msui/test_wms_control.py

@@ -97,7 +97,7 @@ def query_server(self, qtbot, url):
 
 class Test_HSecWMSControlWidget(WMSControlWidgetSetup):
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self._setup("hsec")
         yield
         self._teardown()
@@ -409,7 +409,7 @@ def test_server_no_thread(self, mockthread, qtbot):
 
 class Test_VSecWMSControlWidget(WMSControlWidgetSetup):
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self._setup("vsec")
         yield
         self._teardown()
@@ -495,7 +495,7 @@ class TestWMSControlWidgetSetupSimple:
         <Extent name="ELEVATION" default="900.0"> 500.0,600.0,700.0,900.0 </Extent>"""
 
     @pytest.fixture(autouse=True)
-    def setup(self, qapp):
+    def setup(self, qtbot):
         self.view = HSecViewMockup()
         self.window = wc.HSecWMSControlWidget(view=self.view)
         self.window.show()

tests/fixtures.py

@@ -70,8 +70,16 @@ def close_remaining_widgets():
 
 
 @pytest.fixture
-def qapp(qapp, fail_if_open_message_boxes_left, close_remaining_widgets):
-    yield qapp
+def qtbot(qtbot, fail_if_open_message_boxes_left, close_remaining_widgets):
+    """Fixture that re-defines the qtbot fixture from pytest-qt with additional checks."""
+    yield qtbot
+    # Wait for a while after the requesting test has finished. At time of writing this
+    # is required to (mostly) stabilize the coverage reports, because tests don't
+    # properly close their Qt-related stuff and therefore there is no guarantee about
+    # what the Qt event loop has or hasn't done yet. Waiting just gives it a bit more
+    # time to converge on the same result every time the tests are executed. This is a
+    # band-aid fix, the proper fix is to make sure each test cleans up after itself.
+    qtbot.wait(5000)
 
 
 @pytest.fixture(scope="session")