Try to fix CI, change perfmon test strategy (#3167)

* tick

* simplify

* push comment

* trygithub xvfb

* split backends

* less tests

* just utils

* hide action

* move sys info

* comment

* ignore qt

* just qt no threading

* remove qt notifications

* remove progress

* narrow to just tests

* just dialogs

* just layercontrols

* tick

* just perf

* whoops, just perf

* just widgets

* add back tests

* back to full

* tests and widgets

* tests widgets perf

* add containers

* remove containers again

* full qt again

* comment perf

* add back perf

* update perftest

* restore conftest

* restore progress

* restore notifications

* restore threading

* restore threading progress

* codecov relax

* restore test configs

* update comment

* show viewer

* use xvfb action

* use xvfb action more

.github/workflows/test_comprehensive.yml

@@ -53,6 +53,9 @@ jobs:
     needs: code
     name: ${{ matrix.platform }} py${{ matrix.python }} ${{ matrix.toxenv }} ${{ matrix.MIN_REQ && 'min_req' }}
     runs-on: ${{ matrix.platform }}
+    defaults:
+      run:
+        shell: bash -l {0}
     strategy:
       fail-fast: false
       matrix:
@@ -139,8 +142,9 @@ jobs:
       # in which case we are declaring one specific tox environment to run.
       # see tox.ini for more
       - name: Test with tox
-        shell: bash -l {0}
-        run: tox
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: tox
         env:
           PLATFORM: ${{ matrix.platform }}
           TOXENV: ${{ matrix.toxenv }}

.github/workflows/test_prereleases.yml

@@ -53,7 +53,9 @@ jobs:
 
       - name: Test with tox
         # run tests using pip install --pre
-        run: tox -v --pre
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: tox -v --pre
         env:
           PLATFORM: ${{ matrix.platform }}
           BACKEND: ${{ matrix.backend }}

.github/workflows/test_pull_requests.yml

@@ -140,7 +140,9 @@ jobs:
       # `tox -e py38-linux-pyqt,py38-linux-pyside`
       # see tox.ini for more
       - name: Test with tox
-        run: tox
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: tox
         env:
           PLATFORM: ${{ matrix.platform }}
           BACKEND: ${{ matrix.backend }}
@@ -168,10 +170,13 @@ jobs:
             libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 \
             libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0
 
-      - name: Test
+      - name: Install this commit
         run: |
-          pip install --upgrade pip
-          pip install pytest-xvfb
           # pip install this git sha directly from github
+          pip install --upgrade pip
           pip install git+git://github.com/${{ github.repository }}.git@${{ github.sha }}#egg=napari[all,testing]
-          pytest --pyargs napari -v --color=yes
+
+      - name: Test
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: pytest --pyargs napari --color=yes

codecov.yml

@@ -6,7 +6,7 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 0.5%  # coverage can drop by up to 0.5% while still posting success
+        threshold: 1%  # coverage can drop by up to 1% while still posting success
     patch: off
 comment:
-  require_changes: true  # if true: only post the PR comment if coverage changes
+  require_changes: true  # if true: only post the PR comment if coverage changes

napari/_qt/perf/_tests/test_perf.py

@@ -1,78 +1,51 @@
 import json
 import os
+import subprocess
 import sys
 from pathlib import Path
 
-import numpy as np
-import pytest
-import qtpy
-from qtpy.QtWidgets import QApplication
-
-from napari._qt.perf import qt_event_tracing
-from napari._qt.utils import delete_qapp
-from napari.utils import perf
-
-if os.getenv('CI', '0') != '0' and (
-    sys.version_info >= (3, 9)
-    or (
-        (sys.platform.startswith('linux') or sys.platform.startswith('win'))
-        and qtpy.API_NAME == "PySide2"
-    )
-):
-    # this test is covered by other platforms, and also seems to work locally
-    # on linux
-    pytest.skip(
-        "Perfmon segfaults on linux or windows CI with pyside2, or with python 3.9",
-        allow_module_level=True,
-    )
-
-
-@pytest.fixture(scope="module")
-def qapp():
-    """A modified QApplicationWithTracing just for this test module.
-
-    Note: Because of the difficulty in destroying a QApplication that overrides
-    .notify() like QApplicationWithTracing does, this test must be run last
-    (globally).  So in napari/conftest.py::pytest_collection_modifyitems,
-    we ensure that this test is always last.
-    """
-
-    # before creating QApplicationWithTracing, we need to monkeypatch
-    # the `perf.perf_timer` context manager that gets used in the
-    # qt_event_tracing module
-
-    original_perf_timer = qt_event_tracing.perf.perf_timer
-    _, perf_timer, _, _ = perf._timers._create_timer()
-    qt_event_tracing.perf.perf_timer = perf_timer
-
-    try:
-        if qtpy.API_NAME == 'PySide2' and QApplication.instance():
-            delete_qapp(QApplication.instance())
-        yield qt_event_tracing.QApplicationWithTracing([])
-    finally:
-        qt_event_tracing.perf.perf_timer = original_perf_timer
-
-
-def test_trace_on_start(tmp_path, monkeypatch, make_napari_viewer):
+from napari._tests.utils import skip_local_popups
+
+# NOTE:
+# for some reason, running this test fails in a subprocess with a segfault
+# if you don't show the viewer...
+PERFMON_SCRIPT = """
+import napari
+from qtpy.QtCore import QTimer
+
+v = napari.view_points()
+QTimer.singleShot(100, napari._qt.qt_event_loop.quit_app)
+napari.run()
+"""
+
+CONFIG = {
+    "trace_qt_events": True,
+    "trace_file_on_start": '',
+    "trace_callables": ["chunk_loader"],
+    "callable_lists": {
+        "chunk_loader": [
+            "napari.components.experimental.chunk._loader.ChunkLoader.load_request",
+            "napari.components.experimental.chunk._loader.ChunkLoader._on_done",
+        ]
+    },
+}
+
+
+@skip_local_popups
+def test_trace_on_start(tmp_path: Path):
     """Make sure napari can write a perfmon trace file."""
-
-    timers, _, _, _ = perf._timers._create_timer()
-    monkeypatch.setattr(perf._timers, 'timers', timers)
-
-    # Check perfmon is enabled
     trace_path = tmp_path / "trace.json"
-    timers.start_trace_file(trace_path)
-
-    viewer = make_napari_viewer()
-    data = np.random.random((10, 15))
-    viewer.add_image(data)
-    viewer.close()
+    config_path = tmp_path / "perfmon.json"
+    CONFIG['trace_file_on_start'] = str(trace_path)
+    config_path.write_text(json.dumps(CONFIG))
 
-    timers.stop_trace_file()
+    env = os.environ.copy()
+    env.update({'NAPARI_PERFMON': str(config_path), 'NAPARI_CONFIG': ''})
+    subprocess.run([sys.executable, '-c', PERFMON_SCRIPT], env=env, check=True)
 
     # Make sure file exists and is not empty.
-    assert Path(trace_path).exists(), "Trace file not written"
-    assert Path(trace_path).stat().st_size > 0, "Trace file is empty"
+    assert trace_path.exists(), "Trace file not written"
+    assert trace_path.stat().st_size > 0, "Trace file is empty"
 
     # Assert every event contains every important field.
     with open(trace_path) as infile:

napari/_qt/qt_viewer.py

@@ -326,7 +326,6 @@ def _create_performance_dock_widget(self):
                 QtPerformance(),
                 name=trans._('performance'),
                 area='bottom',
-                shortcut='Ctrl+Shift+P',
             )
         return None
 

napari/_tests/test_sys_info.py

@@ -0,0 +1,16 @@
+from napari.utils.info import sys_info
+
+
+# vispy use_app tries to start Qt, which can cause segfaults when running
+# sys_info on CI unless we provide a pytest Qt app
+def test_sys_info(qapp):
+    str_info = sys_info()
+    assert isinstance(str_info, str)
+    assert '<br>' not in str_info
+    assert '<b>' not in str_info
+    assert "Plugins" in str_info
+
+    html_info = sys_info(as_html=True)
+    assert isinstance(html_info, str)
+    assert '<br>' in html_info
+    assert '<b>' in html_info

napari/conftest.py

@@ -323,17 +323,6 @@ def pytest_internalerror(excinfo):
         raise excinfo.value
 
 
-def pytest_collection_modifyitems(session, config, items):
-
-    put_at_end = ('test_trace_on_start',)
-    at_end = []
-    for i, item in enumerate(items):
-        if item.name in put_at_end:
-            at_end.append(items.pop(i))
-
-    items.extend([x for _, x in sorted(zip(put_at_end, at_end))])
-
-
 @pytest.fixture(autouse=True)
 def fresh_settings(monkeypatch):
     from napari import settings

napari/utils/_tests/test_info.py

@@ -1,26 +1,12 @@
 import subprocess
 from typing import NamedTuple
 
-from .. import info
-from ..info import citation_text, sys_info
-
-
-def test_sys_info():
-    str_info = sys_info()
-    assert isinstance(str_info, str)
-    assert '<br>' not in str_info
-    assert '<b>' not in str_info
-    assert "Plugins" in str_info
-
-    html_info = sys_info(as_html=True)
-    assert isinstance(html_info, str)
-    assert '<br>' in html_info
-    assert '<b>' in html_info
+from napari.utils import info
 
 
 def test_citation_text():
-    assert isinstance(citation_text, str)
-    assert 'doi' in citation_text
+    assert isinstance(info.citation_text, str)
+    assert 'doi' in info.citation_text
 
 
 def test_linux_os_name_file(monkeypatch, tmp_path):

tox.ini

@@ -74,7 +74,6 @@ conda_deps =
 conda_channels =
     conda-forge
 deps =
-    pytest-xvfb ; sys_platform == 'linux'
     pytest-cov
 # use extras specified in setup.cfg for certain test envs
 extras =