FIX: Use upstream/main for tools

.github/workflows/qt_viz_tests.yml

@@ -8,40 +8,75 @@ on:
       branches: [main]
 
 jobs:
+  flake:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install flake8
+      - name: Lint with flake8
+        run: flake8
   pytest:
-    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
+        os: [ubuntu, windows, macos]
+        mne: [main, maint/0.24]
+        opengl: ['[opengl]']
+        python: ['3.9']
+        name: [matrix]
         include:
-        - os: ubuntu-latest
-        - os: windows-latest
-        - os: macos-latest
-    defaults:
-      run:
-        shell: bash
+          - os: ubuntu
+            mne: main
+            opengl: '[opengl]'
+            python: '3.7'
+            name: old 3.7
+          - os: ubuntu
+            mne: main
+            opengl: ''
+            python: '3.9'
+            name: no opengl
+          - os: ubuntu
+            mne: maint/0.24
+            opengl: ''
+            python: '3.9'
+            name: no opengl
+    name: pytest ${{ matrix.name }} / ${{ matrix.os }} / MNE ${{ matrix.mne }}
+    runs-on: ${{ matrix.os }}-latest
     env:
       MNE_LOGGING_LEVEL: 'warning'
       MKL_NUM_THREADS: '1'
       PYTHONUNBUFFERED: '1'
       DISPLAY: ':99.0'
+      MNE_BRANCH: ${{ matrix.mne }}
+      PIP_OPTION: ${{ matrix.opengl }}
+      PYTHON_VERSION: ${{ matrix.python }}
+    defaults:
+      run:
+        shell: bash
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: Install dependencies
         run: |  # use MNE main to ensure test files are available
           set -e
           python -m pip install --upgrade pip
-          git clone -b pg_test --single-branch --depth=1 https://github.com/marsipu/mne-python.git
-          python -m pip install -e ./mne-python .[opengl]
-          python -m pip install -r requirements.txt -r requirements_testing.txt
+          git clone -b ${MNE_BRANCH} --single-branch --depth=1 https://github.com/mne-tools/mne-python.git ../mne-python
+          python -m pip install -qe ../mne-python
+          python -m pip install -ve .${PIP_OPTION} -r requirements.txt -r requirements_testing.txt
       - shell: bash -el {0}
         run: ./tools/get_testing_version.sh
         name: 'Get testing version'
-        working-directory: mne-python
+        working-directory: ../mne-python
       - uses: actions/cache@v2
         with:
           key: ${{ env.TESTING_VERSION }}
@@ -57,30 +92,19 @@ jobs:
         if: runner.os == 'Windows'
       - run: ./tools/setup_xvfb.sh
         name: Setup xvfb on Linux
-        working-directory: mne-python
+        working-directory: ../mne-python
         if: runner.os == 'Linux'
       - name: Show system information
         run: mne sys_info
-      - name: Run MNE-Tests
-        run: |
-          mnePath=$(python -c "import mne; print(mne.__path__[0])")
-          cp $mnePath/viz/tests/test_raw.py ./mne_qt_browser/tests/test_raw.py
-          pytest --cov=mne_qt_browser mne_qt_browser/tests
+      # This can go away if we make OpenGL opt-in in MNE and backport the change
+      # This line also does not work on macOS
+      - run: sed -i'' '/^    error::/a\    ignore:.*PyOpenGL was not found.*:RuntimeWarning' ../mne-python/mne/conftest.py
+        name: Adjust mne conftest
+        if: matrix.opengl == ''
+      - run: pytest -m pgtest --cov=mne_qt_browser --cov-report=xml ../mne-python/mne/viz
+        name: Run MNE-Tests
+      - run: pytest mne_qt_browser/tests
+        name: Run benchmarks
       - uses: codecov/codecov-action@v1
         if: always()
         name: 'Upload coverage to CodeCov'
-
-  flake:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install flake8
-      - name: Lint with flake8
-        run: flake8

.gitignore

@@ -130,3 +130,5 @@ dmypy.json
 
 # PyCharm
 .idea/
+
+junit-results.xml

README.md

@@ -1,4 +1,5 @@
 # mne-qt-browser
+
 #### A new backend based on pyqtgraph for the 2D-Data-Browser in MNE-Python.
 
 This repository hosts the code for an alternative backend for plotting 2D-Data with 
@@ -12,6 +13,7 @@ Currently, only `Raw.plot()` is supported. For the future support for Epochs
 and ICA-Sources is planned.
 
 ### Usage
+
 Import mne-python
 ```python
 import mne
@@ -35,4 +37,23 @@ If you want to try the browser with the sample-dataset from mne-python,
 run `mne-qt-browser` from the terminal.
 
 ### Report Bugs & Feature Requests
+
 Please report bugs and feature requests in the [issues](https://github.com/mne-tools/mne-qt-browser/issues) of this repository.
+
+### Development and testing
+
+You can run a benchmark locally with:
+
+```console
+$ pytest -m benchmark mne_qt_browser
+```
+
+To run tests, clone mne-python, and then run the PyQtGraph tests with e.g.:
+```console
+$ pytest -m pgtest ../mne-python/mne/viz/tests
+```
+If you do not have OpenGL installed, this will currently raise errors, and
+you'll need to add this line to `mne/conftest.py` after the `error::` line:
+```
+    ignore:.*PyOpenGL was not found.*:RuntimeWarning
+```

mne_qt_browser/conftest.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# Author: Eric Larson <larson.eric.d@gmail.com>
+#
+# License: BSD-3-Clause
+
+
+import pytest
+
+_store = dict()
+
+
+def pytest_configure(config):
+    """Configure pytest options."""
+    # Markers
+    for marker in ('benchmark',):
+        config.addinivalue_line('markers', marker)
+
+
+@pytest.fixture(scope='session')
+def store():
+    """Yield our storage object."""
+    yield _store
+
+
+def pytest_sessionfinish(session, exitstatus):
+    """Print our benchmark results (if present)."""
+    if len(_store):
+        from py.io import TerminalWriter
+        writer = TerminalWriter()
+        writer.line()  # newline
+        writer.sep('=', 'benchmark results')
+        for name, vals in _store.items():
+            writer.line(
+                f'{name}:\n'
+                f'    Horizontal: {vals["h"]:6.2f}\n'
+                f'    Vertical:   {vals["v"]:6.2f}')

mne_qt_browser/tests/test_speed.py

@@ -1,6 +1,6 @@
-import sys
 from copy import copy
 from functools import partial
+import sys
 
 import numpy as np
 import pytest
@@ -13,13 +13,28 @@
 v_last_time = None
 
 
-# Skip speed-test for CIs
-@pytest.mark.skip
-@pytest.mark.parametrize('benchmark_param', [{'use_opengl': False},
-                                             {'use_opengl': True},
-                                             {'precompute': False},
-                                             {'precompute': True}])
-def test_scroll_speed(raw_orig, benchmark_param, browser_backend):
+try:
+    import OpenGL  # noqa
+except Exception as exc:
+    has_gl = False
+    reason = str(exc)
+else:
+    has_gl = True
+    reason = ''
+gl_mark = pytest.mark.skipif(
+    not has_gl, reason=f'Requires PyOpengl (got {reason})')
+
+
+@pytest.mark.filterwarnings('ignore:.*PyOpenGL was not found.*:RuntimeWarning')
+@pytest.mark.benchmark
+@pytest.mark.parametrize('benchmark_param', [
+    pytest.param({'use_opengl': False}, id='use_opengl=False'),
+    pytest.param({'use_opengl': True}, marks=gl_mark, id='use_opengl=True'),
+    pytest.param({'precompute': False}, marks=gl_mark, id='precompute=False'),
+    pytest.param({'precompute': True}, marks=gl_mark, id='precompute=True'),
+    pytest.param({}, marks=gl_mark, id='defaults'),
+])
+def test_scroll_speed(raw_orig, benchmark_param, store, pg_backend, request):
     """Test the speed of a parameter."""
     # Remove spaces and get params with values
     from time import perf_counter
@@ -30,7 +45,7 @@ def test_scroll_speed(raw_orig, benchmark_param, browser_backend):
     hscroll_diffs = list()
     vscroll_diffs = list()
 
-    def _ininite_hscroll(pg_fig):
+    def _initiate_hscroll(pg_fig):
         global bm_count
         global hscroll_dir
         global vscroll_dir
@@ -77,9 +92,8 @@ def _ininite_hscroll(pg_fig):
 
             h_mean_fps = 1 / np.median(hscroll_diffs)
             v_mean_fps = 1 / np.median(vscroll_diffs)
-            print(f'The median FPS for {benchmark_param} is:\n'
-                  f'Horizontal = {h_mean_fps:.3f} FPS\n'
-                  f'Vertical = {v_mean_fps:.3f} FPS')
+            store[request.node.callspec.id] = dict(
+                h=h_mean_fps, v=v_mean_fps)
             pg_fig.close()
 
     app = QApplication.instance()
@@ -88,7 +102,7 @@ def _ininite_hscroll(pg_fig):
     fig = raw_orig.plot(duration=5, n_channels=40,
                         show=False, block=False, **benchmark_param)
     timer = QTimer()
-    timer.timeout.connect(partial(_ininite_hscroll, fig))
+    timer.timeout.connect(partial(_initiate_hscroll, fig))
     timer.start(0)
 
     fig.show()

requirements_testing.txt

@@ -1,6 +1,7 @@
 pytest
 pytest-qt
 pytest-cov
+pytest-timeout
 pooch
 pyvista  # for mne sys_info to tell us about OpenGL
 tqdm

setup.cfg

@@ -1,5 +1,4 @@
 [tool:pytest]
 addopts =
-    --durations=20 -ra --cov-report= --tb=short
-    --junit-xml=junit-results.xml
+    -ra --cov-report= --tb=short --junit-xml=junit-results.xml --capture=sys
 junit_family = xunit2