MAINT: Cleaner testing

.circleci/config.yml

@@ -27,6 +27,7 @@ jobs:
               echo "set -e" >> $BASH_ENV
               echo "export OPENBLAS_NUM_THREADS=4" >> $BASH_ENV
               echo "export XDG_RUNTIME_DIR=/tmp/runtime-circleci" >> $BASH_ENV
+              mkdir -p tools
               wget -q -O tools/get_minimal_commands.sh https://raw.githubusercontent.com/mne-tools/mne-python/main/tools/get_minimal_commands.sh
               source tools/get_minimal_commands.sh
               echo "export MNE_3D_BACKEND=pyvistaqt" >> $BASH_ENV
@@ -71,8 +72,7 @@ jobs:
             name: Install dependencies and mne-realtime
             command: |
               python -m pip install --upgrade --progress-bar off pip setuptools wheel
-              python -m pip install --upgrade --progress-bar off -r requirements.txt git+https://github.com/mne-tools/mne-python@main
-              python -m pip install -e .[docs] git+https://github.com/sphinx-gallery/sphinx-gallery@master
+              python -m pip install --upgrade --progress-bar off -r requirements.txt git+https://github.com/mne-tools/mne-python@main -e .[docs] git+https://github.com/sphinx-gallery/sphinx-gallery@master
 
         - save_cache:
             key: pip-cache-1

.github/workflows/circle_artifacts.yml

@@ -8,5 +8,7 @@ jobs:
         uses: larsoner/circleci-artifacts-redirector-action@master
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+          api-token: ${{ secrets.CIRCLECI_TOKEN }}
           artifact-path: 0/dev/index.html
           circleci-jobs: build_docs
+          job-title: Check the rendered docs here!

.github/workflows/tests.yml

@@ -0,0 +1,70 @@
+name: 'Tests'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
+  cancel-in-progress: true
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  pytest:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        kind: [pip]
+        python: ['3.11']
+        include:
+          - os: macos-latest
+            kind: conda
+            python: '3.8'
+    env:
+      CI_OS_NAME: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+        if: ${{ matrix.kind == 'pip' }}
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
+          create-args: python=${{ matrix.python }}
+        if: ${{ matrix.kind == 'conda' }}
+      - run: git clone --single-branch -b main https://github.com/mne-tools/mne-python.git
+        name: Clone MNE-Python
+      - run: |
+          source ./mne-python/tools/get_minimal_commands.sh
+          neuromag2ft --version
+        name: Install neuromag2ft
+        if: ${{ startswith(matrix.os, 'ubuntu') }}
+      - run: |
+          wget -q https://github.com/sccn/liblsl/releases/download/v1.16.2/liblsl-1.16.2-jammy_amd64.deb
+          sudo apt install ./liblsl-*.deb
+        if: ${{ startswith(matrix.os, 'ubuntu') }}
+        name: Install liblsl
+      - run: |
+          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install -r requirements.txt -ve . -e ./mne-python
+        name: Install dependencies and MNE-Realtime
+      - run: mne sys_info
+      - run: ./tools/get_testing_version.sh
+        working-directory: ./mne-python
+      - run: python -c "import mne; print(mne.datasets.testing.data_path(verbose=True))"
+        name: Get test data
+      - uses: actions/cache@v3
+        with:
+          key: ${{ env.TESTING_VERSION }}
+          path: ~/mne_data
+      - run: pytest -vv mne_realtime
+      - uses: codecov/codecov-action@v1
+        if: success()
+        name: 'Upload coverage to CodeCov'

Makefile

@@ -45,9 +45,6 @@ test-full: in
 	rm -f .coverage
 	$(PYTESTS) mne
 
-test-no-network: in
-	sudo unshare -n -- sh -c 'MNE_SKIP_NETWORK_TESTS=1 py.test mne'
-
 test-no-testing-data: in
 	@MNE_SKIP_TESTING_DATASET_TESTS=true \
 	$(PYTESTS) mne

environment.yml

@@ -14,8 +14,8 @@ dependencies:
 - pytest-cov
 - numpydoc
 - flake8
+- liblsl
+- mne
+- pydocstyle
 - pip:
-  - mne>=1.0.0
-  - pydocstyle
-  - codespell
-  - pylsl>=1.12
+  - pylsl

mne_realtime/conftest.py

@@ -13,6 +13,12 @@ def pytest_configure(config):
     for fixture in ('matplotlib_config',):
         config.addinivalue_line('usefixtures', fixture)
 
+    # Marks
+    for marker in (
+        "network_test",
+    ):
+        config.addinivalue_line("markers", marker)
+
     # Warnings
     # - Once SciPy updates not to have non-integer and non-tuple errors (1.2.0)
     #   we should remove them from here.

mne_realtime/tests/test_fieldtrip_client.py

@@ -18,15 +18,16 @@
 
 from mne import Epochs, find_events, pick_types
 from mne.io import read_raw_fif
-from mne.utils import requires_neuromag2ft
 from mne_realtime import FieldTripClient, RtEpochs
-
 from mne_realtime.tests.test_mock_client import _call_base_epochs_public_api
 
 base_dir = op.join(op.dirname(__file__), '..', '..', 'mne_realtime',
                    'tests', 'data')
 raw_fname = op.realpath(op.join(base_dir, 'test_raw.fif'))
 
+if 'NEUROMAG2FT_ROOT' not in os.environ:
+    pytest.skip("NEUROMAG2FT_ROOT env var not found", allow_module_level=True)
+
 
 @pytest.fixture
 def free_tcp_port():
@@ -67,7 +68,6 @@
     return signal_queue
 
 
-@requires_neuromag2ft
 def test_fieldtrip_rtepochs(free_tcp_port, tmpdir):
     """Test FieldTrip RtEpochs."""
     raw_tmax = 7
@@ -125,7 +125,6 @@
         kill_signal.put(False)  # stop the buffer
 
 
-@requires_neuromag2ft
 def test_fieldtrip_client(free_tcp_port):
     """Test fieldtrip_client."""
     kill_signal = _start_buffer_thread(free_tcp_port)

mne_realtime/tests/test_lsl_client.py

@@ -6,7 +6,6 @@
 import time
 import pytest
 
-from mne.utils import requires_pylsl
 from mne.io import read_raw_fif
 from mne.datasets import testing
 import numpy as np
@@ -20,10 +19,10 @@
 raw_fname = op.join(base_dir, 'test_raw.fif')
 
 
-@requires_pylsl
 @testing.requires_testing_data
 def test_lsl_client():
     """Test the LSLClient for connection and data retrieval."""
+    pytest.importorskip('pylsl')
     raw = read_raw_fif(raw_fname)
     raw_info = raw.info
     sfreq = raw_info['sfreq']
@@ -47,22 +46,22 @@ def test_lsl_client():
     assert raw_info['nchan'], sfreq == epoch.get_data().shape[1:]
 
 
-@requires_pylsl
 @testing.requires_testing_data
 def test_lsl_client_nodata():
     """Test that LSLClient gracefully handles no-data from LSL."""
+    pytest.importorskip('pylsl')
     raw = read_raw_fif(raw_fname)
     raw_info = raw.info
     with MockLSLStream(host, raw, ch_type='eeg', status=True):
         with LSLClient(info=raw_info, host=host, wait_max=5) as client:
             epoch = client.get_data_as_epoch(n_samples=0, timeout=0)
             assert epoch is None
 
-@requires_pylsl
-def test_connect(mocker):
+
+def test_connect(monkeypatch):
     """Mock connect to LSL stream."""
     # Import pylsl here so that the test can be skipped if pylsl is not installed
-    pylsl = _check_pylsl_installed(strict=True)
+    pylsl = pytest.importorskip("pylsl")
 
     # Constants
     buffer_size = 17
@@ -71,15 +70,16 @@ def test_connect(mocker):
     numpy_channel_format = np.float32
 
     # Replace pylsl streams with a mock
-    mock_resolve_streams = mocker.patch(
+    monkeypatch.setattr(
         'pylsl.resolve_streams',
-        return_value=[pylsl.StreamInfo(
-            source_id=host,
-            channel_count=n_channels,
-            channel_format=c_channel_format,
-        )],
+        lambda *args, **kwargs: [
+            pylsl.StreamInfo(
+                source_id=host,
+                channel_count=n_channels,
+                channel_format=c_channel_format,
+            )
+        ],
     )
-
     lsl_client = LSLClient(host=host, buffer_size=buffer_size)
     # Mock out the pylsl.resolve_streams
     lsl_client._connect()