diff --git a/.circleci/config.yml b/.circleci/config.yml index 7bad6d80a5b..66a78005dbd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,8 +99,10 @@ jobs: name: Install fonts needed for diagrams command: | mkdir -p $HOME/.fonts + echo "Source Code Pro" curl https://codeload.github.com/adobe-fonts/source-code-pro/tar.gz/2.038R-ro/1.058R-it/1.018R-VAR | tar xz -C $HOME/.fonts - curl https://codeload.github.com/adobe-fonts/source-sans-pro/tar.gz/3.028R | tar xz -C $HOME/.fonts + echo "Source Sans Pro" + curl https://codeload.github.com/adobe-fonts/source-sans/tar.gz/3.028R | tar xz -C $HOME/.fonts fc-cache -f # Load pip cache diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d535e037c0a..93c104576a2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,6 +57,9 @@ jobs: - os: macos-latest python: '3.8' kind: mamba + - os: windows-latest + python: '3.10' + kind: mamba - os: ubuntu-latest python: '3.8' kind: minimal @@ -82,9 +85,9 @@ jobs: # Python (if conda) - uses: conda-incubator/setup-miniconda@v2 with: - activate-environment: ${{ env.CONDA_ACTIVATE_ENV }} python-version: ${{ env.PYTHON_VERSION }} environment-file: ${{ env.CONDA_ENV }} + activate-environment: mne miniforge-version: latest miniforge-variant: Mambaforge use-mamba: ${{ matrix.kind != 'conda' }} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ccea1813875..bc36111280b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -242,10 +242,8 @@ stages: variables: MNE_LOGGING_LEVEL: 'warning' MNE_FORCE_SERIAL: 'true' - OPENBLAS_NUM_THREADS: 1 - MKL_NUM_THREADS: 2 + OPENBLAS_NUM_THREADS: 2 OMP_DYNAMIC: 'false' - MKL_DYNAMIC: 'false' PYTHONUNBUFFERED: 1 PYTHONIOENCODING: 'utf-8' AZURE_CI_WINDOWS: 'true' @@ -254,10 +252,6 @@ stages: strategy: maxParallel: 4 matrix: - 3.10 conda: - PLATFORM: 'x86-64' - TEST_MODE: 'conda' - PYTHON_VERSION: '3.10' 3.9 pip: TEST_MODE: 'pip' PYTHON_VERSION: '3.9' @@ -270,44 +264,16 @@ stages: versionSpec: $(PYTHON_VERSION) architecture: $(PYTHON_ARCH) addToPath: true - condition: in(variables['TEST_MODE'], 'pip', 'pip-pre') displayName: 'Get Python' - # https://docs.microsoft.com/en-us/azure/devops/pipelines/ecosystems/anaconda - # https://github.com/MicrosoftDocs/pipelines-anaconda - # https://github.com/ContinuumIO/anaconda-issues/issues/10949 - - script: | - echo "##vso[task.prependpath]%CONDA%;%CONDA%\condabin;%CONDA%\Scripts;%CONDA%\Library\bin;%PROGRAMFILES%\Git\bin;%SystemRoot%\system32;%SystemRoot%;%SystemRoot%\System32\Wbem;%PROGRAMFILES%\Git\usr\bin" - condition: in(variables['TEST_MODE'], 'conda') - displayName: Add conda to PATH, deal with Qt linking bug - bash: | - set -e + set -eo pipefail git clone --depth 1 https://github.com/pyvista/gl-ci-helpers.git powershell gl-ci-helpers/appveyor/install_opengl.ps1 displayName: Install OpenGL - - bash: | - set -e - ./tools/azure_dependencies.sh - condition: in(variables['TEST_MODE'], 'pip', 'pip-pre') + - bash: ./tools/azure_dependencies.sh displayName: Install dependencies with pip - - script: conda install -c conda-forge "mamba!=1.4.9" - condition: eq(variables['TEST_MODE'], 'conda') - displayName: Get mamba - - script: mamba env update --name base --file environment.yml - condition: eq(variables['TEST_MODE'], 'conda') - displayName: Setup MNE environment - # ipympl is not tested on Windows and even its installation interferes - # with basic matplotlib functionality so it must be uninstalled until fixed - - bash: | - set -e - mamba remove -c conda-forge --force -yq mne ipympl - rm /c/Miniconda/Scripts/mne.exe - condition: eq(variables['TEST_MODE'], 'conda') - displayName: Remove old MNE - script: pip install -e . displayName: 'Install MNE-Python dev' - - script: pip install --progress-bar off -e .[test] - condition: eq(variables['TEST_MODE'], 'conda') - displayName: Install testing requirements - script: mne sys_info -pd displayName: 'Print config' - script: python -c "import numpy; numpy.show_config()" diff --git a/doc/_static/style.css b/doc/_static/style.css index f9ead72d6a9..b10216ddcb3 100644 --- a/doc/_static/style.css +++ b/doc/_static/style.css @@ -103,6 +103,22 @@ html[data-theme="dark"] img { color: var(--pst-color-link); } +/* make versionadded smaller and inline with param name */ +/* don't do for deprecated / versionchanged; they have extra info (too long to fit) */ +div.versionadded > p { + margin-top: 0; + margin-bottom: 0; +} +div.versionadded { + margin: 0; + margin-left: 0.5rem; + display: inline-block; +} +/* when FF supports :has(), change to → dd > p:has(+div.versionadded) */ +dd>p { + display: inline; +} + /* **************************************************** sphinx-gallery fixes */ /* backreference links: restore hover decoration that SG removes */ diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index a08fdaab4b2..f8bbbb8ea29 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -43,6 +43,7 @@ Bugs - Extended test to highlight bug in :func:`mne.stats.permutation_t_test` (:gh:`11575` by :newcontrib:`Joshua Calder-Travis`) - Fix bug that used wrong indices for line/label styles (sometimes causing an ``IndexError``) in :meth:`mne.preprocessing.ICA.plot_sources` under certain conditions (:gh:`11808` by :newcontrib:`Joshua Calder-Travis`) - Fix bug with :func:`~mne.io.read_raw_snirf` to handle files with measurement time containing milliseconds (:gh:`11804` by :newcontrib:`Daniel Tse`) +- Fix bug where :func:`mne.io.read_raw_cnt` imports unnecessary durations (:gh:`11828` by `Jacob Woessner`_) - Fix bug where :meth:`mne.viz.Brain.add_volume_labels` used an incorrect orientation (:gh:`11730` by `Alex Rockhill`_) - Fix bug with :func:`mne.forward.restrict_forward_to_label` where cortical patch information was not adjusted (:gh:`11694` by `Eric Larson`_) - Fix bug with PySide6 compatibility (:gh:`11721` by `Eric Larson`_) @@ -52,11 +53,13 @@ Bugs - Fix bug with :func:`mne.io.read_raw_fil` where datasets without sensor positions would not import (:gh:`11733` by `George O'Neill`_) - Fix bug with :func:`mne.chpi.compute_chpi_snr` where cHPI being off for part of the recording or bad channels being defined led to an error or incorrect behavior (:gh:`11754`, :gh:`11755` by `Eric Larson`_) - Allow int-like for the argument ``id`` of `~mne.make_fixed_length_events` (:gh:`11748` by `Mathieu Scheltienne`_) +- Fix bug where :func:`mne.io.read_raw_egi` did not properly set the EEG reference location for the reference channel itself (:gh:`11822` by `Eric Larson`_) - Fix bug with legacy :meth:`~mne.io.Raw.plot_psd` method where passed axes were not used (:gh:`11778` by `Daniel McCloy`_) - blink :class:`mne.Annotations` read in by :func:`mne.io.read_raw_eyelink` now begin with ``'BAD_'``, i.e. ``'BAD_blink'``, because ocular data are missing during blinks. (:gh:`11746` by `Scott Huberty`_) - Fix bug where :ref:`mne show_fiff` could fail with an ambiguous error if the file is corrupt (:gh:`11800` by `Eric Larson`_) - Fix bug where annotation FIF files lacked an end block tag (:gh:`11800` by `Eric Larson`_) - Fix display of :class:`~mne.Annotations` in `mne.preprocessing.ICA.plot_sources` when the ``raw`` has ``raw.first_samp != 0`` and doesn't have a measurement date (:gh:`11766` by `Mathieu Scheltienne`_) +- Fix bug in read_raw_eyelink, where STATUS information of samples was always assumed to be in the file. Performance and memory improvements were also made. (:gh:`11823` by `Scott Huberty`_) API changes ~~~~~~~~~~~ diff --git a/doc/conf.py b/doc/conf.py index 2f441a0641a..7a4f3df182c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -16,11 +16,11 @@ import numpy as np import matplotlib import sphinx +from sphinx.domains.changeset import versionlabels from sphinx_gallery.sorting import FileNameSortKey, ExplicitOrder from numpydoc import docscrape import mne -from mne.fixes import _compare_version from mne.tests.test_docstring_parameters import error_ignores from mne.utils import ( linkcode_resolve, # noqa, analysis:ignore @@ -752,6 +752,10 @@ def append_attr_meth_examples(app, what, name, obj, options, lines): suppress_warnings = ["image.nonlocal_uri"] # we intentionally link outside +# -- Sphinx hacks / overrides ------------------------------------------------ + +versionlabels["versionadded"] = sphinx.locale._("New in v%s") + # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/doc/install/mne_tools_suite.rst b/doc/install/mne_tools_suite.rst index c9126e5c012..ef6858758e5 100644 --- a/doc/install/mne_tools_suite.rst +++ b/doc/install/mne_tools_suite.rst @@ -51,22 +51,24 @@ MNE-Python, including packages for: (`alphaCSC`_) - independent component analysis (ICA) with good performance on real data (`PICARD`_) +- automatic labeling of ICA components (`MNE-ICAlabel`_) - phase-amplitude coupling (`pactools`_) - representational similarity analysis (`rsa`_) - microstate analysis (`microstate`_) - connectivity analysis using dynamic imaging of coherent sources (DICS) (`conpy`_) +- other connectivity algorithms (`MNE-Connectivity`_) - general-purpose statistical analysis of M/EEG data (`eelbrain`_) - post-hoc modification of linear models (`posthoc`_) - a python implementation of the Preprocessing Pipeline (PREP) for EEG data (`pyprep`_) - automatic multi-dipole localization and uncertainty quantification with the Bayesian algorithm SESAME (`sesameeg`_) -- GLM and group level analysis of near-infrared spectroscopy data (`mne-nirs`_) -- High-level EEG Python library for all kinds of EEG inverse solutions (`invertmeeg`_) +- GLM and group level analysis of near-infrared spectroscopy data (`MNE-NIRS`_) +- high-level EEG Python library for all kinds of EEG inverse solutions (`invertmeeg`_) - All-Resolutions Inference (ARI) for statistically valid circular inference and effect localization (`MNE-ARI`_) - +- real-time analysis (`MNE-Realtime`_) What should I install? ^^^^^^^^^^^^^^^^^^^^^^ @@ -109,6 +111,5 @@ Help with installation is available through the `MNE Forum`_. See the .. _posthoc: https://users.aalto.fi/~vanvlm1/posthoc/python/ .. _pyprep: https://github.com/sappelhoff/pyprep .. _sesameeg: https://pybees.github.io/sesameeg -.. _mne-nirs: https://github.com/mne-tools/mne-nirs .. _invertmeeg: https://github.com/LukeTheHecker/invert .. _MNE-ARI: https://github.com/john-veillette/mne_ari diff --git a/doc/links.inc b/doc/links.inc index 431a4edabc2..8d2db2ee8c8 100644 --- a/doc/links.inc +++ b/doc/links.inc @@ -23,6 +23,9 @@ .. _`MNE-Realtime`: https://mne.tools/mne-realtime .. _`MNE-MATLAB git repository`: https://github.com/mne-tools/mne-matlab .. _`MNE-Docker`: https://github.com/mne-tools/mne-docker +.. _`MNE-ICAlabel`: https://github.com/mne-tools/mne-icalabel +.. _`MNE-Connectivity`: https://github.com/mne-tools/mne-connectivity +.. _`MNE-NIRS`: https://github.com/mne-tools/mne-nirs .. _OpenMEEG: http://openmeeg.github.io .. _EOSS2: https://chanzuckerberg.com/eoss/proposals/improving-usability-of-core-neuroscience-analysis-tools-with-mne-python .. _EOSS4: https://chanzuckerberg.com/eoss/proposals/building-pediatric-and-clinical-data-pipelines-for-mne-python/ diff --git a/mne/channels/montage.py b/mne/channels/montage.py index 13e064ead4a..e75d158d3cf 100644 --- a/mne/channels/montage.py +++ b/mne/channels/montage.py @@ -1171,25 +1171,22 @@ def _backcompat_value(pos, ref_pos): chs = [info["chs"][ii] for ii in picks] non_names = [info["chs"][ii]["ch_name"] for ii in non_picks] del picks - ref_pos = [ch["loc"][3:6] for ch in chs] + ref_pos = np.array([ch["loc"][3:6] for ch in chs]) # keep reference location from EEG-like channels if they # already exist and are all the same. - custom_eeg_ref_dig = False # Note: ref position is an empty list for fieldtrip data - if ref_pos: - if ( - all([np.equal(ref_pos[0], pos).all() for pos in ref_pos]) - and not np.equal(ref_pos[0], [0, 0, 0]).all() - ): - eeg_ref_pos = ref_pos[0] - # since we have an EEG reference position, we have - # to add it into the info['dig'] as EEG000 - custom_eeg_ref_dig = True - if not custom_eeg_ref_dig: + if len(ref_pos) and ref_pos[0].any() and (ref_pos[0] == ref_pos).all(): + eeg_ref_pos = ref_pos[0] + # since we have an EEG reference position, we have + # to add it into the info['dig'] as EEG000 + custom_eeg_ref_dig = True + else: refs = set(ch_pos) & {"EEG000", "REF"} assert len(refs) <= 1 eeg_ref_pos = np.zeros(3) if not refs else ch_pos.pop(refs.pop()) + custom_eeg_ref_dig = False + del ref_pos # This raises based on info being subset/superset of montage info_names = [ch["ch_name"] for ch in chs] diff --git a/mne/conftest.py b/mne/conftest.py index 1bfc74bfe89..327400784b2 100644 --- a/mne/conftest.py +++ b/mne/conftest.py @@ -163,6 +163,10 @@ def pytest_configure(config): warning_line = warning_line.strip() if warning_line and not warning_line.startswith("#"): config.addinivalue_line("filterwarnings", warning_line) + # TODO: Fix this with casts? + # https://github.com/numpy/numpy/pull/22449 + if check_version("numpy", "1.26"): + np.set_printoptions(legacy="1.25") # Have to be careful with autouse=True, but this is just an int comparison diff --git a/mne/cov.py b/mne/cov.py index 15fd043d022..44f730981a8 100644 --- a/mne/cov.py +++ b/mne/cov.py @@ -382,7 +382,15 @@ def plot_topomap( %(sphere_topomap_auto)s %(image_interp_topomap)s %(extrapolate_topomap)s + + .. versionchanged:: 0.21 + + - The default was changed to ``'local'`` for MEG sensors. + - ``'local'`` was changed to use a convex hull mask + - ``'head'`` was changed to extrapolate out to the clipping circle. %(border_topomap)s + + .. versionadded:: 0.20 %(res_topomap)s %(size_topomap)s %(cmap_topomap)s diff --git a/mne/io/base.py b/mne/io/base.py index 9c1ea43a98d..65a5fbb0f2c 100644 --- a/mne/io/base.py +++ b/mne/io/base.py @@ -170,6 +170,8 @@ class BaseRaw( (only needed for types that support on-demand disk reads) """ + _extra_attributes = () + @verbose def __init__( self, diff --git a/mne/io/brainvision/brainvision.py b/mne/io/brainvision/brainvision.py index 495721dfd85..980d5dabef9 100644 --- a/mne/io/brainvision/brainvision.py +++ b/mne/io/brainvision/brainvision.py @@ -61,6 +61,8 @@ class RawBrainVision(BaseRaw): mne.io.Raw : Documentation of attributes and methods. """ + _extra_attributes = ("impedances",) + @verbose def __init__( self, diff --git a/mne/io/bti/bti.py b/mne/io/bti/bti.py index d3f4ea42f4c..0a74b54f568 100644 --- a/mne/io/bti/bti.py +++ b/mne/io/bti/bti.py @@ -1086,7 +1086,7 @@ def __init__( sort_by_ch_name=sort_by_ch_name, eog_ch=eog_ch, ) - self.bti_ch_labels = [c["chan_label"] for c in bti_info["chs"]] + bti_info["bti_ch_labels"] = [c["chan_label"] for c in bti_info["chs"]] # make Raw repr work if we have a BytesIO as input if isinstance(pdf_fname, BytesIO): pdf_fname = repr(pdf_fname) diff --git a/mne/io/bti/tests/test_bti.py b/mne/io/bti/tests/test_bti.py index fef5594cf67..b4ecda3b4f5 100644 --- a/mne/io/bti/tests/test_bti.py +++ b/mne/io/bti/tests/test_bti.py @@ -285,10 +285,12 @@ def test_info_no_rename_no_reorder_no_pdf(): preload=True, ) - sort_idx = [raw1.bti_ch_labels.index(ch) for ch in raw2.bti_ch_labels] + bti_ch_labels_1 = raw1._raw_extras[0]["bti_ch_labels"] + bti_ch_labels_2 = raw2._raw_extras[0]["bti_ch_labels"] + sort_idx = [bti_ch_labels_1.index(ch) for ch in bti_ch_labels_2] raw1._data = raw1._data[sort_idx] assert_array_equal(raw1._data, raw2._data) - assert_array_equal(raw2.bti_ch_labels, raw2.ch_names) + assert_array_equal(bti_ch_labels_2, raw2.ch_names) def test_no_conversion(): diff --git a/mne/io/cnt/cnt.py b/mne/io/cnt/cnt.py index dc52a750b72..80a005911d9 100644 --- a/mne/io/cnt/cnt.py +++ b/mne/io/cnt/cnt.py @@ -145,9 +145,9 @@ def _update_bad_span_onset(accept_reject, onset, duration, description): event_type=type(my_events[0]), data_format=data_format, ) - duration = np.array( - [getattr(e, "Latency", 0.0) for e in my_events], dtype=float - ) + # There is a Latency field but it's not useful for durations, see + # https://github.com/mne-tools/mne-python/pull/11828 + duration = np.zeros(len(my_events), dtype=float) accept_reject = _accept_reject_function( np.array([e.KeyPad_Accept for e in my_events]) ) diff --git a/mne/io/ctf/ctf.py b/mne/io/ctf/ctf.py index 14d55c05cb3..d4b819bcf29 100644 --- a/mne/io/ctf/ctf.py +++ b/mne/io/ctf/ctf.py @@ -193,9 +193,8 @@ def __init__( ) annot = marker_annot if annot is None else annot + marker_annot self.set_annotations(annot) - if clean_names: - self._clean_names() + _clean_names_inst(self) def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): """Read a chunk of raw data.""" @@ -222,15 +221,14 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): _mult_cal_one(data_view, this_data, idx, cals, mult) offset += n_read - def _clean_names(self): - """Clean up CTF suffixes from channel names.""" - mapping = dict(zip(self.ch_names, _clean_names(self.ch_names))) - - self.rename_channels(mapping) - for comp in self.info["comps"]: - for key in ("row_names", "col_names"): - comp["data"][key] = _clean_names(comp["data"][key]) +def _clean_names_inst(inst): + """Clean up CTF suffixes from channel names.""" + mapping = dict(zip(inst.ch_names, _clean_names(inst.ch_names))) + inst.rename_channels(mapping) + for comp in inst.info["comps"]: + for key in ("row_names", "col_names"): + comp["data"][key] = _clean_names(comp["data"][key]) def _get_sample_info(fname, res4, system_clock): diff --git a/mne/io/egi/egi.py b/mne/io/egi/egi.py index 530dcb9179c..a8b9eddc5c1 100644 --- a/mne/io/egi/egi.py +++ b/mne/io/egi/egi.py @@ -178,6 +178,8 @@ def read_raw_egi( class RawEGI(BaseRaw): """Raw object from EGI simple binary file.""" + _extra_attributes = ("event_id",) + @verbose def __init__( self, diff --git a/mne/io/egi/egimff.py b/mne/io/egi/egimff.py index db7247730f8..5f952701fb6 100644 --- a/mne/io/egi/egimff.py +++ b/mne/io/egi/egimff.py @@ -432,6 +432,8 @@ def _read_raw_egi_mff( class RawMff(BaseRaw): """RawMff class.""" + _extra_attributes = ("event_id",) + @verbose def __init__( self, @@ -575,14 +577,9 @@ def __init__( ref_idx = ref_idx.item() ref_coords = info["chs"][int(ref_idx)]["loc"][:3] for chan in info["chs"]: - is_eeg = chan["kind"] == FIFF.FIFFV_EEG_CH - is_not_ref = chan["ch_name"] not in REFERENCE_NAMES - if is_eeg and is_not_ref: + if chan["kind"] == FIFF.FIFFV_EEG_CH: chan["loc"][3:6] = ref_coords - # Cz ref was applied during acquisition, so mark as already set. - with info._unlock(): - info["custom_ref_applied"] = FIFF.FIFFV_MNE_CUSTOM_REF_ON file_bin = op.join(input_fname, egi_info["eeg_fname"]) egi_info["egi_events"] = egi_events @@ -1013,9 +1010,6 @@ def _read_evoked_mff(fname, condition, channel_naming="E%d", verbose=None): info["bads"] = bads # Add EEG reference to info - # Initialize 'custom_ref_applied' to False - with info._unlock(): - info["custom_ref_applied"] = False try: fp = mff.directory.filepointer("history") except (ValueError, FileNotFoundError): # old (<=0.6.3) vs new mffpy @@ -1027,10 +1021,7 @@ def _read_evoked_mff(fname, condition, channel_naming="E%d", verbose=None): if entry["method"] == "Montage Operations Tool": if "Average Reference" in entry["settings"]: # Average reference has been applied - projector, info = setup_proj(info) - else: - # Custom reference has been applied that is not an average - info["custom_ref_applied"] = True + _, info = setup_proj(info) # Get nave from categories.xml try: diff --git a/mne/io/egi/tests/test_egi.py b/mne/io/egi/tests/test_egi.py index 9086253cad5..c6bb86b02b3 100644 --- a/mne/io/egi/tests/test_egi.py +++ b/mne/io/egi/tests/test_egi.py @@ -2,6 +2,7 @@ # simplified BSD-3 license +from copy import deepcopy from pathlib import Path import os import shutil @@ -12,7 +13,7 @@ import pytest from scipy import io as sio -from mne import find_events, pick_types, pick_channels +from mne import find_events, pick_types from mne.io import read_raw_egi, read_evokeds_mff, read_raw_fif from mne.io.constants import FIFF from mne.io.egi.egi import _combine_triggers @@ -161,14 +162,12 @@ def test_io_egi_mff(): assert raw.info["dig"][0]["kind"] == FIFF.FIFFV_POINT_CARDINAL assert raw.info["dig"][3]["kind"] == FIFF.FIFFV_POINT_EEG assert raw.info["dig"][-1]["ident"] == 129 - assert raw.info["custom_ref_applied"] == FIFF.FIFFV_MNE_CUSTOM_REF_ON + # This is not a custom reference, it's consistent across all channels + assert raw.info["custom_ref_applied"] == FIFF.FIFFV_MNE_CUSTOM_REF_OFF ref_loc = raw.info["dig"][-1]["r"] eeg_picks = pick_types(raw.info, eeg=True) assert len(eeg_picks) == n_eeg + n_ref # 129 - # ref channel doesn't store its own loc as ref location - # so don't test it - ref_pick = pick_channels(raw.info["ch_names"], ["VREF"]) - eeg_picks = np.setdiff1d(eeg_picks, ref_pick) + # ref channel should store its own loc as ref location, so't test it for i in eeg_picks: loc = raw.info["chs"][i]["loc"] assert loc[:3].any(), loc[:3] @@ -523,13 +522,26 @@ def test_meas_date(fname, timestamp, utc_offset): (egi_mff_pns_fname, "GSN-HydroCel-257"), # 257 chan EGI file ], ) -def test_set_standard_montage(fname, standard_montage): +def test_set_standard_montage_mff(fname, standard_montage): """Test setting a standard montage.""" raw = read_raw_egi(fname, verbose="warning") - dig_before_mon = raw.info["dig"] + n_eeg = int(standard_montage.split("-")[-1]) + n_dig = n_eeg + 3 + dig_before_mon = deepcopy(raw.info["dig"]) + assert len(dig_before_mon) == n_dig + ref_loc = dig_before_mon[-1]["r"] + picks = pick_types(raw.info, eeg=True) + assert len(picks) == n_eeg + for pick in picks: + assert_allclose(raw.info["chs"][pick]["loc"][3:6], ref_loc) raw.set_montage(standard_montage, match_alias=True, on_missing="ignore") dig_after_mon = raw.info["dig"] # No dig entries should have been dropped while setting montage - assert len(dig_before_mon) == len(dig_after_mon) + assert len(dig_before_mon) == n_dig + assert len(dig_after_mon) == n_dig + + # Check that the reference remained + for pick in picks: + assert_allclose(raw.info["chs"][pick]["loc"][3:6], ref_loc) diff --git a/mne/io/eyelink/_utils.py b/mne/io/eyelink/_utils.py index f5bb910e059..4f30fd2585c 100644 --- a/mne/io/eyelink/_utils.py +++ b/mne/io/eyelink/_utils.py @@ -5,65 +5,7 @@ import re import numpy as np -from ...utils import _check_pandas_installed, _validate_type - - -def _isfloat(token): - """Boolean test for whether string can be of type float. - - Parameters - ---------- - token : str - Single element from tokens list. - """ - _validate_type(token, str, "token") - try: - float(token) - except ValueError: - return False - else: - return True - - -def _convert_types(tokens): - """Convert the type of each token in list. - - The tokens input is a list of string elements. - Posix timestamp strings can be integers, eye gaze position and - pupil size can be floats. flags token ("...") remains as string. - Missing eye/head-target data (indicated by '.' or 'MISSING_DATA') - are replaced by np.nan. - - Parameters - ---------- - Tokens : list - List of string elements. - - Returns - ------- - Tokens list with elements of various types. - """ - return [ - int(token) - if token.isdigit() # execute this before _isfloat() - else float(token) - if _isfloat(token) - else np.nan - if token in (".", "MISSING_DATA") - else token # remains as string - for token in tokens - ] - - -def _parse_line(line): - """Parse tab delminited string from eyelink ASCII file. - - Takes a tab deliminited string from eyelink file, - splits it into a list of tokens, and converts the type - for each token in the list. - """ - tokens = line.split() - return _convert_types(tokens) +from ...utils import _check_pandas_installed def _is_sys_msg(line): @@ -95,7 +37,7 @@ def _is_sys_msg(line): return "!V" in line or "!MODE" in line or ";" in line -def _get_sfreq(rec_info): +def _get_sfreq_from_ascii(rec_info): """Get sampling frequency from Eyelink ASCII file. Parameters @@ -106,12 +48,13 @@ def _get_sfreq(rec_info): Returns ------- - sfreq : int | float + sfreq : float """ - return rec_info[rec_info.index("RATE") + 1] + return float(rec_info[rec_info.index("RATE") + 1]) def _sort_by_time(df, col="time"): + assert col in df.columns df.sort_values(col, ascending=True, inplace=True) df.reset_index(drop=True, inplace=True) @@ -146,7 +89,7 @@ def _convert_times(df, first_samp, col="time"): df[col] /= 1000 -def _fill_times( +def _adjust_times( df, sfreq, time_col="time", @@ -213,7 +156,8 @@ def _find_overlaps(df, max_time=0.05): """ pd = _check_pandas_installed() - df = df.copy() + if not len(df): + return df["overlap_start"] = df.sort_values("time")["time"].diff().lt(max_time) df["overlap_end"] = df["end_time"].diff().abs().lt(max_time) @@ -240,7 +184,7 @@ def _find_overlaps(df, max_time=0.05): return ovrlp.drop(columns=tmp_cols).reset_index(drop=True) -# Used by read_eyelinke_calibration +# Used by read_eyelink_calibration def _find_recording_start(lines): diff --git a/mne/io/eyelink/eyelink.py b/mne/io/eyelink/eyelink.py index 73a16554f96..1d64e5c7d9b 100644 --- a/mne/io/eyelink/eyelink.py +++ b/mne/io/eyelink/eyelink.py @@ -12,17 +12,23 @@ import numpy as np from ._utils import ( _convert_times, - _fill_times, + _adjust_times, _find_overlaps, - _get_sfreq, + _get_sfreq_from_ascii, _is_sys_msg, - _parse_line, ) # helper functions from ..constants import FIFF from ..base import BaseRaw from ..meas_info import create_info from ...annotations import Annotations -from ...utils import _check_fname, _check_pandas_installed, fill_doc, logger, verbose +from ...utils import ( + _check_fname, + _check_pandas_installed, + fill_doc, + logger, + verbose, + warn, +) EYELINK_COLS = { "timestamp": ("time",), @@ -36,9 +42,7 @@ }, "resolution": ("xres", "yres"), "input": ("DIN",), - "flags": ("flags",), "remote": ("x_head", "y_head", "distance"), - "remote_flags": ("head_flags",), "block_num": ("block",), "eye_event": ("eye", "time", "end_time", "duration"), "fixation": ("fix_avg_x", "fix_avg_y", "fix_avg_pupil_size"), @@ -97,12 +101,16 @@ def read_raw_eyelink( times of the left and right eyes are separated by less than 50 ms, then the blink will be merged into a single :class:`mne.Annotations`. gap_description : str (default 'BAD_ACQ_SKIP') - This parameter is deprecated and will be removed in 1.6. - Use :meth:`mne.Annotations.rename` instead. - the annotation that will span across the gap period between the + Label for annotations that span across the gap period between the blocks. Uses ``'BAD_ACQ_SKIP'`` by default so that these time periods will be considered bad by MNE and excluded from operations like epoching. + .. deprecated:: 1.5 + + This parameter is deprecated and will be removed in version 1.6. Use + :meth:`mne.Annotations.rename` if you want something other than + ``BAD_ACQ_SKIP`` as the annotation label. + Returns ------- raw : instance of RawEyelink @@ -179,9 +187,6 @@ class RawEyelink(BaseRaw): ---------- fname : pathlib.Path Eyelink filename - dataframes : dict - Dictionary of pandas DataFrames. One for eyetracking samples, - and one for each type of eyelink event (blinks, messages, etc) See Also -------- @@ -205,14 +210,14 @@ def __init__( self.fname = Path(fname) self._sample_lines = None # sample lines from file self._event_lines = None # event messages from file - self._system_lines = None # unparsed lines of system messages from file self._tracking_mode = None # assigned in self._infer_col_names self._meas_date = None self._rec_info = None + self._ascii_sfreq = None if gap_desc is None: gap_desc = "BAD_ACQ_SKIP" else: - logger.warn( + warn( "gap_description is deprecated in 1.5 and will be removed in 1.6, " "use raw.annotations.rename to use a description other than " "'BAD_ACQ_SKIP'", @@ -221,28 +226,54 @@ def __init__( self._gap_desc = gap_desc self.dataframes = {} + # ======================== Parse ASCII File ========================= self._get_recording_datetime() # sets self._meas_date self._parse_recording_blocks() # sets sample, event, & system lines - sfreq = _get_sfreq(self._event_lines["SAMPLES"][0]) + # ======================== Create DataFrames ======================== + self._create_dataframes() + del self._sample_lines # free up memory + # add column names to dataframes col_names, ch_names = self._infer_col_names() - self._create_dataframes( - col_names, sfreq, find_overlaps=find_overlaps, threshold=overlap_threshold - ) - info = self._create_info(ch_names, sfreq) + self._assign_col_names(col_names) + self._set_df_dtypes() # set dtypes for each dataframe + if "HREF" in self._rec_info: + self._convert_href_samples() + # fill in times between recording blocks with BAD_ACQ_SKIP + n_blocks = len(self._event_lines["START"]) + if n_blocks > 1: + logger.info( + f"There are {n_blocks} recording blocks in this file. Times between" + f" blocks will be annotated with {self._gap_desc}." + ) + self.dataframes["samples"] = _adjust_times( + self.dataframes["samples"], self._ascii_sfreq + ) + # Convert timestamps to seconds + for df in self.dataframes.values(): + first_samp = float(self._event_lines["START"][0][0]) + _convert_times(df, first_samp) + # Find overlaps between left and right eye events + if find_overlaps: + for key in self.dataframes: + if key not in ["blinks", "fixations", "saccades"]: + continue + self.dataframes[key] = _find_overlaps( + self.dataframes[key], max_time=overlap_threshold + ) + + # ======================== Create Raw Object ========================= + info = self._create_info(ch_names, self._ascii_sfreq) eye_ch_data = self.dataframes["samples"][ch_names] eye_ch_data = eye_ch_data.to_numpy().T - - # create mne object super(RawEyelink, self).__init__( info, preload=eye_ch_data, filenames=[self.fname], verbose=verbose ) - # set meas_date self.set_meas_date(self._meas_date) - # Make Annotations + # ======================== Make Annotations ========================= gap_annots = None - if len(self.dataframes["recording_blocks"]) > 1: + if len(self._event_lines["START"]) > 1: gap_annots = self._make_gap_annots() eye_annots = None if create_annotations: @@ -258,6 +289,10 @@ def __init__( else: logger.info("Not creating any annotations") + # Free up memory + del self.dataframes + del self._event_lines + def _parse_recording_blocks(self): """Parse Eyelink ASCII file. @@ -269,7 +304,6 @@ def _parse_recording_blocks(self): messages sent by the stimulus presentation software. """ with self.fname.open() as file: - block_num = 1 self._sample_lines = [] self._event_lines = { "START": [], @@ -291,25 +325,16 @@ def _parse_recording_blocks(self): if line.startswith("START"): # start of recording block is_recording_block = True if is_recording_block: - if _is_sys_msg(line): - self._system_lines.append(line) - continue # system messages don't need to be parsed. - tokens = _parse_line(line) - tokens.append(block_num) # add current block number - if isinstance(tokens[0], (int, float)): # Samples + tokens = line.split() + if tokens[0][0].isnumeric(): # Samples self._sample_lines.append(tokens) elif tokens[0] in self._event_lines.keys(): + if _is_sys_msg(line): + continue # system messages don't need to be parsed. event_key, event_info = tokens[0], tokens[1:] self._event_lines[event_key].append(event_info) - if tokens[0] == "END": # end of recording block - is_recording_block = False - block_num += 1 - if not self._event_lines["START"]: - raise ValueError( - "Could not determine the start of the" - " recording. When converting to ASCII, START" - " events should not be suppressed." - ) + if tokens[0] == "END": # end of recording block + is_recording_block = False if not self._sample_lines: # no samples parsed raise ValueError(f"Couldn't find any samples in {self.fname}") self._validate_data() @@ -317,63 +342,37 @@ def _parse_recording_blocks(self): def _validate_data(self): """Check the incoming data for some known problems that can occur.""" self._rec_info = self._event_lines["SAMPLES"][0] - pupil_info = self._event_lines["PUPIL"][0] - n_blocks = len(self._event_lines["START"]) - sfreq = int(_get_sfreq(self._rec_info)) - first_samp = self._event_lines["START"][0][0] + self._pupil_info = self._event_lines["PUPIL"][0] + self._n_blocks = len(self._event_lines["START"]) + self._ascii_sfreq = _get_sfreq_from_ascii(self._event_lines["SAMPLES"][0]) if ("LEFT" in self._rec_info) and ("RIGHT" in self._rec_info): self._tracking_mode = "binocular" else: self._tracking_mode = "monocular" # Detect the datatypes that are in file. if "GAZE" in self._rec_info: - logger.info("Pixel coordinate data detected.") - logger.warning( + logger.info( + "Pixel coordinate data detected." "Pass `scalings=dict(eyegaze=1e3)` when using plot" " method to make traces more legible." ) + elif "HREF" in self._rec_info: - logger.info("Head-referenced eye angle data detected.") + logger.info("Head-referenced eye-angle (HREF) data detected.") elif "PUPIL" in self._rec_info: - logger.warning("Raw eyegaze coordinates detected. Analyze with" " caution.") - if "AREA" in pupil_info: - logger.info("Pupil-size area reported.") - elif "DIAMETER" in pupil_info: - logger.info("Pupil-size diameter reported.") - # Check sampling frequency. - if sfreq == 2000 and isinstance(first_samp, int): - raise ValueError( - f"The sampling rate is {sfreq}Hz but the" - " timestamps were not output as float values." - " Check the settings in the EDF2ASC application." - ) - elif sfreq != 2000 and isinstance(first_samp, float): - raise ValueError( - "For recordings with a sampling rate less than" - " 2000Hz, timestamps should not be output to the" - " ASCII file as float values. Check the" - " settings in the EDF2ASC application. Got a" - f" sampling rate of {sfreq}Hz." - ) - # If more than 1 recording period, make sure sfreq didn't change. - if n_blocks > 1: - err_msg = ( - "The sampling frequency changed during the recording." - " This file cannot be read into MNE." - ) - for block_info in self._event_lines["SAMPLES"][1:]: - block_sfreq = int(_get_sfreq(block_info)) - if block_sfreq != sfreq: - raise ValueError( - err_msg + f" Got both {sfreq} and {block_sfreq} Hz." - ) + warn("Raw eyegaze coordinates detected. Analyze with caution.") + if "AREA" in self._pupil_info: + logger.info("Pupil-size area detected.") + elif "DIAMETER" in self._pupil_info: + logger.info("Pupil-size diameter detected.") + # If more than 1 recording period, check whether eye being tracked changed. + if self._n_blocks > 1: if self._tracking_mode == "monocular": - assert self._rec_info[1] in ["LEFT", "RIGHT"] eye = self._rec_info[1] blocks_list = self._event_lines["SAMPLES"] eye_per_block = [block_info[1] for block_info in blocks_list] if not all([this_eye == eye for this_eye in eye_per_block]): - logger.warning( + warn( "The eye being tracked changed during the" " recording. The channel names will reflect" " the eye that was tracked at the start of" @@ -400,6 +399,17 @@ def _get_recording_datetime(self): dt_naive = datetime.strptime(dt_str, fmt) dt_aware = dt_naive.replace(tzinfo=tz) self._meas_date = dt_aware + break + + def _convert_href_samples(self): + """Convert HREF eyegaze samples to radians.""" + # grab the xpos and ypos channel names + pos_names = EYELINK_COLS["pos"]["left"][:-1] + EYELINK_COLS["pos"]["right"][:-1] + for col in self.dataframes["samples"].columns: + if col not in pos_names: # 'xpos_left' ... 'ypos_right' + continue + series = self._href_to_radian(self.dataframes["samples"][col]) + self.dataframes["samples"][col] = series def _href_to_radian(self, opposite, f=15_000): """Convert HREF eyegaze samples to radians. @@ -409,7 +419,8 @@ def _href_to_radian(self, opposite, f=15_000): opposite : int The x or y coordinate in an HREF gaze sample. f : int (default 15_000) - distance of plane from the eye. + distance of plane from the eye. Defaults to 15,000 units, which was taken + from the Eyelink 1000 plus user manual. Returns ------- @@ -426,21 +437,21 @@ def _infer_col_names(self): """Build column and channel names for data from Eyelink ASCII file. Returns the expected column names for the sample lines and event - lines, to be passed into pd.DataFrame. Sample and event lines in - eyelink files have a fixed order of columns, but the columns that - are present can vary. The order that col_names is built below should - NOT change. + lines, to be passed into pd.DataFrame. The columns present in an eyelink ASCII + file can vary. The order that col_names are built below should NOT change. """ col_names = {} # initiate the column names for the sample lines - col_names["sample"] = list(EYELINK_COLS["timestamp"]) + col_names["samples"] = list(EYELINK_COLS["timestamp"]) # and for the eye message lines - col_names["blink"] = list(EYELINK_COLS["eye_event"]) - col_names["fixation"] = list( + col_names["blinks"] = list(EYELINK_COLS["eye_event"]) + col_names["fixations"] = list( EYELINK_COLS["eye_event"] + EYELINK_COLS["fixation"] ) - col_names["saccade"] = list(EYELINK_COLS["eye_event"] + EYELINK_COLS["saccade"]) + col_names["saccades"] = list( + EYELINK_COLS["eye_event"] + EYELINK_COLS["saccade"] + ) # Recording was either binocular or monocular # If monocular, find out which eye was tracked and append to ch_name @@ -450,50 +461,39 @@ def _infer_col_names(self): ch_names = list(EYELINK_COLS["pos"][eye]) elif self._tracking_mode == "binocular": ch_names = list(EYELINK_COLS["pos"]["left"] + EYELINK_COLS["pos"]["right"]) - col_names["sample"].extend(ch_names) + col_names["samples"].extend(ch_names) # The order of these if statements should not be changed. if "VEL" in self._rec_info: # If velocity data are reported if self._tracking_mode == "monocular": ch_names.extend(EYELINK_COLS["velocity"][eye]) - col_names["sample"].extend(EYELINK_COLS["velocity"][eye]) + col_names["samples"].extend(EYELINK_COLS["velocity"][eye]) elif self._tracking_mode == "binocular": ch_names.extend( EYELINK_COLS["velocity"]["left"] + EYELINK_COLS["velocity"]["right"] ) - col_names["sample"].extend( + col_names["samples"].extend( EYELINK_COLS["velocity"]["left"] + EYELINK_COLS["velocity"]["right"] ) # if resolution data are reported if "RES" in self._rec_info: ch_names.extend(EYELINK_COLS["resolution"]) - col_names["sample"].extend(EYELINK_COLS["resolution"]) - col_names["fixation"].extend(EYELINK_COLS["resolution"]) - col_names["saccade"].extend(EYELINK_COLS["resolution"]) + col_names["samples"].extend(EYELINK_COLS["resolution"]) + col_names["fixations"].extend(EYELINK_COLS["resolution"]) + col_names["saccades"].extend(EYELINK_COLS["resolution"]) # if digital input port values are reported if "INPUT" in self._rec_info: ch_names.extend(EYELINK_COLS["input"]) - col_names["sample"].extend(EYELINK_COLS["input"]) - - # add flags column - col_names["sample"].extend(EYELINK_COLS["flags"]) + col_names["samples"].extend(EYELINK_COLS["input"]) - # if head target info was reported, add its cols after flags col. + # if head target info was reported, add its cols if "HTARGET" in self._rec_info: ch_names.extend(EYELINK_COLS["remote"]) - col_names["sample"].extend( - EYELINK_COLS["remote"] + EYELINK_COLS["remote_flags"] - ) - - # finally add a column for recording block number - # FYI this column does not exist in the asc file.. - # but it is added during _parse_recording_blocks - for col in col_names.values(): - col.extend(EYELINK_COLS["block_num"]) + col_names["samples"].extend(EYELINK_COLS["remote"]) return col_names, ch_names - def _create_dataframes(self, col_names, sfreq, find_overlaps=False, threshold=0.05): + def _create_dataframes(self): """Create pandas.DataFrame for Eyelink samples and events. Creates a pandas DataFrame for self._sample_lines and for each @@ -501,59 +501,16 @@ def _create_dataframes(self, col_names, sfreq, find_overlaps=False, threshold=0. """ pd = _check_pandas_installed() - # First sample should be the first line of the first recording block - first_samp = self._event_lines["START"][0][0] - # dataframe for samples - self.dataframes["samples"] = pd.DataFrame( - self._sample_lines, columns=col_names["sample"] - ) - if "HREF" in self._rec_info: - pos_names = ( - EYELINK_COLS["pos"]["left"][:-1] + EYELINK_COLS["pos"]["right"][:-1] - ) - for col in self.dataframes["samples"].columns: - if col not in pos_names: # 'xpos_left' ... 'ypos_right' - continue - series = self._href_to_radian(self.dataframes["samples"][col]) - self.dataframes["samples"][col] = series - - n_block = len(self._event_lines["START"]) - if n_block > 1: - logger.info( - f"There are {n_block} recording blocks in this" - " file. Times between blocks will be annotated with" - f" {self._gap_desc}." - ) - # if there is more than 1 recording block we must account for - # the missing timestamps and samples bt the blocks - self.dataframes["samples"] = _fill_times( - self.dataframes["samples"], sfreq=sfreq - ) - _convert_times(self.dataframes["samples"], first_samp) + self.dataframes["samples"] = pd.DataFrame(self._sample_lines) + self._drop_status_col() # Remove STATUS column # dataframe for each type of occular event - for event, columns, label in zip( - ["EFIX", "ESACC", "EBLINK"], - [col_names["fixation"], col_names["saccade"], col_names["blink"]], - ["fixations", "saccades", "blinks"], + for event, label in zip( + ["EFIX", "ESACC", "EBLINK"], ["fixations", "saccades", "blinks"] ): if self._event_lines[event]: # an empty list returns False - self.dataframes[label] = pd.DataFrame( - self._event_lines[event], columns=columns - ) - _convert_times(self.dataframes[label], first_samp) - - if find_overlaps is True: - if self._tracking_mode == "monocular": - raise ValueError( - "find_overlaps is only valid with" - " binocular recordings, this file is" - f" {self._tracking_mode}" - ) - df = _find_overlaps(self.dataframes[label], max_time=threshold) - self.dataframes[label] = df - + self.dataframes[label] = pd.DataFrame(self._event_lines[event]) else: logger.info( f"No {label} were found in this file. " @@ -565,42 +522,89 @@ def _create_dataframes(self, col_names, sfreq, find_overlaps=False, threshold=0. msgs = [] for tokens in self._event_lines["MSG"]: timestamp = tokens[0] - block = tokens[-1] - # if offset token exists, it will be the 1st index - # and is an int or float - if isinstance(tokens[1], (int, float)): - offset = tokens[1] - msg = " ".join(str(x) for x in tokens[2:-1]) + # if offset token exists, it will be the 1st index and is numeric + if tokens[1].lstrip("-").replace(".", "", 1).isnumeric(): + offset = float(tokens[1]) + msg = " ".join(str(x) for x in tokens[2:]) else: # there is no offset token offset = np.nan - msg = " ".join(str(x) for x in tokens[1:-1]) - msgs.append([timestamp, offset, msg, block]) - - cols = ["time", "offset", "event_msg", "block"] - self.dataframes["messages"] = pd.DataFrame(msgs, columns=cols) - _convert_times(self.dataframes["messages"], first_samp) + msg = " ".join(str(x) for x in tokens[1:]) + msgs.append([timestamp, offset, msg]) + self.dataframes["messages"] = pd.DataFrame(msgs) # make dataframe for recording block start, end times - assert len(self._event_lines["START"]) == len(self._event_lines["END"]) - blocks = [ - [bgn[0], end[0], bgn[-1]] # start, end, block_num - for bgn, end in zip(self._event_lines["START"], self._event_lines["END"]) - ] + i = 1 + blocks = list() + for bgn, end in zip(self._event_lines["START"], self._event_lines["END"]): + blocks.append((float(bgn[0]), float(end[0]), i)) + i += 1 cols = ["time", "end_time", "block"] self.dataframes["recording_blocks"] = pd.DataFrame(blocks, columns=cols) - _convert_times(self.dataframes["recording_blocks"], first_samp) # make dataframe for digital input port if self._event_lines["INPUT"]: - cols = ["time", "DIN", "block"] - self.dataframes["DINS"] = pd.DataFrame( - self._event_lines["INPUT"], columns=cols - ) - _convert_times(self.dataframes["DINS"], first_samp) + cols = ["time", "DIN"] + self.dataframes["DINS"] = pd.DataFrame(self._event_lines["INPUT"]) # TODO: Make dataframes for other eyelink events (Buttons) + def _drop_status_col(self): + """Drop STATUS column from samples dataframe. + + see https://github.com/mne-tools/mne-python/issues/11809, and section 4.9.2.1 of + the Eyelink 1000 Plus User Manual, version 1.0.19. We know that the STATUS + column is either 3, 5, 13, or 17 characters long, i.e. "...", ".....", ".C." + """ + status_cols = [] + # we know the first 3 columns will be the time, xpos, ypos + for col in self.dataframes["samples"].columns[3:]: + if self.dataframes["samples"][col][0][0].isnumeric(): + # if the value is numeric, it's not a status column + continue + if len(self.dataframes["samples"][col][0]) in [3, 5, 13, 17]: + status_cols.append(col) + self.dataframes["samples"].drop(columns=status_cols, inplace=True) + + def _assign_col_names(self, col_names): + """Assign column names to dataframes. + + Parameters + ---------- + col_names : dict + Dictionary of column names for each dataframe. + """ + for key, df in self.dataframes.items(): + if key in ("samples", "blinks", "fixations", "saccades"): + df.columns = col_names[key] + elif key == "messages": + cols = ["time", "offset", "event_msg"] + df.columns = cols + elif key == "DINS": + cols = ["time", "DIN"] + df.columns = cols + + def _set_df_dtypes(self): + from ...utils import _set_pandas_dtype + + for key, df in self.dataframes.items(): + if key in ["samples", "DINS"]: + # convert missing position values to NaN + self._set_missing_values(df) + _set_pandas_dtype(df, df.columns, float, verbose="warning") + elif key in ["blinks", "fixations", "saccades"]: + _set_pandas_dtype(df, df.columns[1:], float, verbose="warning") + elif key == "messages": + _set_pandas_dtype(df, ["time"], float, verbose="warning") # timestamp + + def _set_missing_values(self, df): + """Set missing values to NaN. operates in-place.""" + missing_vals = (".", "MISSING_DATA") + for col in df.columns: + if col.startswith(("xpos", "ypos")): + # we explicitly use numpy instead of pd.replace because it is faster + df[col] = np.where(df[col].isin(missing_vals), np.nan, df[col]) + def _create_info(self, ch_names, sfreq): """Create info object for RawEyelink.""" # assign channel type from ch_name @@ -719,8 +723,6 @@ def _make_eyelink_annots(self, df_dict, create_annots, apply_offsets): elif annots: annots += this_annot if not annots: - logger.warning( - f"Annotations for {descs} were requested but" " none could be made." - ) + warn(f"Annotations for {descs} were requested but none could be made.") return return annots diff --git a/mne/io/eyelink/tests/test_eyelink.py b/mne/io/eyelink/tests/test_eyelink.py index ddc2b4b7a1a..a8f11563c2e 100644 --- a/mne/io/eyelink/tests/test_eyelink.py +++ b/mne/io/eyelink/tests/test_eyelink.py @@ -1,13 +1,17 @@ +from pathlib import Path + import pytest import numpy as np from mne.datasets.testing import data_path, requires_testing_data from mne.io import read_raw_eyelink +from mne.io.tests.test_raw import _test_raw_reader from mne.io.constants import FIFF from mne.io.pick import _DATA_CH_TYPES_SPLIT from mne.utils import _check_pandas_installed, requires_pandas + MAPPING = { "left": ["xpos_left", "ypos_left", "pupil_left"], "right": ["xpos_right", "ypos_right", "pupil_right"], @@ -122,7 +126,8 @@ def test_eyelink(fname, create_annotations, find_overlaps, apply_offsets): @pytest.mark.parametrize("fname_href", [(fname_href)]) def test_radian(fname_href): """Test converting HREF position data to radians.""" - raw = read_raw_eyelink(fname_href, create_annotations=["blinks"]) + with pytest.warns(RuntimeWarning, match="Annotations for"): + raw = read_raw_eyelink(fname_href, create_annotations=["blinks"]) # Test channel types assert raw.get_channel_types() == ["eyegaze", "eyegaze", "pupil"] @@ -148,17 +153,17 @@ def test_fill_times(fname): with np.arange don't result in the time columns not merging correctly - i.e. 1560687.0 and 1560687.000001 should merge. """ - from ..eyelink import _fill_times + from ..eyelink import _adjust_times raw = read_raw_eyelink(fname, create_annotations=False) sfreq = raw.info["sfreq"] # just take first 1000 points for testing - df = raw.dataframes["samples"].iloc[:1000].reset_index(drop=True) + df = raw.to_data_frame()[:1000] # even during blinks, pupil val is 0, so there should be no nans # in this column assert not df["pupil_left"].isna().sum() nan_count = df["pupil_left"].isna().sum() # i.e 0 - df_merged = _fill_times(df, sfreq) + df_merged = _adjust_times(df, sfreq) # If times dont merge correctly, there will be additional rows in # in df_merged with all nan values assert df_merged["pupil_left"].isna().sum() == nan_count # i.e. 0 @@ -189,3 +194,103 @@ def test_find_overlaps(): assert len(overlap_df["eye"].unique()) == 3 # ['both', 'left', 'right'] assert len(overlap_df) == 5 # ['both', 'L', 'R', 'L', 'L'] assert overlap_df["eye"].iloc[0] == "both" + + +def _simulate_eye_tracking_data(in_file, out_file): + out_file = Path(out_file) + + new_samples_line = ( + "SAMPLES\tPUPIL\tLEFT\tVEL\tRES\tHTARGET\tRATE\t1000.00" + "\tTRACKING\tCR\tFILTER\t2\tINPUT" + ) + with out_file.open("w") as fp: + in_recording_block = False + events = [] + + for line in Path(in_file).read_text().splitlines(): + if line.startswith("START"): + in_recording_block = True + if in_recording_block: + tokens = line.split() + event_type = tokens[0] + if event_type.isnumeric(): # samples + tokens[4:4] = ["100", "20", "45", "45", "127.0"] # vel, res, DIN + tokens.extend(["1497.0", "5189.0", "512.5", "............."]) + elif event_type in ("EFIX", "ESACC"): + tokens.extend(["45", "45"]) # resolution + elif event_type == "SAMPLES": + tokens[1] = "PUPIL" # simulate raw coordinate data + tokens[3:3] = ["VEL", "RES", "HTARGET"] + tokens.append("INPUT") + elif event_type == "EBLINK": + continue # simulate no blink events + elif event_type == "END": + pass + else: + fp.write("%s\n" % line) + continue + events.append("\t".join(tokens)) + if event_type == "END": + fp.write("\n".join(events) + "\n") + events.clear() + in_recording_block = False + else: + fp.write("%s\n" % line) + + fp.write("%s\n" % "START\t7452389\tRIGHT\tSAMPLES\tEVENTS") + fp.write("%s\n" % new_samples_line) + + for timestamp in np.arange(7452389, 7453390): # simulate a second block + fp.write( + "%s\n" + % ( + f"{timestamp}\t-2434.0\t-1760.0\t840.0\t100\t20\t45\t45\t127.0\t" + "...\t1497\t5189\t512.5\t............." + ) + ) + + fp.write("%s\n" % "END\t7453390\tRIGHT\tSAMPLES\tEVENTS") + + +@requires_testing_data +@requires_pandas +@pytest.mark.parametrize("fname", [fname_href]) +def test_multi_block_misc_channels(fname, tmp_path): + """Test an eyelink file with multiple blocks and additional misc channels.""" + out_file = tmp_path / "tmp_eyelink.asc" + _simulate_eye_tracking_data(fname, out_file) + + with pytest.warns(RuntimeWarning, match="Raw eyegaze coordinates"): + raw = read_raw_eyelink(out_file) + + chs_in_file = [ + "xpos_right", + "ypos_right", + "pupil_right", + "xvel_right", + "yvel_right", + "xres", + "yres", + "DIN", + "x_head", + "y_head", + "distance", + ] + + assert raw.ch_names == chs_in_file + assert raw.annotations.description[1] == "SYNCTIME" + assert raw.annotations.description[-1] == "BAD_ACQ_SKIP" + assert np.isclose(raw.annotations.onset[-1], 1.001) + assert np.isclose(raw.annotations.duration[-1], 0.1) + + data, times = raw.get_data(return_times=True) + assert not np.isnan(data[0, np.where(times < 1)[0]]).any() + assert np.isnan(data[0, np.logical_and(times > 1, times <= 1.1)]).all() + + +@pytest.mark.xfail(reason="Attributes and test_preloading fail") +@requires_testing_data +@pytest.mark.parametrize("this_fname", (fname, fname_href)) +def test_basics(this_fname): + """Test basics of reading.""" + _test_raw_reader(read_raw_eyelink, fname=this_fname) diff --git a/mne/io/fiff/raw.py b/mne/io/fiff/raw.py index 30b9fb826a7..bc3de3c9620 100644 --- a/mne/io/fiff/raw.py +++ b/mne/io/fiff/raw.py @@ -77,6 +77,12 @@ class Raw(BaseRaw): %(verbose)s """ + _extra_attributes = ( + "fix_mag_coil_types", + "acqparser", + "_read_raw_file", # this would be ugly to move, but maybe we should + ) + @verbose def __init__( self, diff --git a/mne/io/hitachi/hitachi.py b/mne/io/hitachi/hitachi.py index 51027642442..99be691b965 100644 --- a/mne/io/hitachi/hitachi.py +++ b/mne/io/hitachi/hitachi.py @@ -79,7 +79,7 @@ def __init__(self, fname, preload=False, *, verbose=None): S_offset = D_offset = 0 ignore_names = ["Time"] for this_fname in fname: - info, extra, last_samp, offsets = self._get_hitachi_info( + info, extra, last_samp, offsets = _get_hitachi_info( this_fname, S_offset, D_offset, ignore_names ) ignore_names = list(set(ignore_names + info["ch_names"])) @@ -111,191 +111,6 @@ def __init__(self, fname, preload=False, *, verbose=None): verbose=verbose, ) - # This could be a function, but for the sake of indentation, let's make it - # a method instead - def _get_hitachi_info(self, fname, S_offset, D_offset, ignore_names): - logger.info("Loading %s" % fname) - raw_extra = dict(fname=fname) - info_extra = dict() - subject_info = dict() - ch_wavelengths = dict() - fnirs_wavelengths = [None, None] - meas_date = age = ch_names = sfreq = None - with open(fname, "rb") as fid: - lines = fid.read() - lines = lines.decode("latin-1").rstrip("\r\n") - oldlen = len(lines) - assert len(lines) == oldlen - bounds = [0] - end = "\n" if "\n" in lines else "\r" - bounds.extend(a.end() for a in re.finditer(end, lines)) - bounds.append(len(lines)) - lines = lines.split(end) - assert len(bounds) == len(lines) + 1 - line = lines[0].rstrip(",\r\n") - _check_bad(line != "Header", "no header found") - li = 0 - mode = None - for li, line in enumerate(lines[1:], 1): - # Newer format has some blank lines - if len(line) == 0: - continue - parts = line.rstrip(",\r\n").split(",") - if len(parts) == 0: # some header lines are blank - continue - kind, parts = parts[0], parts[1:] - if len(parts) == 0: - parts = [""] # some fields (e.g., Comment) meaningfully blank - if kind == "File Version": - logger.info(f"Reading Hitachi fNIRS file version {parts[0]}") - elif kind == "AnalyzeMode": - _check_bad(parts != ["Continuous"], f"not continuous data ({parts})") - elif kind == "Sampling Period[s]": - sfreq = 1 / float(parts[0]) - elif kind == "Exception": - raise NotImplementedError(kind) - elif kind == "Comment": - info_extra["description"] = parts[0] - elif kind == "ID": - subject_info["his_id"] = parts[0] - elif kind == "Name": - if len(parts): - name = parts[0].split(" ") - if len(name): - subject_info["first_name"] = name[0] - subject_info["last_name"] = " ".join(name[1:]) - elif kind == "Age": - age = int(parts[0].rstrip("y")) - elif kind == "Mode": - mode = parts[0] - elif kind in ("HPF[Hz]", "LPF[Hz]"): - try: - freq = float(parts[0]) - except ValueError: - pass - else: - info_extra[ - {"HPF[Hz]": "highpass", "LPF[Hz]": "lowpass"}[kind] - ] = freq - elif kind == "Date": - # 5/17/04 5:14 - try: - mdy, HM = parts[0].split(" ") - H, M = HM.split(":") - if len(H) == 1: - H = f"0{H}" - mdyHM = " ".join([mdy, ":".join([H, M])]) - for fmt in ("%m/%d/%y %H:%M", "%Y/%m/%d %H:%M"): - try: - meas_date = dt.datetime.strptime(mdyHM, fmt) - except Exception: - pass - else: - break - else: - raise RuntimeError # unknown format - except Exception: - warn( - "Extraction of measurement date failed. " - "Please report this as a github issue. " - "The date is being set to January 1st, 2000, " - f"instead of {repr(parts[0])}" - ) - elif kind == "Sex": - try: - subject_info["sex"] = dict( - female=FIFF.FIFFV_SUBJ_SEX_FEMALE, male=FIFF.FIFFV_SUBJ_SEX_MALE - )[parts[0].lower()] - except KeyError: - pass - elif kind == "Wave[nm]": - fnirs_wavelengths[:] = [int(part) for part in parts] - elif kind == "Wave Length": - ch_regex = re.compile(r"^(.*)\(([0-9\.]+)\)$") - for ent in parts: - _, v = ch_regex.match(ent).groups() - ch_wavelengths[ent] = float(v) - elif kind == "Data": - break - fnirs_wavelengths = np.array(fnirs_wavelengths, int) - assert len(fnirs_wavelengths) == 2 - ch_names = lines[li + 1].rstrip(",\r\n").split(",") - # cull to correct ones - raw_extra["keep_mask"] = ~np.in1d(ch_names, list(ignore_names)) - for ci, ch_name in enumerate(ch_names): - if re.match("Probe[0-9]+", ch_name): - raw_extra["keep_mask"][ci] = False - # set types - ch_names = [ - ch_name for ci, ch_name in enumerate(ch_names) if raw_extra["keep_mask"][ci] - ] - ch_types = [ - "fnirs_cw_amplitude" if ch_name.startswith("CH") else "stim" - for ch_name in ch_names - ] - # get locations - nirs_names = [ - ch_name - for ch_name, ch_type in zip(ch_names, ch_types) - if ch_type == "fnirs_cw_amplitude" - ] - n_nirs = len(nirs_names) - assert n_nirs % 2 == 0 - names = { - "3x3": "ETG-100", - "3x5": "ETG-7000", - "4x4": "ETG-7000", - "3x11": "ETG-4000", - } - _check_option("Hitachi mode", mode, sorted(names)) - n_row, n_col = [int(x) for x in mode.split("x")] - logger.info(f"Constructing pairing matrix for {names[mode]} ({mode})") - pairs = _compute_pairs(n_row, n_col, n=1 + (mode == "3x3")) - assert n_nirs == len(pairs) * 2 - locs = np.zeros((len(ch_names), 12)) - locs[:, :9] = np.nan - idxs = np.where(np.array(ch_types, "U") == "fnirs_cw_amplitude")[0] - for ii, idx in enumerate(idxs): - ch_name = ch_names[idx] - # Use the actual/accurate wavelength in loc - acc_freq = ch_wavelengths[ch_name] - locs[idx][9] = acc_freq - # Rename channel based on standard naming scheme, using the - # nominal wavelength - sidx, didx = pairs[ii // 2] - nom_freq = fnirs_wavelengths[ - np.argmin(np.abs(acc_freq - fnirs_wavelengths)) - ] - ch_names[idx] = ( - f"S{S_offset + sidx + 1}_" f"D{D_offset + didx + 1} " f"{nom_freq}" - ) - offsets = np.array(pairs, int).max(axis=0) + 1 - - # figure out bounds - bounds = raw_extra["bounds"] = bounds[li + 2 :] - last_samp = len(bounds) - 2 - - if age is not None and meas_date is not None: - subject_info["birthday"] = ( - meas_date.year - age, - meas_date.month, - meas_date.day, - ) - if meas_date is None: - meas_date = dt.datetime(2000, 1, 1, 0, 0, 0) - meas_date = meas_date.replace(tzinfo=dt.timezone.utc) - if subject_info: - info_extra["subject_info"] = subject_info - - # Create mne structure - info = create_info(ch_names, sfreq, ch_types=ch_types) - with info._unlock(): - info.update(info_extra) - info["meas_date"] = meas_date - for li, loc in enumerate(locs): - info["chs"][li]["loc"][:] = loc - return info, raw_extra, last_samp, offsets - def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): """Read a segment of data from a file.""" this_data = list() @@ -319,6 +134,186 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): return data +def _get_hitachi_info(fname, S_offset, D_offset, ignore_names): + logger.info("Loading %s" % fname) + raw_extra = dict(fname=fname) + info_extra = dict() + subject_info = dict() + ch_wavelengths = dict() + fnirs_wavelengths = [None, None] + meas_date = age = ch_names = sfreq = None + with open(fname, "rb") as fid: + lines = fid.read() + lines = lines.decode("latin-1").rstrip("\r\n") + oldlen = len(lines) + assert len(lines) == oldlen + bounds = [0] + end = "\n" if "\n" in lines else "\r" + bounds.extend(a.end() for a in re.finditer(end, lines)) + bounds.append(len(lines)) + lines = lines.split(end) + assert len(bounds) == len(lines) + 1 + line = lines[0].rstrip(",\r\n") + _check_bad(line != "Header", "no header found") + li = 0 + mode = None + for li, line in enumerate(lines[1:], 1): + # Newer format has some blank lines + if len(line) == 0: + continue + parts = line.rstrip(",\r\n").split(",") + if len(parts) == 0: # some header lines are blank + continue + kind, parts = parts[0], parts[1:] + if len(parts) == 0: + parts = [""] # some fields (e.g., Comment) meaningfully blank + if kind == "File Version": + logger.info(f"Reading Hitachi fNIRS file version {parts[0]}") + elif kind == "AnalyzeMode": + _check_bad(parts != ["Continuous"], f"not continuous data ({parts})") + elif kind == "Sampling Period[s]": + sfreq = 1 / float(parts[0]) + elif kind == "Exception": + raise NotImplementedError(kind) + elif kind == "Comment": + info_extra["description"] = parts[0] + elif kind == "ID": + subject_info["his_id"] = parts[0] + elif kind == "Name": + if len(parts): + name = parts[0].split(" ") + if len(name): + subject_info["first_name"] = name[0] + subject_info["last_name"] = " ".join(name[1:]) + elif kind == "Age": + age = int(parts[0].rstrip("y")) + elif kind == "Mode": + mode = parts[0] + elif kind in ("HPF[Hz]", "LPF[Hz]"): + try: + freq = float(parts[0]) + except ValueError: + pass + else: + info_extra[{"HPF[Hz]": "highpass", "LPF[Hz]": "lowpass"}[kind]] = freq + elif kind == "Date": + # 5/17/04 5:14 + try: + mdy, HM = parts[0].split(" ") + H, M = HM.split(":") + if len(H) == 1: + H = f"0{H}" + mdyHM = " ".join([mdy, ":".join([H, M])]) + for fmt in ("%m/%d/%y %H:%M", "%Y/%m/%d %H:%M"): + try: + meas_date = dt.datetime.strptime(mdyHM, fmt) + except Exception: + pass + else: + break + else: + raise RuntimeError # unknown format + except Exception: + warn( + "Extraction of measurement date failed. " + "Please report this as a github issue. " + "The date is being set to January 1st, 2000, " + f"instead of {repr(parts[0])}" + ) + elif kind == "Sex": + try: + subject_info["sex"] = dict( + female=FIFF.FIFFV_SUBJ_SEX_FEMALE, male=FIFF.FIFFV_SUBJ_SEX_MALE + )[parts[0].lower()] + except KeyError: + pass + elif kind == "Wave[nm]": + fnirs_wavelengths[:] = [int(part) for part in parts] + elif kind == "Wave Length": + ch_regex = re.compile(r"^(.*)\(([0-9\.]+)\)$") + for ent in parts: + _, v = ch_regex.match(ent).groups() + ch_wavelengths[ent] = float(v) + elif kind == "Data": + break + fnirs_wavelengths = np.array(fnirs_wavelengths, int) + assert len(fnirs_wavelengths) == 2 + ch_names = lines[li + 1].rstrip(",\r\n").split(",") + # cull to correct ones + raw_extra["keep_mask"] = ~np.in1d(ch_names, list(ignore_names)) + for ci, ch_name in enumerate(ch_names): + if re.match("Probe[0-9]+", ch_name): + raw_extra["keep_mask"][ci] = False + # set types + ch_names = [ + ch_name for ci, ch_name in enumerate(ch_names) if raw_extra["keep_mask"][ci] + ] + ch_types = [ + "fnirs_cw_amplitude" if ch_name.startswith("CH") else "stim" + for ch_name in ch_names + ] + # get locations + nirs_names = [ + ch_name + for ch_name, ch_type in zip(ch_names, ch_types) + if ch_type == "fnirs_cw_amplitude" + ] + n_nirs = len(nirs_names) + assert n_nirs % 2 == 0 + names = { + "3x3": "ETG-100", + "3x5": "ETG-7000", + "4x4": "ETG-7000", + "3x11": "ETG-4000", + } + _check_option("Hitachi mode", mode, sorted(names)) + n_row, n_col = [int(x) for x in mode.split("x")] + logger.info(f"Constructing pairing matrix for {names[mode]} ({mode})") + pairs = _compute_pairs(n_row, n_col, n=1 + (mode == "3x3")) + assert n_nirs == len(pairs) * 2 + locs = np.zeros((len(ch_names), 12)) + locs[:, :9] = np.nan + idxs = np.where(np.array(ch_types, "U") == "fnirs_cw_amplitude")[0] + for ii, idx in enumerate(idxs): + ch_name = ch_names[idx] + # Use the actual/accurate wavelength in loc + acc_freq = ch_wavelengths[ch_name] + locs[idx][9] = acc_freq + # Rename channel based on standard naming scheme, using the + # nominal wavelength + sidx, didx = pairs[ii // 2] + nom_freq = fnirs_wavelengths[np.argmin(np.abs(acc_freq - fnirs_wavelengths))] + ch_names[idx] = ( + f"S{S_offset + sidx + 1}_" f"D{D_offset + didx + 1} " f"{nom_freq}" + ) + offsets = np.array(pairs, int).max(axis=0) + 1 + + # figure out bounds + bounds = raw_extra["bounds"] = bounds[li + 2 :] + last_samp = len(bounds) - 2 + + if age is not None and meas_date is not None: + subject_info["birthday"] = ( + meas_date.year - age, + meas_date.month, + meas_date.day, + ) + if meas_date is None: + meas_date = dt.datetime(2000, 1, 1, 0, 0, 0) + meas_date = meas_date.replace(tzinfo=dt.timezone.utc) + if subject_info: + info_extra["subject_info"] = subject_info + + # Create mne structure + info = create_info(ch_names, sfreq, ch_types=ch_types) + with info._unlock(): + info.update(info_extra) + info["meas_date"] = meas_date + for li, loc in enumerate(locs): + info["chs"][li]["loc"][:] = loc + return info, raw_extra, last_samp, offsets + + def _compute_pairs(n_rows, n_cols, n=1): n_tot = n_rows * n_cols sd_idx = (np.arange(n_tot) // 2).reshape(n_rows, n_cols) diff --git a/mne/io/kit/kit.py b/mne/io/kit/kit.py index 082202707fb..a182ebaf913 100644 --- a/mne/io/kit/kit.py +++ b/mne/io/kit/kit.py @@ -116,6 +116,8 @@ class RawKIT(BaseRaw): mne.io.Raw : Documentation of attributes and methods. """ + _extra_attributes = ("read_stim_ch",) + @verbose def __init__( self, @@ -147,7 +149,7 @@ def __init__( last_samps = [kit_info["n_samples"] - 1] self._raw_extras = [kit_info] - self._set_stimchannels(info, stim, stim_code) + _set_stimchannels(self, info, stim, stim_code) super(RawKIT, self).__init__( info, preload, @@ -187,75 +189,6 @@ def read_stim_ch(self, buffer_size=1e5): return stim_ch - @fill_doc - def _set_stimchannels(self, info, stim, stim_code): - """Specify how the trigger channel is synthesized from analog channels. - - Has to be done before loading data. For a RawKIT instance that has been - created with preload=True, this method will raise a - NotImplementedError. - - Parameters - ---------- - %(info_not_none)s - stim : list of int | '<' | '>' - Can be submitted as list of trigger channels. - If a list is not specified, the default triggers extracted from - misc channels will be used with specified directionality. - '<' means that largest values assigned to the first channel - in sequence. - '>' means the largest trigger assigned to the last channel - in sequence. - stim_code : 'binary' | 'channel' - How to decode trigger values from stim channels. 'binary' read stim - channel events as binary code, 'channel' encodes channel number. - """ - if self.preload: - raise NotImplementedError("Can't change stim channel after " "loading data") - _check_option("stim_code", stim_code, ["binary", "channel"]) - - if stim is not None: - if isinstance(stim, str): - picks = _default_stim_chs(info) - if stim == "<": - stim = picks[::-1] - elif stim == ">": - stim = picks - else: - raise ValueError( - "stim needs to be list of int, '>' or " - "'<', not %r" % str(stim) - ) - else: - stim = np.asarray(stim, int) - if stim.max() >= self._raw_extras[0]["nchan"]: - raise ValueError( - "Got stim=%s, but sqd file only has %i channels" - % (stim, self._raw_extras[0]["nchan"]) - ) - - # modify info - nchan = self._raw_extras[0]["nchan"] + 1 - info["chs"].append( - dict( - cal=KIT.CALIB_FACTOR, - logno=nchan, - scanno=nchan, - range=1.0, - unit=FIFF.FIFF_UNIT_NONE, - unit_mul=FIFF.FIFF_UNITM_NONE, - ch_name="STI 014", - coil_type=FIFF.FIFFV_COIL_NONE, - loc=np.full(12, np.nan), - kind=FIFF.FIFFV_STIM_CH, - coord_frame=FIFF.FIFFV_COORD_UNKNOWN, - ) - ) - info._update_redundant() - - self._raw_extras[0]["stim"] = stim - self._raw_extras[0]["stim_code"] = stim_code - def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): """Read a chunk of raw data.""" sqd = self._raw_extras[fi] @@ -295,6 +228,74 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): # cals are all unity, so can be ignored +def _set_stimchannels(inst, info, stim, stim_code): + """Specify how the trigger channel is synthesized from analog channels. + + Has to be done before loading data. For a RawKIT instance that has been + created with preload=True, this method will raise a + NotImplementedError. + + Parameters + ---------- + %(info_not_none)s + stim : list of int | '<' | '>' + Can be submitted as list of trigger channels. + If a list is not specified, the default triggers extracted from + misc channels will be used with specified directionality. + '<' means that largest values assigned to the first channel + in sequence. + '>' means the largest trigger assigned to the last channel + in sequence. + stim_code : 'binary' | 'channel' + How to decode trigger values from stim channels. 'binary' read stim + channel events as binary code, 'channel' encodes channel number. + """ + if inst.preload: + raise NotImplementedError("Can't change stim channel after loading data") + _check_option("stim_code", stim_code, ["binary", "channel"]) + + if stim is not None: + if isinstance(stim, str): + picks = _default_stim_chs(info) + if stim == "<": + stim = picks[::-1] + elif stim == ">": + stim = picks + else: + raise ValueError( + "stim needs to be list of int, '>' or " "'<', not %r" % str(stim) + ) + else: + stim = np.asarray(stim, int) + if stim.max() >= inst._raw_extras[0]["nchan"]: + raise ValueError( + "Got stim=%s, but sqd file only has %i channels" + % (stim, inst._raw_extras[0]["nchan"]) + ) + + # modify info + nchan = inst._raw_extras[0]["nchan"] + 1 + info["chs"].append( + dict( + cal=KIT.CALIB_FACTOR, + logno=nchan, + scanno=nchan, + range=1.0, + unit=FIFF.FIFF_UNIT_NONE, + unit_mul=FIFF.FIFF_UNITM_NONE, + ch_name="STI 014", + coil_type=FIFF.FIFFV_COIL_NONE, + loc=np.full(12, np.nan), + kind=FIFF.FIFFV_STIM_CH, + coord_frame=FIFF.FIFFV_COORD_UNKNOWN, + ) + ) + info._update_redundant() + + inst._raw_extras[0]["stim"] = stim + inst._raw_extras[0]["stim_code"] = stim_code + + def _default_stim_chs(info): """Return default stim channels for SQD files.""" return pick_types(info, meg=False, ref_meg=False, misc=True, exclude=[])[:8] diff --git a/mne/io/nihon/nihon.py b/mne/io/nihon/nihon.py index 36b6d68d578..6198678db03 100644 --- a/mne/io/nihon/nihon.py +++ b/mne/io/nihon/nihon.py @@ -410,8 +410,6 @@ def __init__(self, fname, preload=False, verbose=None): ] raw_extras = dict(cal=cal, offsets=offsets, gains=gains, header=header) - self._header = header - for i_ch, ch_name in enumerate(info["ch_names"]): t_range = chs[ch_name]["phys_max"] - chs[ch_name]["phys_min"] info["chs"][i_ch]["range"] = t_range @@ -430,7 +428,7 @@ def __init__(self, fname, preload=False, verbose=None): annots = _read_nihon_annotations(fname) # Annotate acquisition skips - controlblock = self._header["controlblocks"][0] + controlblock = header["controlblocks"][0] cur_sample = 0 if controlblock["n_datablocks"] > 1: for i_block in range(controlblock["n_datablocks"] - 1): diff --git a/mne/io/proj.py b/mne/io/proj.py index 1cf2e61a0e1..1cae5d05ea8 100644 --- a/mne/io/proj.py +++ b/mne/io/proj.py @@ -143,6 +143,8 @@ def plot_topomap( .. versionadded:: 1.2 %(border_topomap)s + + .. versionadded:: 0.20 %(res_topomap)s %(size_topomap)s %(cmap_topomap)s @@ -427,7 +429,15 @@ def plot_projs_topomap( %(extrapolate_topomap)s .. versionadded:: 0.20 + + .. versionchanged:: 0.21 + + - The default was changed to ``'local'`` for MEG sensors. + - ``'local'`` was changed to use a convex hull mask + - ``'head'`` was changed to extrapolate out to the clipping circle. %(border_topomap)s + + .. versionadded:: 0.20 %(res_topomap)s %(size_topomap)s Only applies when plotting multiple topomaps at a time. diff --git a/mne/io/tests/test_raw.py b/mne/io/tests/test_raw.py index 048d23efdbb..c3e7243fd9e 100644 --- a/mne/io/tests/test_raw.py +++ b/mne/io/tests/test_raw.py @@ -64,6 +64,25 @@ def assert_named_constants(info): assert re.match(check, r, re.DOTALL) is not None, (check, r) +def assert_attributes(raw): + """Assert that the instance keeps all its extra attributes in _raw_extras.""" + __tracebackhide__ = True + assert isinstance(raw, BaseRaw) + base_attrs = set(dir(BaseRaw(create_info(1, 1000.0, "eeg"), last_samps=[1]))) + base_attrs = base_attrs.union( + [ + "_data", # in the case of preloaded data + "__slotnames__", # something about being decorated (?) + ] + ) + for attr in raw._extra_attributes: + assert attr not in base_attrs + base_attrs.add(attr) + got_attrs = set(dir(raw)) + extra = got_attrs.difference(base_attrs) + assert extra == set() + + def test_orig_units(): """Test the error handling for original units.""" # Should work fine @@ -155,7 +174,7 @@ def _test_raw_reader( # test projection vs cals and data units other_raw = reader(preload=False, **kwargs) other_raw.del_proj() - eeg = meg = fnirs = seeg = False + eeg = meg = fnirs = seeg = eyetrack = False if "eeg" in raw: eeg, atol = True, 1e-18 elif "grad" in raw: @@ -168,12 +187,21 @@ def _test_raw_reader( fnirs, atol = "hbr", 1e-10 elif "fnirs_cw_amplitude" in raw: fnirs, atol = "fnirs_cw_amplitude", 1e-10 + elif "eyegaze" in raw: + eyetrack = "eyegaze", 1e-3 else: # e.g., https://github.com/mne-tools/mne-python/pull/11432/files assert "seeg" in raw, "New channel type necessary? See gh-11432 for example" seeg, atol = True, 1e-18 - picks = pick_types(other_raw.info, meg=meg, eeg=eeg, fnirs=fnirs, seeg=seeg) + picks = pick_types( + other_raw.info, + meg=meg, + eeg=eeg, + fnirs=fnirs, + seeg=seeg, + eyetrack=eyetrack, + ) col_names = [other_raw.ch_names[pick] for pick in picks] proj = np.ones((1, len(picks))) proj /= np.sqrt(proj.shape[1]) @@ -284,6 +312,7 @@ def _test_raw_reader( raw = reader(**kwargs) n_samp = len(raw.times) assert_named_constants(raw.info) + assert_attributes(raw) # smoke test for gh #9743 ids = [id(ch["loc"]) for ch in raw.info["chs"]] assert len(set(ids)) == len(ids) diff --git a/mne/time_frequency/tfr.py b/mne/time_frequency/tfr.py index 367112de01b..b56dcc8acc1 100644 --- a/mne/time_frequency/tfr.py +++ b/mne/time_frequency/tfr.py @@ -2530,14 +2530,26 @@ def plot_topomap( ('zlogratio') %(sensors_topomap)s %(show_names_topomap)s + + .. versionadded:: 1.2 %(mask_evoked_topomap)s + + .. versionadded:: 1.2 %(mask_params_topomap)s + + .. versionadded:: 1.2 %(contours_topomap)s %(outlines_topomap)s %(sphere_topomap_auto)s %(image_interp_topomap)s + + .. versionadded:: 1.2 %(extrapolate_topomap)s + + .. versionadded:: 1.2 %(border_topomap)s + + .. versionadded:: 0.20 %(res_topomap)s %(size_topomap)s %(cmap_topomap)s diff --git a/mne/utils/docs.py b/mne/utils/docs.py index 0fd8b7de4e5..dadc853e9bd 100644 --- a/mne/utils/docs.py +++ b/mne/utils/docs.py @@ -492,8 +492,6 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75): border : float | 'mean' Value to extrapolate to on the topomap borders. If ``'mean'`` (default), then each extrapolated point has the average value of its neighbours. - - .. versionadded:: 0.20 """ docdict[ @@ -1473,12 +1471,6 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75): the head circle when the sensors are contained within the head circle, but it can extend beyond the head when sensors are plotted outside the head circle. - - .. versionchanged:: 0.21 - - - The default was changed to ``'local'`` for MEG sensors. - - ``'local'`` was changed to use a convex hull mask - - ``'head'`` was changed to extrapolate out to the clipping circle. """ # %% diff --git a/mne/viz/_figure.py b/mne/viz/_figure.py index 392228785a9..43efd89bb4f 100644 --- a/mne/viz/_figure.py +++ b/mne/viz/_figure.py @@ -85,17 +85,18 @@ def __init__(self, **kwargs): self.mne.whitened_ch_names = list() if hasattr(self.mne, "noise_cov"): self.mne.use_noise_cov = self.mne.noise_cov is not None + # allow up to 10000 zorder levels for annotations self.mne.zorder = dict( patch=0, grid=1, ann=2, - events=3, - bads=4, - data=5, - mag=6, - grad=7, - scalebar=8, - vline=9, + events=10003, + bads=10004, + data=10005, + mag=10006, + grad=10007, + scalebar=10008, + vline=10009, ) # additional params for epochs (won't affect raw / ICA) self.mne.epoch_traces = list() diff --git a/mne/viz/_mpl_figure.py b/mne/viz/_mpl_figure.py index 323b9bc6948..d84f8a20b2b 100644 --- a/mne/viz/_mpl_figure.py +++ b/mne/viz/_mpl_figure.py @@ -788,6 +788,8 @@ def _keypress(self, event): def _buttonpress(self, event): """Handle mouse clicks.""" + from matplotlib.collections import PolyCollection + butterfly = self.mne.butterfly annotating = self.mne.fig_annotation is not None ax_main = self.mne.ax_main @@ -828,16 +830,34 @@ def _buttonpress(self, event): self._toggle_help_fig(event) else: # right-click (secondary) if annotating: - if any(c.contains(event)[0] for c in ax_main.collections): + spans = [ + span + for span in ax_main.collections + if isinstance(span, PolyCollection) + ] + if any(span.contains(event)[0] for span in spans): xdata = event.xdata - self.mne.first_time start = _sync_onset(inst, inst.annotations.onset) end = start + inst.annotations.duration - ann_idx = np.where((xdata > start) & (xdata < end))[0] - for idx in sorted(ann_idx)[::-1]: - # only remove visible annotation spans - descr = inst.annotations[idx]["description"] - if self.mne.visible_annotations[descr]: - inst.annotations.delete(idx) + is_onscreen = self.mne.onscreen_annotations # boolean array + was_clicked = (xdata > start) & (xdata < end) & is_onscreen + # determine which annotation label is "selected" + buttons = self.mne.fig_annotation.mne.radio_ax.buttons + current_label = buttons.value_selected + is_active_label = inst.annotations.description == current_label + # use z-order as tiebreaker (or if click wasn't on an active span) + # (ax_main.collections only includes *visible* annots, so we offset) + visible_zorders = [span.zorder for span in spans] + zorders = np.zeros_like(is_onscreen).astype(int) + offset = np.where(is_onscreen)[0][0] + zorders[offset : (offset + len(visible_zorders))] = visible_zorders + # among overlapping clicked spans, prefer removing spans whose label + # is the active label; then fall back to zorder as deciding factor + active_clicked = was_clicked & is_active_label + mask = active_clicked if any(active_clicked) else was_clicked + highest = zorders == zorders[mask].max() + idx = np.where(highest)[0] + inst.annotations.delete(idx) self._remove_annotation_hover_line() self._draw_annotations() self.canvas.draw_idle() @@ -1392,13 +1412,15 @@ def _draw_annotations(self): self._clear_annotations() self._update_annotation_segments() segments = self.mne.annotation_segments + onscreen_annotations = np.zeros(len(segments), dtype=bool) times = self.mne.times ax = self.mne.ax_main ylim = ax.get_ylim() for idx, (start, end) in enumerate(segments): descr = self.mne.inst.annotations.description[idx] segment_color = self.mne.annotation_segment_colors[descr] - kwargs = dict(color=segment_color, alpha=0.3, zorder=self.mne.zorder["ann"]) + zorder = self.mne.zorder["ann"] + idx + kwargs = dict(color=segment_color, alpha=0.3, zorder=zorder) if self.mne.visible_annotations[descr]: # draw all segments on ax_hscroll annot = self.mne.ax_hscroll.fill_betweenx((0, 1), start, end, **kwargs) @@ -1408,6 +1430,7 @@ def _draw_annotations(self): if np.diff(visible_segment) > 0: annot = ax.fill_betweenx(ylim, *visible_segment, **kwargs) self.mne.annotations.append(annot) + onscreen_annotations[idx] = True xy = (visible_segment.mean(), ylim[1]) text = ax.annotate( descr, @@ -1419,6 +1442,7 @@ def _draw_annotations(self): color=segment_color, ) self.mne.annotation_texts.append(text) + self.mne.onscreen_annotations = onscreen_annotations # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # CHANNEL SELECTION GUI diff --git a/mne/viz/epochs.py b/mne/viz/epochs.py index 5e4a4d76874..c1ee9fa2f5b 100644 --- a/mne/viz/epochs.py +++ b/mne/viz/epochs.py @@ -767,7 +767,7 @@ def plot_epochs( n_epochs=20, n_channels=20, title=None, - events=None, + events=False, event_color=None, order=None, show=True, @@ -816,6 +816,10 @@ def plot_epochs( align with the data. .. versionadded:: 0.14.0 + + .. versionchanged:: 1.5 + Passing ``events=None`` is deprecated and will be removed in version 1.6. + The new equivalent is ``events=False``. %(event_color)s Defaults to ``None``. order : array of str | None diff --git a/mne/viz/tests/test_raw.py b/mne/viz/tests/test_raw.py index 9a779d9d5da..e3eabfecd97 100644 --- a/mne/viz/tests/test_raw.py +++ b/mne/viz/tests/test_raw.py @@ -23,6 +23,7 @@ get_config, set_config, _assert_no_instances, + check_version, ) from mne.viz import plot_raw, plot_sensors from mne.viz.utils import _fake_click, _fake_keypress @@ -745,6 +746,63 @@ def test_plot_annotations(raw, browser_backend): assert fig.mne.regions[0].isVisible() +@pytest.mark.parametrize("active_annot_idx", (0, 1, 2)) +def test_overlapping_annotation_deletion(raw, browser_backend, active_annot_idx): + """Test deletion of annotations via right-click.""" + ismpl = browser_backend.name == "matplotlib" + if not ismpl and not check_version("mne_qt_browser", "0.5.2"): + pytest.xfail("Old mne-qt-browser") + with raw.info._unlock(): + raw.info["lowpass"] = 10.0 + annot_labels = list("abc") + # the test applies to the middle three annotations; those before and after are + # there to ensure our bookkeeping works + annot = Annotations( + onset=[3, 3.4, 3.7, 13, 13.4, 13.7, 19, 19.4, 19.7], + duration=[2, 1, 3] * 3, + description=annot_labels * 3, + ) + raw.set_annotations(annot) + start = 10 + duration = 8 + fig = raw.plot(start=start, duration=duration) + + def _get_visible_labels(fig_dot_mne): + if ismpl: + # MPL backend's `fig.mne.annotation_texts` → only the visible ones + visible_labels = [x.get_text() for x in fig_dot_mne.annotation_texts] + else: + # PyQtGraph backend's `fig.mne.regions` → all annots (even offscreen ones) + # so we need to (1) get annotations from fig.mne.inst, and (2) compute + # ourselves which ones are visible. + _annot = fig_dot_mne.inst.annotations + _start = start + fig_dot_mne.inst.first_time + _end = _start + duration + visible_indices = np.nonzero( + np.logical_and(_annot.onset > _start, _annot.onset < _end) + ) + visible_labels = np.array( + [x.label_item.toPlainText() for x in fig_dot_mne.regions] + )[visible_indices].tolist() + return visible_labels + + assert annot_labels == _get_visible_labels(fig.mne) + fig._fake_keypress("a") # start annotation mode + if ismpl: + buttons = fig.mne.fig_annotation.mne.radio_ax.buttons + buttons.set_active(active_annot_idx) + current_active = buttons.value_selected + else: + buttons = fig.mne.fig_annotation.description_cmbx + buttons.setCurrentIndex(active_annot_idx) + current_active = buttons.currentText() + assert current_active == annot_labels[active_annot_idx] + # x value of 14 is in area that overlaps all 3 visible annotations + fig._fake_click((14, 1.0), xform="data", button=3) + expected = set(annot_labels) - set(annot_labels[active_annot_idx]) + assert expected == set(_get_visible_labels(fig.mne)) + + @pytest.mark.parametrize("hide_which", ([], [0], [1], [0, 1])) def test_remove_annotations(raw, hide_which, browser_backend): """Test that right-click doesn't remove hidden annotation spans.""" @@ -763,7 +821,9 @@ def test_remove_annotations(raw, hide_which, browser_backend): hide_key = descriptions[hide_idx] fig.mne.visible_annotations[hide_key] = False fig._update_regions_visible() - fig._fake_click((2.5, 0.1), xform="data", button=3) + # always click twice: should not affect hidden annotation spans + for _ in descriptions: + fig._fake_click((2.5, 0.1), xform="data", button=3) assert len(raw.annotations) == len(hide_which) diff --git a/mne/viz/topomap.py b/mne/viz/topomap.py index 3ad6891eb0c..de9b8304740 100644 --- a/mne/viz/topomap.py +++ b/mne/viz/topomap.py @@ -376,7 +376,15 @@ def plot_projs_topomap( %(extrapolate_topomap)s .. versionadded:: 0.20 + + .. versionchanged:: 0.21 + + - The default was changed to ``'local'`` for MEG sensors. + - ``'local'`` was changed to use a convex hull mask + - ``'head'`` was changed to extrapolate out to the clipping circle. %(border_topomap)s + + .. versionadded:: 0.20 %(res_topomap)s %(size_topomap)s %(cmap_topomap)s @@ -936,7 +944,15 @@ def plot_topomap( %(extrapolate_topomap)s .. versionadded:: 0.18 + + .. versionchanged:: 0.21 + + - The default was changed to ``'local'`` for MEG sensors. + - ``'local'`` was changed to use a convex hull mask + - ``'head'`` was changed to extrapolate out to the clipping circle. %(border_topomap)s + + .. versionadded:: 0.20 %(res_topomap)s %(size_topomap)s %(cmap_topomap_simple)s @@ -1792,7 +1808,15 @@ def plot_tfr_topomap( %(sphere_topomap_auto)s %(image_interp_topomap)s %(extrapolate_topomap)s + + .. versionchanged:: 0.21 + + - The default was changed to ``'local'`` for MEG sensors. + - ``'local'`` was changed to use a convex hull mask + - ``'head'`` was changed to extrapolate out to the clipping circle. %(border_topomap)s + + .. versionadded:: 0.20 %(res_topomap)s %(size_topomap)s %(cmap_topomap)s @@ -1984,7 +2008,15 @@ def plot_evoked_topomap( %(extrapolate_topomap)s .. versionadded:: 0.18 + + .. versionchanged:: 0.21 + + - The default was changed to ``'local'`` for MEG sensors. + - ``'local'`` was changed to use a convex hull mask + - ``'head'`` was changed to extrapolate out to the clipping circle. %(border_topomap)s + + .. versionadded:: 0.20 %(res_topomap)s %(size_topomap)s %(cmap_topomap)s @@ -2517,7 +2549,15 @@ def plot_epochs_psd_topomap( %(sphere_topomap_auto)s %(image_interp_topomap)s %(extrapolate_topomap)s + + .. versionchanged:: 0.21 + + - The default was changed to ``'local'`` for MEG sensors. + - ``'local'`` was changed to use a convex hull mask + - ``'head'`` was changed to extrapolate out to the clipping circle. %(border_topomap)s + + .. versionadded:: 0.20 %(res_topomap)s %(size_topomap)s %(cmap_topomap)s @@ -2610,7 +2650,15 @@ def plot_psds_topomap( %(sphere_topomap_auto)s %(image_interp_topomap)s %(extrapolate_topomap)s + + .. versionchanged:: 0.21 + + - The default was changed to ``'local'`` for MEG sensors. + - ``'local'`` was changed to use a convex hull mask + - ``'head'`` was changed to extrapolate out to the clipping circle. %(border_topomap)s + + .. versionadded:: 0.20 %(res_topomap)s %(size_topomap)s %(cmap_topomap)s @@ -3421,6 +3469,12 @@ def plot_arrowmap( %(extrapolate_topomap)s .. versionadded:: 0.18 + + .. versionchanged:: 0.21 + + - The default was changed to ``'local'`` for MEG sensors. + - ``'local'`` was changed to use a convex hull mask + - ``'head'`` was changed to extrapolate out to the clipping circle. %(sphere_topomap_auto)s Returns @@ -3864,7 +3918,15 @@ def plot_regression_weights( %(sphere_topomap_auto)s %(image_interp_topomap)s %(extrapolate_topomap)s + + .. versionchanged:: 0.21 + + - The default was changed to ``'local'`` for MEG sensors. + - ``'local'`` was changed to use a convex hull mask + - ``'head'`` was changed to extrapolate out to the clipping circle. %(border_topomap)s + + .. versionadded:: 0.20 %(res_topomap)s %(size_topomap)s %(cmap_topomap)s diff --git a/tools/azure_dependencies.sh b/tools/azure_dependencies.sh index 380113d1127..cc618c52f53 100755 --- a/tools/azure_dependencies.sh +++ b/tools/azure_dependencies.sh @@ -5,7 +5,7 @@ if [ "${TEST_MODE}" == "pip" ]; then python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade --only-binary="numba,llvmlite,numpy,scipy,vtk" -r requirements.txt elif [ "${TEST_MODE}" == "pip-pre" ]; then - python -m pip install --progress-bar off --upgrade pip setuptools wheel packaging + python -m pip install --progress-bar off --upgrade pip setuptools wheel packaging setuptools_scm python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --extra-index-url "https://www.riverbankcomputing.com/pypi/simple" PyQt6 PyQt6-sip PyQt6-Qt6 python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" numpy scipy statsmodels pandas scikit-learn matplotlib python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --extra-index-url "https://pypi.anaconda.org/scipy-wheels-nightly/simple" dipy diff --git a/tools/github_actions_dependencies.sh b/tools/github_actions_dependencies.sh index cec75a75e1a..9ac14002c37 100755 --- a/tools/github_actions_dependencies.sh +++ b/tools/github_actions_dependencies.sh @@ -4,7 +4,8 @@ STD_ARGS="--progress-bar off --upgrade" EXTRA_ARGS="" if [ ! -z "$CONDA_ENV" ]; then echo "Uninstalling MNE for CONDA_ENV=${CONDA_ENV}" - pip uninstall -yq mne + conda remove -c conda-forge --force -yq mne + python -m pip uninstall -y mne elif [ ! -z "$CONDA_DEPENDENCIES" ]; then echo "Using Mamba to install CONDA_DEPENDENCIES=${CONDA_DEPENDENCIES}" mamba install -y $CONDA_DEPENDENCIES @@ -37,13 +38,19 @@ else pip install --progress-bar off git+https://github.com/mne-tools/mne-qt-browser EXTRA_ARGS="--pre" fi +echo "" + # for compat_minimal and compat_old, we don't want to --upgrade if [ ! -z "$CONDA_DEPENDENCIES" ]; then - pip install -r requirements_base.txt -r requirements_testing.txt + echo "Installing dependencies for conda" + python -m pip install -r requirements_base.txt -r requirements_testing.txt else - pip install $STD_ARGS $EXTRA_ARGS -r requirements_base.txt -r requirements_testing.txt -r requirements_hdf5.txt + echo "Installing dependencies using pip" + python -m pip install $STD_ARGS $EXTRA_ARGS -r requirements_base.txt -r requirements_testing.txt -r requirements_hdf5.txt fi +echo "" if [ "${DEPS}" != "minimal" ]; then - pip install $STD_ARGS $EXTRA_ARGS -r requirements_testing_extra.txt + echo "Installing non-minimal dependencies" + python -m pip install $STD_ARGS $EXTRA_ARGS -r requirements_testing_extra.txt fi diff --git a/tools/github_actions_env_vars.sh b/tools/github_actions_env_vars.sh index cf1dbdb45a5..6b479c76b34 100755 --- a/tools/github_actions_env_vars.sh +++ b/tools/github_actions_env_vars.sh @@ -4,23 +4,19 @@ set -eo pipefail -x # old and minimal use conda if [[ "$MNE_CI_KIND" == "old" ]]; then echo "Setting conda env vars for old" - echo "CONDA_ACTIVATE_ENV=true" >> $GITHUB_ENV echo "CONDA_DEPENDENCIES=numpy=1.20.2 scipy=1.6.3 matplotlib=3.4 pandas=1.2.4 scikit-learn=0.24.2" >> $GITHUB_ENV echo "MNE_IGNORE_WARNINGS_IN_TESTS=true" >> $GITHUB_ENV echo "MNE_SKIP_NETWORK_TESTS=1" >> $GITHUB_ENV elif [[ "$MNE_CI_KIND" == "minimal" ]]; then echo "Setting conda env vars for minimal" - echo "CONDA_ACTIVATE_ENV=true" >> $GITHUB_ENV echo "CONDA_DEPENDENCIES=numpy scipy matplotlib" >> $GITHUB_ENV elif [[ "$MNE_CI_KIND" == "notebook" ]]; then echo "CONDA_ENV=environment.yml" >> $GITHUB_ENV - echo "CONDA_ACTIVATE_ENV=mne" >> $GITHUB_ENV # TODO: This should work but breaks stuff... # echo "MNE_3D_BACKEND=notebook" >> $GITHUB_ENV elif [[ "$MNE_CI_KIND" != "pip"* ]]; then # conda, mamba (use warning level for completeness) echo "Setting conda env vars for $MNE_CI_KIND" echo "CONDA_ENV=environment.yml" >> $GITHUB_ENV - echo "CONDA_ACTIVATE_ENV=mne" >> $GITHUB_ENV echo "MNE_QT_BACKEND=PyQt5" >> $GITHUB_ENV echo "MNE_LOGGING_LEVEL=warning" >> $GITHUB_ENV else # pip-like diff --git a/tools/github_actions_test.sh b/tools/github_actions_test.sh index d4fa197dad4..1f89d926fed 100755 --- a/tools/github_actions_test.sh +++ b/tools/github_actions_test.sh @@ -2,7 +2,7 @@ set -eo pipefail -if [[ "${CI_OS_NAME}" != "macos"* ]]; then +if [[ "${CI_OS_NAME}" == "ubuntu"* ]]; then if [[ "${MNE_CI_KIND}" == "pip-pre" ]]; then CONDITION="not (slowtest or pgtest)" else