Allow preprocessing on dicts of recordings

.github/scripts/check_kilosort4_releases.py

@@ -1,9 +1,10 @@
 import os
+import re
 from pathlib import Path
 import requests
 import json
 from packaging.version import parse
-
+import spikeinterface
 
 def get_pypi_versions(package_name):
     """
@@ -15,10 +16,8 @@ def get_pypi_versions(package_name):
     response.raise_for_status()
     data = response.json()
     versions = list(sorted(data["releases"].keys()))
-    # Filter out versions that are less than 4.0.16 and different from 4.0.26 and 4.0.27
-    # (buggy - https://github.com/MouseLand/Kilosort/releases/tag/v4.0.26)
-    versions = [ver for ver in versions if parse(ver) >= parse("4.0.16") and
-                parse(ver) not in [parse("4.0.26"), parse("4.0.27")]]
+    # Filter out versions that are less than 4.0.16
+    versions = [ver for ver in versions if parse(ver) >= parse("4.0.16")]
     return versions
 
 

.github/scripts/test_kilosort4_ci.py

@@ -258,22 +258,17 @@ def test_compute_preprocessing_arguments(self):
         self._check_arguments(compute_preprocessing, ["ops", "device", "tic0", "file_object"])
 
     def test_compute_drift_location_arguments(self):
-        expected_arguments = ["ops", "device", "tic0", "progress_bar", "file_object", "clear_cache"]
-        if parse(kilosort.__version__) >= parse("4.0.28"):
-            expected_arguments += ["verbose"]
-        self._check_arguments(compute_drift_correction, expected_arguments)
+        self._check_arguments(
+            compute_drift_correction, ["ops", "device", "tic0", "progress_bar", "file_object", "clear_cache"]
+        )
 
     def test_detect_spikes_arguments(self):
-        expected_arguments = ["ops", "device", "bfile", "tic0", "progress_bar", "clear_cache"]
-        if parse(kilosort.__version__) >= parse("4.0.28"):
-            expected_arguments += ["verbose"]
-        self._check_arguments(detect_spikes, expected_arguments)
+        self._check_arguments(detect_spikes, ["ops", "device", "bfile", "tic0", "progress_bar", "clear_cache"])
 
     def test_cluster_spikes_arguments(self):
-        expected_arguments = ["st", "tF", "ops", "device", "bfile", "tic0", "progress_bar", "clear_cache"]
-        if parse(kilosort.__version__) >= parse("4.0.28"):
-            expected_arguments += ["verbose"]
-        self._check_arguments(cluster_spikes, expected_arguments)
+        self._check_arguments(
+            cluster_spikes, ["st", "tF", "ops", "device", "bfile", "tic0", "progress_bar", "clear_cache"]
+        )
 
     def test_save_sorting_arguments(self):
         expected_arguments = ["ops", "results_dir", "st", "clu", "tF", "Wall", "imin", "tic0", "save_extra_vars"]

.github/workflows/all-tests.yml

@@ -10,7 +10,9 @@ on:
       - main
 
 env:
-  KACHERY_API_KEY: ${{ secrets.KACHERY_API_KEY }}
+  KACHERY_CLOUD_CLIENT_ID: ${{ secrets.KACHERY_CLOUD_CLIENT_ID }}
+  KACHERY_CLOUD_PRIVATE_KEY: ${{ secrets.KACHERY_CLOUD_PRIVATE_KEY }}
+  KACHERY_ZONE: ${{ secrets.KACHERY_ZONE }}
 
 concurrency:  # Cancel previous workflows on the same pull request
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -74,9 +76,6 @@ jobs:
           pip install -e .[test_core]
         shell: bash
 
-      - name: Pip list
-        run: pip list
-
       - name: Test core
         run: pytest -m "core"
         shell: bash
@@ -137,109 +136,93 @@ jobs:
         if: env.RUN_EXTRACTORS_TESTS == 'true'
         run: |
           pip install -e .[extractors,streaming_extractors,test_extractors]
-          pip list
           ./.github/run_tests.sh "extractors and not streaming_extractors" --no-virtual-env
 
       - name: Test streaming extractors
         shell: bash
         if: env.RUN_STREAMING_EXTRACTORS_TESTS == 'true'
         run: |
           pip install -e .[streaming_extractors,test_extractors]
-          pip list
           ./.github/run_tests.sh "streaming_extractors" --no-virtual-env
 
       - name: Test preprocessing
         shell: bash
         if: env.RUN_PREPROCESSING_TESTS == 'true'
         run: |
           pip install -e .[preprocessing,test_preprocessing]
-          pip list
           ./.github/run_tests.sh "preprocessing and not deepinterpolation" --no-virtual-env
 
       - name: Install remaining testing dependencies # TODO: Remove this step once we have better modularization
         shell: bash
         run: |
           pip install -e .[test]
-          pip list
 
       - name: Test postprocessing
         shell: bash
         if: env.RUN_POSTPROCESSING_TESTS == 'true'
         run: |
           pip install -e .[full]
-          pip list
           ./.github/run_tests.sh postprocessing --no-virtual-env
 
       - name: Test quality metrics
         shell: bash
         if: env.RUN_QUALITYMETRICS_TESTS == 'true'
         run: |
           pip install -e .[qualitymetrics]
-          pip list
           ./.github/run_tests.sh qualitymetrics --no-virtual-env
 
       - name: Test comparison
         shell: bash
         if: env.RUN_COMPARISON_TESTS == 'true'
         run: |
           pip install -e .[full]
-          pip list
           ./.github/run_tests.sh comparison --no-virtual-env
 
       - name: Test core sorters
         shell: bash
         if: env.RUN_SORTERS_TESTS == 'true'
         run: |
           pip install -e .[full]
-          pip list
           ./.github/run_tests.sh sorters --no-virtual-env
 
       - name: Test internal sorters
         shell: bash
         if: env.RUN_INTERNAL_SORTERS_TESTS == 'true'
         run: |
           pip install -e .[full]
-          pip list
           ./.github/run_tests.sh sorters_internal --no-virtual-env
 
       - name: Test curation
         shell: bash
         if: env.RUN_CURATION_TESTS == 'true'
         run: |
           pip install -e .[full]
-          pip list
           ./.github/run_tests.sh curation --no-virtual-env
 
       - name: Test widgets
         shell: bash
         if: env.RUN_WIDGETS_TESTS == 'true'
-        env:
-          KACHERY_ZONE: "scratch"
         run: |
           pip install -e .[full,widgets]
-          pip list
           ./.github/run_tests.sh widgets --no-virtual-env
 
       - name: Test exporters
         shell: bash
         if: env.RUN_EXPORTERS_TESTS == 'true'
         run: |
           pip install -e .[full]
-          pip list
           ./.github/run_tests.sh exporters --no-virtual-env
 
       - name: Test sortingcomponents
         shell: bash
         if: env.RUN_SORTINGCOMPONENTS_TESTS == 'true'
         run: |
           pip install -e .[full]
-          pip list
           ./.github/run_tests.sh sortingcomponents --no-virtual-env
 
       - name: Test generation
         shell: bash
         if: env.RUN_GENERATION_TESTS == 'true'
         run: |
           pip install -e .[full]
-          pip list
           ./.github/run_tests.sh generation --no-virtual-env

.github/workflows/deepinterpolation.yml

@@ -43,7 +43,12 @@ jobs:
           pip install tensorflow==2.7.0
           pip install deepinterpolation@git+https://github.com/AllenInstitute/deepinterpolation.git
           pip install protobuf==3.20.*
+          pip install numpy==1.26.4
           pip install -e .[full,test_core]
+      - name: Pip list
+        if: ${{ steps.modules-changed.outputs.DEEPINTERPOLATION_CHANGED == 'true' }}
+        run: |
+          pip list
       - name: Test DeepInterpolation with pytest
         if: ${{ steps.modules-changed.outputs.DEEPINTERPOLATION_CHANGED == 'true' }}
         run: |

.github/workflows/full-test-with-codecov.yml

@@ -6,7 +6,9 @@ on:
     - cron: "0 12 * * *"  # Daily at noon UTC
 
 env:
-  KACHERY_API_KEY: ${{ secrets.KACHERY_API_KEY }}
+  KACHERY_CLOUD_CLIENT_ID: ${{ secrets.KACHERY_CLOUD_CLIENT_ID }}
+  KACHERY_CLOUD_PRIVATE_KEY: ${{ secrets.KACHERY_CLOUD_PRIVATE_KEY }}
+  KACHERY_ZONE: ${{ secrets.KACHERY_ZONE }}
 
 jobs:
   full-tests-with-codecov:
@@ -39,8 +41,6 @@ jobs:
           restore-keys: ${{ runner.os }}-datasets
       - name: Install packages
         uses: ./.github/actions/build-test-environment
-      - name: Pip list
-        run: pip list
       - name: run tests
         env:
           HDF5_PLUGIN_PATH: ${{ github.workspace }}/hdf5_plugin_path_maxwell

doc/api.rst

@@ -22,8 +22,6 @@ spikeinterface.core
     .. autofunction:: estimate_sparsity
     .. autoclass:: ChannelSparsity
         :members:
-    .. autoclass:: Motion
-        :members:
     .. autoclass:: BinaryRecordingExtractor
     .. autoclass:: ZarrRecordingExtractor
     .. autoclass:: BinaryFolderRecording
@@ -279,6 +277,8 @@ spikeinterface.comparison
 
     .. autoclass:: CollisionGTComparison
     .. autoclass:: CorrelogramGTComparison
+    .. autoclass:: CollisionGTStudy
+    .. autoclass:: CorrelogramGTStudy
 
 
 
@@ -449,6 +449,7 @@ Motion Correction
 ~~~~~~~~~~~~~~~~~
 .. automodule:: spikeinterface.sortingcomponents.motion
 
+    .. autoclass:: Motion
     .. autofunction:: estimate_motion
     .. autofunction:: interpolate_motion
     .. autofunction:: correct_motion_on_peaks

doc/conf.py

@@ -31,12 +31,14 @@
 folders =  [
     '../examples/tutorials/core/my_recording',
     '../examples/tutorials/core/my_sorting',
-    '../examples/tutorials/core/analyzer_folder',
-    '../examples/tutorials/core/analyzer_some_units',
-    '../examples/tutorials/core/analyzer.zarr',
-    '../examples/tutorials/curation/my_folder',
-    '../examples/tutorials/qualitymetrics/curated_sorting',
-    '../examples/tutorials/qualitymetrics/clean_analyzer.zarr',
+    '../examples/tutorials/core/waveform_folder',
+    '../examples/tutorials/core/waveform_folder_parallel',
+    '../examples/tutorials/core/waveform_folder_sparse',
+    '../examples/tutorials/core/waveform_folder_sparse_direct',
+    '../examples/tutorials/core/waveform_folder2',
+    '../examples/tutorials/core/waveform_folder',
+    '../examples/tutorials/qualitymetrics/waveforms_mearec',
+    '../examples/tutorials/qualitymetrics/wfs_mearec',
     '../examples/tutorials/widgets/waveforms_mearec',
 
 ]
@@ -99,6 +101,7 @@
     import sphinx_rtd_theme
 
     html_theme = "sphinx_rtd_theme"
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 except ImportError:
     print("RTD theme not installed, using default")
     html_theme = 'alabaster'
@@ -107,9 +110,8 @@
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 # html_static_path = ['_static']
-# html_css_files = ['custom.css']
-html_favicon = "images/logo.png"
-html_logo = "images/logo.png"
+
+html_favicon = "images/favicon-32x32.png"
 
 
 from sphinx_gallery.sorting import ExplicitOrder

doc/how_to/load_your_data_into_sorting.rst

@@ -65,8 +65,8 @@ the requested unit_ids).
 
     # in this case we are making a monosegment sorting
     # we have four spikes that are spread among two neurons
-    my_sorting = NumpySorting.from_samples_and_labels(
-        samples_list=[
+    my_sorting = NumpySorting.from_times_labels(
+        times_list=[
             np.array([1000,12000,15000,22000])   # Note these are samples/frames not times in seconds
             ],
         labels_list=[
@@ -120,7 +120,7 @@ Loading multisegment data into a :code:`Sorting`
 One of the great advantages of SpikeInterface :code:`Sorting` objects is that they can also handle
 multisegment recordings and sortings (e.g. you have a baseline, stimulus, post-stimulus). The
 exact same machinery can be used to generate your sorting, but in this case we do a list of arrays instead of
-a single list. Let's go through one example for using :code:`from_samples_and_labels`:
+a single list. Let's go through one example for using :code:`from_times_labels`:
 
 .. code-block:: python
 
@@ -130,8 +130,8 @@ a single list. Let's go through one example for using :code:`from_samples_and_la
     # in this case we are making three-segment sorting
     # we have four spikes that are spread among two neurons
     # in each segment
-    my_sorting = NumpySorting.from_samples_and_labels(
-        samples_list=[
+    my_sorting = NumpySorting.from_times_labels(
+        times_list=[
             np.array([1000,12000,15000,22000]),
             np.array([30000,33000, 41000, 47000]),
             np.array([50000,53000,64000,70000]),

doc/index.rst

@@ -2,6 +2,11 @@ Welcome to SpikeInterface's documentation!
 ==========================================
 
 
+.. image:: images/logo.png
+  :scale: 100 %
+  :align: center
+
+
 SpikeInterface is a Python module to analyze extracellular electrophysiology data.
 
 With a few lines of code, SpikeInterface enables you to load and pre-process the recording, run several

doc/modules/comparison.rst

@@ -26,7 +26,7 @@ For spike train comparison, there are three use cases:
 
 A ground-truth dataset can be a paired recording, in which a neuron is recorded both extracellularly and with
 a patch or juxtacellular electrode (either **in vitro** or **in vivo**), or it can be a simulated dataset
-(**in silico**) using spiking activity simulators such as `MEArec <https://github.com/SpikeInterface/MEArec>`_.
+(**in silico**) using spiking activity simulators such as `MEArec`_.
 
 The comparison to ground-truth datasets is useful to benchmark spike sorting algorithms.
 

doc/modules/core.rst

@@ -711,13 +711,8 @@ In this example, we create a recording and a sorting object from numpy objects:
         spike_trains += spike_trains_i
         labels += labels_i
 
-    # construct a mono-segment
-    samples_list = [np.array(spike_trains)]
-    labels_list = [np.array(labels)]
-
-    sorting_memory = NumpySorting.from_samples_and_labels(
-        samples_list=samples_list, labels_list=labels_list, sampling_frequency=sampling_frequency
-    )
+    sorting_memory = NumpySorting.from_times_labels(times=spike_trains, labels=labels,
+                                                    sampling_frequency=sampling_frequency)
 
 
 Any sorting object can be transformed into a :py:class:`~spikeinterface.core.NumpySorting` or

doc/modules/index.rst

@@ -17,4 +17,3 @@ Modules documentation
     sortingcomponents
     motion_correction
     generation
-    benchmark

doc/modules/sorters.rst

@@ -473,6 +473,8 @@ Here is the list of external sorters accessible using the run_sorter wrapper:
 * **HDSort** :code:`run_sorter(sorter_name='hdsort')`
 * **YASS** :code:`run_sorter(sorter_name='yass')`
 
+Internal Sorters
+----------------
 
 Here a list of internal sorter based on `spikeinterface.sortingcomponents`; they are totally
 experimental for now:

examples/how_to/drift_with_lfp.py

@@ -47,7 +47,7 @@
 
 # the dataset has been downloaded locally
 base_folder = Path("/mnt/data/sam/DataSpikeSorting/")
-np_data_drift = base_folder / "human_neuropixel" / "Pt02"
+np_data_drift = base_folder / 'human_neuropixel" / "Pt02"
 
 # ### Read the spikeglx file
 

examples/tutorials/core/plot_2_sorting_extractor.py

@@ -40,7 +40,7 @@
 ##############################################################################
 # And instantiate a :py:class:`~spikeinterface.core.NumpySorting` object:
 
-sorting = se.NumpySorting.from_samples_and_labels([times0, times1], [labels0, labels1], sampling_frequency)
+sorting = se.NumpySorting.from_times_labels([times0, times1], [labels0, labels1], sampling_frequency)
 print(sorting)
 
 ##############################################################################