diff --git a/.github/workflows/_license-check.yml b/.github/workflows/_license-check.yml index eeade0c4..cc83a40a 100644 --- a/.github/workflows/_license-check.yml +++ b/.github/workflows/_license-check.yml @@ -26,3 +26,4 @@ jobs: uv run pip-licenses --allow-only="MIT;Apache-2.0;Apache 2.0;Apache Software License;BSD-2-Clause;BSD-3-Clause;BSD License;BSD;ISC;LGPL-3.0-only;MPL-2.0;Mozilla Public License;Python-2.0;PSF-2.0;Python Software Foundation License;Unlicense" --partial-match + --ignore-packages quantities diff --git a/pyproject.toml b/pyproject.toml index f823eb44..241b35fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,8 @@ postgresql = ["psycopg[binary]>=3.0"] spiraldb = [ "pyspiral>=0.14.0", ] -all = ["orcapod[redis]", "orcapod[ray]", "orcapod[postgresql]", "orcapod[spiraldb]"] +spikeinterface = ["spikeinterface>=0.101"] +all = ["orcapod[redis]", "orcapod[ray]", "orcapod[postgresql]", "orcapod[spiraldb]", "orcapod[spikeinterface]"] [tool.hatch.version] @@ -89,6 +90,7 @@ dev = [ "tqdm>=4.67.1", "mkdocs-material>=9.7.5", "mkdocstrings[python]>=1.0.3", + "spikeinterface>=0.101", ] diff --git a/src/orcapod/contexts/data/v0.1.json b/src/orcapod/contexts/data/v0.1.json index c8bab5d9..5c893d3a 100644 --- a/src/orcapod/contexts/data/v0.1.json +++ b/src/orcapod/contexts/data/v0.1.json @@ -45,6 +45,11 @@ "_class": "orcapod.extension_types.numpy_type.LogicalNumpyArray", "_config": {} }, + { + "_class": "orcapod.extension_types.spikeinterface_types.LogicalSIRecording", + "_config": {}, + "_optional": true + }, { "_class": "orcapod.extension_types.pandas_type.LogicalPandasDataFrame", "_config": {} @@ -102,6 +107,7 @@ [{"_type": "typing._SpecialForm"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.SpecialFormHandler", "_config": {}}], [{"_type": "pyarrow.Table"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.ArrowTableHandler", "_config": {}}], [{"_type": "pyarrow.RecordBatch"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.ArrowTableHandler", "_config": {}}], + [{"_type": "spikeinterface.core.BaseRecording", "_optional": true}, {"_class": "orcapod.extension_types.spikeinterface_types.SIRecordingHandler", "_config": {}, "_optional": true}], [{"_type": "numpy.ndarray"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.NumpyArrayHandler", "_config": {}}], [{"_type": "pandas.DataFrame"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.PandasDataFrameHandler", "_config": {}}], [{"_type": "pandas.Series"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.PandasSeriesHandler", "_config": {}}] @@ -138,6 +144,7 @@ "Migrated LogicalFile Arrow storage from plain path string to JSON {\"path\": \"...\"} for consistency with LogicalDirectory (ITL-451)", "Added orcapod.Directory content-identified type with recursive Merkle tree hashing, LogicalDirectory Arrow extension, ignore filter support, and DirectoryHandler (ITL-451)", "Added numpy.ndarray as a native value type via LogicalNumpyArray (large_binary/.npy) and NumpyArrayHandler; object-dtype arrays are rejected eagerly (ITL-460)", + "Added spikeinterface.BaseRecording as a native value type via LogicalSIRecording (large_string/JSON) and SIRecordingHandler (SHA-256 of JSON bytes); auto-registered when spikeinterface is installed via _optional entries in v0.1.json; added _optional flag support to parse_objectspec for optional-extras types (ITL-459)", "Added pandas.DataFrame and pandas.Series as native value types via LogicalPandasDataFrame and LogicalPandasSeries (Arrow IPC / large_binary), with index preservation and PandasDataFrameHandler / PandasSeriesHandler for content hashing using StarfixArrowHasher (PLT-1869)" ] } diff --git a/src/orcapod/extension_types/__init__.py b/src/orcapod/extension_types/__init__.py index 24a4079c..cbb7641e 100644 --- a/src/orcapod/extension_types/__init__.py +++ b/src/orcapod/extension_types/__init__.py @@ -30,6 +30,13 @@ from .numpy_type import LogicalNumpyArray # ITL-460 from .pandas_type import LogicalPandasDataFrame, LogicalPandasSeries # PLT-1869 +# ITL-459 — SpikeInterface support (optional; requires pip install orcapod[spikeinterface]) +try: + from .spikeinterface_types import LogicalSIRecording, register_spikeinterface_types + _SI_AVAILABLE = True +except ImportError: + _SI_AVAILABLE = False + __all__ = [ "LogicalTypeProtocol", "LogicalTypeFactoryProtocol", @@ -57,6 +64,8 @@ "LogicalDirectory", # ITL-460 "LogicalNumpyArray", + # ITL-459 (conditional — only present when spikeinterface is installed) + *( ["LogicalSIRecording", "register_spikeinterface_types"] if _SI_AVAILABLE else [] ), # PLT-1869 "LogicalPandasDataFrame", "LogicalPandasSeries", diff --git a/src/orcapod/extension_types/spikeinterface_types.py b/src/orcapod/extension_types/spikeinterface_types.py new file mode 100644 index 00000000..7362369a --- /dev/null +++ b/src/orcapod/extension_types/spikeinterface_types.py @@ -0,0 +1,268 @@ +"""SpikeInterface LogicalType and handler for orcapod (ITL-459). + +`LogicalSIRecording` maps `spikeinterface.core.BaseRecording` ↔ Arrow +`large_string` using SpikeInterface's own `to_dict(recursive=True, +include_annotations=True, include_properties=False)` JSON dump (encoded +via `SIJsonEncoder`) as the storage envelope. `SIRecordingHandler` hashes +the same JSON bytes via SHA-256 for content identity. + +This module requires the optional `spikeinterface` extras group: +`pip install orcapod[spikeinterface]` + +Register SI types into the default orcapod context before using them in +pods: call `register_spikeinterface_types()` once at startup. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +from typing import TYPE_CHECKING, Any + +import polars as pl +import pyarrow as pa + +from orcapod.extension_types.base_logical_type import BaseLogicalType +from orcapod.extension_types.registry import make_arrow_extension_type, make_polars_extension_type +from orcapod.types import ContentHash + +if TYPE_CHECKING: + from orcapod.extension_types.protocols import TypeConverterProtocol + from orcapod.protocols.hashing_protocols import SemanticHasherProtocol + +try: + from spikeinterface.core import BaseRecording +except ImportError as _exc: + raise ImportError( + "spikeinterface is not installed. " + "Install it with: pip install orcapod[spikeinterface]" + ) from _exc + +logger = logging.getLogger(__name__) + + +class LogicalSIRecording(BaseLogicalType): + """Logical type for `spikeinterface.core.BaseRecording`. + + Stores `BaseRecording` instances as Arrow `large_string` columns + tagged with extension name `"spikeinterface.recording"`. The stored + value is SpikeInterface's own `to_dict(recursive=True, + include_annotations=True, include_properties=False)` output, encoded + via `SIJsonEncoder`. Loading reconstructs the recording via + `spikeinterface.core.load(dict)`. + + Only recordings whose `check_serializability("json")` returns `True` + are accepted. Lazy recordings built on top of file-backed data (zarr, + binary folder, etc.) qualify. In-memory `NumpyRecording` objects do + not and raise `ValueError` with clear save instructions. + + Example: + >>> import tempfile, numpy as np + >>> import spikeinterface.core as si + >>> from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + >>> lt = LogicalSIRecording() + >>> with tempfile.TemporaryDirectory() as tmp: + ... rec = si.NumpyRecording([np.zeros((100, 4), dtype="float32")], 30000) + ... saved = rec.save_to_folder(tmp + "/rec") + ... storage = lt.python_to_storage(saved) + ... recovered = lt.storage_to_python(storage) + ... saved.get_traces(segment_index=0).shape == recovered.get_traces(segment_index=0).shape + True + """ + + _arrow_ext_class = make_arrow_extension_type("spikeinterface.recording", pa.large_string()) + _arrow_ext: pa.ExtensionType | None = None + _polars_ext_class = make_polars_extension_type("spikeinterface.recording", pa.large_string()) + _polars_ext: pl.BaseExtension | None = None + + logical_type_name: str = "spikeinterface.recording" + python_type: type = BaseRecording + + def get_arrow_extension_type(self) -> pa.ExtensionType: + """Return the cached Arrow extension type for `BaseRecording`. + + Returns: + A `pa.ExtensionType` with extension name + `"spikeinterface.recording"` and storage type `pa.large_string()`. + """ + if LogicalSIRecording._arrow_ext is None: + LogicalSIRecording._arrow_ext = LogicalSIRecording._arrow_ext_class() + return LogicalSIRecording._arrow_ext + + def get_polars_extension_type(self) -> pl.BaseExtension: + """Return the cached Polars extension type for `BaseRecording`. + + Returns: + A `pl.BaseExtension` registered under `"spikeinterface.recording"`. + """ + if LogicalSIRecording._polars_ext is None: + LogicalSIRecording._polars_ext = LogicalSIRecording._polars_ext_class() + return LogicalSIRecording._polars_ext + + def python_to_storage( + self, value: Any, converter: TypeConverterProtocol | None = None + ) -> str: + """Serialise a `BaseRecording` to its JSON storage representation. + + Args: + value: A `BaseRecording` instance whose + `check_serializability("json")` returns `True`. + converter: Ignored. Present for protocol conformance. + + Returns: + A JSON string produced by `recording.to_dict(recursive=True, + include_annotations=True, include_properties=False)` encoded + via `SIJsonEncoder`. + + Raises: + ValueError: If the recording is not JSON-serialisable (e.g. an + in-memory `NumpyRecording`). + """ + if not value.check_serializability("json"): + raise ValueError( + "This BaseRecording is not JSON-serializable and cannot be stored " + "by orcapod. This typically means it holds data in memory (e.g. " + "NumpyRecording). Lazy recordings built on top of file-backed data " + "(zarr, binary folder, etc.) are fine and do not need to be " + "materialized first. If your recording is in-memory, call " + "recording.save_to_zarr(path) or recording.save_to_folder(path) " + "first, then pass the returned extractor to the pod." + ) + from spikeinterface.core.core_tools import SIJsonEncoder + return json.dumps( + value.to_dict( + include_annotations=True, + include_properties=False, + recursive=True, + ), + cls=SIJsonEncoder, + ) + + def storage_to_python( + self, storage_value: Any, converter: TypeConverterProtocol | None = None + ) -> BaseRecording: + """Reconstruct a `BaseRecording` from its JSON storage string. + + Args: + storage_value: A JSON string as stored in Arrow. + converter: Ignored. Present for protocol conformance. + + Returns: + A `BaseRecording` instance reconstructed via + `spikeinterface.core.load`. + + Raises: + ValueError: If `storage_value` is not valid JSON. + FileNotFoundError: If the backing zarr/folder no longer exists + (raised by SpikeInterface, propagated as-is). + """ + from spikeinterface.core import load as si_load + try: + si_dict = json.loads(storage_value) + except (json.JSONDecodeError, TypeError) as exc: + raise ValueError( + f"LogicalSIRecording: cannot deserialise storage value " + f"{storage_value!r}; expected a JSON string." + ) from exc + return si_load(si_dict) + + +class SIRecordingHandler: + """Semantic hash handler for `spikeinterface.core.BaseRecording`. + + Computes a SHA-256 `ContentHash` of the JSON bytes produced by + `recording.to_dict(recursive=True, include_annotations=True, + include_properties=False)` encoded via `SIJsonEncoder`. This is + identical to the bytes that `LogicalSIRecording` stores in Arrow, so + hash input and storage representation are always consistent. + + The `hasher` argument is accepted for protocol conformance but not used — + hashing is done directly via `hashlib.sha256` to avoid overhead. + """ + + def handle(self, obj: Any, hasher: SemanticHasherProtocol | None) -> ContentHash: + """Return a SHA-256 `ContentHash` of the recording's JSON dump. + + Args: + obj: A `BaseRecording` instance. + hasher: Accepted for protocol conformance; not used. + + Returns: + A `ContentHash` with `method="sha256"` and digest equal to the + SHA-256 of the JSON bytes from `to_dict(recursive=True, + include_annotations=True, include_properties=False)` encoded + via `SIJsonEncoder`. + + Raises: + TypeError: If `obj` is not a `BaseRecording`. + ValueError: If the recording is not JSON-serialisable (in-memory). + """ + if not isinstance(obj, BaseRecording): + raise TypeError( + f"SIRecordingHandler: expected BaseRecording, got {type(obj)!r}" + ) + if not obj.check_serializability("json"): + raise ValueError( + "Cannot hash an in-memory BaseRecording " + "(check_serializability('json') is False). " + "Save it to disk first with save_to_zarr() or save_to_folder()." + ) + # TODO(ITL-467): phase 2 — also hash backing source directory contents + from spikeinterface.core.core_tools import SIJsonEncoder + json_bytes = json.dumps( + obj.to_dict(include_annotations=True, include_properties=False, recursive=True), + cls=SIJsonEncoder, + ).encode() + logger.debug("SIRecordingHandler: hashing %d JSON bytes", len(json_bytes)) + return ContentHash( + method="sha256", + digest=hashlib.sha256(json_bytes).digest(), + ) + + +def register_spikeinterface_types(context: Any = None) -> None: + """Register SpikeInterface LogicalTypes into an orcapod `DataContext`. + + For the default context this is called automatically at startup (the + default `v0.1.json` context config lists `LogicalSIRecording` and + `SIRecordingHandler` with `"_optional": true`, so they are wired in + whenever `spikeinterface` is installed). Call this function explicitly + only when you are working with a custom `DataContext` that was not + constructed from the default config. + + If `context` is `None`, the default context (from + `orcapod.contexts.get_default_context()`) is used. The function is + idempotent — calling it more than once on the same context is safe. + + Args: + context: A `DataContext` instance, or `None` to use the default. + + Example: + >>> from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types + >>> register_spikeinterface_types() # no-op if default context already has SI types + """ + if context is None: + from orcapod.contexts import get_default_context + context = get_default_context() + + lt = LogicalSIRecording() + try: + context.type_converter.register_logical_type(lt) + except ValueError as exc: + # A different LogicalSIRecording instance is already registered (e.g. + # auto-registered from v0.1.json at context creation time). That is + # fine — both instances are equivalent. Any other ValueError propagates. + if "already bound to" not in str(exc): + raise + logger.debug( + "register_spikeinterface_types: LogicalSIRecording already registered, skipping" + ) + else: + logger.debug( + "register_spikeinterface_types: registered LogicalSIRecording and SIRecordingHandler" + ) + + # SIRecordingHandler registration silently replaces an existing entry, so + # this call is always safe regardless of prior registration state. + context.semantic_hasher.type_handler_registry.register(BaseRecording, SIRecordingHandler()) diff --git a/src/orcapod/hashing/semantic_hashing/type_handler_registry.py b/src/orcapod/hashing/semantic_hashing/type_handler_registry.py index 6389b501..77b3d0d4 100644 --- a/src/orcapod/hashing/semantic_hashing/type_handler_registry.py +++ b/src/orcapod/hashing/semantic_hashing/type_handler_registry.py @@ -44,8 +44,15 @@ def __init__( self._handlers: dict[type, "PythonTypeHandlerProtocol"] = {} self._lock = threading.RLock() if handlers: - for target_type, handler in handlers: - self.register(target_type, handler) + for entry in handlers: + # Skip empty or incomplete pairs. When a handler pair in the context + # JSON carries "_optional": true on its elements and the backing module + # is absent, parse_objectspec filters those elements out of the inner + # list, leaving an empty list [] here. Also skip pairs where either + # element resolved to None. + if entry and len(entry) == 2 and all(x is not None for x in entry): + target_type, handler = entry + self.register(target_type, handler) def register(self, target_type: type, handler: "PythonTypeHandlerProtocol") -> None: """Register a hasher for a specific Python type. diff --git a/src/orcapod/utils/object_spec.py b/src/orcapod/utils/object_spec.py index 652170f5..e7e22d78 100644 --- a/src/orcapod/utils/object_spec.py +++ b/src/orcapod/utils/object_spec.py @@ -29,30 +29,59 @@ def parse_objectspec( } elif isinstance(obj_spec, (list, tuple)): - processed = [parse_objectspec(item, ref_lut, validate) for item in obj_spec] + processed = [] + for item in obj_spec: + result = parse_objectspec(item, ref_lut, validate) + # Omit None results produced by dict items with "_optional": true that + # failed to resolve due to a missing dependency. This allows entries + # like LogicalSIRecording (backed by an optional extras group) to be + # listed in a context JSON without breaking startup when the extras are + # absent — they are simply absent from the resulting list. + if result is None and isinstance(item, dict) and item.get("_optional", False): + continue + processed.append(result) return tuple(processed) if isinstance(obj_spec, tuple) else processed else: return obj_spec -def _resolve_type_from_spec(spec: dict[str, Any]) -> type: +def _resolve_type_from_spec(spec: dict[str, Any]) -> type | None: """Resolve a ``{"_type": "module.ClassName"}`` spec to the actual Python type. Bare names without a dot (e.g. ``"bytes"``) are resolved from ``builtins``. + + When ``"_optional": true`` is present in the spec and the module cannot be + imported (``ImportError``), returns ``None`` instead of raising. This + allows optional-dependency types to be silently skipped at context + construction time. """ + optional: bool = spec.get("_optional", False) type_str: str = spec["_type"] if "." not in type_str: type_str = f"builtins.{type_str}" module_name, _, attr_name = type_str.rpartition(".") - module = importlib.import_module(module_name) - return getattr(module, attr_name) + try: + module = importlib.import_module(module_name) + return getattr(module, attr_name) + except ImportError: + if optional: + return None + raise def _create_instance_from_spec( spec: dict[str, Any], ref_lut: dict[str, Any], validate: bool ) -> Any: - """Create instance with better error handling.""" + """Create instance with better error handling. + + When ``"_optional": true`` is present in the spec and the class module + cannot be imported (``ImportError``), returns ``None`` instead of raising. + This allows optional-dependency types (e.g. those backed by an extras + group such as ``orcapod[spikeinterface]``) to be listed in context JSON + without breaking startup for users who have not installed the extras group. + """ + optional: bool = spec.get("_optional", False) try: class_path = spec["_class"] config = spec.get("_config", {}) @@ -71,6 +100,10 @@ def _create_instance_from_spec( return cls(**processed_config) + except ImportError: + if optional: + return None + raise except Exception as e: raise ValueError(f"Failed to create instance from spec {spec}: {e}") from e diff --git a/superpowers/plans/2026-07-01-itl-459-spikeinterface-baserecording.md b/superpowers/plans/2026-07-01-itl-459-spikeinterface-baserecording.md new file mode 100644 index 00000000..8faa0f99 --- /dev/null +++ b/superpowers/plans/2026-07-01-itl-459-spikeinterface-baserecording.md @@ -0,0 +1,764 @@ +# SpikeInterface BaseRecording LogicalType — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add `LogicalSIRecording` so that pods can accept and return SpikeInterface `BaseRecording` objects natively — stored as SI's JSON dump in Arrow, content-hashed by SHA-256 of that JSON. + +**Architecture:** `spikeinterface_types.py` imports `BaseRecording` at module level (raises `ImportError` if SI absent, caught gracefully by `extension_types/__init__.py`). `LogicalSIRecording` stores `to_dict(recursive=True)` as JSON `large_string`. `SIRecordingHandler` hashes those same JSON bytes via SHA-256. Registration into the default context is explicit via `register_spikeinterface_types()`. No changes to `v0.1.json` or `builtin_handlers.py`. + +**Tech Stack:** SpikeInterface ≥0.101, PyArrow, Polars, Python 3.12, pytest, uv + +--- + +## File Map + +| Action | Path | Responsibility | +|--------|------|----------------| +| Create | `src/orcapod/extension_types/spikeinterface_types.py` | `LogicalSIRecording`, `SIRecordingHandler`, `register_spikeinterface_types()` | +| Modify | `src/orcapod/extension_types/__init__.py` | Conditionally export `LogicalSIRecording`, `register_spikeinterface_types` | +| Modify | `pyproject.toml` | Add `spikeinterface` optional extras group | +| Create | `tests/test_extension_types/test_spikeinterface_types.py` | All SI type tests | + +`builtin_handlers.py` and `contexts/data/v0.1.json` are **not touched**. + +--- + +## Task 1: Add optional extras group to `pyproject.toml` + +**Files:** +- Modify: `pyproject.toml` + +- [ ] **Step 1: Add the extras group** + +In `pyproject.toml`, find the `[project.optional-dependencies]` section and make these two changes: + +```toml +spikeinterface = ["spikeinterface>=0.101"] +all = ["orcapod[redis]", "orcapod[ray]", "orcapod[postgresql]", "orcapod[spiraldb]", "orcapod[spikeinterface]"] +``` + +The `all` line replaces the existing `all` line (add `"orcapod[spikeinterface]"` to the end of the existing list). + +- [ ] **Step 2: Verify the extras install correctly** + +```bash +uv run --with spikeinterface python -c "import spikeinterface; print(spikeinterface.__version__)" +``` + +Expected output: a version string like `0.104.8` (no errors). + +- [ ] **Step 3: Commit** + +```bash +git add pyproject.toml +git commit -m "chore(deps): add spikeinterface optional extras group (ITL-459)" +``` + +--- + +## Task 2: Create `spikeinterface_types.py` with `LogicalSIRecording` + +**Files:** +- Create: `src/orcapod/extension_types/spikeinterface_types.py` +- Create: `tests/test_extension_types/test_spikeinterface_types.py` + +- [ ] **Step 1: Write the failing import test** + +```python +# tests/test_extension_types/test_spikeinterface_types.py +"""Tests for LogicalSIRecording and SIRecordingHandler (ITL-459).""" + +from __future__ import annotations + +import json + +import numpy as np +import pytest + + +def test_spikeinterface_not_installed_raises_import_error(monkeypatch): + """Importing spikeinterface_types when SI is absent raises ImportError.""" + import sys + import importlib + + # Remove any cached spikeinterface modules + si_keys = [k for k in sys.modules if k == "spikeinterface" or k.startswith("spikeinterface.")] + saved = {k: sys.modules.pop(k) for k in si_keys} + # Block re-import + monkeypatch.setitem(sys.modules, "spikeinterface", None) + monkeypatch.setitem(sys.modules, "spikeinterface.core", None) + + try: + # Remove the cached spikeinterface_types module so it re-imports + si_types_key = "orcapod.extension_types.spikeinterface_types" + saved_si_types = sys.modules.pop(si_types_key, None) + with pytest.raises(ImportError, match="pip install orcapod\\[spikeinterface\\]"): + importlib.import_module(si_types_key) + finally: + # Restore everything + for k in list(sys.modules): + if k == "spikeinterface" or k.startswith("spikeinterface."): + del sys.modules[k] + sys.modules.update(saved) + if saved_si_types is not None: + sys.modules[si_types_key] = saved_si_types + else: + sys.modules.pop(si_types_key, None) + importlib.import_module(si_types_key) + + +# All tests below require spikeinterface — skip if not installed +si = pytest.importorskip("spikeinterface", reason="spikeinterface not installed") +import spikeinterface.core as si_core # noqa: E402 + + +def _make_numpy_recording() -> si_core.NumpyRecording: + """Create a small in-memory NumpyRecording for use as a test source.""" + rng = np.random.default_rng(42) + traces = rng.standard_normal((200, 4)).astype("float32") + return si_core.NumpyRecording([traces], sampling_frequency=30_000) + + +def test_logical_si_recording_importable(): + from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + import pyarrow as pa + + lt = LogicalSIRecording() + assert lt.logical_type_name == "spikeinterface.recording" + assert lt.python_type is si_core.BaseRecording + assert lt.get_arrow_extension_type().storage_type == pa.large_string() +``` + +- [ ] **Step 2: Run to confirm failure** + +```bash +uv run --with spikeinterface pytest tests/test_extension_types/test_spikeinterface_types.py::test_logical_si_recording_importable -v +``` + +Expected: `FAILED` — `ModuleNotFoundError: No module named 'orcapod.extension_types.spikeinterface_types'` + +- [ ] **Step 3: Create `spikeinterface_types.py`** + +```python +# src/orcapod/extension_types/spikeinterface_types.py +"""SpikeInterface LogicalType and handler for orcapod (ITL-459). + +``LogicalSIRecording`` maps ``spikeinterface.core.BaseRecording`` ↔ Arrow +``large_string`` using SpikeInterface's own ``to_dict(recursive=True)`` JSON +dump as the storage envelope. ``SIRecordingHandler`` hashes the same JSON +bytes via SHA-256 for content identity. + +This module requires the optional ``spikeinterface`` extras group:: + + pip install orcapod[spikeinterface] + +Register SI types into the default orcapod context before using them in +pods:: + + from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types + register_spikeinterface_types() +""" + +from __future__ import annotations + +import hashlib +import json +import logging +from typing import TYPE_CHECKING, Any + +import polars as pl +import pyarrow as pa + +from orcapod.extension_types.base_logical_type import BaseLogicalType +from orcapod.extension_types.registry import make_arrow_extension_type, make_polars_extension_type +from orcapod.types import ContentHash + +if TYPE_CHECKING: + from orcapod.extension_types.protocols import TypeConverterProtocol + from orcapod.protocols.hashing_protocols import SemanticHasherProtocol + +try: + from spikeinterface.core import BaseRecording +except ImportError as _exc: + raise ImportError( + "spikeinterface is not installed. " + "Install it with: pip install orcapod[spikeinterface]" + ) from _exc + +logger = logging.getLogger(__name__) + + +class LogicalSIRecording(BaseLogicalType): + """Logical type for ``spikeinterface.core.BaseRecording``. + + Stores ``BaseRecording`` instances as Arrow ``large_string`` columns + tagged with extension name ``"spikeinterface.recording"``. The stored + value is SpikeInterface's own ``to_dict(recursive=True)`` output, + JSON-serialised. Loading reconstructs the recording via + ``spikeinterface.core.load_extractor(dict)``. + + Only recordings whose ``check_serializability("json")`` returns ``True`` + are accepted. Lazy recordings built on top of file-backed data (zarr, + binary folder, etc.) qualify. In-memory ``NumpyRecording`` objects do + not and raise ``ValueError`` with clear save instructions. + + Example: + >>> import tempfile, numpy as np + >>> import spikeinterface.core as si + >>> from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + >>> lt = LogicalSIRecording() + >>> with tempfile.TemporaryDirectory() as tmp: + ... rec = si.NumpyRecording([np.zeros((100, 4), dtype="float32")], 30000) + ... saved = rec.save_to_folder(tmp + "/rec") + ... storage = lt.python_to_storage(saved) + ... recovered = lt.storage_to_python(storage) + ... saved.get_traces(segment_index=0).shape == recovered.get_traces(segment_index=0).shape + True + """ + + _arrow_ext_class = make_arrow_extension_type("spikeinterface.recording", pa.large_string()) + _arrow_ext: pa.ExtensionType | None = None + _polars_ext_class = make_polars_extension_type("spikeinterface.recording", pa.large_string()) + _polars_ext: pl.BaseExtension | None = None + + logical_type_name: str = "spikeinterface.recording" + python_type: type = BaseRecording + + def get_arrow_extension_type(self) -> pa.ExtensionType: + """Return the cached Arrow extension type for ``BaseRecording``. + + Returns: + A ``pa.ExtensionType`` with extension name + ``"spikeinterface.recording"`` and storage type ``pa.large_string()``. + """ + if LogicalSIRecording._arrow_ext is None: + LogicalSIRecording._arrow_ext = LogicalSIRecording._arrow_ext_class() + return LogicalSIRecording._arrow_ext + + def get_polars_extension_type(self) -> pl.BaseExtension: + """Return the cached Polars extension type for ``BaseRecording``. + + Returns: + A ``pl.BaseExtension`` registered under ``"spikeinterface.recording"``. + """ + if LogicalSIRecording._polars_ext is None: + LogicalSIRecording._polars_ext = LogicalSIRecording._polars_ext_class() + return LogicalSIRecording._polars_ext + + def python_to_storage( + self, value: Any, converter: TypeConverterProtocol | None = None + ) -> str: + """Serialise a ``BaseRecording`` to its JSON storage representation. + + Args: + value: A ``BaseRecording`` instance whose + ``check_serializability("json")`` returns ``True``. + converter: Ignored. Present for protocol conformance. + + Returns: + A JSON string produced by ``recording.to_dict(recursive=True)``. + + Raises: + ValueError: If the recording is not JSON-serialisable (e.g. an + in-memory ``NumpyRecording``). + """ + if not value.check_serializability("json"): + raise ValueError( + "This BaseRecording is not JSON-serializable and cannot be stored " + "by orcapod. This typically means it holds data in memory (e.g. " + "NumpyRecording). Lazy recordings built on top of file-backed data " + "(zarr, binary folder, etc.) are fine and do not need to be " + "materialized first. If your recording is in-memory, call " + "recording.save_to_zarr(path) or recording.save_to_folder(path) " + "first, then pass the returned extractor to the pod." + ) + return json.dumps(value.to_dict(recursive=True)) + + def storage_to_python( + self, storage_value: Any, converter: TypeConverterProtocol | None = None + ) -> BaseRecording: + """Reconstruct a ``BaseRecording`` from its JSON storage string. + + Args: + storage_value: A JSON string as stored in Arrow. + converter: Ignored. Present for protocol conformance. + + Returns: + A ``BaseRecording`` instance reconstructed via + ``spikeinterface.core.load_extractor``. + + Raises: + ValueError: If ``storage_value`` is not valid JSON. + FileNotFoundError: If the backing zarr/folder no longer exists + (raised by SpikeInterface, propagated as-is). + """ + from spikeinterface.core import load_extractor + try: + si_dict = json.loads(storage_value) + except (json.JSONDecodeError, TypeError) as exc: + raise ValueError( + f"LogicalSIRecording: cannot deserialise storage value " + f"{storage_value!r}; expected a JSON string." + ) from exc + return load_extractor(si_dict) + + +class SIRecordingHandler: + """Semantic hash handler for ``spikeinterface.core.BaseRecording``. + + Computes a SHA-256 ``ContentHash`` of the JSON bytes produced by + ``recording.to_dict(recursive=True)``. This is identical to the bytes + that ``LogicalSIRecording`` stores in Arrow, so hash input and storage + representation are always consistent. + + The ``hasher`` argument is accepted for protocol conformance but not used — + hashing is done directly via ``hashlib.sha256`` to avoid overhead. + + Phase 2 (deferred — ITL-467): hash will additionally cover backing source + directory contents once efficient directory hashing infrastructure lands. + """ + + def handle(self, obj: Any, hasher: SemanticHasherProtocol | None) -> ContentHash: + """Return a SHA-256 ``ContentHash`` of the recording's JSON dump. + + Args: + obj: A ``BaseRecording`` instance. + hasher: Accepted for protocol conformance; not used. + + Returns: + A ``ContentHash`` with ``method="sha256"`` and digest equal to the + SHA-256 of ``json.dumps(recording.to_dict(recursive=True)).encode()``. + + Raises: + TypeError: If ``obj`` is not a ``BaseRecording``. + ValueError: If the recording is not JSON-serialisable (in-memory). + """ + if not isinstance(obj, BaseRecording): + raise TypeError( + f"SIRecordingHandler: expected BaseRecording, got {type(obj)!r}" + ) + if not obj.check_serializability("json"): + raise ValueError( + "Cannot hash an in-memory BaseRecording " + "(check_serializability('json') is False). " + "Save it to disk first with save_to_zarr() or save_to_folder()." + ) + json_bytes = json.dumps(obj.to_dict(recursive=True)).encode() + logger.debug("SIRecordingHandler: hashing %d JSON bytes", len(json_bytes)) + return ContentHash( + method="sha256", + digest=hashlib.sha256(json_bytes).digest(), + ) + + +def register_spikeinterface_types(context: Any = None) -> None: + """Register SpikeInterface LogicalTypes into an orcapod ``DataContext``. + + Call this once at startup, before any pods that use SpikeInterface types + are declared or executed. If ``context`` is ``None``, the default context + (from ``orcapod.contexts.get_default_context()``) is used. + + Args: + context: A ``DataContext`` instance, or ``None`` to use the default. + + Example: + >>> from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types + >>> register_spikeinterface_types() # registers into the default context + """ + if context is None: + from orcapod.contexts import get_default_context + context = get_default_context() + + lt = LogicalSIRecording() + context.type_converter._logical_type_registry.register_logical_type(lt) + context.semantic_hasher.type_handler_registry.register(BaseRecording, SIRecordingHandler()) + logger.debug( + "register_spikeinterface_types: registered LogicalSIRecording and SIRecordingHandler" + ) +``` + +- [ ] **Step 4: Run both import tests** + +```bash +uv run --with spikeinterface pytest tests/test_extension_types/test_spikeinterface_types.py::test_logical_si_recording_importable tests/test_extension_types/test_spikeinterface_types.py::test_spikeinterface_not_installed_raises_import_error -v +``` + +Expected: both `PASSED` + +- [ ] **Step 5: Commit** + +```bash +git add src/orcapod/extension_types/spikeinterface_types.py tests/test_extension_types/test_spikeinterface_types.py +git commit -m "feat(extension_types): add LogicalSIRecording, SIRecordingHandler, register_spikeinterface_types (ITL-459)" +``` + +--- + +## Task 3: `python_to_storage` — in-memory recording raises `ValueError` + +**Files:** +- Modify: `tests/test_extension_types/test_spikeinterface_types.py` + +- [ ] **Step 1: Add the test** + +Add to `tests/test_extension_types/test_spikeinterface_types.py` (after the `importorskip` line): + +```python +def test_in_memory_recording_raises(): + """NumpyRecording (json=False) raises ValueError with clear instructions.""" + from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + + rec = _make_numpy_recording() + lt = LogicalSIRecording() + + with pytest.raises(ValueError, match="not JSON-serializable"): + lt.python_to_storage(rec) + + # The error message must distinguish in-memory from lazy file-backed recordings + with pytest.raises(ValueError, match="file-backed"): + lt.python_to_storage(rec) +``` + +- [ ] **Step 2: Run to confirm it passes (implementation already present in the skeleton)** + +```bash +uv run --with spikeinterface pytest tests/test_extension_types/test_spikeinterface_types.py::test_in_memory_recording_raises -v +``` + +Expected: `PASSED` + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_extension_types/test_spikeinterface_types.py +git commit -m "test(spikeinterface_types): in-memory NumpyRecording raises ValueError (ITL-459)" +``` + +--- + +## Task 4: Round-trip tests — folder-backed, zarr-backed, ephemeral + +**Files:** +- Modify: `tests/test_extension_types/test_spikeinterface_types.py` + +- [ ] **Step 1: Add the three round-trip tests** + +Add to `tests/test_extension_types/test_spikeinterface_types.py`: + +```python +def test_folder_recording_round_trip(tmp_path): + """Binary-folder-backed recording round-trips through python_to_storage / storage_to_python.""" + from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + + saved = _make_numpy_recording().save_to_folder(str(tmp_path / "rec")) + lt = LogicalSIRecording() + + storage = lt.python_to_storage(saved) + assert isinstance(storage, str) + data = json.loads(storage) + assert "class" in data # SI dict always has a "class" key + + recovered = lt.storage_to_python(storage) + np.testing.assert_array_equal( + saved.get_traces(segment_index=0), + recovered.get_traces(segment_index=0), + ) + + +def test_zarr_recording_round_trip(tmp_path): + """Zarr-backed recording round-trips through python_to_storage / storage_to_python.""" + from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + + saved = _make_numpy_recording().save_to_zarr(str(tmp_path / "rec.zarr")) + lt = LogicalSIRecording() + + storage = lt.python_to_storage(saved) + assert isinstance(storage, str) + + recovered = lt.storage_to_python(storage) + np.testing.assert_array_equal( + saved.get_traces(segment_index=0), + recovered.get_traces(segment_index=0), + ) + + +def test_ephemeral_recording_round_trip(tmp_path): + """A lazy preprocessing chain on a file-backed recording round-trips correctly. + + The preprocessed recording is NOT materialized to zarr/folder but IS + JSON-serializable because its root is file-backed. On reload, + SpikeInterface re-applies the preprocessing chain lazily. + """ + import spikeinterface.preprocessing as spre + from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + + # Save the source to disk; apply a lazy preprocessing step on top + source = _make_numpy_recording().save_to_folder(str(tmp_path / "source")) + preprocessed = spre.bandpass_filter(source, freq_min=300.0, freq_max=6000.0) + + assert preprocessed.check_serializability("json"), ( + "Preprocessed recording on folder-backed source must be JSON-serializable" + ) + + lt = LogicalSIRecording() + storage = lt.python_to_storage(preprocessed) + recovered = lt.storage_to_python(storage) + + np.testing.assert_array_equal( + preprocessed.get_traces(segment_index=0), + recovered.get_traces(segment_index=0), + ) +``` + +- [ ] **Step 2: Run all three round-trip tests** + +```bash +uv run --with spikeinterface pytest \ + tests/test_extension_types/test_spikeinterface_types.py::test_folder_recording_round_trip \ + tests/test_extension_types/test_spikeinterface_types.py::test_zarr_recording_round_trip \ + tests/test_extension_types/test_spikeinterface_types.py::test_ephemeral_recording_round_trip \ + -v +``` + +Expected: all three `PASSED` + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_extension_types/test_spikeinterface_types.py +git commit -m "test(spikeinterface_types): round-trip tests for folder, zarr, ephemeral recordings (ITL-459)" +``` + +--- + +## Task 5: Hash stability and content-change tests + +**Files:** +- Modify: `tests/test_extension_types/test_spikeinterface_types.py` + +- [ ] **Step 1: Add hash tests** + +Add to `tests/test_extension_types/test_spikeinterface_types.py`: + +```python +def test_si_recording_handler_hash_stability(tmp_path): + """Same recording produces identical ContentHash across two calls.""" + from orcapod.extension_types.spikeinterface_types import SIRecordingHandler + from orcapod.types import ContentHash + + saved = _make_numpy_recording().save_to_folder(str(tmp_path / "rec")) + handler = SIRecordingHandler() + + h1 = handler.handle(saved, hasher=None) + h2 = handler.handle(saved, hasher=None) + + assert isinstance(h1, ContentHash) + assert h1 == h2 + + +def test_si_recording_handler_hash_changes_with_content(tmp_path): + """Different recordings produce different ContentHash values.""" + from orcapod.extension_types.spikeinterface_types import SIRecordingHandler + + rng = np.random.default_rng(0) + rec_a = si_core.NumpyRecording( + [rng.standard_normal((200, 4)).astype("float32")], sampling_frequency=30_000 + ).save_to_folder(str(tmp_path / "rec_a")) + rec_b = si_core.NumpyRecording( + [rng.standard_normal((200, 4)).astype("float32")], sampling_frequency=30_000 + ).save_to_folder(str(tmp_path / "rec_b")) + + handler = SIRecordingHandler() + assert handler.handle(rec_a, hasher=None) != handler.handle(rec_b, hasher=None) + + +def test_si_recording_handler_in_memory_raises(): + """SIRecordingHandler raises ValueError for in-memory recordings.""" + from orcapod.extension_types.spikeinterface_types import SIRecordingHandler + + rec = _make_numpy_recording() + handler = SIRecordingHandler() + with pytest.raises(ValueError, match="in-memory"): + handler.handle(rec, hasher=None) +``` + +- [ ] **Step 2: Run hash tests** + +```bash +uv run --with spikeinterface pytest \ + tests/test_extension_types/test_spikeinterface_types.py::test_si_recording_handler_hash_stability \ + tests/test_extension_types/test_spikeinterface_types.py::test_si_recording_handler_hash_changes_with_content \ + tests/test_extension_types/test_spikeinterface_types.py::test_si_recording_handler_in_memory_raises \ + -v +``` + +Expected: all three `PASSED` + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_extension_types/test_spikeinterface_types.py +git commit -m "test(spikeinterface_types): hash stability and content-change tests (ITL-459)" +``` + +--- + +## Task 6: Integration test for `register_spikeinterface_types()` + +**Files:** +- Modify: `tests/test_extension_types/test_spikeinterface_types.py` + +- [ ] **Step 1: Add the integration test** + +Add to `tests/test_extension_types/test_spikeinterface_types.py`: + +```python +def test_register_spikeinterface_types(tmp_path): + """register_spikeinterface_types() wires LogicalSIRecording and SIRecordingHandler + into the default context so they are found by type lookup.""" + from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types + from orcapod.contexts import get_default_context + + register_spikeinterface_types() + ctx = get_default_context() + + saved = _make_numpy_recording().save_to_folder(str(tmp_path / "rec")) + + # LogicalType registered: type_converter can find it + arrow_type = ctx.type_converter.python_type_to_arrow_type(type(saved)) + assert arrow_type.extension_name == "spikeinterface.recording" + + # Handler registered: semantic_hasher can find it + from orcapod.types import ContentHash + handler = ctx.semantic_hasher.type_handler_registry.get_handler(saved) + assert handler is not None + result = handler.handle(saved, hasher=None) + assert isinstance(result, ContentHash) + + # Full round-trip through python_to_storage / storage_to_python + storage = ctx.type_converter.python_to_storage(saved, type(saved)) + recovered = ctx.type_converter.storage_to_python(storage, type(saved)) + np.testing.assert_array_equal( + saved.get_traces(segment_index=0), + recovered.get_traces(segment_index=0), + ) +``` + +- [ ] **Step 2: Run the integration test** + +```bash +uv run --with spikeinterface pytest tests/test_extension_types/test_spikeinterface_types.py::test_register_spikeinterface_types -v +``` + +Expected: `PASSED` + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_extension_types/test_spikeinterface_types.py +git commit -m "test(spikeinterface_types): integration test for register_spikeinterface_types (ITL-459)" +``` + +--- + +## Task 7: Export from `extension_types/__init__.py` + +**Files:** +- Modify: `src/orcapod/extension_types/__init__.py` + +- [ ] **Step 1: Add conditional import and export** + +In `src/orcapod/extension_types/__init__.py`, add after the `from .numpy_type import LogicalNumpyArray # ITL-460` line: + +```python +# ITL-459 — SpikeInterface support (optional; requires pip install orcapod[spikeinterface]) +try: + from .spikeinterface_types import LogicalSIRecording, register_spikeinterface_types + _SI_AVAILABLE = True +except ImportError: + _SI_AVAILABLE = False +``` + +And at the end of `__all__`, add: + +```python + # ITL-459 (conditional — only present when spikeinterface is installed) + *( ["LogicalSIRecording", "register_spikeinterface_types"] if _SI_AVAILABLE else [] ), +``` + +- [ ] **Step 2: Verify import works with SI installed** + +```bash +uv run --with spikeinterface python -c " +from orcapod.extension_types import LogicalSIRecording, register_spikeinterface_types, __all__ +assert 'LogicalSIRecording' in __all__ +print('OK with SI:', LogicalSIRecording, register_spikeinterface_types) +" +``` + +Expected: prints the class and function without error. + +- [ ] **Step 3: Verify import works without SI installed** + +```bash +uv run python -c " +from orcapod.extension_types import __all__ +assert 'LogicalSIRecording' not in __all__, f'should not be exported, got: {__all__}' +print('OK without SI: LogicalSIRecording not in __all__') +" +``` + +Expected: prints the OK message without error. + +- [ ] **Step 4: Commit** + +```bash +git add src/orcapod/extension_types/__init__.py +git commit -m "feat(extension_types): conditionally export LogicalSIRecording when SI available (ITL-459)" +``` + +--- + +## Task 8: Full suite check and changelog + +**Files:** +- Modify: `src/orcapod/contexts/data/v0.1.json` + +- [ ] **Step 1: Run all spikeinterface tests** + +```bash +uv run --with spikeinterface pytest tests/test_extension_types/test_spikeinterface_types.py -v +``` + +Expected: all 10 tests `PASSED` (1 not-installed test + 9 SI tests). + +- [ ] **Step 2: Run the full extension_types suite without SI to verify no regressions** + +```bash +uv run pytest tests/test_extension_types/ -v +``` + +Expected: all pre-existing tests pass; SI tests that need SI are `SKIPPED` (not `FAILED`). The `test_spikeinterface_not_installed_raises_import_error` test runs and passes even without SI. + +- [ ] **Step 3: Run the full test suite** + +```bash +uv run pytest tests/ -v --timeout=60 +``` + +Expected: all pre-existing tests pass; new SI tests either `PASSED` (with SI) or `SKIPPED` (without SI). + +- [ ] **Step 4: Update `v0.1.json` changelog** + +In `src/orcapod/contexts/data/v0.1.json`, append to the `"changelog"` array: + +```json +"Added spikeinterface.BaseRecording as a native value type via LogicalSIRecording (large_string/JSON) and SIRecordingHandler (SHA-256 of JSON bytes); requires pip install orcapod[spikeinterface] and a register_spikeinterface_types() call (ITL-459)" +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/orcapod/contexts/data/v0.1.json +git commit -m "docs(context): record LogicalSIRecording in v0.1.json changelog (ITL-459)" +``` diff --git a/superpowers/specs/2026-07-01-spikeinterface-baserecording-design.md b/superpowers/specs/2026-07-01-spikeinterface-baserecording-design.md new file mode 100644 index 00000000..87aa9100 --- /dev/null +++ b/superpowers/specs/2026-07-01-spikeinterface-baserecording-design.md @@ -0,0 +1,190 @@ +# SpikeInterface BaseRecording LogicalType — Design Spec + +**Issue:** ITL-459 +**Date:** 2026-07-01 +**Status:** Approved + +--- + +## Overview + +SpikeInterface `BaseRecording` objects are not orcapod-serializable today — they hit +`ValueError: Unsupported Python type` in `universal_converter.py`. This forces every pod in +the `orcapod-spikesorting` pipeline to manually persist recordings to disk and return +`op.Directory` paths, with bespoke save/reload code in each pod. + +This spec adds a `LogicalSIRecording` registered `LogicalType` that makes `BaseRecording` a +first-class orcapod type. The key insight: recordings that are backed by on-disk files (zarr, binary folder, npz folder, +etc.) are fully captured by SpikeInterface's own `to_dict()` JSON dump — the dump contains the +class name, the path to the backing artifact, and all constructor kwargs needed to reconstruct +it. orcapod stores that JSON string and delegates all disk I/O to SI. + +--- + +## Goals & Success Criteria + +- A `LogicalSIRecording` registered in the type registry so pods can annotate parameters and + return values with `BaseRecording` without errors. +- `python_to_storage()` serializes via `recording.to_dict(recursive=True)` → JSON string. + Raises a clear `ValueError` for in-memory recordings (`check_serializability("json") == False`). +- `storage_to_python()` reconstructs via `spikeinterface.core.load(dict)`. +- A `SIRecordingHandler` registered in the semantic hasher that hashes the JSON string + (phase 1 — source-path content hashing is deferred to a follow-up issue). +- `spikeinterface` available as `pip install orcapod[spikeinterface]`; imports are lazy so + users without the extras group pay no import cost. +- Tests: round-trip for zarr-backed and folder-backed recordings; clear error for + in-memory `NumpyRecording`. + +--- + +## Architecture + +### Package location + +New file: `src/orcapod/extension_types/spikeinterface_types.py` + +`spikeinterface_types.py` imports `BaseRecording` at module level using a `try/except ImportError` +block that re-raises with a clear pip-install message if SI is absent. The +`extension_types/__init__.py` wraps the import in its own `try/except ImportError` so that +`import orcapod` never fails when SI is not installed — the SI types are silently omitted from +`__all__` in that case. + +### Optional dependency + +`pyproject.toml` gains a new extras group: + +```toml +[project.optional-dependencies] +spikeinterface = ["spikeinterface>=0.101"] +``` + +### Registration + +`LogicalSIRecording` and `SIRecordingHandler` are registered in `contexts/data/v0.1.json` with +`"_optional": true`. The `parse_objectspec` loader catches `ImportError` on optional entries and +returns `None`; both `LogicalTypeRegistry` and `PythonTypeHandlerRegistry` silently skip `None` +entries at construction time. This means: + +- **SI installed** — types are auto-registered when the default context is first constructed. + No explicit call is required. +- **SI absent** — entries resolve to `None` and are skipped. Context construction succeeds and + all non-SI types continue to work normally. + +`register_spikeinterface_types()` is retained for custom `DataContext` instances that are not +constructed from `v0.1.json`. It is idempotent — calling it on a context that already has SI +types registered (e.g. the default context) is safe and logs a debug message. + +--- + +## Components + +### `LogicalSIRecording` + +Follows the same `BaseLogicalType` pattern as `LogicalDirectory` and `LogicalNumpyArray`. + +**Arrow storage type:** `pa.large_string()` — stores a JSON string, consistent with +`LogicalFile` and `LogicalDirectory`. + +**`logical_type_name`:** `"spikeinterface.recording"` + +**`python_type`:** `BaseRecording` — the registry's MRO lookup automatically handles all +subclasses (`ZarrRecordingExtractor`, `BinaryFolderRecording`, `NumpyFolderRecording`, etc.) +without requiring a factory. This is the same binding strategy used by `LogicalNumpyArray` +for `np.ndarray`. + +**`python_to_storage(value, converter)`:** + +1. Calls `value.check_serializability("json")`. If `False`, raises: + ``` + ValueError: This BaseRecording is not JSON-serializable and cannot be stored by orcapod. + This typically means it holds data in memory (e.g. NumpyRecording). Lazy recordings + built on top of file-backed data are fine and do not need to be materialized first. + If your recording is in-memory, call recording.save_to_zarr(path) or + recording.save_to_folder(path) first, then pass the returned extractor to the pod. + ``` +2. Calls `value.to_dict(recursive=True)` to obtain the SI dict representation. + `recursive=True` ensures nested extractors (e.g. a recording composed from multiple + sub-recordings) are also serialized by value, not by reference. +3. Returns `json.dumps(si_dict)`. + +**`storage_to_python(storage_value, converter)`:** + +1. Parses `json.loads(storage_value)` to recover the SI dict. +2. Calls `spikeinterface.core.load(si_dict)` and returns the result. +3. If the backing zarr/folder has been moved or deleted, SI raises naturally — no + additional wrapping needed (same behaviour as `LogicalDirectory` with a deleted path). + +### `SIRecordingHandler` + +Registered in the semantic hasher alongside `FileHandler` and `DirectoryHandler`. + +**Phase 1 (this issue):** Hash the JSON string produced by `to_dict(recursive=True)`. + +```python +class SIRecordingHandler: + def handle(self, obj: BaseRecording, hasher) -> ContentHash: + if not obj.check_serializability("json"): + raise ValueError( + "Cannot hash an in-memory BaseRecording. Save it to disk first." + ) + json_bytes = json.dumps(obj.to_dict(recursive=True)).encode() + # Exact hashing API (hash_bytes or equivalent) to be confirmed + # against the SemanticHasher interface during implementation. + return hasher.hash_bytes(json_bytes) +``` + +**Phase 2 (deferred — separate issue):** Hash = hash(JSON string) + hash(backing source +directory contents). This requires the database-level artifact storage / efficient directory +hashing infrastructure to land first (ITL-467). + +--- + +## Error Handling + +| Situation | Behaviour | +|-----------|-----------| +| `NumpyRecording` or any recording with `check_serializability("json") == False` | `ValueError` clarifying that lazy file-backed recordings are fine but in-memory ones are not, with save instructions | +| `spikeinterface` not installed | `ImportError` on first use of `LogicalSIRecording`, message: `"Install spikeinterface: pip install orcapod[spikeinterface]"` | +| Backing zarr/folder deleted after storage | SI raises `FileNotFoundError` or similar on `load()` — propagates as-is | +| Corrupt JSON in storage | `json.JSONDecodeError` raised with the raw value in the message | + +--- + +## Tests + +File: `tests/test_extension_types/test_spikeinterface_types.py` + +All tests that actually load/create SI recordings use `pytest.importorskip("spikeinterface")` +so they are skipped automatically when SI is not installed. + +| Test | What it checks | +|------|----------------| +| `test_zarr_recording_round_trip` | zarr-backed recording → `python_to_storage` → `storage_to_python` → same data | +| `test_folder_recording_round_trip` | binary-folder-backed recording → same | +| `test_ephemeral_recording_round_trip` | lazy preprocessing chain on top of file-backed data (not materialized to zarr/folder) → round-trips correctly; `get_traces()` returns same values | +| `test_in_memory_recording_raises` | `NumpyRecording` (explicit `json=False`) → `ValueError` with clear message | +| `test_hash_stability` | same recording hashes identically across two calls | +| `test_hash_changes_with_content` | different recordings produce different hashes | +| `test_spikeinterface_not_installed` | `ImportError` raised when SI unavailable (mock import) | + +--- + +## Out of Scope + +- `BaseSorting`, `SortingAnalyzer`, `Motion` support (ITL-468, ITL-469, ITL-470) +- Phase 2 hashing (JSON + source-path content hash) — deferred to follow-up +- Auto-saving in-memory recordings to disk — requires database-level artifact storage + protocol (ITL-467) +- Remote/S3 zarr storage — works transparently if SI supports it, no orcapod changes needed +- Relative path portability — recordings store absolute paths; portability is a future concern + +--- + +## References + +- `src/orcapod/extension_types/directory_type.py` — pattern for JSON-storage LogicalType +- `src/orcapod/extension_types/numpy_type.py` — pattern for direct python_type binding + hash handler +- `src/orcapod/extension_types/spikeinterface_types.py` — implementation (LogicalType, handler, registration) +- `src/orcapod/contexts/data/v0.1.json` — context config changelog entry (LogicalType/handler NOT wired in statically) +- SpikeInterface `BaseExtractor.to_dict()` / `check_serializability()`: `spikeinterface/core/base.py` +- ITL-467 — database-level artifact storage (phase 2 dependency) diff --git a/tests/test_extension_types/test_spikeinterface_types.py b/tests/test_extension_types/test_spikeinterface_types.py new file mode 100644 index 00000000..bf9c331d --- /dev/null +++ b/tests/test_extension_types/test_spikeinterface_types.py @@ -0,0 +1,227 @@ +"""Tests for LogicalSIRecording and SIRecordingHandler (ITL-459).""" + +from __future__ import annotations + +import json + +import numpy as np +import pytest + + +def test_spikeinterface_not_installed_raises_import_error(monkeypatch): + """Importing spikeinterface_types when SI is absent raises ImportError. + + This test runs regardless of whether spikeinterface is installed — it + simulates absence by blocking the import via sys.modules. + """ + import sys + import importlib + + # Remove any cached spikeinterface modules + si_keys = [k for k in sys.modules if k == "spikeinterface" or k.startswith("spikeinterface.")] + saved = {k: sys.modules.pop(k) for k in si_keys} + # Block re-import + monkeypatch.setitem(sys.modules, "spikeinterface", None) + monkeypatch.setitem(sys.modules, "spikeinterface.core", None) + + try: + # Remove the cached spikeinterface_types module so it re-imports + si_types_key = "orcapod.extension_types.spikeinterface_types" + saved_si_types = sys.modules.pop(si_types_key, None) + with pytest.raises(ImportError, match="pip install orcapod\\[spikeinterface\\]"): + importlib.import_module(si_types_key) + finally: + # Restore everything + for k in list(sys.modules): + if k == "spikeinterface" or k.startswith("spikeinterface."): + del sys.modules[k] + sys.modules.update(saved) + if saved_si_types is not None: + sys.modules[si_types_key] = saved_si_types + else: + sys.modules.pop(si_types_key, None) + # Re-import only if SI is genuinely available; if not, there is + # nothing to restore and the ImportError is expected. + try: + importlib.import_module(si_types_key) + except ImportError: + pass + + +def _make_numpy_recording(): + """Create a small in-memory NumpyRecording for use as a test source.""" + import spikeinterface.core as si_core + rng = np.random.default_rng(42) + traces = rng.standard_normal((200, 4)).astype("float32") + return si_core.NumpyRecording([traces], sampling_frequency=30_000) + + +def test_logical_si_recording_importable(): + si_core = pytest.importorskip("spikeinterface.core", reason="spikeinterface not installed") + from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + import pyarrow as pa + + lt = LogicalSIRecording() + assert lt.logical_type_name == "spikeinterface.recording" + assert lt.python_type is si_core.BaseRecording + assert lt.get_arrow_extension_type().storage_type == pa.large_string() + + +def test_in_memory_recording_raises(): + """NumpyRecording (json=False) raises ValueError with clear instructions.""" + pytest.importorskip("spikeinterface", reason="spikeinterface not installed") + from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + + rec = _make_numpy_recording() + lt = LogicalSIRecording() + + with pytest.raises(ValueError, match="not JSON-serializable"): + lt.python_to_storage(rec) + + # The error message must distinguish in-memory from lazy file-backed recordings + with pytest.raises(ValueError, match="file-backed"): + lt.python_to_storage(rec) + + +def test_folder_recording_round_trip(tmp_path): + """Binary-folder-backed recording round-trips through python_to_storage / storage_to_python.""" + pytest.importorskip("spikeinterface", reason="spikeinterface not installed") + from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + + saved = _make_numpy_recording().save_to_folder(str(tmp_path / "rec")) + lt = LogicalSIRecording() + + storage = lt.python_to_storage(saved) + assert isinstance(storage, str) + data = json.loads(storage) + assert "class" in data # SI dict always has a "class" key + + recovered = lt.storage_to_python(storage) + np.testing.assert_array_equal( + saved.get_traces(segment_index=0), + recovered.get_traces(segment_index=0), + ) + + +def test_zarr_recording_round_trip(tmp_path): + """Zarr-backed recording round-trips through python_to_storage / storage_to_python.""" + pytest.importorskip("spikeinterface", reason="spikeinterface not installed") + from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + + saved = _make_numpy_recording().save_to_zarr(str(tmp_path / "rec.zarr")) + lt = LogicalSIRecording() + + storage = lt.python_to_storage(saved) + assert isinstance(storage, str) + + recovered = lt.storage_to_python(storage) + np.testing.assert_array_equal( + saved.get_traces(segment_index=0), + recovered.get_traces(segment_index=0), + ) + + +def test_ephemeral_recording_round_trip(tmp_path): + """A lazy preprocessing chain on a file-backed recording round-trips correctly. + + The preprocessed recording is NOT materialized to zarr/folder but IS + JSON-serializable because its root is file-backed. On reload, + SpikeInterface re-applies the preprocessing chain lazily (using zscore + which requires no optional dependencies and is fully deterministic). + """ + pytest.importorskip("spikeinterface", reason="spikeinterface not installed") + import spikeinterface.preprocessing as spre + from orcapod.extension_types.spikeinterface_types import LogicalSIRecording + + # Save the source to disk; apply a lazy preprocessing step on top + source = _make_numpy_recording().save_to_folder(str(tmp_path / "source")) + preprocessed = spre.zscore(source) + + assert preprocessed.check_serializability("json"), ( + "Preprocessed recording on folder-backed source must be JSON-serializable" + ) + + lt = LogicalSIRecording() + storage = lt.python_to_storage(preprocessed) + recovered = lt.storage_to_python(storage) + + np.testing.assert_array_equal( + preprocessed.get_traces(segment_index=0), + recovered.get_traces(segment_index=0), + ) + + +def test_si_recording_handler_hash_stability(tmp_path): + """Same recording produces identical ContentHash across two calls.""" + pytest.importorskip("spikeinterface", reason="spikeinterface not installed") + from orcapod.extension_types.spikeinterface_types import SIRecordingHandler + from orcapod.types import ContentHash + + saved = _make_numpy_recording().save_to_folder(str(tmp_path / "rec")) + handler = SIRecordingHandler() + + h1 = handler.handle(saved, hasher=None) + h2 = handler.handle(saved, hasher=None) + + assert isinstance(h1, ContentHash) + assert h1 == h2 + + +def test_si_recording_handler_hash_changes_with_content(tmp_path): + """Different recordings produce different ContentHash values.""" + si_core = pytest.importorskip("spikeinterface.core", reason="spikeinterface not installed") + from orcapod.extension_types.spikeinterface_types import SIRecordingHandler + + rng = np.random.default_rng(0) + rec_a = si_core.NumpyRecording( + [rng.standard_normal((200, 4)).astype("float32")], sampling_frequency=30_000 + ).save_to_folder(str(tmp_path / "rec_a")) + rec_b = si_core.NumpyRecording( + [rng.standard_normal((200, 4)).astype("float32")], sampling_frequency=30_000 + ).save_to_folder(str(tmp_path / "rec_b")) + + handler = SIRecordingHandler() + assert handler.handle(rec_a, hasher=None) != handler.handle(rec_b, hasher=None) + + +def test_si_recording_handler_in_memory_raises(): + """SIRecordingHandler raises ValueError for in-memory recordings.""" + pytest.importorskip("spikeinterface", reason="spikeinterface not installed") + from orcapod.extension_types.spikeinterface_types import SIRecordingHandler + + rec = _make_numpy_recording() + handler = SIRecordingHandler() + with pytest.raises(ValueError, match="in-memory"): + handler.handle(rec, hasher=None) + + +def test_register_spikeinterface_types(tmp_path): + """register_spikeinterface_types() wires LogicalSIRecording and SIRecordingHandler + into the default context so they are found by type lookup.""" + pytest.importorskip("spikeinterface", reason="spikeinterface not installed") + from orcapod.extension_types.spikeinterface_types import register_spikeinterface_types + from orcapod.contexts import get_default_context + + register_spikeinterface_types() + ctx = get_default_context() + + saved = _make_numpy_recording().save_to_folder(str(tmp_path / "rec")) + + # LogicalType registered: type_converter can find it + arrow_type = ctx.type_converter.python_type_to_arrow_type(type(saved)) + assert arrow_type.extension_name == "spikeinterface.recording" + + # Handler registered: semantic_hasher can find it + from orcapod.types import ContentHash + handler = ctx.semantic_hasher.type_handler_registry.get_handler(saved) + assert handler is not None + result = handler.handle(saved, hasher=None) + assert isinstance(result, ContentHash) + + # Full round-trip through python_to_storage / storage_to_python + storage = ctx.type_converter.python_to_storage(saved, type(saved)) + recovered = ctx.type_converter.storage_to_python(storage, type(saved)) + np.testing.assert_array_equal( + saved.get_traces(segment_index=0), + recovered.get_traces(segment_index=0), + ) diff --git a/tests/test_utils/test_object_spec.py b/tests/test_utils/test_object_spec.py new file mode 100644 index 00000000..e72cd793 --- /dev/null +++ b/tests/test_utils/test_object_spec.py @@ -0,0 +1,231 @@ +"""Tests for parse_objectspec, focusing on the ``_optional`` flag (ITL-459).""" + +from __future__ import annotations + +import sys +import importlib +from typing import Any + +import pytest + +from orcapod.utils.object_spec import parse_objectspec, _create_instance_from_spec, _resolve_type_from_spec + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +class _Sentinel: + """Trivial class used as a resolvable target in tests.""" + def __init__(self, value: Any = None) -> None: + self.value = value + + +# --------------------------------------------------------------------------- +# _resolve_type_from_spec — _optional flag +# --------------------------------------------------------------------------- + +class TestResolveTypeFromSpecOptional: + def test_resolves_present_type(self): + spec = {"_type": "builtins.int"} + assert _resolve_type_from_spec(spec) is int + + def test_missing_module_no_optional_raises(self, monkeypatch): + monkeypatch.setitem(sys.modules, "nonexistent_pkg_xyz", None) + spec = {"_type": "nonexistent_pkg_xyz.SomeClass"} + with pytest.raises(ImportError): + _resolve_type_from_spec(spec) + + def test_missing_module_with_optional_returns_none(self, monkeypatch): + monkeypatch.setitem(sys.modules, "nonexistent_pkg_xyz", None) + spec = {"_type": "nonexistent_pkg_xyz.SomeClass", "_optional": True} + assert _resolve_type_from_spec(spec) is None + + def test_optional_false_still_raises(self, monkeypatch): + monkeypatch.setitem(sys.modules, "nonexistent_pkg_xyz", None) + spec = {"_type": "nonexistent_pkg_xyz.SomeClass", "_optional": False} + with pytest.raises(ImportError): + _resolve_type_from_spec(spec) + + +# --------------------------------------------------------------------------- +# _create_instance_from_spec — _optional flag +# --------------------------------------------------------------------------- + +class TestCreateInstanceFromSpecOptional: + def test_creates_instance_normally(self): + spec = { + "_class": "tests.test_utils.test_object_spec._Sentinel", + "_config": {"value": 42}, + } + obj = _create_instance_from_spec(spec, ref_lut={}, validate=False) + assert isinstance(obj, _Sentinel) + assert obj.value == 42 + + def test_missing_module_no_optional_raises(self, monkeypatch): + monkeypatch.setitem(sys.modules, "nonexistent_pkg_xyz", None) + spec = {"_class": "nonexistent_pkg_xyz.SomeClass", "_config": {}} + with pytest.raises((ImportError, ValueError)): + _create_instance_from_spec(spec, ref_lut={}, validate=False) + + def test_missing_module_with_optional_returns_none(self, monkeypatch): + monkeypatch.setitem(sys.modules, "nonexistent_pkg_xyz", None) + spec = { + "_class": "nonexistent_pkg_xyz.SomeClass", + "_config": {}, + "_optional": True, + } + assert _create_instance_from_spec(spec, ref_lut={}, validate=False) is None + + def test_optional_false_still_raises(self, monkeypatch): + monkeypatch.setitem(sys.modules, "nonexistent_pkg_xyz", None) + spec = { + "_class": "nonexistent_pkg_xyz.SomeClass", + "_config": {}, + "_optional": False, + } + with pytest.raises((ImportError, ValueError)): + _create_instance_from_spec(spec, ref_lut={}, validate=False) + + +# --------------------------------------------------------------------------- +# parse_objectspec — _optional integration +# --------------------------------------------------------------------------- + +class TestParseObjectspecOptional: + def test_list_with_optional_missing_drops_entry(self, monkeypatch): + """Optional dict items that fail to resolve are dropped from the list entirely.""" + monkeypatch.setitem(sys.modules, "nonexistent_pkg_xyz", None) + specs = [ + {"_type": "builtins.int"}, + {"_type": "nonexistent_pkg_xyz.SomeClass", "_optional": True}, + {"_type": "builtins.str"}, + ] + result = parse_objectspec(specs) + # The missing optional entry is dropped; the list has 2 elements, not 3. + assert result == [int, str] + + def test_class_spec_with_optional_missing_returns_none(self, monkeypatch): + monkeypatch.setitem(sys.modules, "nonexistent_pkg_xyz", None) + spec = { + "_class": "nonexistent_pkg_xyz.Whatever", + "_config": {}, + "_optional": True, + } + assert parse_objectspec(spec) is None + + def test_class_spec_without_optional_raises(self, monkeypatch): + monkeypatch.setitem(sys.modules, "nonexistent_pkg_xyz", None) + spec = {"_class": "nonexistent_pkg_xyz.Whatever", "_config": {}} + with pytest.raises((ImportError, ValueError)): + parse_objectspec(spec) + + +# --------------------------------------------------------------------------- +# LogicalTypeRegistry — None filtering +# --------------------------------------------------------------------------- + +class TestParseObjectspecOptionalListFiltering: + def test_optional_class_spec_filtered_from_list(self, monkeypatch): + """_optional _class items that fail to resolve are dropped from the list.""" + monkeypatch.setitem(sys.modules, "nonexistent_pkg_xyz", None) + specs = [ + {"_type": "builtins.int"}, + {"_class": "nonexistent_pkg_xyz.SomeClass", "_config": {}, "_optional": True}, + {"_type": "builtins.str"}, + ] + result = parse_objectspec(specs) + assert result == [int, str] + + def test_optional_class_spec_included_when_present(self): + """_optional _class items that resolve successfully are included.""" + specs = [ + {"_type": "builtins.int"}, + {"_class": "builtins.list", "_config": {}, "_optional": True}, + {"_type": "builtins.str"}, + ] + result = parse_objectspec(specs) + assert result[0] is int + assert isinstance(result[1], list) + assert result[2] is str + + +# --------------------------------------------------------------------------- +# PythonTypeHandlerRegistry — None filtering +# --------------------------------------------------------------------------- + +class TestPythonTypeHandlerRegistryNoneFiltering: + def test_none_pair_in_handlers_is_skipped(self): + from orcapod.hashing.semantic_hashing.type_handler_registry import PythonTypeHandlerRegistry + from orcapod.hashing.semantic_hashing.builtin_handlers import BytesHandler + + # A pair containing None (simulating optional missing dep) should be skipped. + handlers = [ + (bytes, BytesHandler()), + [None, None], # optional pair where both resolved to None + [None, BytesHandler()], # partial None also skipped + ] + registry = PythonTypeHandlerRegistry(handlers=handlers) + handler = registry.get_handler(b"hello") + assert handler is not None + + def test_empty_pair_in_handlers_is_skipped(self): + """Empty list entries (optional pair where all elements were filtered) are skipped.""" + from orcapod.hashing.semantic_hashing.type_handler_registry import PythonTypeHandlerRegistry + from orcapod.hashing.semantic_hashing.builtin_handlers import BytesHandler + + # parse_objectspec filters _optional elements from the inner list, leaving []. + handlers = [ + (bytes, BytesHandler()), + [], # inner list became empty after optional filtering + ] + registry = PythonTypeHandlerRegistry(handlers=handlers) + assert registry.get_handler(b"x") is not None + + def test_none_entry_itself_skipped(self): + from orcapod.hashing.semantic_hashing.type_handler_registry import PythonTypeHandlerRegistry + from orcapod.hashing.semantic_hashing.builtin_handlers import BytesHandler + + handlers = [None, (bytes, BytesHandler())] + registry = PythonTypeHandlerRegistry(handlers=handlers) + assert registry.get_handler(b"x") is not None + + +# --------------------------------------------------------------------------- +# v0.1.json default context — SI auto-registration +# --------------------------------------------------------------------------- + +class TestDefaultContextSIAutoRegistration: + """Verify that the default context auto-registers SI types when SI is installed, + and starts cleanly when SI is absent.""" + + def test_default_context_loads_without_spikeinterface(self, monkeypatch): + """Default context construction must not raise even when SI is absent.""" + # Block SI imports + si_keys = [k for k in sys.modules if k == "spikeinterface" or k.startswith("spikeinterface.")] + saved = {k: sys.modules.pop(k) for k in si_keys} + monkeypatch.setitem(sys.modules, "spikeinterface", None) + monkeypatch.setitem(sys.modules, "spikeinterface.core", None) + + # Also force the context to be re-created by clearing the cached default + from orcapod import contexts as _ctx_mod + saved_default = getattr(_ctx_mod, "_default_context", None) + try: + _ctx_mod._default_context = None + ctx = _ctx_mod.get_default_context() + assert ctx is not None + finally: + _ctx_mod._default_context = saved_default + for k in list(sys.modules): + if k == "spikeinterface" or k.startswith("spikeinterface."): + del sys.modules[k] + sys.modules.update(saved) + + def test_default_context_has_si_types_when_installed(self): + """When SI is installed, BaseRecording is recognised by the default context.""" + si_core = pytest.importorskip("spikeinterface.core", reason="spikeinterface not installed") + from orcapod.contexts import get_default_context + + ctx = get_default_context() + arrow_type = ctx.type_converter.python_type_to_arrow_type(si_core.BaseRecording) + assert arrow_type.extension_name == "spikeinterface.recording" diff --git a/uv.lock b/uv.lock index a10a9e5e..279b0113 100644 --- a/uv.lock +++ b/uv.lock @@ -321,6 +321,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f3/af/0d591453490941e7cd2524ccac0398824eabafa745d0a25a758b1de2e361/arro3_core-0.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:bb0b13975c5394cb6a9887495aaf06cad8993893f99911c8aa2b827cd55dd6a8", size = 2612300, upload-time = "2025-05-31T23:20:54.249Z" }, ] +[[package]] +name = "asciitree" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/6a/885bc91484e1aa8f618f6f0228d76d0e67000b0fdd6090673b777e311913/asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e", size = 3951, upload-time = "2016-09-05T19:10:42.681Z" } + [[package]] name = "asttokens" version = "3.0.0" @@ -913,6 +919,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/8f/c4d9bafc34ad7ad5d8dc16dd1347ee0e507a52c3adb6bfa8887e1c6a26ba/executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa", size = 26702, upload-time = "2025-01-22T15:41:25.929Z" }, ] +[[package]] +name = "fasteners" +version = "0.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/18/7881a99ba5244bfc82f06017316ffe93217dbbbcfa52b887caa1d4f2a6d3/fasteners-0.20.tar.gz", hash = "sha256:55dce8792a41b56f727ba6e123fcaee77fd87e638a6863cec00007bfea84c8d8", size = 25087, upload-time = "2025-08-11T10:19:37.785Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/ac/e5d886f892666d2d1e5cb8c1a41146e1d79ae8896477b1153a21711d3b44/fasteners-0.20-py3-none-any.whl", hash = "sha256:9422c40d1e350e4259f509fb2e608d6bc43c0136f79a00db1b49046029d0b3b7", size = 18702, upload-time = "2025-08-11T10:19:35.716Z" }, +] + [[package]] name = "filelock" version = "3.18.0" @@ -2094,6 +2109,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/5d/e17845bb0fa76334477d5de38654d27946d5b5d3695443987a094a71b440/multidict-6.4.4-py3-none-any.whl", hash = "sha256:bd4557071b561a8b3b6075c3ce93cf9bfb6182cb241805c3d66ced3b75eff4ac", size = 10481, upload-time = "2025-05-19T14:16:36.024Z" }, ] +[[package]] +name = "neo" +version = "0.14.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, + { name = "quantities" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f0/ee/168b94170399e21d68cba4efb79f532e81b2cb3ce747d642d95370da8882/neo-0.14.5.tar.gz", hash = "sha256:5485668c5ab4fc9e384e5e645edc9e560987374d24ba8b53665b7a93cd4dcd37", size = 5101786, upload-time = "2026-06-25T09:08:40.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c0/7d482249c9678902b7cef16b7262d91d2fe7492010f170149ad3abfeee87/neo-0.14.5-py3-none-any.whl", hash = "sha256:2b54b5617ff08852486244974c462a892414d3581c3b0eaa8c3e1f9a000dba44", size = 712507, upload-time = "2026-06-25T09:08:38.921Z" }, +] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -2137,6 +2166,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/91/23/1f904bc9cbd8eece393e20840c08ba3ac03440090c3a4e95168fa6d2709f/nodejs_wheel_binaries-24.14.0-py2.py3-none-win_arm64.whl", hash = "sha256:78a9bd1d6b11baf1433f9fb84962ff8aa71c87d48b6434f98224bc49a2253a6e", size = 38926103, upload-time = "2026-02-27T02:57:27.458Z" }, ] +[[package]] +name = "numcodecs" +version = "0.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/fc/bb532969eb8236984ba65e4f0079a7da885b8ac0ce1f0835decbb3938a62/numcodecs-0.15.1.tar.gz", hash = "sha256:eeed77e4d6636641a2cc605fbc6078c7a8f2cc40f3dfa2b3f61e52e6091b04ff", size = 6267275, upload-time = "2025-02-10T10:23:33.254Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/fc/410f1cacaef0931f5daf06813b1b8a2442f7418ee284ec73fe5e830dca48/numcodecs-0.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:698f1d59511488b8fe215fadc1e679a4c70d894de2cca6d8bf2ab770eed34dfd", size = 1649501, upload-time = "2025-02-10T10:23:01.828Z" }, + { url = "https://files.pythonhosted.org/packages/85/29/dff62fae04323035912c419a82dc9624fad7d08541dbfcd9ab78a3a40074/numcodecs-0.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bef8c8e64fab76677324a07672b10c31861775d03fc63ed5012ca384144e4bb9", size = 1187306, upload-time = "2025-02-10T10:23:04.569Z" }, + { url = "https://files.pythonhosted.org/packages/a6/a8/908a226632ffabf19caf8c99f1b2898f2f22aac02795a6fe9d018fd6d9dd/numcodecs-0.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdfaef9f5f2ed8f65858db801f1953f1007c9613ee490a1c56233cd78b505ed5", size = 8891971, upload-time = "2025-02-10T10:23:07.689Z" }, + { url = "https://files.pythonhosted.org/packages/2b/e8/058aac43e1300d588e99b2d0d5b771c8a43fa92ce9c9517da596869fc146/numcodecs-0.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:e2547fa3a7ffc9399cfd2936aecb620a3db285f2630c86c8a678e477741a4b3c", size = 840035, upload-time = "2025-02-10T10:23:10.761Z" }, + { url = "https://files.pythonhosted.org/packages/e7/7e/f12fc32d3beedc6a8f1ec69ea0ba72e93cb99c0350feed2cff5d04679bc3/numcodecs-0.15.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b0a9d9cd29a0088220682dda4a9898321f7813ff7802be2bbb545f6e3d2f10ff", size = 1691889, upload-time = "2025-02-10T10:23:12.934Z" }, + { url = "https://files.pythonhosted.org/packages/81/38/88e40d40288b73c3b3a390ed5614a34b0661d00255bdd4cfb91c32101364/numcodecs-0.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a34f0fe5e5f3b837bbedbeb98794a6d4a12eeeef8d4697b523905837900b5e1c", size = 1189149, upload-time = "2025-02-10T10:23:15.803Z" }, + { url = "https://files.pythonhosted.org/packages/28/7d/7527d9180bc76011d6163c848c9cf02cd28a623c2c66cf543e1e86de7c5e/numcodecs-0.15.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3a09e22140f2c691f7df26303ff8fa2dadcf26d7d0828398c0bc09b69e5efa3", size = 8879163, upload-time = "2025-02-10T10:23:18.582Z" }, + { url = "https://files.pythonhosted.org/packages/ab/bc/b6c3cde91c754860a3467a8c058dcf0b1a5ca14d82b1c5397c700cf8b1eb/numcodecs-0.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:daed6066ffcf40082da847d318b5ab6123d69ceb433ba603cb87c323a541a8bc", size = 836785, upload-time = "2025-02-10T10:23:22.314Z" }, + { url = "https://files.pythonhosted.org/packages/78/57/acbc54b3419e5be65015e47177c76c0a73e037fd3ae2cde5808169194d4d/numcodecs-0.15.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e3d82b70500cf61e8d115faa0d0a76be6ecdc24a16477ee3279d711699ad85f3", size = 1688220, upload-time = "2025-02-10T10:23:23.79Z" }, + { url = "https://files.pythonhosted.org/packages/b6/56/9863fa6dc679f40a31bea5e9713ee5507a31dcd3ee82ea4b1a9268ce52e8/numcodecs-0.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1d471a1829ce52d3f365053a2bd1379e32e369517557c4027ddf5ac0d99c591e", size = 1180294, upload-time = "2025-02-10T10:23:25.533Z" }, + { url = "https://files.pythonhosted.org/packages/fa/91/d96999b41e3146b6c0ce6bddc5ad85803cb4d743c95394562c2a4bb8cded/numcodecs-0.15.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1dfdea4a67108205edfce99c1cb6cd621343bc7abb7e16a041c966776920e7de", size = 8834323, upload-time = "2025-02-10T10:23:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/c3/32/233e5ede6568bdb044e6f99aaa9fa39827ff3109c6487fc137315f733586/numcodecs-0.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:a4f7bdb26f1b34423cb56d48e75821223be38040907c9b5954eeb7463e7eb03c", size = 831955, upload-time = "2025-02-10T10:23:30.601Z" }, +] + [[package]] name = "numpy" version = "2.2.6" @@ -2322,6 +2375,7 @@ all = [ { name = "pyspiral" }, { name = "ray", extra = ["default"] }, { name = "redis" }, + { name = "spikeinterface" }, ] postgresql = [ { name = "psycopg", extra = ["binary"] }, @@ -2333,6 +2387,9 @@ ray = [ redis = [ { name = "redis" }, ] +spikeinterface = [ + { name = "spikeinterface" }, +] spiraldb = [ { name = "pyspiral" }, ] @@ -2362,6 +2419,7 @@ dev = [ { name = "ray", extra = ["default"] }, { name = "redis" }, { name = "ruff" }, + { name = "spikeinterface" }, { name = "testcontainers", extra = ["minio"] }, { name = "tqdm" }, ] @@ -2394,6 +2452,8 @@ requires-dist = [ { name = "redis", marker = "extra == 'all'", specifier = ">=6.2.0" }, { name = "redis", marker = "extra == 'redis'", specifier = ">=6.2.0" }, { name = "s3fs", specifier = ">=2025.12.0" }, + { name = "spikeinterface", marker = "extra == 'all'", specifier = ">=0.101" }, + { name = "spikeinterface", marker = "extra == 'spikeinterface'", specifier = ">=0.101" }, { name = "starfix", specifier = "~=0.4.0" }, { name = "typing-extensions" }, { name = "tzdata", specifier = ">=2024.1" }, @@ -2401,7 +2461,7 @@ requires-dist = [ { name = "uuid-utils", specifier = ">=0.11.1" }, { name = "xxhash" }, ] -provides-extras = ["all", "postgresql", "ray", "redis", "spiraldb"] +provides-extras = ["all", "postgresql", "ray", "redis", "spikeinterface", "spiraldb"] [package.metadata.requires-dev] dev = [ @@ -2428,6 +2488,7 @@ dev = [ { name = "ray", extras = ["default"], specifier = "==2.48.0" }, { name = "redis", specifier = ">=6.2.0" }, { name = "ruff", specifier = ">=0.14.4" }, + { name = "spikeinterface", specifier = ">=0.101" }, { name = "testcontainers", extras = ["minio"], specifier = ">=4.0.0" }, { name = "tqdm", specifier = ">=4.67.1" }, ] @@ -2764,6 +2825,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/8c/83087ebc47ab0396ce092363001fa37c17153119ee282700c0713a195853/prettytable-3.17.0-py3-none-any.whl", hash = "sha256:aad69b294ddbe3e1f95ef8886a060ed1666a0b83018bbf56295f6f226c43d287", size = 34433, upload-time = "2025-11-14T17:33:19.093Z" }, ] +[[package]] +name = "probeinterface" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/b6/149976dac9228c39a80fd1dee1743b3ba8e878b8c425df4304a7ac395aa1/probeinterface-0.3.2.tar.gz", hash = "sha256:8cf02afb1f344be027e1f29e7a2c54fd35b38573ad77890fc5076f957ddf3db5", size = 6681139, upload-time = "2026-03-20T16:06:09.401Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/c9/7cd477fa3cc6e31b643c1ca8d13a7b398fa8c2b6a9c5926cab916729cf43/probeinterface-0.3.2-py3-none-any.whl", hash = "sha256:f288d013697a73836384a9ee8db6bc979764814b981b6730d463d3bd04d5f317", size = 68822, upload-time = "2026-03-20T16:06:08.059Z" }, +] + [[package]] name = "prometheus-client" version = "0.22.1" @@ -3636,6 +3711,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/9c/d8073bd898eb896e94c679abe82e47506e2b750eb261cf6010ced869797c/pyzmq-26.4.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a222ad02fbe80166b0526c038776e8042cd4e5f0dec1489a006a1df47e9040e0", size = 555371, upload-time = "2025-04-04T12:05:20.702Z" }, ] +[[package]] +name = "quantities" +version = "0.16.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/50/19194d2cf428239ecbda876d81bbfddb76697a248c5a7478ae83d0b98fee/quantities-0.16.4.tar.gz", hash = "sha256:f5b6deb74f6ed814ec56b204dcf7b5d3c526456157de48e5a78681064f98bf36", size = 100726, upload-time = "2026-01-16T10:43:48.858Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/1f/9751abbc14422092754423d41c89b0884dba3abc236125adc427d108216a/quantities-0.16.4-py3-none-any.whl", hash = "sha256:cd8e82339eb553d8bee0bda0e41a80b91e21795e1174e7d8a82db8e6de2663f6", size = 104009, upload-time = "2026-01-16T10:43:47.252Z" }, +] + [[package]] name = "questionary" version = "2.1.1" @@ -3986,6 +4073,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] +[[package]] +name = "spikeinterface" +version = "0.104.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "neo" }, + { name = "numcodecs" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "probeinterface" }, + { name = "pydantic" }, + { name = "threadpoolctl" }, + { name = "tqdm" }, + { name = "zarr" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/89/bf/d039a0ba89e529f1406c8ec94a79f24f21e896bc78c3c3632202a19807b7/spikeinterface-0.104.8.tar.gz", hash = "sha256:ebd5b3ff19066c795ab7c424a7c06bbc3a312e59807c01647821d3c68b00b24b", size = 907430, upload-time = "2026-06-30T13:49:39.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/83/cd37e77b850628353069f2f82cd9402fe286bd397b22bdf7095f566d359e/spikeinterface-0.104.8-py3-none-any.whl", hash = "sha256:e99a9b3c4e87b9ce33eef6d262c96a57af38efdb25b2cbae3719126009a979ae", size = 1129221, upload-time = "2026-06-30T13:49:38.251Z" }, +] + [[package]] name = "stack-data" version = "0.6.3" @@ -4066,6 +4173,15 @@ minio = [ { name = "minio" }, ] +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + [[package]] name = "tomli" version = "2.2.1" @@ -4488,6 +4604,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" }, ] +[[package]] +name = "zarr" +version = "2.18.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asciitree" }, + { name = "fasteners", marker = "sys_platform != 'emscripten'" }, + { name = "numcodecs" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/1d/01cf9e3ab2d85190278efc3fca9f68563de35ae30ee59e7640e3af98abe3/zarr-2.18.7.tar.gz", hash = "sha256:b2b8f66f14dac4af66b180d2338819981b981f70e196c9a66e6bfaa9e59572f5", size = 3604558, upload-time = "2025-04-09T07:59:28.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/d8/9ffd8c237b3559945bb52103cf0eed64ea098f7b7f573f8d2962ef27b4b2/zarr-2.18.7-py3-none-any.whl", hash = "sha256:ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223", size = 211273, upload-time = "2025-04-09T07:59:27.039Z" }, +] + [[package]] name = "zipp" version = "3.23.0"