From 38340b30fcd4cb42d82d86324a3d0a87bf4e4a5a Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 8 Oct 2025 18:57:32 -0600 Subject: [PATCH 1/3] delay memory access till event access on spikeglx reader --- neo/rawio/spikeglxrawio.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 3837cdfcf..056079ff9 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -246,12 +246,14 @@ def _parse_header(self): chan_name = local_chan chan_id = f"{stream_name}#{chan_name}" event_channels.append((chan_name, chan_id, "event")) - # add events_memmap - data = np.memmap(info["bin_file"], dtype="int16", mode="r", offset=0, order="C") - data = data.reshape(-1, info["num_chan"]) - # The digital word is usually the last channel, after all the individual analog channels - extracted_word = data[:, len(info["analog_channels"])] - self._events_memmap = np.unpackbits(extracted_word.astype(np.uint8)[:, None], axis=1) + # Create memmap for digital word but defer unpacking until needed + # The digital word is stored as the last channel, after all the individual analog channels + # For example: if there are 8 analog channels (indices 0-7), the digital word is at index 8 + num_samples = info["sample_length"] + num_chan = info["num_chan"] + data = np.memmap(info["bin_file"], dtype="int16", mode="r", shape=(num_samples, num_chan), order="C") + digital_word_channel_index = len(info["analog_channels"]) + self._events_memmap_digital_word = data[:, digital_word_channel_index] event_channels = np.array(event_channels, dtype=_event_channel_dtype) # No spikes @@ -342,7 +344,9 @@ def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_s info = self.signals_info_dict[0, "nidq"] # There are no events that are not in the nidq stream dig_ch = info["digital_channels"] if len(dig_ch) > 0: - event_data = self._events_memmap + # Unpack bits on-demand - this is when memory allocation happens + event_data = np.unpackbits(self._events_memmap_digital_word.astype(np.uint8)[:, None], axis=1) + channel = dig_ch[event_channel_index] ch_idx = 7 - int(channel[2:]) # They are in the reverse order this_stream = event_data[:, ch_idx] From 2bcc45cb7707de3a7cdc5d69d368349c209f4979 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 8 Oct 2025 19:29:41 -0600 Subject: [PATCH 2/3] fix tests --- neo/rawio/spikeglxrawio.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 056079ff9..5c3212673 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -250,8 +250,8 @@ def _parse_header(self): # The digital word is stored as the last channel, after all the individual analog channels # For example: if there are 8 analog channels (indices 0-7), the digital word is at index 8 num_samples = info["sample_length"] - num_chan = info["num_chan"] - data = np.memmap(info["bin_file"], dtype="int16", mode="r", shape=(num_samples, num_chan), order="C") + num_channels = info["num_chan"] + data = np.memmap(info["bin_file"], dtype="int16", mode="r", shape=(num_samples, num_channels), order="C") digital_word_channel_index = len(info["analog_channels"]) self._events_memmap_digital_word = data[:, digital_word_channel_index] event_channels = np.array(event_channels, dtype=_event_channel_dtype) @@ -686,7 +686,12 @@ def extract_stream_info(meta_file, meta): info["sampling_rate"] = float(meta[k]) info["num_chan"] = num_chan - info["sample_length"] = int(meta["fileSizeBytes"]) // 2 // num_chan + # Calculate sample_length from actual file size instead of metadata to handle stub/modified files + # The metadata fileSizeBytes may not match the actual .bin file (e.g., in test stub files) + # Original calculation (only correct for unmodified files): + # info["sample_length"] = int(meta["fileSizeBytes"]) // 2 // num_chan + actual_file_size = Path(meta_file).with_suffix(".bin").stat().st_size + info["sample_length"] = actual_file_size // 2 // num_chan # 2 bytes per int16 sample info["gate_num"] = gate_num info["trigger_num"] = trigger_num info["device"] = device From 6a1897e05e63489ebd4691735d23ddac559e9ec2 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 8 Oct 2025 19:30:06 -0600 Subject: [PATCH 3/3] black --- neo/rawio/spikeglxrawio.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 5c3212673..5a25e5d29 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -251,7 +251,13 @@ def _parse_header(self): # For example: if there are 8 analog channels (indices 0-7), the digital word is at index 8 num_samples = info["sample_length"] num_channels = info["num_chan"] - data = np.memmap(info["bin_file"], dtype="int16", mode="r", shape=(num_samples, num_channels), order="C") + data = np.memmap( + info["bin_file"], + dtype="int16", + mode="r", + shape=(num_samples, num_channels), + order="C", + ) digital_word_channel_index = len(info["analog_channels"]) self._events_memmap_digital_word = data[:, digital_word_channel_index] event_channels = np.array(event_channels, dtype=_event_channel_dtype)