From 581caf75c604dd0f64be0070cb4e2f853c8cefe1 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Tue, 8 Sep 2020 11:52:10 -0700 Subject: [PATCH 01/36] Use NLX_Base_Class_Type to determine recording type. --- neo/rawio/neuralynxrawio.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 305049b18..46f930b06 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -661,6 +661,7 @@ def _to_bool(txt): ('ApplicationName', '', None), # also include version number when present ('AcquisitionSystem', '', None), ('ReferenceChannel', '', None), + ('NLX_Base_Class_Type','',None) # in version 4 and earlier versions of Cheetah ] # Filename and datetime may appear in header lines starting with # at @@ -809,6 +810,37 @@ def buildForFile(filename): return info + def typeOfRecording(self): + """ + Determines type of recording in Ncs file with this header. + + RETURN: + one of 'PRE4','BML','DIGITALLYNX','DIGITALLYNXSX','UNKNOWN' + """ + + if 'NLX_Base_Class_Type' in self: + + # older style standard neuralynx acquisition with rounded sampling frequency + if self['NLX_Base_Class_Type'] == 'CscAcqEnt': + return 'PRE4' + + # BML style with fractional frequency and microsPerSamp + elif self['NLX_Base_Class_Type'] == 'BmlAcq': + return 'BML' + + elif 'HardwareSubsystemType' in self: + + # DigitalLynx + if self['HardwareSubsystemType'] == 'DigitalLynx': + return 'DIGITALLYNX' + + # DigitalLynxSX + elif self['HardwareSubsystemType'] == 'DigitalLynxSX': + return 'DIGITALLYNXSX' + + else: + return 'UNKNOWN' + class NcsHeader(): """ From 2bbc68f0269f6a90771ae549f68c79dc768c5fa2 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Tue, 8 Sep 2020 11:52:45 -0700 Subject: [PATCH 02/36] Initial test of Ncs recording type from header. --- neo/test/rawiotest/test_neuralynxrawio.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 1c5a1aa92..b13c9e689 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -2,6 +2,7 @@ from neo.rawio.neuralynxrawio import NeuralynxRawIO from neo.test.rawiotest.common_rawio_test import BaseTestRawIO +from neo.rawio.neuralynxrawio import NlxHeader import logging @@ -14,7 +15,8 @@ class TestNeuralynxRawIO(BaseTestRawIO, unittest.TestCase, ): 'Cheetah_v5.5.1/original_data', 'Cheetah_v5.6.3/original_data', 'Cheetah_v5.7.4/original_data', - 'Cheetah_v6.3.2/incomplete_blocks' + 'Cheetah_v6.3.2/incomplete_blocks', + 'Cheetah_v4.0.2/original_data' ] files_to_download = [ 'Cheetah_v5.5.1/original_data/CheetahLogFile.txt', @@ -58,7 +60,20 @@ class TestNeuralynxRawIO(BaseTestRawIO, unittest.TestCase, ): 'Cheetah_v5.7.4/README.txt', 'Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs', 'Cheetah_v6.3.2/incomplete_blocks/Events.nev', - 'Cheetah_v6.3.2/incomplete_blocks/README.txt'] + 'Cheetah_v6.3.2/incomplete_blocks/README.txt', + 'Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs'] + +class TestNcsRecordingType(TestNeuralynxRawIO, unittest.TestCase): + """ + Test of decoding of NlxHeader for type of recording. + """ + + def test_recording_types(self): + + filename = self.get_filename_path('Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs') + hdr = NlxHeader.buildForFile(filename) + self.assertEqual(hdr.typeOfRecording(),'PRE4') + if __name__ == "__main__": From c421a307bf6a77ea5d3b6412d2019ffda7c5f350 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Thu, 8 Oct 2020 11:07:16 -0700 Subject: [PATCH 03/36] Decode headers from recent file types and test them. --- neo/rawio/neuralynxrawio.py | 15 ++++++++++--- neo/test/rawiotest/test_neuralynxrawio.py | 27 +++++++++++++++++------ 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 46f930b06..4dd9f4bac 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -828,16 +828,25 @@ def typeOfRecording(self): elif self['NLX_Base_Class_Type'] == 'BmlAcq': return 'BML' - elif 'HardwareSubsystemType' in self: + else: return 'UNKNOWN' + + elif 'HardwareSubSystemType' in self: # DigitalLynx - if self['HardwareSubsystemType'] == 'DigitalLynx': + if self['HardwareSubSystemType'] == 'DigitalLynx': return 'DIGITALLYNX' # DigitalLynxSX - elif self['HardwareSubsystemType'] == 'DigitalLynxSX': + elif self['HardwareSubSystemType'] == 'DigitalLynxSX': return 'DIGITALLYNXSX' + elif 'FileType' in self: + + if self['FileVersion'] in ['3.3','3.4']: + return self['AcquisitionSystem'].split()[1].upper() + + else: return 'UNKNOWN' + else: return 'UNKNOWN' diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index b13c9e689..0985d7635 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -12,13 +12,14 @@ class TestNeuralynxRawIO(BaseTestRawIO, unittest.TestCase, ): rawioclass = NeuralynxRawIO entities_to_test = [ + 'Cheetah_v4.0.2/original_data', 'Cheetah_v5.5.1/original_data', 'Cheetah_v5.6.3/original_data', 'Cheetah_v5.7.4/original_data', - 'Cheetah_v6.3.2/incomplete_blocks', - 'Cheetah_v4.0.2/original_data' + 'Cheetah_v6.3.2/incomplete_blocks' ] files_to_download = [ + 'Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs', 'Cheetah_v5.5.1/original_data/CheetahLogFile.txt', 'Cheetah_v5.5.1/original_data/CheetahLostADRecords.txt', 'Cheetah_v5.5.1/original_data/Events.nev', @@ -60,20 +61,32 @@ class TestNeuralynxRawIO(BaseTestRawIO, unittest.TestCase, ): 'Cheetah_v5.7.4/README.txt', 'Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs', 'Cheetah_v6.3.2/incomplete_blocks/Events.nev', - 'Cheetah_v6.3.2/incomplete_blocks/README.txt', - 'Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs'] + 'Cheetah_v6.3.2/incomplete_blocks/README.txt' + ] + class TestNcsRecordingType(TestNeuralynxRawIO, unittest.TestCase): """ Test of decoding of NlxHeader for type of recording. """ + ncsTypeTestFiles = [ + ('Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs','PRE4'), + ('Cheetah_v5.5.1/original_data/STet3a.nse','DIGITALLYNXSX'), + ('Cheetah_v5.5.1/original_data/Tet3a.ncs','DIGITALLYNXSX'), + ('Cheetah_v5.6.3/original_data/CSC1.ncs','DIGITALLYNXSX'), + ('Cheetah_v5.6.3/original_data/TT1.ntt','DIGITALLYNXSX'), + ('Cheetah_v5.7.4/original_data/CSC1.ncs','DIGITALLYNXSX'), + ('Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs','DIGITALLYNXSX') + ] + def test_recording_types(self): - filename = self.get_filename_path('Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs') - hdr = NlxHeader.buildForFile(filename) - self.assertEqual(hdr.typeOfRecording(),'PRE4') + for typeTest in self.ncsTypeTestFiles: + filename = self.get_filename_path(typeTest[0]) + hdr = NlxHeader.buildForFile(filename) + self.assertEqual(hdr.typeOfRecording(),typeTest[1]) if __name__ == "__main__": From 5e224b62ab56e64f8fa423d17ab467df1f66fa70 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Thu, 8 Oct 2020 11:08:07 -0700 Subject: [PATCH 04/36] Add self to authors file. --- doc/source/authors.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/authors.rst b/doc/source/authors.rst index f48515458..bddb41844 100644 --- a/doc/source/authors.rst +++ b/doc/source/authors.rst @@ -52,6 +52,7 @@ and may not be the current affiliation of a contributor. * rishidhingra@github * Hugo van Kemenade * Aitor Morales-Gregorio [13] +* Peter N Steinmetz [22] 1. Centre de Recherche en Neuroscience de Lyon, CNRS UMR5292 - INSERM U1028 - Universite Claude Bernard Lyon 1 2. Unité de Neuroscience, Information et Complexité, CNRS UPR 3293, Gif-sur-Yvette, France @@ -74,6 +75,7 @@ and may not be the current affiliation of a contributor. 19. IAL Developmental Neurobiology, Kazan Federal University, Kazan, Russia 20. Harden Technologies, LLC 21. Institut des Neurosciences Paris-Saclay, CNRS UMR 9197 - Université Paris-Sud, Gif-sur-Yvette, France +22. Neurtex Brain Research Institute, Dallas, TX, USAs If we've somehow missed you off the list we're very sorry - please let us know. From 1a5a3fc976c5b6bf1283d5a6ed4787322704c6cc Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Thu, 8 Oct 2020 11:11:08 -0700 Subject: [PATCH 05/36] Add 4.0.2 test data. --- neo/test/iotest/test_neuralynxio.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/test_neuralynxio.py b/neo/test/iotest/test_neuralynxio.py index adc97ba6a..a6b82d5f4 100644 --- a/neo/test/iotest/test_neuralynxio.py +++ b/neo/test/iotest/test_neuralynxio.py @@ -22,12 +22,14 @@ class CommonNeuralynxIOTest(BaseTestIO, unittest.TestCase, ): ioclass = NeuralynxIO files_to_test = [ + 'Cheetah_v4.0.2/original_data', 'Cheetah_v5.5.1/original_data', 'Cheetah_v5.6.3/original_data', 'Cheetah_v5.7.4/original_data', 'Pegasus_v2.1.1', 'Cheetah_v6.3.2/incomplete_blocks'] files_to_download = [ + 'Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs', 'Cheetah_v5.5.1/original_data/CheetahLogFile.txt', 'Cheetah_v5.5.1/original_data/CheetahLostADRecords.txt', 'Cheetah_v5.5.1/original_data/Events.nev', @@ -71,7 +73,8 @@ class CommonNeuralynxIOTest(BaseTestIO, unittest.TestCase, ): 'Pegasus_v2.1.1/Events_0008.nev', 'Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs', 'Cheetah_v6.3.2/incomplete_blocks/Events.nev', - 'Cheetah_v6.3.2/incomplete_blocks/README.txt'] + 'Cheetah_v6.3.2/incomplete_blocks/README.txt' + ] class TestCheetah_v551(CommonNeuralynxIOTest, unittest.TestCase): @@ -336,7 +339,6 @@ def test_gap_handling_v563(self): self.assertEqual(len(block.channel_indexes[0].analogsignals), n_gaps + 1) self.assertEqual(len(block.channel_indexes[-1].units[0].spiketrains), n_gaps + 1) - def compare_old_and_new_neuralynxio(): base = '/tmp/files_for_testing_neo/neuralynx/' dirname = base + 'Cheetah_v5.5.1/original_data/' From 7008f9c5e73e1a3604fcdcc328c177520326a016 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Fri, 9 Oct 2020 09:32:24 -0700 Subject: [PATCH 06/36] Do not fully test the 4.0.2 data yet. Allows full normal testing to run, using the 4.0.2 data only to test the parsing of ncs recording from the header information. --- neo/test/rawiotest/test_neuralynxrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 0985d7635..3a8641a97 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -12,7 +12,7 @@ class TestNeuralynxRawIO(BaseTestRawIO, unittest.TestCase, ): rawioclass = NeuralynxRawIO entities_to_test = [ - 'Cheetah_v4.0.2/original_data', + # 'Cheetah_v4.0.2/original_data', 'Cheetah_v5.5.1/original_data', 'Cheetah_v5.6.3/original_data', 'Cheetah_v5.7.4/original_data', From b6f8153b14baa8a2c34b89a4cf6a3e9789648f31 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Fri, 9 Oct 2020 09:43:07 -0700 Subject: [PATCH 07/36] =?UTF-8?q?Don=E2=80=99t=20test=204.0.2=20in=20test?= =?UTF-8?q?=20of=20neuralynxio=20either.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- neo/test/iotest/test_neuralynxio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/test/iotest/test_neuralynxio.py b/neo/test/iotest/test_neuralynxio.py index a6b82d5f4..3bf29d12f 100644 --- a/neo/test/iotest/test_neuralynxio.py +++ b/neo/test/iotest/test_neuralynxio.py @@ -22,7 +22,7 @@ class CommonNeuralynxIOTest(BaseTestIO, unittest.TestCase, ): ioclass = NeuralynxIO files_to_test = [ - 'Cheetah_v4.0.2/original_data', + # 'Cheetah_v4.0.2/original_data', 'Cheetah_v5.5.1/original_data', 'Cheetah_v5.6.3/original_data', 'Cheetah_v5.7.4/original_data', From b0deaf287a6fd8d55ce6d11175e6f97e3a238f21 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Fri, 9 Oct 2020 09:46:17 -0700 Subject: [PATCH 08/36] Add blank line for PEP8. --- neo/test/iotest/test_neuralynxio.py | 1 + 1 file changed, 1 insertion(+) diff --git a/neo/test/iotest/test_neuralynxio.py b/neo/test/iotest/test_neuralynxio.py index 3bf29d12f..85667034d 100644 --- a/neo/test/iotest/test_neuralynxio.py +++ b/neo/test/iotest/test_neuralynxio.py @@ -339,6 +339,7 @@ def test_gap_handling_v563(self): self.assertEqual(len(block.channel_indexes[0].analogsignals), n_gaps + 1) self.assertEqual(len(block.channel_indexes[-1].units[0].spiketrains), n_gaps + 1) + def compare_old_and_new_neuralynxio(): base = '/tmp/files_for_testing_neo/neuralynx/' dirname = base + 'Cheetah_v5.5.1/original_data/' From 51daa9325ca5c34ebbfaa4e65199f11ac132cfd7 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Fri, 9 Oct 2020 10:05:21 -0700 Subject: [PATCH 09/36] Clean up more PEP8 style issues. --- neo/rawio/neuralynxrawio.py | 10 ++++++---- neo/test/iotest/test_neuralynxio.py | 3 +-- neo/test/rawiotest/test_neuralynxrawio.py | 21 +++++++++------------ 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 4dd9f4bac..4e08d120c 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -661,7 +661,7 @@ def _to_bool(txt): ('ApplicationName', '', None), # also include version number when present ('AcquisitionSystem', '', None), ('ReferenceChannel', '', None), - ('NLX_Base_Class_Type','',None) # in version 4 and earlier versions of Cheetah + ('NLX_Base_Class_Type', '', None) # in version 4 and earlier versions of Cheetah ] # Filename and datetime may appear in header lines starting with # at @@ -828,7 +828,8 @@ def typeOfRecording(self): elif self['NLX_Base_Class_Type'] == 'BmlAcq': return 'BML' - else: return 'UNKNOWN' + else: + return 'UNKNOWN' elif 'HardwareSubSystemType' in self: @@ -842,10 +843,11 @@ def typeOfRecording(self): elif 'FileType' in self: - if self['FileVersion'] in ['3.3','3.4']: + if self['FileVersion'] in ['3.3', '3.4']: return self['AcquisitionSystem'].split()[1].upper() - else: return 'UNKNOWN' + else: + return 'UNKNOWN' else: return 'UNKNOWN' diff --git a/neo/test/iotest/test_neuralynxio.py b/neo/test/iotest/test_neuralynxio.py index 85667034d..116515940 100644 --- a/neo/test/iotest/test_neuralynxio.py +++ b/neo/test/iotest/test_neuralynxio.py @@ -73,8 +73,7 @@ class CommonNeuralynxIOTest(BaseTestIO, unittest.TestCase, ): 'Pegasus_v2.1.1/Events_0008.nev', 'Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs', 'Cheetah_v6.3.2/incomplete_blocks/Events.nev', - 'Cheetah_v6.3.2/incomplete_blocks/README.txt' - ] + 'Cheetah_v6.3.2/incomplete_blocks/README.txt'] class TestCheetah_v551(CommonNeuralynxIOTest, unittest.TestCase): diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 3a8641a97..ba20bdb83 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -16,8 +16,7 @@ class TestNeuralynxRawIO(BaseTestRawIO, unittest.TestCase, ): 'Cheetah_v5.5.1/original_data', 'Cheetah_v5.6.3/original_data', 'Cheetah_v5.7.4/original_data', - 'Cheetah_v6.3.2/incomplete_blocks' - ] + 'Cheetah_v6.3.2/incomplete_blocks'] files_to_download = [ 'Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs', 'Cheetah_v5.5.1/original_data/CheetahLogFile.txt', @@ -61,8 +60,7 @@ class TestNeuralynxRawIO(BaseTestRawIO, unittest.TestCase, ): 'Cheetah_v5.7.4/README.txt', 'Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs', 'Cheetah_v6.3.2/incomplete_blocks/Events.nev', - 'Cheetah_v6.3.2/incomplete_blocks/README.txt' - ] + 'Cheetah_v6.3.2/incomplete_blocks/README.txt'] class TestNcsRecordingType(TestNeuralynxRawIO, unittest.TestCase): @@ -71,14 +69,13 @@ class TestNcsRecordingType(TestNeuralynxRawIO, unittest.TestCase): """ ncsTypeTestFiles = [ - ('Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs','PRE4'), - ('Cheetah_v5.5.1/original_data/STet3a.nse','DIGITALLYNXSX'), - ('Cheetah_v5.5.1/original_data/Tet3a.ncs','DIGITALLYNXSX'), - ('Cheetah_v5.6.3/original_data/CSC1.ncs','DIGITALLYNXSX'), - ('Cheetah_v5.6.3/original_data/TT1.ntt','DIGITALLYNXSX'), - ('Cheetah_v5.7.4/original_data/CSC1.ncs','DIGITALLYNXSX'), - ('Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs','DIGITALLYNXSX') - ] + ('Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs', 'PRE4'), + ('Cheetah_v5.5.1/original_data/STet3a.nse', 'DIGITALLYNXSX'), + ('Cheetah_v5.5.1/original_data/Tet3a.ncs', 'DIGITALLYNXSX'), + ('Cheetah_v5.6.3/original_data/CSC1.ncs', 'DIGITALLYNXSX'), + ('Cheetah_v5.6.3/original_data/TT1.ntt', 'DIGITALLYNXSX'), + ('Cheetah_v5.7.4/original_data/CSC1.ncs', 'DIGITALLYNXSX'), + ('Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs', 'DIGITALLYNXSX')] def test_recording_types(self): From ea95d7065c632c722776e28e4a0998ad9e0b6e4b Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Fri, 9 Oct 2020 10:13:13 -0700 Subject: [PATCH 10/36] Clean up another whitespace issue for PEP8. --- neo/test/rawiotest/test_neuralynxrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index ba20bdb83..dce7357ac 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -83,7 +83,7 @@ def test_recording_types(self): filename = self.get_filename_path(typeTest[0]) hdr = NlxHeader.buildForFile(filename) - self.assertEqual(hdr.typeOfRecording(),typeTest[1]) + self.assertEqual(hdr.typeOfRecording(), typeTest[1]) if __name__ == "__main__": From 0a2b9590c8cac3a40911927597210d128c4c420c Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Fri, 9 Oct 2020 11:15:52 -0700 Subject: [PATCH 11/36] Move constants into class so accessible for testing. --- neo/rawio/neuralynxrawio.py | 63 ++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 4e08d120c..380e2cc3c 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -29,7 +29,6 @@ import datetime from collections import OrderedDict -BLOCK_SIZE = 512 # nb sample per signal block class NeuralynxRawIO(BaseRawIO): @@ -51,6 +50,10 @@ class NeuralynxRawIO(BaseRawIO): extensions = ['nse', 'ncs', 'nev', 'ntt'] rawmode = 'one-dir' + _BLOCK_SIZE = 512 # nb sample per signal block + _ncs_dtype = [('timestamp', 'uint64'), ('channel_id', 'uint32'), ('sample_rate', 'uint32'), + ('nb_valid', 'uint32'), ('samples', 'int16', (_BLOCK_SIZE,))] + def __init__(self, dirname='', keep_original_times=False, **kargs): """ Parameters @@ -348,8 +351,8 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, chann if i_stop is None: i_stop = self._sigs_length[seg_index] - block_start = i_start // BLOCK_SIZE - block_stop = i_stop // BLOCK_SIZE + 1 + block_start = i_start // self._BLOCK_SIZE + block_stop = i_stop // self._BLOCK_SIZE + 1 sl0 = i_start % 512 sl1 = sl0 + (i_stop - i_start) @@ -492,11 +495,11 @@ def read_ncs_files(self, ncs_filenames): if len(ncs_filenames) == 0: return None - good_delta = int(BLOCK_SIZE * 1e6 / self._sigs_sampling_rate) + good_delta = int(self._BLOCK_SIZE * 1e6 / self._sigs_sampling_rate) chan_uid0 = list(ncs_filenames.keys())[0] filename0 = ncs_filenames[chan_uid0] - data0 = np.memmap(filename0, dtype=ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) + data0 = np.memmap(filename0, dtype=self.ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) gap_indexes = None lost_indexes = None @@ -535,7 +538,7 @@ def read_ncs_files(self, ncs_filenames): # is not strictly necessary as all channels might have same partially filled # blocks at the end. - lost_indexes, = np.nonzero(data0['nb_valid'] < BLOCK_SIZE) + lost_indexes, = np.nonzero(data0['nb_valid'] < self._BLOCK_SIZE) if self.use_cache: self.add_in_cache(lost_indexes=lost_indexes) @@ -562,7 +565,8 @@ def read_ncs_files(self, ncs_filenames): # create segment with subdata block/t_start/t_stop/length for chan_uid, ncs_filename in self.ncs_filenames.items(): - data = np.memmap(ncs_filename, dtype=ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) + data = np.memmap(ncs_filename, dtype=self.ncs_dtype, mode='r', + offset=NlxHeader.HEADER_SIZE) assert data.size == data0.size, 'ncs files do not have the same data length' for seg_index, (i0, i1) in enumerate(gap_pairs): @@ -578,21 +582,20 @@ def read_ncs_files(self, ncs_filenames): if chan_uid == chan_uid0: ts0 = subdata[0]['timestamp'] ts1 = subdata[-1]['timestamp'] +\ - np.uint64(BLOCK_SIZE / self._sigs_sampling_rate * 1e6) + np.uint64(self._BLOCK_SIZE / self._sigs_sampling_rate * 1e6) self._timestamp_limits.append((ts0, ts1)) t_start = ts0 / 1e6 self._sigs_t_start.append(t_start) t_stop = ts1 / 1e6 self._sigs_t_stop.append(t_stop) - length = subdata.size * BLOCK_SIZE + length = subdata.size * self._BLOCK_SIZE self._sigs_length.append(length) class NcsBlocks(): """ Contains information regarding the blocks of records in an Ncs file. - Factory methods perform parsing of this information from an Ncs file or - confirmation that file agrees with block structure. + Factory methods perform parsing of this information from an Ncs file. """ startBlocks = [] @@ -601,6 +604,41 @@ class NcsBlocks(): microsPerSampUsed = 0 +class NcsBlocksFactory(): + """ + Class for factory methods which perform parsing of blocks in Ncs files. + + Moved here since algorithm covering all 3 header styles and types used is + more complicated. Copied from Java code on Sept 7, 2020. + """ + + _tolerance = 0.001 # tolerance for drift of timestamps within one NcsBlock + _maxGapLength = 5 # maximum gap between predicted and actual block timestamps + # still considered within one NcsBlock + + def _parseGivenActualFrequency(sampsMemMap,ncsBlocks,chanNum,reqFreq,blkOnePredTime): + """ + Parse blocks in file when microsPerSampUsed and sampFreqUsed are known, + filling in an NcsBlocks object. + + PARAMETERS + sampsMemMap: + memmap of Ncs file + ncsBlocks: + result with microsPerSamp and sampFreqUsed set correctly + chanNum: + channel number that should be present in all records + reqFreq: + rounded frequency that all records should contain + blkOnePredTime: + predicted starting time of first block + + RETURN + NcsBlocks object with block locations marked + """ + + + class NlxHeader(OrderedDict): """ Representation of basic information in all 16 kbytes Neuralynx file headers, @@ -860,9 +898,6 @@ class NcsHeader(): """ -ncs_dtype = [('timestamp', 'uint64'), ('channel_id', 'uint32'), ('sample_rate', 'uint32'), - ('nb_valid', 'uint32'), ('samples', 'int16', (BLOCK_SIZE,))] - nev_dtype = [ ('reserved', ' Date: Mon, 12 Oct 2020 11:52:31 -0700 Subject: [PATCH 12/36] Test of building NcsBlocks. --- neo/test/rawiotest/test_neuralynxrawio.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index dce7357ac..64a29988b 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -1,8 +1,10 @@ import unittest +import numpy as np + from neo.rawio.neuralynxrawio import NeuralynxRawIO -from neo.test.rawiotest.common_rawio_test import BaseTestRawIO from neo.rawio.neuralynxrawio import NlxHeader +from neo.test.rawiotest.common_rawio_test import BaseTestRawIO import logging @@ -85,6 +87,17 @@ def test_recording_types(self): hdr = NlxHeader.buildForFile(filename) self.assertEqual(hdr.typeOfRecording(), typeTest[1]) +class TestNcsBlocksFactory(TestNeuralynxRawIO, unittest.TestCase): + """ + Test building NcsBlocks for files of different revisions. + """ + + def test_ncsblocks_partial(self): + filename = self.get_filename_path('Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs') + data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', + offset=NlxHeader.HEADER_SIZE) + self.assertEqual(data0.shape[0],6690) + self.assertEqual(data0['timestamp'][6689],8515800549) # timestamp of last record if __name__ == "__main__": unittest.main() From 409ea4117e2ed035fd1364c3896ba08bfc34fe84 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Tue, 13 Oct 2020 11:52:40 -0700 Subject: [PATCH 13/36] Handle old files with truncated frequency in header and test. --- neo/rawio/neuralynxrawio.py | 146 ++++++++++++++++++++-- neo/test/rawiotest/test_neuralynxrawio.py | 18 +++ 2 files changed, 156 insertions(+), 8 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 380e2cc3c..b24cefe2c 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -592,21 +592,71 @@ def read_ncs_files(self, ncs_filenames): self._sigs_length.append(length) +class WholeMicrosTimePositionBlock(): + """ + Map of time to sample positions. + + Times are rounded to nearest microsecond. Model here is that times + from start of a sample until just before the next sample are included, + that is, closed lower bound and open upper bound on intervals. A + channel with no samples is empty and contains no time intervals. + """ + + _sampFrequency = 0 + _startTime = 0 + _size = 0 + _microsPerSamp = 0 + + @staticmethod + def getMicrosPerSampForFreq(sampFreq): + """ + Compute fractional microseconds per sample. + """ + return 1e6 / sampFreq + + @staticmethod + def calcSampleTime(sampFr, startTime, posn): + """ + Calculate time rounded to microseconds for sample given frequency, + start time, and sample position. + """ + return round(startTime+ + WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(sampFr)*posn) + +class CscRecordHeader(): + """ + Information in header of each Ncs record, excluding sample values themselves. + """ + timestamp = 0 + channel_id = 0 + sample_rate = 0 + nb_valid = 0 + + def __init__(self,ncsMemMap,recn): + """ + Construct a record header for a given record in a memory map for an NcsFile. + """ + self.timestamp = ncsMemMap['timestamp'][recn] + self.channel_id = ncsMemMap['channel_id'][recn] + self.sample_rate = ncsMemMap['sample_rate'][recn] + self.nb_valid = ncsMemMap['nb_valid'][recn] + class NcsBlocks(): """ - Contains information regarding the blocks of records in an Ncs file. + Contains information regarding the contiguous blocks of records in an Ncs file. Factory methods perform parsing of this information from an Ncs file. """ startBlocks = [] endBlocks = [] - sampFreqUsed = 0 - microsPerSampUsed = 0 + sampFreqUsed = 0 # actual sampling frequency of samples + microsPerSampUsed = 0 # microseconds per sample class NcsBlocksFactory(): """ - Class for factory methods which perform parsing of blocks in Ncs files. + Class for factory methods which perform parsing of contiguous blocks of records + in Ncs files. Moved here since algorithm covering all 3 header styles and types used is more complicated. Copied from Java code on Sept 7, 2020. @@ -616,13 +666,13 @@ class NcsBlocksFactory(): _maxGapLength = 5 # maximum gap between predicted and actual block timestamps # still considered within one NcsBlock - def _parseGivenActualFrequency(sampsMemMap,ncsBlocks,chanNum,reqFreq,blkOnePredTime): + def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePredTime): """ - Parse blocks in file when microsPerSampUsed and sampFreqUsed are known, + Parse blocks in memory mapped file when microsPerSampUsed and sampFreqUsed are known, filling in an NcsBlocks object. PARAMETERS - sampsMemMap: + ncsMemMap: memmap of Ncs file ncsBlocks: result with microsPerSamp and sampFreqUsed set correctly @@ -636,8 +686,88 @@ def _parseGivenActualFrequency(sampsMemMap,ncsBlocks,chanNum,reqFreq,blkOnePredT RETURN NcsBlocks object with block locations marked """ - + startBlockPredTime = blkOnePredTime + blkLen = 0 + for recn in range(1, ncsMemMap.shape[0]): + hdr = CscRecordHeader(ncsMemMap, recn) + if hdr.channel_id!=chanNum | hdr.sample_rate!=reqFreq: + raise IOError('Channel number or sampling frequency changed in records within file') + predTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, + startBlockPredTime, blkLen) + nValidSamps = hdr.nb_valid + if hdr.timestamp != predTime: + ncsBlocks.endBlocks.append(recn-1) + ncsBlocks.startBlocks.append(recn) + startBlockPredTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, + hdr.timestamp, + nValidSamps) + blklen = 0 + else: + blkLen += nValidSamps + ncsBlocks.endBlocks.append(ncsMemMap.shape[0] - 1) + + return ncsBlocks + + + def _buildGivenActualFrequency(ncsMemMap, ncsBlocks, reqFreq): + """ + Build NcsBlocks object for file given actual sampling frequency. + + Requires that frequency in each record agrees with requested frequency. This is + normally obtained by rounding the header frequency; however, this value may be different + from the rounded actual frequency used in the recording, since the underlying + requirement in older Ncs files was that the rounded number of whole microseconds + per sample be the same for all records in a block. + + PARAMETERS + ncsMemMap: + memmap of Ncs file + ncsBlocks: + containing the actual sampling frequency used and microsPerSamp for the result + reqFreq: + frequency to require in records + RETURN: + NcsBlocks object + """ + # check frequency in first record + rh0 = CscRecordHeader(ncsMemMap, 0) + if rh0.sample_rate != reqFreq: + raise IOError("Sampling frequency in first record doesn't agree with header.") + chanNum = rh0.channel_id + + # check if file is one block of records, which is often the case, and avoid full parse + lastBlkI = ncsMemMap.shape[0] - 1 + rhl = CscRecordHeader(ncsMemMap, lastBlkI) + predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, rh0.timestamp, + NeuralynxRawIO._BLOCK_SIZE * lastBlkI) + if rhl.channel_id == chanNum and rhl.sample_rate == reqFreq and rhl.timestamp == predLastBlockStartTime: + ncsBlocks.startBlocks.append(0) + ncsBlocks.endBlocks.append(lastBlkI) + return ncsBlocks + + # otherwise need to scan looking for breaks + else: + blkOnePredTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, rh0.timestamp, + rh0.nb_valid) + return NcsBlocksFactory._parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePredTime) + + + def _parseForMaxGap(ncsMemMap, hdr, nomFreq, maxGapLen): + """ + Parse blocks of records from file, allowing a maximum gap in timestamps between records + in blocks. Estimates frequency being used based on timestamps. + + PARAMETERS + ncsMemMap: + memmap of Ncs file + hdr: + CSC record headr information + nomFreq: + nominal frequency to use in computing time for samples (Hz) + maxGapLen: + maximum difference within a block between predicted time of start of record and recorded time + """ class NlxHeader(OrderedDict): """ diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 64a29988b..8c092aed9 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -4,6 +4,8 @@ from neo.rawio.neuralynxrawio import NeuralynxRawIO from neo.rawio.neuralynxrawio import NlxHeader +from neo.rawio.neuralynxrawio import NcsBlocksFactory +from neo.rawio.neuralynxrawio import NcsBlocks from neo.test.rawiotest.common_rawio_test import BaseTestRawIO import logging @@ -99,5 +101,21 @@ def test_ncsblocks_partial(self): self.assertEqual(data0.shape[0],6690) self.assertEqual(data0['timestamp'][6689],8515800549) # timestamp of last record + def testBuildGivenActualFrequency(self): + + # Test early files where the frequency listed in the header is + # floor(1e6/(actual number of microseconds between samples) + filename = self.get_filename_path('Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs') + data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', + offset=NlxHeader.HEADER_SIZE) + ncsBlocks = NcsBlocks() + ncsBlocks.sampFreqUsed = 1/(35e-6) + ncsBlocks.microsPerSampUsed = 35 + ncsBlocks = NcsBlocksFactory._buildGivenActualFrequency(data0, ncsBlocks, 27789) + self.assertEqual(len(ncsBlocks.startBlocks), 1) + self.assertEqual(ncsBlocks.startBlocks[0], 0) + self.assertEqual(len(ncsBlocks.endBlocks), 1) + self.assertEqual(ncsBlocks.endBlocks[0], 9) + if __name__ == "__main__": unittest.main() From 2c3f80b8848c901b5f1af12550eccf1496addabf Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Fri, 16 Oct 2020 11:38:07 -0700 Subject: [PATCH 14/36] Change interface on parse versus build. --- neo/rawio/neuralynxrawio.py | 157 ++++++++++++++++++++-- neo/test/rawiotest/test_neuralynxrawio.py | 2 +- 2 files changed, 144 insertions(+), 15 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index b24cefe2c..be52bfe14 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -28,6 +28,7 @@ import distutils.version import datetime from collections import OrderedDict +import math @@ -110,7 +111,7 @@ def _parse_header(self): self._empty_ncs.append(filename) continue - # All file have more or less the same header structure + # All files have more or less the same header structure info = NlxHeader.buildForFile(filename) chan_names = info['channel_names'] chan_ids = info['channel_ids'] @@ -666,6 +667,7 @@ class NcsBlocksFactory(): _maxGapLength = 5 # maximum gap between predicted and actual block timestamps # still considered within one NcsBlock + @staticmethod def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePredTime): """ Parse blocks in memory mapped file when microsPerSampUsed and sampFreqUsed are known, @@ -675,7 +677,7 @@ def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePre ncsMemMap: memmap of Ncs file ncsBlocks: - result with microsPerSamp and sampFreqUsed set correctly + NcsBlocks with actual sampFreqUsed correct chanNum: channel number that should be present in all records reqFreq: @@ -709,7 +711,8 @@ def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePre return ncsBlocks - def _buildGivenActualFrequency(ncsMemMap, ncsBlocks, reqFreq): + @staticmethod + def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): """ Build NcsBlocks object for file given actual sampling frequency. @@ -736,24 +739,30 @@ def _buildGivenActualFrequency(ncsMemMap, ncsBlocks, reqFreq): raise IOError("Sampling frequency in first record doesn't agree with header.") chanNum = rh0.channel_id + nb = NcsBlocks() + nb.sampFreqUsed = actualSampFreq + nb.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(actualSampFreq) + # check if file is one block of records, which is often the case, and avoid full parse lastBlkI = ncsMemMap.shape[0] - 1 rhl = CscRecordHeader(ncsMemMap, lastBlkI) - predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, rh0.timestamp, + predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(actualSampFreq, rh0.timestamp, NeuralynxRawIO._BLOCK_SIZE * lastBlkI) if rhl.channel_id == chanNum and rhl.sample_rate == reqFreq and rhl.timestamp == predLastBlockStartTime: - ncsBlocks.startBlocks.append(0) - ncsBlocks.endBlocks.append(lastBlkI) - return ncsBlocks + nb = NcsBlocks() + nb.startBlocks.append(0) + nb.endBlocks.append(lastBlkI) + return nb # otherwise need to scan looking for breaks else: - blkOnePredTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, rh0.timestamp, + blkOnePredTime = WholeMicrosTimePositionBlock.calcSampleTime(actualSampFreq, rh0.timestamp, rh0.nb_valid) - return NcsBlocksFactory._parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePredTime) + return NcsBlocksFactory._parseGivenActualFrequency(ncsMemMap, nb, chanNum, reqFreq, blkOnePredTime) - def _parseForMaxGap(ncsMemMap, hdr, nomFreq, maxGapLen): + @staticmethod + def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): """ Parse blocks of records from file, allowing a maximum gap in timestamps between records in blocks. Estimates frequency being used based on timestamps. @@ -761,13 +770,133 @@ def _parseForMaxGap(ncsMemMap, hdr, nomFreq, maxGapLen): PARAMETERS ncsMemMap: memmap of Ncs file - hdr: - CSC record headr information - nomFreq: - nominal frequency to use in computing time for samples (Hz) + ncsBlocks: + NcsBlocks object with sampFreqUsed set to nominal frequency to use in computing time for samples (Hz) maxGapLen: maximum difference within a block between predicted time of start of record and recorded time + + RETURN: + NcsBlocks object with sampFreqUsed and microsPerSamp set based on estimate from largest block + """ + + # track frequency of each block and use estimate with longest block + maxBlkLen = 0 + maxBlkFreqEstimate = 0 + + # Parse the record sequence, finding blocks of continuous time with no more than maxGapLength + # and same channel number + rh0 = CscRecordHeader(ncsMemMap, 0) + chanNum = rh0.channel_id + + startBlockTime = rh0.timestamp + blkLen = rh0.nb_valid + lastRecTime = rh0.timestamp + lastRecNumSamps = rh0.nb_valid + recFreq = rh0.sample_rate + + ncsBlocks.startBlocks.append(0) + for recn in range(1, ncsMemMap.shape[0]): + hdr = CscRecordHeader(ncsMemMap, recn) + if hdr.channel_id != chanNum | hdr.sample_rate != recFreq: + raise IOError('Channel number or sampling frequency changed in records within file') + predTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, lastRecTime, + lastRecNumSamps) + if (abs(hdr.timestamp - predTime) > maxGapLen): + ncsBlocks.endBlocks.append(recn-1) + ncsBlocks.startBlocks.append(recn) + if blkLen > maxBlkLen: + maxBlkLen = blkLen + maxBlkFreqEstimate = (blkLen - lastRecNumSamps) * 1e6 / (lastRecTime - startBlockTime) + startBlockTime = hdr.timestamp + blkLen = hdr.nb_valid + else: + blkLen += hdr.nb_valid + ncsBlocks.append(ncsMemMap.shape[0] - 1) + + ncsBlocks.sampFreqUsed = maxBlkFreqEstimate + ncsBlocks.setMicrosPerSamp = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(maxBlkFreqEstimate) + + return ncsBlocks + + + @staticmethod + def _buildForToleranceAndMaxGap(ncsMemMap, nomFreq): """ + Determine blocks of records in memory mapped Ncs file given a nominal frequency of the file, + using the default values of frequency tolerance and maximum gap between blocks. + + PARAMETERS + ncsMemMap: + memmap of Ncs file + nomFreq: + nominal sampling frequency used, normally from header of file + + RETURN: + NcsBlocks object + """ + nb = NcsBlocks() + + numRecs = ncsMemMap.shape[0] + if numRecs < 1: + return nb + + rh0 = CscRecordHeader(ncsMemMap, 0) + chanNum = rh0.channel_id + + lastBlkI = numRecs - 1 + rhl = CscRecordHeader(ncsMemMap,lastBlkI) + + # check if file is one block of records, to within tolerance, which is often the case + numSampsForPred = NeuralynxRawIO._BLOCK_SIZE * lastBlkI + predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(nomFreq,rh0.timestamp, + numSampsForPred) + freqInFile = math.floor(nomFreq) + if abs(rhl.timestamp - predLastBlockStartTime) / rhl.timestamp < NcsBlocks._tolerance and \ + rhl.channel_id == chanNum and rhl.sample_rate == freqInFile: + nb.endBlocks.append(lastBlkI) + nb.sampFreqUsed = numSampsForPred / (rhl.timestamp - rh0.timestamp) / 1e6 + nb.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(nb.sampFreqUsed) + + # otherwise parse records to determine blocks using default maximum gap length + else: + nb.sampFreqUsed = nomFreq + nb = NcsBlocks._parseForMaxGap(ncsMemMap, nb, NcsBlocks._maxGapLength) + + return nb + + @staticmethod + def buildForNcsFile(ncsMemMap,nlxHdr): + """ + Build an NcsBlocks object for an NcsFile, given as a memmap and NlxHeader, + handling gap detection appropriately given the file type as specified by the header. + + PARAMETERS + ncsMemMap: + memory map of file + acqType: + string specifying type of data acquisition used, one of types returned by NlxHeader.typeOfRecording() + """ + acqType = nlxHdr.typeOfRecording() + + # old Neuralynx style with rounded whole microseconds for the samples + if acqType == "PRE4": + freq = nlxHdr['SamplingFrequency'] + sampFreqUsed = WholeMicrosTimePositionBlock.getFreqForMicrosPerSamp(microsPerSampUsed) + nb = NcsBlocks._buildGivenActualFrequency(ncsMemMap, sampFreqUsed, math.floor(freq)) + nb.microsPerSampUsed = math.floor(WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(freq)) + + # digital lynx style with fractional frequency and micros per samp determined from block times + elif acqType == "DIGITALLYNX" or acqType == "DIGITALLYNXSX": + nomFreq = nlxHdr['SamplingFrequency'] + nb = NcsBlocks._buildForToleranceAndMaxGap(ncsMemMap, nomFreq) + + # BML style with fractional frequency and micros per samp + elif acqType == "BML": + sampFreqUsed = nlxHdr['SamplingFrequency'] + nb = NcsBlocks._buildGivenActualFrequency(ncsMemMap, sampFreqUsed, math.floor(sampFreqUsed)) + + else: + raise TypeError("Unknown Ncs file type from header.") class NlxHeader(OrderedDict): """ diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 8c092aed9..653ae2a68 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -111,7 +111,7 @@ def testBuildGivenActualFrequency(self): ncsBlocks = NcsBlocks() ncsBlocks.sampFreqUsed = 1/(35e-6) ncsBlocks.microsPerSampUsed = 35 - ncsBlocks = NcsBlocksFactory._buildGivenActualFrequency(data0, ncsBlocks, 27789) + ncsBlocks = NcsBlocksFactory._buildGivenActualFrequency(data0, ncsBlocks.sampFreqUsed, 27789) self.assertEqual(len(ncsBlocks.startBlocks), 1) self.assertEqual(ncsBlocks.startBlocks[0], 0) self.assertEqual(len(ncsBlocks.endBlocks), 1) From 60006871a0fe2126adefd1a8d6a93243bd8500c3 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Mon, 19 Oct 2020 11:37:57 -0700 Subject: [PATCH 15/36] Tests for PRE4 type and code corrections. Tests for v5.5.1 still failing. --- neo/rawio/neuralynxrawio.py | 39 +++++++++++++++-------- neo/test/rawiotest/test_neuralynxrawio.py | 31 +++++++++++++++++- 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index be52bfe14..e778f8f02 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -609,11 +609,18 @@ class WholeMicrosTimePositionBlock(): _microsPerSamp = 0 @staticmethod - def getMicrosPerSampForFreq(sampFreq): + def getFreqForMicrosPerSamp(micros): """ - Compute fractional microseconds per sample. + Compute fractional sampling frequency, given microseconds per sample. """ - return 1e6 / sampFreq + return 1e6 / micros + + @staticmethod + def getMicrosPerSampForFreq(sampFr): + """ + Calculate fractional microseconds per sample, given the sampling frequency (Hz). + """ + return 1e6 / sampFr @staticmethod def calcSampleTime(sampFr, startTime, posn): @@ -811,7 +818,7 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): blkLen = hdr.nb_valid else: blkLen += hdr.nb_valid - ncsBlocks.append(ncsMemMap.shape[0] - 1) + ncsBlocks.endBlocks.append(ncsMemMap.shape[0] - 1) ncsBlocks.sampFreqUsed = maxBlkFreqEstimate ncsBlocks.setMicrosPerSamp = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(maxBlkFreqEstimate) @@ -851,7 +858,7 @@ def _buildForToleranceAndMaxGap(ncsMemMap, nomFreq): predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(nomFreq,rh0.timestamp, numSampsForPred) freqInFile = math.floor(nomFreq) - if abs(rhl.timestamp - predLastBlockStartTime) / rhl.timestamp < NcsBlocks._tolerance and \ + if abs(rhl.timestamp - predLastBlockStartTime) / rhl.timestamp < NcsBlocksFactory._tolerance and \ rhl.channel_id == chanNum and rhl.sample_rate == freqInFile: nb.endBlocks.append(lastBlkI) nb.sampFreqUsed = numSampsForPred / (rhl.timestamp - rh0.timestamp) / 1e6 @@ -860,7 +867,8 @@ def _buildForToleranceAndMaxGap(ncsMemMap, nomFreq): # otherwise parse records to determine blocks using default maximum gap length else: nb.sampFreqUsed = nomFreq - nb = NcsBlocks._parseForMaxGap(ncsMemMap, nb, NcsBlocks._maxGapLength) + nb.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(nb.sampFreqUsed) + nb = NcsBlocksFactory._parseForMaxGap(ncsMemMap, nb, NcsBlocksFactory._maxGapLength) return nb @@ -880,24 +888,29 @@ def buildForNcsFile(ncsMemMap,nlxHdr): # old Neuralynx style with rounded whole microseconds for the samples if acqType == "PRE4": - freq = nlxHdr['SamplingFrequency'] + freq = nlxHdr['sampling_rate'] + microsPerSampUsed = math.floor(WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(freq)) sampFreqUsed = WholeMicrosTimePositionBlock.getFreqForMicrosPerSamp(microsPerSampUsed) - nb = NcsBlocks._buildGivenActualFrequency(ncsMemMap, sampFreqUsed, math.floor(freq)) - nb.microsPerSampUsed = math.floor(WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(freq)) + nb = NcsBlocksFactory._buildGivenActualFrequency(ncsMemMap, sampFreqUsed, math.floor(freq)) + nb.sampFreqUsed = sampFreqUsed + nb.microsPerSampUsed = microsPerSampUsed # digital lynx style with fractional frequency and micros per samp determined from block times elif acqType == "DIGITALLYNX" or acqType == "DIGITALLYNXSX": - nomFreq = nlxHdr['SamplingFrequency'] - nb = NcsBlocks._buildForToleranceAndMaxGap(ncsMemMap, nomFreq) + nomFreq = nlxHdr['sampling_rate'] + nb = NcsBlocksFactory._buildForToleranceAndMaxGap(ncsMemMap, nomFreq) # BML style with fractional frequency and micros per samp elif acqType == "BML": - sampFreqUsed = nlxHdr['SamplingFrequency'] - nb = NcsBlocks._buildGivenActualFrequency(ncsMemMap, sampFreqUsed, math.floor(sampFreqUsed)) + sampFreqUsed = nlxHdr['sampling_rate'] + nb = NcsBlocksFactory._buildGivenActualFrequency(ncsMemMap, sampFreqUsed, math.floor(sampFreqUsed)) else: raise TypeError("Unknown Ncs file type from header.") + return nb + + class NlxHeader(OrderedDict): """ Representation of basic information in all 16 kbytes Neuralynx file headers, diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 653ae2a68..1a0a40302 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -109,7 +109,7 @@ def testBuildGivenActualFrequency(self): data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) ncsBlocks = NcsBlocks() - ncsBlocks.sampFreqUsed = 1/(35e-6) + ncsBlocks.sampFreqUsed = 1/35e-6 ncsBlocks.microsPerSampUsed = 35 ncsBlocks = NcsBlocksFactory._buildGivenActualFrequency(data0, ncsBlocks.sampFreqUsed, 27789) self.assertEqual(len(ncsBlocks.startBlocks), 1) @@ -117,5 +117,34 @@ def testBuildGivenActualFrequency(self): self.assertEqual(len(ncsBlocks.endBlocks), 1) self.assertEqual(ncsBlocks.endBlocks[0], 9) + + def testBuildUsingHeaderAndScanning(self): + + # Test early files where the frequency listed in the header is + # floor(1e6/(actual number of microseconds between samples) + filename = self.get_filename_path('Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs') + hdr = NlxHeader.buildForFile(filename) + data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', + offset=NlxHeader.HEADER_SIZE) + nb = NcsBlocksFactory.buildForNcsFile(data0, hdr) + + self.assertEqual(nb.sampFreqUsed, 1/35e-6) + self.assertEqual(nb.microsPerSampUsed, 35) + self.assertEqual(len(nb.startBlocks), 1) + self.assertEqual(nb.startBlocks[0], 0) + self.assertEqual(len(nb.endBlocks), 1) + self.assertEqual(nb.endBlocks[0], 9) + + # test Cheetah 5.5.1, which is DigitalLynxSX + filename = self.get_filename_path('Cheetah_v5.5.1/original_data/Tet3a.ncs') + hdr = NlxHeader.buildForFile(filename) + data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', + offset=NlxHeader.HEADER_SIZE) + nb = NcsBlocksFactory.buildForNcsFile(data0, hdr) + self.assertEqual(nb.sampFreqUsed, 32000) + self.assertEqual(nb.microsPerSampUsed, 31.25) + + + if __name__ == "__main__": unittest.main() From c4e05ae922209e08cb92a818d4554798cd0888ce Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Mon, 26 Oct 2020 15:35:43 -0700 Subject: [PATCH 16/36] Fix initializer, update loop vars. --- neo/rawio/neuralynxrawio.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index e778f8f02..2905f7225 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -652,13 +652,14 @@ def __init__(self,ncsMemMap,recn): class NcsBlocks(): """ Contains information regarding the contiguous blocks of records in an Ncs file. - Factory methods perform parsing of this information from an Ncs file. + Methods of NcsBlocksFactory perform parsing of this information from an Ncs file. """ - startBlocks = [] - endBlocks = [] - sampFreqUsed = 0 # actual sampling frequency of samples - microsPerSampUsed = 0 # microseconds per sample + def __init__(self): + self.startBlocks = [] + self.endBlocks = [] + self.sampFreqUsed = 0 # actual sampling frequency of samples + self.microsPerSampUsed = 0 # microseconds per sample class NcsBlocksFactory(): @@ -818,6 +819,9 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): blkLen = hdr.nb_valid else: blkLen += hdr.nb_valid + lastRecTime = hdr.timestamp + lastRecNumSamps = hdr.nb_valid + ncsBlocks.endBlocks.append(ncsMemMap.shape[0] - 1) ncsBlocks.sampFreqUsed = maxBlkFreqEstimate @@ -858,7 +862,7 @@ def _buildForToleranceAndMaxGap(ncsMemMap, nomFreq): predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(nomFreq,rh0.timestamp, numSampsForPred) freqInFile = math.floor(nomFreq) - if abs(rhl.timestamp - predLastBlockStartTime) / rhl.timestamp < NcsBlocksFactory._tolerance and \ + if abs(rhl.timestamp - predLastBlockStartTime) / (rhl.timestamp - rh0.timestamp) < NcsBlocksFactory._tolerance and \ rhl.channel_id == chanNum and rhl.sample_rate == freqInFile: nb.endBlocks.append(lastBlkI) nb.sampFreqUsed = numSampsForPred / (rhl.timestamp - rh0.timestamp) / 1e6 From 85a05e8244b9a7921958c0ced27b3d784768f4ef Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Mon, 26 Oct 2020 15:40:47 -0700 Subject: [PATCH 17/36] Add additional tests on v5.5.1 with 2 blocks --- neo/test/rawiotest/test_neuralynxrawio.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 1a0a40302..6b999328b 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -135,7 +135,8 @@ def testBuildUsingHeaderAndScanning(self): self.assertEqual(len(nb.endBlocks), 1) self.assertEqual(nb.endBlocks[0], 9) - # test Cheetah 5.5.1, which is DigitalLynxSX + # test Cheetah 5.5.1, which is DigitalLynxSX and has two blocks of records + # with a fairly large gap filename = self.get_filename_path('Cheetah_v5.5.1/original_data/Tet3a.ncs') hdr = NlxHeader.buildForFile(filename) data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', @@ -143,6 +144,12 @@ def testBuildUsingHeaderAndScanning(self): nb = NcsBlocksFactory.buildForNcsFile(data0, hdr) self.assertEqual(nb.sampFreqUsed, 32000) self.assertEqual(nb.microsPerSampUsed, 31.25) + self.assertEqual(len(nb.startBlocks), 2) + self.assertEqual(nb.startBlocks[0], 0) + self.assertEqual(nb.startBlocks[1], 2498) + self.assertEqual(len(nb.endBlocks), 2) + self.assertEqual(nb.endBlocks[0], 2497) + self.assertEqual(nb.endBlocks[1], 3331) From 846edb055b91907519a47980cf1fc7a608a2ab5d Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Mon, 26 Oct 2020 16:22:08 -0700 Subject: [PATCH 18/36] Tests of block construction for incomplete blocks --- neo/test/rawiotest/test_neuralynxrawio.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 6b999328b..c3b3e6bc8 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -101,6 +101,15 @@ def test_ncsblocks_partial(self): self.assertEqual(data0.shape[0],6690) self.assertEqual(data0['timestamp'][6689],8515800549) # timestamp of last record + hdr = NlxHeader.buildForFile(filename) + nb = NcsBlocksFactory.buildForNcsFile(data0, hdr) + self.assertEqual(nb.sampFreqUsed, 32009.05084744305) + self.assertEqual(nb.microsPerSampUsed, 31.241163781021083) + self.assertEqual(len(nb.startBlocks), 1) + self.assertEqual(nb.startBlocks[0], 0) + self.assertEqual(len(nb.endBlocks), 1) + self.assertEqual(nb.endBlocks[0], 6689) + def testBuildGivenActualFrequency(self): # Test early files where the frequency listed in the header is From 5ba611370640164438b5f2eeb96385927d46953e Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Mon, 26 Oct 2020 16:22:29 -0700 Subject: [PATCH 19/36] Fix up single block case. --- neo/rawio/neuralynxrawio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 2905f7225..73e8e4666 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -864,8 +864,9 @@ def _buildForToleranceAndMaxGap(ncsMemMap, nomFreq): freqInFile = math.floor(nomFreq) if abs(rhl.timestamp - predLastBlockStartTime) / (rhl.timestamp - rh0.timestamp) < NcsBlocksFactory._tolerance and \ rhl.channel_id == chanNum and rhl.sample_rate == freqInFile: + nb.startBlocks.append(0) nb.endBlocks.append(lastBlkI) - nb.sampFreqUsed = numSampsForPred / (rhl.timestamp - rh0.timestamp) / 1e6 + nb.sampFreqUsed = numSampsForPred / (rhl.timestamp - rh0.timestamp) * 1e6 nb.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(nb.sampFreqUsed) # otherwise parse records to determine blocks using default maximum gap length From 153446dc38d0d2c41ad71fc8add52609876812c8 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Tue, 27 Oct 2020 08:40:18 -0700 Subject: [PATCH 20/36] Remove unneeded classes. Clean up style. --- neo/rawio/neuralynxrawio.py | 78 ++++++++++++++----------------------- 1 file changed, 29 insertions(+), 49 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 73e8e4666..a95b5c05e 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -31,7 +31,6 @@ import math - class NeuralynxRawIO(BaseRawIO): """" Class for reading datasets recorded by Neuralynx. @@ -53,7 +52,7 @@ class NeuralynxRawIO(BaseRawIO): _BLOCK_SIZE = 512 # nb sample per signal block _ncs_dtype = [('timestamp', 'uint64'), ('channel_id', 'uint32'), ('sample_rate', 'uint32'), - ('nb_valid', 'uint32'), ('samples', 'int16', (_BLOCK_SIZE,))] + ('nb_valid', 'uint32'), ('samples', 'int16', (_BLOCK_SIZE,))] def __init__(self, dirname='', keep_original_times=False, **kargs): """ @@ -235,7 +234,7 @@ def _parse_header(self): # :TODO: current algorithm depends on side-effect of read_ncs_files on # self._sigs_memmap, self._sigs_t_start, self._sigs_t_stop, # self._sigs_length, self._nb_segment, self._timestamp_limits - ncsBlocks = self.read_ncs_files(self.ncs_filenames) + self.read_ncs_files(self.ncs_filenames) # Determine timestamp limits in nev, nse file by scanning them. ts0, ts1 = None, None @@ -470,9 +469,9 @@ def _rescale_event_timestamp(self, event_timestamps, dtype): def read_ncs_files(self, ncs_filenames): """ - Given a list of ncs files, return a dictionary of NcsBlocks indexed by channel uid. + Given a list of ncs files, read their basic structure and setup the following + attributes: - :TODO: Current algorithm has side effects on following attributes: * self._sigs_memmap = [ {} for seg_index in range(self._nb_segment) ] * self._sigs_t_start = [] * self._sigs_t_stop = [] @@ -567,7 +566,7 @@ def read_ncs_files(self, ncs_filenames): for chan_uid, ncs_filename in self.ncs_filenames.items(): data = np.memmap(ncs_filename, dtype=self.ncs_dtype, mode='r', - offset=NlxHeader.HEADER_SIZE) + offset=NlxHeader.HEADER_SIZE) assert data.size == data0.size, 'ncs files do not have the same data length' for seg_index, (i0, i1) in enumerate(gap_pairs): @@ -595,7 +594,7 @@ def read_ncs_files(self, ncs_filenames): class WholeMicrosTimePositionBlock(): """ - Map of time to sample positions. + Wrapper of static calculations of time to sample positions. Times are rounded to nearest microsecond. Model here is that times from start of a sample until just before the next sample are included, @@ -603,11 +602,6 @@ class WholeMicrosTimePositionBlock(): channel with no samples is empty and contains no time intervals. """ - _sampFrequency = 0 - _startTime = 0 - _size = 0 - _microsPerSamp = 0 - @staticmethod def getFreqForMicrosPerSamp(micros): """ @@ -628,19 +622,16 @@ def calcSampleTime(sampFr, startTime, posn): Calculate time rounded to microseconds for sample given frequency, start time, and sample position. """ - return round(startTime+ + return round(startTime + WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(sampFr)*posn) + class CscRecordHeader(): """ Information in header of each Ncs record, excluding sample values themselves. """ - timestamp = 0 - channel_id = 0 - sample_rate = 0 - nb_valid = 0 - def __init__(self,ncsMemMap,recn): + def __init__(self, ncsMemMap, recn): """ Construct a record header for a given record in a memory map for an NcsFile. """ @@ -649,6 +640,7 @@ def __init__(self,ncsMemMap,recn): self.sample_rate = ncsMemMap['sample_rate'][recn] self.nb_valid = ncsMemMap['nb_valid'][recn] + class NcsBlocks(): """ Contains information regarding the contiguous blocks of records in an Ncs file. @@ -659,10 +651,10 @@ def __init__(self): self.startBlocks = [] self.endBlocks = [] self.sampFreqUsed = 0 # actual sampling frequency of samples - self.microsPerSampUsed = 0 # microseconds per sample + self.microsPerSampUsed = 0 # microseconds per sample -class NcsBlocksFactory(): +class NcsBlocksFactory: """ Class for factory methods which perform parsing of contiguous blocks of records in Ncs files. @@ -671,9 +663,8 @@ class NcsBlocksFactory(): more complicated. Copied from Java code on Sept 7, 2020. """ - _tolerance = 0.001 # tolerance for drift of timestamps within one NcsBlock - _maxGapLength = 5 # maximum gap between predicted and actual block timestamps - # still considered within one NcsBlock + _tolerance = 0.001 # tolerance for drift of timestamps within one NcsBlock + _maxGapLength = 5 # maximum gap between predicted and actual block timestamps still considered within one NcsBlock @staticmethod def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePredTime): @@ -700,7 +691,7 @@ def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePre blkLen = 0 for recn in range(1, ncsMemMap.shape[0]): hdr = CscRecordHeader(ncsMemMap, recn) - if hdr.channel_id!=chanNum | hdr.sample_rate!=reqFreq: + if hdr.channel_id != chanNum | hdr.sample_rate != reqFreq: raise IOError('Channel number or sampling frequency changed in records within file') predTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, startBlockPredTime, blkLen) @@ -718,7 +709,6 @@ def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePre return ncsBlocks - @staticmethod def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): """ @@ -755,7 +745,7 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): lastBlkI = ncsMemMap.shape[0] - 1 rhl = CscRecordHeader(ncsMemMap, lastBlkI) predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(actualSampFreq, rh0.timestamp, - NeuralynxRawIO._BLOCK_SIZE * lastBlkI) + NeuralynxRawIO._BLOCK_SIZE * lastBlkI) if rhl.channel_id == chanNum and rhl.sample_rate == reqFreq and rhl.timestamp == predLastBlockStartTime: nb = NcsBlocks() nb.startBlocks.append(0) @@ -765,10 +755,9 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): # otherwise need to scan looking for breaks else: blkOnePredTime = WholeMicrosTimePositionBlock.calcSampleTime(actualSampFreq, rh0.timestamp, - rh0.nb_valid) + rh0.nb_valid) return NcsBlocksFactory._parseGivenActualFrequency(ncsMemMap, nb, chanNum, reqFreq, blkOnePredTime) - @staticmethod def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): """ @@ -808,8 +797,8 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): if hdr.channel_id != chanNum | hdr.sample_rate != recFreq: raise IOError('Channel number or sampling frequency changed in records within file') predTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, lastRecTime, - lastRecNumSamps) - if (abs(hdr.timestamp - predTime) > maxGapLen): + lastRecNumSamps) + if abs(hdr.timestamp - predTime) > maxGapLen: ncsBlocks.endBlocks.append(recn-1) ncsBlocks.startBlocks.append(recn) if blkLen > maxBlkLen: @@ -829,7 +818,6 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): return ncsBlocks - @staticmethod def _buildForToleranceAndMaxGap(ncsMemMap, nomFreq): """ @@ -855,15 +843,16 @@ def _buildForToleranceAndMaxGap(ncsMemMap, nomFreq): chanNum = rh0.channel_id lastBlkI = numRecs - 1 - rhl = CscRecordHeader(ncsMemMap,lastBlkI) + rhl = CscRecordHeader(ncsMemMap, lastBlkI) # check if file is one block of records, to within tolerance, which is often the case numSampsForPred = NeuralynxRawIO._BLOCK_SIZE * lastBlkI - predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(nomFreq,rh0.timestamp, + predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(nomFreq, rh0.timestamp, numSampsForPred) freqInFile = math.floor(nomFreq) - if abs(rhl.timestamp - predLastBlockStartTime) / (rhl.timestamp - rh0.timestamp) < NcsBlocksFactory._tolerance and \ - rhl.channel_id == chanNum and rhl.sample_rate == freqInFile: + if abs(rhl.timestamp - predLastBlockStartTime) / \ + (rhl.timestamp - rh0.timestamp) < NcsBlocksFactory._tolerance and \ + rhl.channel_id == chanNum and rhl.sample_rate == freqInFile: nb.startBlocks.append(0) nb.endBlocks.append(lastBlkI) nb.sampFreqUsed = numSampsForPred / (rhl.timestamp - rh0.timestamp) * 1e6 @@ -871,14 +860,14 @@ def _buildForToleranceAndMaxGap(ncsMemMap, nomFreq): # otherwise parse records to determine blocks using default maximum gap length else: - nb.sampFreqUsed = nomFreq - nb.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(nb.sampFreqUsed) - nb = NcsBlocksFactory._parseForMaxGap(ncsMemMap, nb, NcsBlocksFactory._maxGapLength) + nb.sampFreqUsed = nomFreq + nb.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(nb.sampFreqUsed) + nb = NcsBlocksFactory._parseForMaxGap(ncsMemMap, nb, NcsBlocksFactory._maxGapLength) return nb @staticmethod - def buildForNcsFile(ncsMemMap,nlxHdr): + def buildForNcsFile(ncsMemMap, nlxHdr): """ Build an NcsBlocks object for an NcsFile, given as a memmap and NlxHeader, handling gap detection appropriately given the file type as specified by the header. @@ -931,7 +920,7 @@ def _to_bool(txt): elif txt == 'False': return False else: - raise Exception('Can not convert %s to bool' % (txt)) + raise Exception('Can not convert %s to bool' % txt) # keys that may be present in header which we parse txt_header_keys = [ @@ -1110,8 +1099,6 @@ def buildForFile(filename): else: hpd = NlxHeader.header_pattern_dicts['def'] - original_filename = re.search(hpd['filename_regex'], txt_header).groupdict()['filename'] - # opening time dt1 = re.search(hpd['datetime1_regex'], txt_header).groupdict() info['recording_opened'] = datetime.datetime.strptime( @@ -1168,13 +1155,6 @@ def typeOfRecording(self): return 'UNKNOWN' -class NcsHeader(): - """ - Representation of information in Ncs file headers, including exact - recording type. - """ - - nev_dtype = [ ('reserved', ' Date: Tue, 27 Oct 2020 09:03:56 -0700 Subject: [PATCH 21/36] Use private dtype by new private name. --- neo/rawio/neuralynxrawio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index a95b5c05e..9bf80804c 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -499,7 +499,7 @@ def read_ncs_files(self, ncs_filenames): chan_uid0 = list(ncs_filenames.keys())[0] filename0 = ncs_filenames[chan_uid0] - data0 = np.memmap(filename0, dtype=self.ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) + data0 = np.memmap(filename0, dtype=self._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) gap_indexes = None lost_indexes = None @@ -565,7 +565,7 @@ def read_ncs_files(self, ncs_filenames): # create segment with subdata block/t_start/t_stop/length for chan_uid, ncs_filename in self.ncs_filenames.items(): - data = np.memmap(ncs_filename, dtype=self.ncs_dtype, mode='r', + data = np.memmap(ncs_filename, dtype=self._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) assert data.size == data0.size, 'ncs files do not have the same data length' From b6ecbcd0b78147c80e82eb19324acb5693bec228 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Tue, 27 Oct 2020 09:48:42 -0700 Subject: [PATCH 22/36] Add test of side effects of read_ncs_files --- neo/test/rawiotest/test_neuralynxrawio.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index c3b3e6bc8..2a6000bbb 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -66,6 +66,21 @@ class TestNeuralynxRawIO(BaseTestRawIO, unittest.TestCase, ): 'Cheetah_v6.3.2/incomplete_blocks/Events.nev', 'Cheetah_v6.3.2/incomplete_blocks/README.txt'] + def test_read_ncs_files_sideeffects(self): + + # Test Cheetah 5.5.1, which is DigitalLynxSX and has two blocks of records + # with a fairly large gap. + rawio = NeuralynxRawIO(self.get_filename_path('Cheetah_v5.5.1/original_data')) + rawio.parse_header() + self.assertEqual(rawio._nb_segment, 2) + self.assertListEqual(rawio._timestamp_limits,[(26122557633, 26162525633), + (26366360633, 26379704633)]) + self.assertListEqual(rawio._sigs_length,[1278976, 427008]) + self.assertListEqual(rawio._sigs_t_stop,[26162.525633, 26379.704633]) + self.assertListEqual(rawio._sigs_t_start,[26122.557633, 26366.360633]) + self.assertEqual(len(rawio._sigs_memmap),2) # check only that there are 2 memmaps + + class TestNcsRecordingType(TestNeuralynxRawIO, unittest.TestCase): """ From 6982c597f8b56b872e3d781569143609d63f2342 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Tue, 27 Oct 2020 11:34:40 -0700 Subject: [PATCH 23/36] Use NcsBlocksFactory and logical or. --- neo/rawio/neuralynxrawio.py | 121 ++++++++++++++---------------------- 1 file changed, 45 insertions(+), 76 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 9bf80804c..c0dec5e1f 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -479,116 +479,85 @@ def read_ncs_files(self, ncs_filenames): * self._nb_segment * self._timestamp_limits - The first file is read entirely to detect gaps in timestamp. - each gap lead to a new segment. - - Other files are not read entirely but we check than gaps - are at the same place. - - - gap_indexes can be given (when cached) to avoid full read. - + Files will be scanned to determine the blocks of records. If file is a single block of records, + this scan is brief, otherwise it will check each record which may take some time. """ + # :TODO: Needs to account for gaps and start and end times potentially # being different in different groups of channels. These groups typically # correspond to the channels collected by a single ADC card. if len(ncs_filenames) == 0: return None - good_delta = int(self._BLOCK_SIZE * 1e6 / self._sigs_sampling_rate) chan_uid0 = list(ncs_filenames.keys())[0] filename0 = ncs_filenames[chan_uid0] + # parse the structure of the first file data0 = np.memmap(filename0, dtype=self._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) - - gap_indexes = None - lost_indexes = None - - if self.use_cache: - gap_indexes = self._cache.get('gap_indexes') - lost_indexes = self._cache.get('lost_indexes') - - # detect gaps on first file - if (gap_indexes is None) or (lost_indexes is None): - - # this can be long!!!! - timestamps0 = data0['timestamp'] - deltas0 = np.diff(timestamps0) - - # :TODO: This algorithm needs to account for older style files which had a rounded - # off sampling rate in the header. - # - # It should be that: - # gap_indexes, = np.nonzero(deltas0!=good_delta) - # but for a file I have found many deltas0==15999, 16000, 16001 (for sampling at 32000) - # I guess this is a round problem - # So this is the same with a tolerance of 1 or 2 ticks - max_tolerance = 2 - mask = np.abs((deltas0 - good_delta).astype('int64')) > max_tolerance - - gap_indexes, = np.nonzero(mask) - - if self.use_cache: - self.add_in_cache(gap_indexes=gap_indexes) - - # update for lost_indexes - # Sometimes NLX writes a faulty block, but it then validates how much samples it wrote - # the validation field is in delta0['nb_valid'], it should be equal to BLOCK_SIZE - # :TODO: this algorithm ignores samples in partially filled blocks, which - # is not strictly necessary as all channels might have same partially filled - # blocks at the end. - - lost_indexes, = np.nonzero(data0['nb_valid'] < self._BLOCK_SIZE) - - if self.use_cache: - self.add_in_cache(lost_indexes=lost_indexes) - - gap_candidates = np.unique([0] - + [data0.size] - + (gap_indexes + 1).tolist() - + lost_indexes.tolist()) # linear - - gap_pairs = np.vstack([gap_candidates[:-1], gap_candidates[1:]]).T # 2D (n_segments, 2) + hdr0 = NlxHeader.buildForFile(filename0) + nb0 = NcsBlocksFactory.buildForNcsFile(data0,hdr0) # construct proper gap ranges free of lost samples artifacts minimal_segment_length = 1 # in blocks - goodpairs = np.diff(gap_pairs, 1).reshape(-1) > minimal_segment_length - gap_pairs = gap_pairs[goodpairs] # ensures a segment is at least a block wide - self._nb_segment = len(gap_pairs) + self._nb_segment = len(nb0.startBlocks) self._sigs_memmap = [{} for seg_index in range(self._nb_segment)] self._sigs_t_start = [] self._sigs_t_stop = [] self._sigs_length = [] self._timestamp_limits = [] - # create segment with subdata block/t_start/t_stop/length + # create segment with subdata block/t_start/t_stop/length for each channel for chan_uid, ncs_filename in self.ncs_filenames.items(): - data = np.memmap(ncs_filename, dtype=self._ncs_dtype, mode='r', - offset=NlxHeader.HEADER_SIZE) - assert data.size == data0.size, 'ncs files do not have the same data length' + if chan_uid == chan_uid0: + data = data0 + hdr = hdr0 + nb = nb0 + else: + data = np.memmap(ncs_filename, dtype=self._ncs_dtype, mode='r', + offset=NlxHeader.HEADER_SIZE) + hdr = NlxHeader.buildForFile(ncs_filename) + nb = NcsBlocksFactory.buildForNcsFile(data, hdr) + + # Check that record block structure of each file is identical to the first. + if len(nb.startBlocks) != len(nb0.startBlocks) or len(nb.endBlocks) != len(nb0.endBlocks): + raise IOError('ncs files have different numbers of blocks of records') + + for i, sbi in enumerate(nb.startBlocks): + if (sbi != nb0.startBlocks[i]): + raise IOError('ncs files have different start block structure') + + for i, ebi in enumerate(nb.endBlocks): + if (ebi != nb0.endBlocks[i]): + raise IOError('ncs files have different end block structure') - for seg_index, (i0, i1) in enumerate(gap_pairs): + # create a memmap for each record block + for seg_index in range(len(nb.startBlocks)): - assert data[i0]['timestamp'] == data0[i0][ - 'timestamp'], 'ncs files do not have the same gaps' - assert data[i1 - 1]['timestamp'] == data0[i1 - 1][ - 'timestamp'], 'ncs files do not have the same gaps' + if (data[nb.startBlocks[seg_index]]['timestamp'] != + data0[nb0.startBlocks[seg_index]]['timestamp'] or + data[nb.endBlocks[seg_index]]['timestamp'] != + data0[nb0.endBlocks[seg_index]]['timestamp']) : + raise IOError('ncs files have different timestamp structure') - subdata = data[i0:i1] + subdata = data[nb.startBlocks[seg_index]:nb.endBlocks[seg_index]] self._sigs_memmap[seg_index][chan_uid] = subdata if chan_uid == chan_uid0: + numSampsLastBlock = subdata[-1]['nb_valid'] ts0 = subdata[0]['timestamp'] - ts1 = subdata[-1]['timestamp'] +\ - np.uint64(self._BLOCK_SIZE / self._sigs_sampling_rate * 1e6) + ts1 = WholeMicrosTimePositionBlock.calcSampleTime(nb0.sampFreqUsed, + subdata[-1]['timestamp'], + numSampsLastBlock) self._timestamp_limits.append((ts0, ts1)) t_start = ts0 / 1e6 self._sigs_t_start.append(t_start) t_stop = ts1 / 1e6 self._sigs_t_stop.append(t_stop) - length = subdata.size * self._BLOCK_SIZE + # :TODO: this should really be the total of block lengths, but this allows + # the last block to be shorter, the most common case + length = (subdata.size - 1) * self._BLOCK_SIZE + numSampsLastBlock self._sigs_length.append(length) @@ -794,7 +763,7 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): ncsBlocks.startBlocks.append(0) for recn in range(1, ncsMemMap.shape[0]): hdr = CscRecordHeader(ncsMemMap, recn) - if hdr.channel_id != chanNum | hdr.sample_rate != recFreq: + if hdr.channel_id != chanNum or hdr.sample_rate != recFreq: raise IOError('Channel number or sampling frequency changed in records within file') predTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, lastRecTime, lastRecNumSamps) From d9ade17b0201883f4e909845232973b30cd63f1f Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Wed, 28 Oct 2020 15:36:46 -0700 Subject: [PATCH 24/36] Fix off by one in range for list. Comments. --- neo/rawio/neuralynxrawio.py | 18 ++++++++++-------- neo/test/rawiotest/test_neuralynxrawio.py | 1 + 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index c0dec5e1f..02cfdbade 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -541,22 +541,24 @@ def read_ncs_files(self, ncs_filenames): data0[nb0.endBlocks[seg_index]]['timestamp']) : raise IOError('ncs files have different timestamp structure') - subdata = data[nb.startBlocks[seg_index]:nb.endBlocks[seg_index]] + subdata = data[nb.startBlocks[seg_index]:(nb.endBlocks[seg_index]+1)] self._sigs_memmap[seg_index][chan_uid] = subdata if chan_uid == chan_uid0: numSampsLastBlock = subdata[-1]['nb_valid'] ts0 = subdata[0]['timestamp'] - ts1 = WholeMicrosTimePositionBlock.calcSampleTime(nb0.sampFreqUsed, - subdata[-1]['timestamp'], - numSampsLastBlock) + ts1 = subdata[-1]['timestamp'] +\ + np.uint64(self._BLOCK_SIZE / self._sigs_sampling_rate * 1e6) + # ts1 = WholeMicrosTimePositionBlock.calcSampleTime(nb0.sampFreqUsed, + # subdata[-1]['timestamp'], + # numSampsLastBlock) self._timestamp_limits.append((ts0, ts1)) t_start = ts0 / 1e6 self._sigs_t_start.append(t_start) t_stop = ts1 / 1e6 self._sigs_t_stop.append(t_stop) - # :TODO: this should really be the total of block lengths, but this allows - # the last block to be shorter, the most common case + # :TODO: this should really be the total of nb_valid in records, but this allows + # the last record of a block to be shorter, the most common case length = (subdata.size - 1) * self._BLOCK_SIZE + numSampsLastBlock self._sigs_length.append(length) @@ -617,8 +619,8 @@ class NcsBlocks(): """ def __init__(self): - self.startBlocks = [] - self.endBlocks = [] + self.startBlocks = [] # index of starting record for each block + self.endBlocks = [] # index of last record (inclusive) for each block self.sampFreqUsed = 0 # actual sampling frequency of samples self.microsPerSampUsed = 0 # microseconds per sample diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 2a6000bbb..c15bfa822 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -73,6 +73,7 @@ def test_read_ncs_files_sideeffects(self): rawio = NeuralynxRawIO(self.get_filename_path('Cheetah_v5.5.1/original_data')) rawio.parse_header() self.assertEqual(rawio._nb_segment, 2) + # test values here from direct inspection of .ncs files self.assertListEqual(rawio._timestamp_limits,[(26122557633, 26162525633), (26366360633, 26379704633)]) self.assertListEqual(rawio._sigs_length,[1278976, 427008]) From 4af130c0bbc2767cfb414407bb3d53b8afcb9798 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Wed, 28 Oct 2020 15:40:27 -0700 Subject: [PATCH 25/36] Use standard time calculation for last time of block. --- neo/rawio/neuralynxrawio.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 02cfdbade..6c7ae89fe 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -547,18 +547,17 @@ def read_ncs_files(self, ncs_filenames): if chan_uid == chan_uid0: numSampsLastBlock = subdata[-1]['nb_valid'] ts0 = subdata[0]['timestamp'] - ts1 = subdata[-1]['timestamp'] +\ - np.uint64(self._BLOCK_SIZE / self._sigs_sampling_rate * 1e6) - # ts1 = WholeMicrosTimePositionBlock.calcSampleTime(nb0.sampFreqUsed, - # subdata[-1]['timestamp'], - # numSampsLastBlock) + ts1 = WholeMicrosTimePositionBlock.calcSampleTime(nb0.sampFreqUsed, + subdata[-1]['timestamp'], + numSampsLastBlock) self._timestamp_limits.append((ts0, ts1)) t_start = ts0 / 1e6 self._sigs_t_start.append(t_start) t_stop = ts1 / 1e6 self._sigs_t_stop.append(t_stop) - # :TODO: this should really be the total of nb_valid in records, but this allows - # the last record of a block to be shorter, the most common case + # :TODO: This should really be the total of nb_valid in records, but this allows + # the last record of a block to be shorter, the most common case. Have never + # seen a block of records with not full records before the last. length = (subdata.size - 1) * self._BLOCK_SIZE + numSampsLastBlock self._sigs_length.append(length) From 88880e019607aff8a8a8bedcdc7e38d52d076a12 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Wed, 28 Oct 2020 17:52:04 -0700 Subject: [PATCH 26/36] Remove test with tolerance over whole length. Fix microsPerSampUsed assignement. Using a tolerance over a longer experiment is not sensitive enough to detect blocks where perhaps a large amount of samples are dropped and there is a small gap afterwards. --- neo/rawio/neuralynxrawio.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 6c7ae89fe..34ad5975d 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -633,7 +633,6 @@ class NcsBlocksFactory: more complicated. Copied from Java code on Sept 7, 2020. """ - _tolerance = 0.001 # tolerance for drift of timestamps within one NcsBlock _maxGapLength = 5 # maximum gap between predicted and actual block timestamps still considered within one NcsBlock @staticmethod @@ -784,12 +783,12 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): ncsBlocks.endBlocks.append(ncsMemMap.shape[0] - 1) ncsBlocks.sampFreqUsed = maxBlkFreqEstimate - ncsBlocks.setMicrosPerSamp = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(maxBlkFreqEstimate) + ncsBlocks.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(maxBlkFreqEstimate) return ncsBlocks @staticmethod - def _buildForToleranceAndMaxGap(ncsMemMap, nomFreq): + def _buildForMaxGap(ncsMemMap, nomFreq): """ Determine blocks of records in memory mapped Ncs file given a nominal frequency of the file, using the default values of frequency tolerance and maximum gap between blocks. @@ -815,13 +814,12 @@ def _buildForToleranceAndMaxGap(ncsMemMap, nomFreq): lastBlkI = numRecs - 1 rhl = CscRecordHeader(ncsMemMap, lastBlkI) - # check if file is one block of records, to within tolerance, which is often the case + # check if file is one block of records, with exact timestamp match, which may be the case numSampsForPred = NeuralynxRawIO._BLOCK_SIZE * lastBlkI predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(nomFreq, rh0.timestamp, numSampsForPred) freqInFile = math.floor(nomFreq) - if abs(rhl.timestamp - predLastBlockStartTime) / \ - (rhl.timestamp - rh0.timestamp) < NcsBlocksFactory._tolerance and \ + if abs(rhl.timestamp - predLastBlockStartTime) == 0 and \ rhl.channel_id == chanNum and rhl.sample_rate == freqInFile: nb.startBlocks.append(0) nb.endBlocks.append(lastBlkI) @@ -862,7 +860,7 @@ def buildForNcsFile(ncsMemMap, nlxHdr): # digital lynx style with fractional frequency and micros per samp determined from block times elif acqType == "DIGITALLYNX" or acqType == "DIGITALLYNXSX": nomFreq = nlxHdr['sampling_rate'] - nb = NcsBlocksFactory._buildForToleranceAndMaxGap(ncsMemMap, nomFreq) + nb = NcsBlocksFactory._buildForMaxGap(ncsMemMap, nomFreq) # BML style with fractional frequency and micros per samp elif acqType == "BML": From 6b8b64a3f1638261d9b306c78fb6d48e597edb22 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Wed, 28 Oct 2020 17:53:37 -0700 Subject: [PATCH 27/36] Tests of raw io for incomplete records multiple block case. --- neo/test/rawiotest/test_neuralynxrawio.py | 26 +++++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index c15bfa822..2feded960 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -72,8 +72,8 @@ def test_read_ncs_files_sideeffects(self): # with a fairly large gap. rawio = NeuralynxRawIO(self.get_filename_path('Cheetah_v5.5.1/original_data')) rawio.parse_header() - self.assertEqual(rawio._nb_segment, 2) # test values here from direct inspection of .ncs files + self.assertEqual(rawio._nb_segment, 2) self.assertListEqual(rawio._timestamp_limits,[(26122557633, 26162525633), (26366360633, 26379704633)]) self.assertListEqual(rawio._sigs_length,[1278976, 427008]) @@ -81,6 +81,19 @@ def test_read_ncs_files_sideeffects(self): self.assertListEqual(rawio._sigs_t_start,[26122.557633, 26366.360633]) self.assertEqual(len(rawio._sigs_memmap),2) # check only that there are 2 memmaps + # Test Cheetah 6.3.2, the incomplete_blocks test. This is a DigitalLynxSX with + # three blocks of records. Gaps are on the order of 16 ms or so. + rawio = NeuralynxRawIO(self.get_filename_path('Cheetah_v6.3.2/incomplete_blocks')) + rawio.parse_header() + # test values here from direct inspection of .ncs file + self.assertEqual(rawio._nb_segment, 3) + self.assertListEqual(rawio._timestamp_limits,[(8408806811, 8427831990), + (8427832053, 8487768498), + (8487768561, 8515816549)]) + self.assertListEqual(rawio._sigs_length,[608806, 1917967, 897536]) + self.assertListEqual(rawio._sigs_t_stop,[8427.831990, 8487.768498, 8515.816549]) + self.assertListEqual(rawio._sigs_t_start,[8408.806811, 8427.832053, 8487.768561]) + self.assertEqual(len(rawio._sigs_memmap),3) # check only that there are 3 memmaps class TestNcsRecordingType(TestNeuralynxRawIO, unittest.TestCase): @@ -105,6 +118,7 @@ def test_recording_types(self): hdr = NlxHeader.buildForFile(filename) self.assertEqual(hdr.typeOfRecording(), typeTest[1]) + class TestNcsBlocksFactory(TestNeuralynxRawIO, unittest.TestCase): """ Test building NcsBlocks for files of different revisions. @@ -119,12 +133,10 @@ def test_ncsblocks_partial(self): hdr = NlxHeader.buildForFile(filename) nb = NcsBlocksFactory.buildForNcsFile(data0, hdr) - self.assertEqual(nb.sampFreqUsed, 32009.05084744305) - self.assertEqual(nb.microsPerSampUsed, 31.241163781021083) - self.assertEqual(len(nb.startBlocks), 1) - self.assertEqual(nb.startBlocks[0], 0) - self.assertEqual(len(nb.endBlocks), 1) - self.assertEqual(nb.endBlocks[0], 6689) + self.assertEqual(nb.sampFreqUsed, 32000.012813673042) + self.assertEqual(nb.microsPerSampUsed, 31.249987486652431) + self.assertListEqual(nb.startBlocks, [0, 1190, 4937]) + self.assertListEqual(nb.endBlocks, [1189, 4936, 6689]) def testBuildGivenActualFrequency(self): From 81f14b22cdd2d7f9d09d6bd785218ce3ab168522 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Wed, 28 Oct 2020 18:04:40 -0700 Subject: [PATCH 28/36] Update stop times to include time for samples in partially filled records. --- neo/test/iotest/test_neuralynxio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/test/iotest/test_neuralynxio.py b/neo/test/iotest/test_neuralynxio.py index 116515940..cfa4dd384 100644 --- a/neo/test/iotest/test_neuralynxio.py +++ b/neo/test/iotest/test_neuralynxio.py @@ -308,7 +308,7 @@ def test_incomplete_block_handling_v632(self): for t, gt in zip(nio._sigs_t_start, [8408.806811, 8427.832053, 8487.768561]): self.assertEqual(np.round(t, 4), np.round(gt, 4)) - for t, gt in zip(nio._sigs_t_stop, [8427.830803, 8487.768029, 8515.816549]): + for t, gt in zip(nio._sigs_t_stop, [8427.831990, 8487.768498, 8515.816549]): self.assertEqual(np.round(t, 4), np.round(gt, 4)) From 4ed56794657ca4481a504869ca41e7c135c752b8 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Wed, 28 Oct 2020 18:21:35 -0700 Subject: [PATCH 29/36] PEP and style cleanup. Corrected gap comment. --- neo/rawio/neuralynxrawio.py | 20 +++++------ neo/test/rawiotest/test_neuralynxrawio.py | 42 +++++++++++------------ 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 34ad5975d..1f4edbf38 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -164,7 +164,7 @@ def _parse_header(self): dtype = get_nse_or_ntt_dtype(info, ext) - if (os.path.getsize(filename) <= NlxHeader.HEADER_SIZE): + if os.path.getsize(filename) <= NlxHeader.HEADER_SIZE: self._empty_nse_ntt.append(filename) data = np.zeros((0,), dtype=dtype) else: @@ -197,7 +197,7 @@ def _parse_header(self): # each ('event_id', 'ttl_input') give a new event channel self.nev_filenames[chan_id] = filename - if (os.path.getsize(filename) <= NlxHeader.HEADER_SIZE): + if os.path.getsize(filename) <= NlxHeader.HEADER_SIZE: self._empty_nev.append(filename) data = np.zeros((0,), dtype=nev_dtype) internal_ids = [] @@ -495,7 +495,7 @@ def read_ncs_files(self, ncs_filenames): # parse the structure of the first file data0 = np.memmap(filename0, dtype=self._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) hdr0 = NlxHeader.buildForFile(filename0) - nb0 = NcsBlocksFactory.buildForNcsFile(data0,hdr0) + nb0 = NcsBlocksFactory.buildForNcsFile(data0, hdr0) # construct proper gap ranges free of lost samples artifacts minimal_segment_length = 1 # in blocks @@ -525,11 +525,11 @@ def read_ncs_files(self, ncs_filenames): raise IOError('ncs files have different numbers of blocks of records') for i, sbi in enumerate(nb.startBlocks): - if (sbi != nb0.startBlocks[i]): + if sbi != nb0.startBlocks[i]: raise IOError('ncs files have different start block structure') for i, ebi in enumerate(nb.endBlocks): - if (ebi != nb0.endBlocks[i]): + if ebi != nb0.endBlocks[i]: raise IOError('ncs files have different end block structure') # create a memmap for each record block @@ -538,7 +538,7 @@ def read_ncs_files(self, ncs_filenames): if (data[nb.startBlocks[seg_index]]['timestamp'] != data0[nb0.startBlocks[seg_index]]['timestamp'] or data[nb.endBlocks[seg_index]]['timestamp'] != - data0[nb0.endBlocks[seg_index]]['timestamp']) : + data0[nb0.endBlocks[seg_index]]['timestamp']): raise IOError('ncs files have different timestamp structure') subdata = data[nb.startBlocks[seg_index]:(nb.endBlocks[seg_index]+1)] @@ -548,7 +548,7 @@ def read_ncs_files(self, ncs_filenames): numSampsLastBlock = subdata[-1]['nb_valid'] ts0 = subdata[0]['timestamp'] ts1 = WholeMicrosTimePositionBlock.calcSampleTime(nb0.sampFreqUsed, - subdata[-1]['timestamp'], + subdata[-1]['timestamp'], numSampsLastBlock) self._timestamp_limits.append((ts0, ts1)) t_start = ts0 / 1e6 @@ -562,7 +562,7 @@ def read_ncs_files(self, ncs_filenames): self._sigs_length.append(length) -class WholeMicrosTimePositionBlock(): +class WholeMicrosTimePositionBlock: """ Wrapper of static calculations of time to sample positions. @@ -596,7 +596,7 @@ def calcSampleTime(sampFr, startTime, posn): WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(sampFr)*posn) -class CscRecordHeader(): +class CscRecordHeader: """ Information in header of each Ncs record, excluding sample values themselves. """ @@ -611,7 +611,7 @@ def __init__(self, ncsMemMap, recn): self.nb_valid = ncsMemMap['nb_valid'][recn] -class NcsBlocks(): +class NcsBlocks: """ Contains information regarding the contiguous blocks of records in an Ncs file. Methods of NcsBlocksFactory perform parsing of this information from an Ncs file. diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 2feded960..a11996cef 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -74,26 +74,26 @@ def test_read_ncs_files_sideeffects(self): rawio.parse_header() # test values here from direct inspection of .ncs files self.assertEqual(rawio._nb_segment, 2) - self.assertListEqual(rawio._timestamp_limits,[(26122557633, 26162525633), - (26366360633, 26379704633)]) - self.assertListEqual(rawio._sigs_length,[1278976, 427008]) - self.assertListEqual(rawio._sigs_t_stop,[26162.525633, 26379.704633]) - self.assertListEqual(rawio._sigs_t_start,[26122.557633, 26366.360633]) - self.assertEqual(len(rawio._sigs_memmap),2) # check only that there are 2 memmaps + self.assertListEqual(rawio._timestamp_limits, [(26122557633, 26162525633), + (26366360633, 26379704633)]) + self.assertListEqual(rawio._sigs_length, [1278976, 427008]) + self.assertListEqual(rawio._sigs_t_stop, [26162.525633, 26379.704633]) + self.assertListEqual(rawio._sigs_t_start, [26122.557633, 26366.360633]) + self.assertEqual(len(rawio._sigs_memmap), 2) # check only that there are 2 memmaps # Test Cheetah 6.3.2, the incomplete_blocks test. This is a DigitalLynxSX with - # three blocks of records. Gaps are on the order of 16 ms or so. + # three blocks of records. Gaps are on the order of 60 microseconds or so. rawio = NeuralynxRawIO(self.get_filename_path('Cheetah_v6.3.2/incomplete_blocks')) rawio.parse_header() # test values here from direct inspection of .ncs file self.assertEqual(rawio._nb_segment, 3) - self.assertListEqual(rawio._timestamp_limits,[(8408806811, 8427831990), - (8427832053, 8487768498), - (8487768561, 8515816549)]) - self.assertListEqual(rawio._sigs_length,[608806, 1917967, 897536]) - self.assertListEqual(rawio._sigs_t_stop,[8427.831990, 8487.768498, 8515.816549]) - self.assertListEqual(rawio._sigs_t_start,[8408.806811, 8427.832053, 8487.768561]) - self.assertEqual(len(rawio._sigs_memmap),3) # check only that there are 3 memmaps + self.assertListEqual(rawio._timestamp_limits, [(8408806811, 8427831990), + (8427832053, 8487768498), + (8487768561, 8515816549)]) + self.assertListEqual(rawio._sigs_length, [608806, 1917967, 897536]) + self.assertListEqual(rawio._sigs_t_stop, [8427.831990, 8487.768498, 8515.816549]) + self.assertListEqual(rawio._sigs_t_start, [8408.806811, 8427.832053, 8487.768561]) + self.assertEqual(len(rawio._sigs_memmap), 3) # check only that there are 3 memmaps class TestNcsRecordingType(TestNeuralynxRawIO, unittest.TestCase): @@ -127,9 +127,9 @@ class TestNcsBlocksFactory(TestNeuralynxRawIO, unittest.TestCase): def test_ncsblocks_partial(self): filename = self.get_filename_path('Cheetah_v6.3.2/incomplete_blocks/CSC1_reduced.ncs') data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', - offset=NlxHeader.HEADER_SIZE) - self.assertEqual(data0.shape[0],6690) - self.assertEqual(data0['timestamp'][6689],8515800549) # timestamp of last record + offset=NlxHeader.HEADER_SIZE) + self.assertEqual(data0.shape[0], 6690) + self.assertEqual(data0['timestamp'][6689], 8515800549) # timestamp of last record hdr = NlxHeader.buildForFile(filename) nb = NcsBlocksFactory.buildForNcsFile(data0, hdr) @@ -144,7 +144,7 @@ def testBuildGivenActualFrequency(self): # floor(1e6/(actual number of microseconds between samples) filename = self.get_filename_path('Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs') data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', - offset=NlxHeader.HEADER_SIZE) + offset=NlxHeader.HEADER_SIZE) ncsBlocks = NcsBlocks() ncsBlocks.sampFreqUsed = 1/35e-6 ncsBlocks.microsPerSampUsed = 35 @@ -154,7 +154,6 @@ def testBuildGivenActualFrequency(self): self.assertEqual(len(ncsBlocks.endBlocks), 1) self.assertEqual(ncsBlocks.endBlocks[0], 9) - def testBuildUsingHeaderAndScanning(self): # Test early files where the frequency listed in the header is @@ -162,7 +161,7 @@ def testBuildUsingHeaderAndScanning(self): filename = self.get_filename_path('Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs') hdr = NlxHeader.buildForFile(filename) data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', - offset=NlxHeader.HEADER_SIZE) + offset=NlxHeader.HEADER_SIZE) nb = NcsBlocksFactory.buildForNcsFile(data0, hdr) self.assertEqual(nb.sampFreqUsed, 1/35e-6) @@ -177,7 +176,7 @@ def testBuildUsingHeaderAndScanning(self): filename = self.get_filename_path('Cheetah_v5.5.1/original_data/Tet3a.ncs') hdr = NlxHeader.buildForFile(filename) data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', - offset=NlxHeader.HEADER_SIZE) + offset=NlxHeader.HEADER_SIZE) nb = NcsBlocksFactory.buildForNcsFile(data0, hdr) self.assertEqual(nb.sampFreqUsed, 32000) self.assertEqual(nb.microsPerSampUsed, 31.25) @@ -189,6 +188,5 @@ def testBuildUsingHeaderAndScanning(self): self.assertEqual(nb.endBlocks[1], 3331) - if __name__ == "__main__": unittest.main() From 82c93d93564a28c120991f82b390030347905b1d Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Thu, 29 Oct 2020 08:18:56 -0700 Subject: [PATCH 30/36] Line shortening for PEP8. --- neo/rawio/neuralynxrawio.py | 92 +++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 1f4edbf38..473d7e6a7 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -479,8 +479,9 @@ def read_ncs_files(self, ncs_filenames): * self._nb_segment * self._timestamp_limits - Files will be scanned to determine the blocks of records. If file is a single block of records, - this scan is brief, otherwise it will check each record which may take some time. + Files will be scanned to determine the blocks of records. If file is a single + block of records, this scan is brief, otherwise it will check each record which may + take some time. """ # :TODO: Needs to account for gaps and start and end times potentially @@ -521,7 +522,8 @@ def read_ncs_files(self, ncs_filenames): nb = NcsBlocksFactory.buildForNcsFile(data, hdr) # Check that record block structure of each file is identical to the first. - if len(nb.startBlocks) != len(nb0.startBlocks) or len(nb.endBlocks) != len(nb0.endBlocks): + if len(nb.startBlocks) != len(nb0.startBlocks) or len(nb.endBlocks) != \ + len(nb0.endBlocks): raise IOError('ncs files have different numbers of blocks of records') for i, sbi in enumerate(nb.startBlocks): @@ -555,9 +557,9 @@ def read_ncs_files(self, ncs_filenames): self._sigs_t_start.append(t_start) t_stop = ts1 / 1e6 self._sigs_t_stop.append(t_stop) - # :TODO: This should really be the total of nb_valid in records, but this allows - # the last record of a block to be shorter, the most common case. Have never - # seen a block of records with not full records before the last. + # :TODO: This should really be the total of nb_valid in records, but this + # allows the last record of a block to be shorter, the most common case. + # Have never seen a block of records with not full records before the last. length = (subdata.size - 1) * self._BLOCK_SIZE + numSampsLastBlock self._sigs_length.append(length) @@ -633,7 +635,8 @@ class NcsBlocksFactory: more complicated. Copied from Java code on Sept 7, 2020. """ - _maxGapLength = 5 # maximum gap between predicted and actual block timestamps still considered within one NcsBlock + _maxGapLength = 5 # maximum gap between predicted and actual block timestamps still + # considered within one NcsBlock @staticmethod def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePredTime): @@ -661,16 +664,18 @@ def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePre for recn in range(1, ncsMemMap.shape[0]): hdr = CscRecordHeader(ncsMemMap, recn) if hdr.channel_id != chanNum | hdr.sample_rate != reqFreq: - raise IOError('Channel number or sampling frequency changed in records within file') + raise IOError('Channel number or sampling frequency changed in ' + + 'records within file') predTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, startBlockPredTime, blkLen) nValidSamps = hdr.nb_valid if hdr.timestamp != predTime: ncsBlocks.endBlocks.append(recn-1) ncsBlocks.startBlocks.append(recn) - startBlockPredTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, - hdr.timestamp, - nValidSamps) + startBlockPredTime = WholeMicrosTimePositionBlock.calcSampleTime( + ncsBlocks.sampFreqUsed, + hdr.timestamp, + nValidSamps) blklen = 0 else: blkLen += nValidSamps @@ -713,9 +718,11 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): # check if file is one block of records, which is often the case, and avoid full parse lastBlkI = ncsMemMap.shape[0] - 1 rhl = CscRecordHeader(ncsMemMap, lastBlkI) - predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(actualSampFreq, rh0.timestamp, - NeuralynxRawIO._BLOCK_SIZE * lastBlkI) - if rhl.channel_id == chanNum and rhl.sample_rate == reqFreq and rhl.timestamp == predLastBlockStartTime: + predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(actualSampFreq, + rh0.timestamp, + NeuralynxRawIO._BLOCK_SIZE * lastBlkI) + if rhl.channel_id == chanNum and rhl.sample_rate == reqFreq and \ + rhl.timestamp == predLastBlockStartTime: nb = NcsBlocks() nb.startBlocks.append(0) nb.endBlocks.append(lastBlkI) @@ -723,9 +730,11 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): # otherwise need to scan looking for breaks else: - blkOnePredTime = WholeMicrosTimePositionBlock.calcSampleTime(actualSampFreq, rh0.timestamp, + blkOnePredTime = WholeMicrosTimePositionBlock.calcSampleTime(actualSampFreq, + rh0.timestamp, rh0.nb_valid) - return NcsBlocksFactory._parseGivenActualFrequency(ncsMemMap, nb, chanNum, reqFreq, blkOnePredTime) + return NcsBlocksFactory._parseGivenActualFrequency(ncsMemMap, nb, chanNum, reqFreq, + blkOnePredTime) @staticmethod def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): @@ -737,20 +746,23 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): ncsMemMap: memmap of Ncs file ncsBlocks: - NcsBlocks object with sampFreqUsed set to nominal frequency to use in computing time for samples (Hz) + NcsBlocks object with sampFreqUsed set to nominal frequency to use in computing time + for samples (Hz) maxGapLen: - maximum difference within a block between predicted time of start of record and recorded time + maximum difference within a block between predicted time of start of record and + recorded time RETURN: - NcsBlocks object with sampFreqUsed and microsPerSamp set based on estimate from largest block + NcsBlocks object with sampFreqUsed and microsPerSamp set based on estimate from + largest block """ # track frequency of each block and use estimate with longest block maxBlkLen = 0 maxBlkFreqEstimate = 0 - # Parse the record sequence, finding blocks of continuous time with no more than maxGapLength - # and same channel number + # Parse the record sequence, finding blocks of continuous time with no more than + # maxGapLength and same channel number rh0 = CscRecordHeader(ncsMemMap, 0) chanNum = rh0.channel_id @@ -764,15 +776,17 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): for recn in range(1, ncsMemMap.shape[0]): hdr = CscRecordHeader(ncsMemMap, recn) if hdr.channel_id != chanNum or hdr.sample_rate != recFreq: - raise IOError('Channel number or sampling frequency changed in records within file') - predTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, lastRecTime, - lastRecNumSamps) + raise IOError('Channel number or sampling frequency changed in ' + + 'records within file') + predTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, + lastRecTime, lastRecNumSamps) if abs(hdr.timestamp - predTime) > maxGapLen: ncsBlocks.endBlocks.append(recn-1) ncsBlocks.startBlocks.append(recn) if blkLen > maxBlkLen: maxBlkLen = blkLen - maxBlkFreqEstimate = (blkLen - lastRecNumSamps) * 1e6 / (lastRecTime - startBlockTime) + maxBlkFreqEstimate = (blkLen - lastRecNumSamps) * 1e6 / \ + (lastRecTime - startBlockTime) startBlockTime = hdr.timestamp blkLen = hdr.nb_valid else: @@ -783,15 +797,16 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): ncsBlocks.endBlocks.append(ncsMemMap.shape[0] - 1) ncsBlocks.sampFreqUsed = maxBlkFreqEstimate - ncsBlocks.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(maxBlkFreqEstimate) + ncsBlocks.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq( + maxBlkFreqEstimate) return ncsBlocks @staticmethod def _buildForMaxGap(ncsMemMap, nomFreq): """ - Determine blocks of records in memory mapped Ncs file given a nominal frequency of the file, - using the default values of frequency tolerance and maximum gap between blocks. + Determine blocks of records in memory mapped Ncs file given a nominal frequency of + the file, using the default values of frequency tolerance and maximum gap between blocks. PARAMETERS ncsMemMap: @@ -816,7 +831,8 @@ def _buildForMaxGap(ncsMemMap, nomFreq): # check if file is one block of records, with exact timestamp match, which may be the case numSampsForPred = NeuralynxRawIO._BLOCK_SIZE * lastBlkI - predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(nomFreq, rh0.timestamp, + predLastBlockStartTime = WholeMicrosTimePositionBlock.calcSampleTime(nomFreq, + rh0.timestamp, numSampsForPred) freqInFile = math.floor(nomFreq) if abs(rhl.timestamp - predLastBlockStartTime) == 0 and \ @@ -824,12 +840,14 @@ def _buildForMaxGap(ncsMemMap, nomFreq): nb.startBlocks.append(0) nb.endBlocks.append(lastBlkI) nb.sampFreqUsed = numSampsForPred / (rhl.timestamp - rh0.timestamp) * 1e6 - nb.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(nb.sampFreqUsed) + nb.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq( + nb.sampFreqUsed) # otherwise parse records to determine blocks using default maximum gap length else: nb.sampFreqUsed = nomFreq - nb.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(nb.sampFreqUsed) + nb.microsPerSampUsed = WholeMicrosTimePositionBlock.getMicrosPerSampForFreq( + nb.sampFreqUsed) nb = NcsBlocksFactory._parseForMaxGap(ncsMemMap, nb, NcsBlocksFactory._maxGapLength) return nb @@ -844,16 +862,19 @@ def buildForNcsFile(ncsMemMap, nlxHdr): ncsMemMap: memory map of file acqType: - string specifying type of data acquisition used, one of types returned by NlxHeader.typeOfRecording() + string specifying type of data acquisition used, one of types returned by + NlxHeader.typeOfRecording() """ acqType = nlxHdr.typeOfRecording() # old Neuralynx style with rounded whole microseconds for the samples if acqType == "PRE4": freq = nlxHdr['sampling_rate'] - microsPerSampUsed = math.floor(WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(freq)) + microsPerSampUsed = math.floor(WholeMicrosTimePositionBlock.getMicrosPerSampForFreq( + freq)) sampFreqUsed = WholeMicrosTimePositionBlock.getFreqForMicrosPerSamp(microsPerSampUsed) - nb = NcsBlocksFactory._buildGivenActualFrequency(ncsMemMap, sampFreqUsed, math.floor(freq)) + nb = NcsBlocksFactory._buildGivenActualFrequency(ncsMemMap, sampFreqUsed, + math.floor(freq)) nb.sampFreqUsed = sampFreqUsed nb.microsPerSampUsed = microsPerSampUsed @@ -865,7 +886,8 @@ def buildForNcsFile(ncsMemMap, nlxHdr): # BML style with fractional frequency and micros per samp elif acqType == "BML": sampFreqUsed = nlxHdr['sampling_rate'] - nb = NcsBlocksFactory._buildGivenActualFrequency(ncsMemMap, sampFreqUsed, math.floor(sampFreqUsed)) + nb = NcsBlocksFactory._buildGivenActualFrequency(ncsMemMap, sampFreqUsed, + math.floor(sampFreqUsed)) else: raise TypeError("Unknown Ncs file type from header.") From ba82efe4580e74c321e04cf110be27c050ce77d5 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Thu, 29 Oct 2020 08:25:20 -0700 Subject: [PATCH 31/36] More PEP8 items. --- neo/rawio/neuralynxrawio.py | 11 ++++++----- neo/test/rawiotest/test_neuralynxrawio.py | 7 ++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 473d7e6a7..8ba6aae49 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -543,7 +543,7 @@ def read_ncs_files(self, ncs_filenames): data0[nb0.endBlocks[seg_index]]['timestamp']): raise IOError('ncs files have different timestamp structure') - subdata = data[nb.startBlocks[seg_index]:(nb.endBlocks[seg_index]+1)] + subdata = data[nb.startBlocks[seg_index]:(nb.endBlocks[seg_index] + 1)] self._sigs_memmap[seg_index][chan_uid] = subdata if chan_uid == chan_uid0: @@ -595,7 +595,7 @@ def calcSampleTime(sampFr, startTime, posn): start time, and sample position. """ return round(startTime + - WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(sampFr)*posn) + WholeMicrosTimePositionBlock.getMicrosPerSampForFreq(sampFr) * posn) class CscRecordHeader: @@ -636,7 +636,7 @@ class NcsBlocksFactory: """ _maxGapLength = 5 # maximum gap between predicted and actual block timestamps still - # considered within one NcsBlock + # considered within one NcsBlock @staticmethod def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePredTime): @@ -781,7 +781,7 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): predTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, lastRecTime, lastRecNumSamps) if abs(hdr.timestamp - predTime) > maxGapLen: - ncsBlocks.endBlocks.append(recn-1) + ncsBlocks.endBlocks.append(recn - 1) ncsBlocks.startBlocks.append(recn) if blkLen > maxBlkLen: maxBlkLen = blkLen @@ -878,7 +878,8 @@ def buildForNcsFile(ncsMemMap, nlxHdr): nb.sampFreqUsed = sampFreqUsed nb.microsPerSampUsed = microsPerSampUsed - # digital lynx style with fractional frequency and micros per samp determined from block times + # digital lynx style with fractional frequency and micros per samp determined from + # block times elif acqType == "DIGITALLYNX" or acqType == "DIGITALLYNXSX": nomFreq = nlxHdr['sampling_rate'] nb = NcsBlocksFactory._buildForMaxGap(ncsMemMap, nomFreq) diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index a11996cef..10c49a87a 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -146,9 +146,10 @@ def testBuildGivenActualFrequency(self): data0 = np.memmap(filename, dtype=NeuralynxRawIO._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) ncsBlocks = NcsBlocks() - ncsBlocks.sampFreqUsed = 1/35e-6 + ncsBlocks.sampFreqUsed = 1 / 35e-6 ncsBlocks.microsPerSampUsed = 35 - ncsBlocks = NcsBlocksFactory._buildGivenActualFrequency(data0, ncsBlocks.sampFreqUsed, 27789) + ncsBlocks = NcsBlocksFactory._buildGivenActualFrequency(data0, ncsBlocks.sampFreqUsed, + 27789) self.assertEqual(len(ncsBlocks.startBlocks), 1) self.assertEqual(ncsBlocks.startBlocks[0], 0) self.assertEqual(len(ncsBlocks.endBlocks), 1) @@ -164,7 +165,7 @@ def testBuildUsingHeaderAndScanning(self): offset=NlxHeader.HEADER_SIZE) nb = NcsBlocksFactory.buildForNcsFile(data0, hdr) - self.assertEqual(nb.sampFreqUsed, 1/35e-6) + self.assertEqual(nb.sampFreqUsed, 1 / 35e-6) self.assertEqual(nb.microsPerSampUsed, 35) self.assertEqual(len(nb.startBlocks), 1) self.assertEqual(nb.startBlocks[0], 0) From b7d87122b1975e2e5d48f924963ee9c8a0f64207 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Thu, 29 Oct 2020 10:25:12 -0700 Subject: [PATCH 32/36] Correct column error in data tested. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Strange this hasn’t caused issues before with other file sets. --- neo/test/iotest/test_neuralynxio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/test/iotest/test_neuralynxio.py b/neo/test/iotest/test_neuralynxio.py index cfa4dd384..32b2969fd 100644 --- a/neo/test/iotest/test_neuralynxio.py +++ b/neo/test/iotest/test_neuralynxio.py @@ -262,7 +262,7 @@ def test_ncs(self): chuid = (chname, chid) filename = nio.ncs_filenames[chuid][:-3] + 'txt' filename = filename.replace('original_data', 'plain_data') - plain_data = np.loadtxt(filename)[:, 5:].flatten() # first columns are meta info + plain_data = np.loadtxt(filename)[:, 4:].flatten() # first 4 columns are meta info overlap = 512 * 500 gain_factor_0 = plain_data[0] / anasig.magnitude[0, 0] np.testing.assert_allclose(plain_data[:overlap], From 84af1cddcea1a31b36074b0f31f83e467c037ef1 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Thu, 29 Oct 2020 10:26:15 -0700 Subject: [PATCH 33/36] Update comment on PRE4 file limitations. --- neo/rawio/neuralynxrawio.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 8ba6aae49..4258dd7b6 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -691,8 +691,9 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): Requires that frequency in each record agrees with requested frequency. This is normally obtained by rounding the header frequency; however, this value may be different from the rounded actual frequency used in the recording, since the underlying - requirement in older Ncs files was that the rounded number of whole microseconds - per sample be the same for all records in a block. + requirement in older Ncs files was that the number of microseconds per sample in the + records is the inverse of the sampling frequency stated in the header truncated to + whole microseconds. PARAMETERS ncsMemMap: @@ -867,7 +868,8 @@ def buildForNcsFile(ncsMemMap, nlxHdr): """ acqType = nlxHdr.typeOfRecording() - # old Neuralynx style with rounded whole microseconds for the samples + # Old Neuralynx style with truncated whole microseconds for actual sampling. This + # restriction arose from the sampling being based on a master 1 MHz clock. if acqType == "PRE4": freq = nlxHdr['sampling_rate'] microsPerSampUsed = math.floor(WholeMicrosTimePositionBlock.getMicrosPerSampForFreq( From 9008a53fe9eefe872b34d4f496cb4c4cd8056f83 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Thu, 29 Oct 2020 10:27:17 -0700 Subject: [PATCH 34/36] Tests for PRE4 file type in Cheetah v4.0.2. --- neo/test/iotest/test_neuralynxio.py | 4 +++- neo/test/rawiotest/test_neuralynxrawio.py | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/test_neuralynxio.py b/neo/test/iotest/test_neuralynxio.py index 32b2969fd..6f67fddc9 100644 --- a/neo/test/iotest/test_neuralynxio.py +++ b/neo/test/iotest/test_neuralynxio.py @@ -22,7 +22,7 @@ class CommonNeuralynxIOTest(BaseTestIO, unittest.TestCase, ): ioclass = NeuralynxIO files_to_test = [ - # 'Cheetah_v4.0.2/original_data', + 'Cheetah_v4.0.2/original_data', 'Cheetah_v5.5.1/original_data', 'Cheetah_v5.6.3/original_data', 'Cheetah_v5.7.4/original_data', @@ -30,6 +30,8 @@ class CommonNeuralynxIOTest(BaseTestIO, unittest.TestCase, ): 'Cheetah_v6.3.2/incomplete_blocks'] files_to_download = [ 'Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs', + 'Cheetah_v4.0.2/plain_data/CSC14_trunc.txt', + 'Cheetah_v4.0.2/README.txt', 'Cheetah_v5.5.1/original_data/CheetahLogFile.txt', 'Cheetah_v5.5.1/original_data/CheetahLostADRecords.txt', 'Cheetah_v5.5.1/original_data/Events.nev', diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index 10c49a87a..c6d37e921 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -16,13 +16,15 @@ class TestNeuralynxRawIO(BaseTestRawIO, unittest.TestCase, ): rawioclass = NeuralynxRawIO entities_to_test = [ - # 'Cheetah_v4.0.2/original_data', + 'Cheetah_v4.0.2/original_data', 'Cheetah_v5.5.1/original_data', 'Cheetah_v5.6.3/original_data', 'Cheetah_v5.7.4/original_data', 'Cheetah_v6.3.2/incomplete_blocks'] files_to_download = [ 'Cheetah_v4.0.2/original_data/CSC14_trunc.Ncs', + 'Cheetah_v4.0.2/plain_data/CSC14_trunc.txt', + 'Cheetah_v4.0.2/README.txt', 'Cheetah_v5.5.1/original_data/CheetahLogFile.txt', 'Cheetah_v5.5.1/original_data/CheetahLostADRecords.txt', 'Cheetah_v5.5.1/original_data/Events.nev', @@ -68,6 +70,19 @@ class TestNeuralynxRawIO(BaseTestRawIO, unittest.TestCase, ): def test_read_ncs_files_sideeffects(self): + # Test Cheetah 4.0.2, which is PRE4 type with frequency in header and + # no microsPerSamp. Number of microseconds per sample in file is inverse of + # sampling frequency in header trucated to microseconds. + rawio = NeuralynxRawIO(self.get_filename_path('Cheetah_v4.0.2/original_data')) + rawio.parse_header() + # test values here from direct inspection of .ncs files + self.assertEqual(rawio._nb_segment, 1) + self.assertListEqual(rawio._timestamp_limits, [(266982936, 267162136)]) + self.assertEqual(rawio._sigs_length[0], 5120) + self.assertEqual(rawio._sigs_t_start[0], 266.982936) + self.assertEqual(rawio._sigs_t_stop[0], 267.162136) + self.assertEqual(len(rawio._sigs_memmap), 1) + # Test Cheetah 5.5.1, which is DigitalLynxSX and has two blocks of records # with a fairly large gap. rawio = NeuralynxRawIO(self.get_filename_path('Cheetah_v5.5.1/original_data')) From 0671428aacf55caa8ee49cda683d28a5859da0a6 Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Thu, 29 Oct 2020 10:43:16 -0700 Subject: [PATCH 35/36] PEP8 indent corrections. --- neo/rawio/neuralynxrawio.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 4258dd7b6..80c0235df 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -665,7 +665,7 @@ def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePre hdr = CscRecordHeader(ncsMemMap, recn) if hdr.channel_id != chanNum | hdr.sample_rate != reqFreq: raise IOError('Channel number or sampling frequency changed in ' + - 'records within file') + 'records within file') predTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, startBlockPredTime, blkLen) nValidSamps = hdr.nb_valid @@ -778,7 +778,7 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): hdr = CscRecordHeader(ncsMemMap, recn) if hdr.channel_id != chanNum or hdr.sample_rate != recFreq: raise IOError('Channel number or sampling frequency changed in ' + - 'records within file') + 'records within file') predTime = WholeMicrosTimePositionBlock.calcSampleTime(ncsBlocks.sampFreqUsed, lastRecTime, lastRecNumSamps) if abs(hdr.timestamp - predTime) > maxGapLen: @@ -787,7 +787,7 @@ def _parseForMaxGap(ncsMemMap, ncsBlocks, maxGapLen): if blkLen > maxBlkLen: maxBlkLen = blkLen maxBlkFreqEstimate = (blkLen - lastRecNumSamps) * 1e6 / \ - (lastRecTime - startBlockTime) + (lastRecTime - startBlockTime) startBlockTime = hdr.timestamp blkLen = hdr.nb_valid else: From 8589105d8446aa3e1fc6cba3e0636855c33f14fe Mon Sep 17 00:00:00 2001 From: "Peter N. Steinmetz" Date: Thu, 29 Oct 2020 10:51:37 -0700 Subject: [PATCH 36/36] Small PEP8 correctin. --- neo/rawio/neuralynxrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py index 80c0235df..c7f6ac42b 100644 --- a/neo/rawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio.py @@ -670,7 +670,7 @@ def _parseGivenActualFrequency(ncsMemMap, ncsBlocks, chanNum, reqFreq, blkOnePre startBlockPredTime, blkLen) nValidSamps = hdr.nb_valid if hdr.timestamp != predTime: - ncsBlocks.endBlocks.append(recn-1) + ncsBlocks.endBlocks.append(recn - 1) ncsBlocks.startBlocks.append(recn) startBlockPredTime = WholeMicrosTimePositionBlock.calcSampleTime( ncsBlocks.sampFreqUsed,