From c35e832efe873545cc1c3fca14dc1707e510e26e Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Tue, 4 Aug 2020 10:24:03 -0700
Subject: [PATCH 01/16] Ignore virtual environments.

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index f9242537b..debc97c68 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,3 +64,4 @@ Thumbs.db
 ###################################
 neo/test/io/neurosharemergeio.py
 files_for_testing_neo
+/venv

From 693b46cb78d09ebf387d3096d08ef787fb6837c1 Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Thu, 6 Aug 2020 11:52:27 -0700
Subject: [PATCH 02/16] Permit filename extensions in other than lowercase.

---
 neo/rawio/neuralynxrawio.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index c18fd8ee8..993054d94 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -87,6 +87,7 @@ def _parse_header(self):
 
             _, ext = os.path.splitext(filename)
             ext = ext[1:]  # remove dot
+            ext = ext.lower() # make lower case for comparisons
             if ext not in self.extensions:
                 continue
 

From db6846ce96f250b52dbc9b248f2fcf4ebf35462c Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Fri, 7 Aug 2020 11:27:32 -0700
Subject: [PATCH 03/16] Changes to handle headers from earlier Cheetah versions
 and BML.

---
 neo/rawio/neuralynxrawio.py | 97 ++++++++++++++++++++++++-------------
 1 file changed, 64 insertions(+), 33 deletions(-)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index 993054d94..c447cfedf 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -577,23 +577,51 @@ def _to_bool(txt):
     (r'Feature \w+ \d+', '', None),
     ('SessionUUID', '', None),
     ('FileUUID', '', None),
-    ('CheetahRev', '', None),  # only for old version
+    ('CheetahRev', '', None),  # only for older versions of Cheetah
     ('ProbeName', '', None),
     ('OriginalFileName', '', None),
     ('TimeCreated', '', None),
     ('TimeClosed', '', None),
-    ('ApplicationName', '', None),  # also include version number
+    ('ApplicationName', '', None),  # also include version number when present
     ('AcquisitionSystem', '', None),
     ('ReferenceChannel', '', None),
 ]
 
 
+# Filename and datetime may appear in header lines starting with # at
+# beginning of header or in later versions as a property. The exact format
+# used depends on the application name and its version as well as the
+# -FileVersion property.
+#
+# There are 3 styles understood by this code and the patterns used for parsing
+# the items within each are stored in a dictionary. Each dictionary is then
+# stored in main dictionary keyed by an abbreviation for the style.
+header_pattern_dicts = {
+    # Cheetah before version 5 and BML
+    'bv5': dict(
+        datetime1_regex=r'## Time Opened \(m/d/y\): (?P<date>\S+)  At Time: (?P<time>\S+)',
+        filename_regex = r'## File Name: (?P<filename>\S+)',
+        datetimeformat = '%m/%d/%Y %H:%M:%S.%f'),
+    # Cheetah version 5 before and including v 5.6.4
+    'bv5.6.4': dict(
+        datetime1_regex = r'## Time Opened \(m/d/y\): (?P<date>\S+)  \(h:m:s\.ms\) (?P<time>\S+)',
+        datetime2_regex = r'## Time Closed \(m/d/y\): (?P<date>\S+)  \(h:m:s\.ms\) (?P<time>\S+)',
+        filename_regex = r'## File Name (?P<filename>\S+)',
+        datetimeformat = '%m/%d/%Y %H:%M:%S.%f'),
+    # Cheetah after v 5.6.4 and default for others such as Pegasus
+    'def': dict(
+        datetime1_regex=r'-TimeCreated (?P<date>\S+) (?P<time>\S+)',
+        datetime2_regex = r'-TimeClosed (?P<date>\S+) (?P<time>\S+)',
+        filename_regex = r'-OriginalFileName "?(?P<filename>\S+)"?',
+        datetimeformat = '%Y/%m/%d %H:%M:%S')
+    }
+
+
 def read_txt_header(filename):
     """
-    All file in neuralynx contains a 16kB hedaer in txt
-    format.
-    This function parse it to create info dict.
-    This include datetime
+    All file in neuralynx contains a 16kB header in txt format.
+    This function parses it to create info dict.
+    This includes datetime.
     """
     with open(filename, 'rb') as f:
         txt_header = f.read(HEADER_SIZE)
@@ -635,16 +663,22 @@ def read_txt_header(filename):
         info['channel_names'] = [name] * len(info['channel_ids'])
 
     # version and application name
+    # older Cheetah versions with CheetahRev property
     if 'CheetahRev' in info:
         assert 'ApplicationName' not in info
         info['ApplicationName'] = 'Cheetah'
         app_version = info['CheetahRev']
-    else:
-        assert 'ApplicationName' in info
+    # new file version 3.4 does not contain CheetahRev property, but ApplicationName instead
+    elif 'ApplicationName' in info:
         pattern = r'(\S*) "([\S ]*)"'
         match = re.findall(pattern, info['ApplicationName'])
         assert len(match) == 1, 'impossible to find application name and version'
         info['ApplicationName'], app_version = match[0]
+    # BML Ncs file writing contained neither property, assume BML version 2
+    else:
+        info['ApplicationName'] = 'BML'
+        app_version = "2.0"
+
     info['ApplicationVersion'] = distutils.version.LooseVersion(app_version)
 
     # convert bit_to_microvolt
@@ -665,36 +699,33 @@ def read_txt_header(filename):
         assert len(info['InputRange']) == len(chid_entries), \
             'Number of channel ids does not match input range values.'
 
-    # filename and datetime depend on app name and its version
-    if info['ApplicationName'] == 'Cheetah':
-        if info['ApplicationVersion'] <= '5.6.4':
-            old_date_format = True
+    # Filename and datetime depend on app name, app version, and -FileVersion
+    an = info['ApplicationName']
+    if an == 'Cheetah':
+        av = info['ApplicationVersion']
+        if av < '5':
+            hpd = header_pattern_dicts['bv5']
+        elif av <= '5.6.4':
+            hpd = header_pattern_dicts['bv5.6.4']
         else:
-            old_date_format = False
-    else:
-        # for other version (pegasus, ..) I don't known the rules
-        old_date_format = (r'## Time Opened' in txt_header)
-
-    if old_date_format:
-        datetime1_regex = r'## Time Opened \(m/d/y\): (?P<date>\S+)  \(h:m:s\.ms\) (?P<time>\S+)'
-        datetime2_regex = r'## Time Closed \(m/d/y\): (?P<date>\S+)  \(h:m:s\.ms\) (?P<time>\S+)'
-        filename_regex = r'## File Name (?P<filename>\S+)'
-        datetimeformat = '%m/%d/%Y %H:%M:%S.%f'
+            hpd = header_pattern_dicts['def']
+    elif an == 'BML':
+        hpd = header_pattern_dicts['bv5']
     else:
-        datetime1_regex = r'-TimeCreated (?P<date>\S+) (?P<time>\S+)'
-        datetime2_regex = r'-TimeClosed (?P<date>\S+) (?P<time>\S+)'
-        filename_regex = r'-OriginalFileName "?(?P<filename>\S+)"?'
-        datetimeformat = '%Y/%m/%d %H:%M:%S'
+        hpd = header_pattern_dicts['def']
 
-    original_filename = re.search(filename_regex, txt_header).groupdict()['filename']
-
-    dt1 = re.search(datetime1_regex, txt_header).groupdict()
-    dt2 = re.search(datetime2_regex, txt_header).groupdict()
+    original_filename = re.search(hpd['filename_regex'], txt_header).groupdict()['filename']
 
+    # opening time
+    dt1 = re.search(hpd['datetime1_regex'], txt_header).groupdict()
     info['recording_opened'] = datetime.datetime.strptime(
-        dt1['date'] + ' ' + dt1['time'], datetimeformat)
-    info['recording_closed'] = datetime.datetime.strptime(
-        dt2['date'] + ' ' + dt2['time'], datetimeformat)
+        dt1['date'] + ' ' + dt1['time'], hpd['datetimeformat'])
+
+    # close time, if available
+    if 'datetime2_regex' in hpd:
+        dt2 = re.search(hpd['datetime2_regex'], txt_header).groupdict()
+        info['recording_closed'] = datetime.datetime.strptime(
+            dt2['date'] + ' ' + dt2['time'], hpd['datetimeformat'])
 
     return info
 

From b188dc0ab7f00c6065446ec47dc9cdc8e0aebe54 Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Sun, 9 Aug 2020 13:53:15 -0700
Subject: [PATCH 04/16] Add missing colon in header.

---
 neo/rawio/neuralynxrawio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index c447cfedf..f8e76d62b 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -599,7 +599,7 @@ def _to_bool(txt):
 header_pattern_dicts = {
     # Cheetah before version 5 and BML
     'bv5': dict(
-        datetime1_regex=r'## Time Opened \(m/d/y\): (?P<date>\S+)  At Time: (?P<time>\S+)',
+        datetime1_regex=r'## Time Opened: \(m/d/y\): (?P<date>\S+)  At Time: (?P<time>\S+)',
         filename_regex = r'## File Name: (?P<filename>\S+)',
         datetimeformat = '%m/%d/%Y %H:%M:%S.%f'),
     # Cheetah version 5 before and including v 5.6.4

From 72e87412cc8d5fee2981b9920eb7c3b3a0e3b0d1 Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Tue, 11 Aug 2020 15:56:04 -0700
Subject: [PATCH 05/16] Allow any whitespace between key and value in header.

---
 neo/rawio/neuralynxrawio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index f8e76d62b..e68e08011 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -630,7 +630,7 @@ def read_txt_header(filename):
     # find keys
     info = OrderedDict()
     for k1, k2, type_ in txt_header_keys:
-        pattern = r'-(?P<name>' + k1 + r') (?P<value>[\S ]*)'
+        pattern = r'-(?P<name>' + k1 + r')\s+(?P<value>[\S ]*)'
         matches = re.findall(pattern, txt_header)
         for match in matches:
             if k2 == '':

From 5dd146f1c4a05dcabe499939a05d5058c9746a37 Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Tue, 11 Aug 2020 16:04:17 -0700
Subject: [PATCH 06/16] Allow input_inverted to be missing which means no
 inversion of gain.

---
 neo/rawio/neuralynxrawio.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index e68e08011..fcc8ba6a4 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -108,7 +108,7 @@ def _parse_header(self):
                     # a signal channels
                     units = 'uV'
                     gain = info['bit_to_microVolt'][idx]
-                    if info['input_inverted']:
+                    if info.get('input_inverted', False):
                         gain *= -1
                     offset = 0.
                     group_id = 0
@@ -166,7 +166,7 @@ def _parse_header(self):
                         unit_id = '{}'.format(unit_id)
                         wf_units = 'uV'
                         wf_gain = info['bit_to_microVolt'][idx]
-                        if info['input_inverted']:
+                        if info.get('input_inverted',False):
                             wf_gain *= -1
                         wf_offset = 0.
                         wf_left_sweep = -1  # NOT KNOWN

From 46903ca5ba0ff9077f688dab2e2fa79fcadd5d7c Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Tue, 11 Aug 2020 16:42:04 -0700
Subject: [PATCH 07/16] Add :TODO: about older file blocks.

---
 neo/rawio/neuralynxrawio.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index fcc8ba6a4..3864e7469 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -11,7 +11,7 @@
 
 NCS can contains gaps that can be detected in inregularity
 in timestamps of data blocks. Each gap lead to one new segment.
-NCS files need to be read entirely to detect that gaps.... too bad....
+NCS files need to be read entirely to detect those gaps.... too bad....
 
 
 Author: Julia Sprenger, Carlos Canova, Samuel Garcia
@@ -419,7 +419,7 @@ def _rescale_event_timestamp(self, event_timestamps, dtype):
 
     def read_ncs_files(self, ncs_filenames):
         """
-        Given a list of ncs files contrsuct:
+        Given a list of ncs files construct:
             * self._sigs_memmap = [ {} for seg_index in range(self._nb_segment) ]
             * self._sigs_t_start = []
             * self._sigs_t_stop = []
@@ -437,6 +437,7 @@ def read_ncs_files(self, ncs_filenames):
         gap_indexes can be given (when cached) to avoid full read.
 
         """
+        # :TODO: Needs to account for gaps being different in different blocks of channels.
         if len(ncs_filenames) == 0:
             self._nb_segment = 1
             self._timestamp_limits = None
@@ -462,6 +463,9 @@ def read_ncs_files(self, ncs_filenames):
             timestamps0 = data0['timestamp']
             deltas0 = np.diff(timestamps0)
 
+            # :TODO: This algorithm needs to account for older style files which had a rounded
+            # :TODO: off sampling rate in the header.
+            #
             # It should be that:
             # gap_indexes, = np.nonzero(deltas0!=good_delta)
             # but for a file I have found many deltas0==15999, 16000, 16001 (for sampling at 32000)

From 8f3ebd7aae8e0a3c590eb50bc6469bdb4ce92960 Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Thu, 20 Aug 2020 11:09:51 -0700
Subject: [PATCH 08/16] Refactored header code into NlxHeader class.

---
 neo/rawio/neuralynxrawio.py | 408 +++++++++++++++++++-----------------
 1 file changed, 214 insertions(+), 194 deletions(-)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index 3864e7469..bc36949cd 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -30,7 +30,6 @@
 from collections import OrderedDict
 
 BLOCK_SIZE = 512  # nb sample per signal block
-HEADER_SIZE = 2 ** 14  # file have a txt header of 16kB
 
 
 class NeuralynxRawIO(BaseRawIO):
@@ -91,12 +90,12 @@ def _parse_header(self):
             if ext not in self.extensions:
                 continue
 
-            if (os.path.getsize(filename) <= HEADER_SIZE) and (ext in ['ncs']):
+            if (os.path.getsize(filename) <= NlxHeader.HEADER_SIZE) and (ext in ['ncs']):
                 self._empty_ncs.append(filename)
                 continue
 
             # All file have more or less the same header structure
-            info = read_txt_header(filename)
+            info = NlxHeader.buildForFile(filename)
             chan_names = info['channel_names']
             chan_ids = info['channel_ids']
 
@@ -149,11 +148,11 @@ def _parse_header(self):
 
                     dtype = get_nse_or_ntt_dtype(info, ext)
 
-                    if (os.path.getsize(filename) <= HEADER_SIZE):
+                    if (os.path.getsize(filename) <= NlxHeader.HEADER_SIZE):
                         self._empty_nse_ntt.append(filename)
                         data = np.zeros((0,), dtype=dtype)
                     else:
-                        data = np.memmap(filename, dtype=dtype, mode='r', offset=HEADER_SIZE)
+                        data = np.memmap(filename, dtype=dtype, mode='r', offset=NlxHeader.HEADER_SIZE)
 
                     self._spike_memmap[chan_uid] = data
 
@@ -181,12 +180,12 @@ def _parse_header(self):
                     # each ('event_id',  'ttl_input') give a new event channel
                     self.nev_filenames[chan_id] = filename
 
-                    if (os.path.getsize(filename) <= HEADER_SIZE):
+                    if (os.path.getsize(filename) <= NlxHeader.HEADER_SIZE):
                         self._empty_nev.append(filename)
                         data = np.zeros((0,), dtype=nev_dtype)
                         internal_ids = []
                     else:
-                        data = np.memmap(filename, dtype=nev_dtype, mode='r', offset=HEADER_SIZE)
+                        data = np.memmap(filename, dtype=nev_dtype, mode='r', offset=NlxHeader.HEADER_SIZE)
                         internal_ids = np.unique(data[['event_id', 'ttl_input']]).tolist()
                     for internal_event_id in internal_ids:
                         if internal_event_id not in self.internal_event_ids:
@@ -447,7 +446,7 @@ def read_ncs_files(self, ncs_filenames):
         chan_uid0 = list(ncs_filenames.keys())[0]
         filename0 = ncs_filenames[chan_uid0]
 
-        data0 = np.memmap(filename0, dtype=ncs_dtype, mode='r', offset=HEADER_SIZE)
+        data0 = np.memmap(filename0, dtype=ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE)
 
         gap_indexes = None
         lost_indexes = None
@@ -510,7 +509,7 @@ def read_ncs_files(self, ncs_filenames):
         # create segment with subdata block/t_start/t_stop/length
         for chan_uid, ncs_filename in self.ncs_filenames.items():
 
-            data = np.memmap(ncs_filename, dtype=ncs_dtype, mode='r', offset=HEADER_SIZE)
+            data = np.memmap(ncs_filename, dtype=ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE)
             assert data.size == data0.size, 'ncs files do not have the same data length'
 
             for seg_index, (i0, i1) in enumerate(gap_pairs):
@@ -535,204 +534,225 @@ def read_ncs_files(self, ncs_filenames):
                     length = subdata.size * BLOCK_SIZE
                     self._sigs_length.append(length)
 
+class NcsBlocks():
+    """
+    Contains information regarding the blocks of records in an Ncs file.
+    Factory methods perform parsing of this information from an Ncs file or
+    confirmation that file agrees with block structure.
+    """
 
-# Keys funcitons
-def _to_bool(txt):
-    if txt == 'True':
-        return True
-    elif txt == 'False':
-        return False
-    else:
-        raise Exception('Can not convert %s to bool' % (txt))
-
-
-# keys in
-txt_header_keys = [
-    ('AcqEntName', 'channel_names', None),  # used
-    ('FileType', '', None),
-    ('FileVersion', '', None),
-    ('RecordSize', '', None),
-    ('HardwareSubSystemName', '', None),
-    ('HardwareSubSystemType', '', None),
-    ('SamplingFrequency', 'sampling_rate', float),  # used
-    ('ADMaxValue', '', None),
-    ('ADBitVolts', 'bit_to_microVolt', None),  # used
-    ('NumADChannels', '', None),
-    ('ADChannel', 'channel_ids', None),  # used
-    ('InputRange', '', None),
-    ('InputInverted', 'input_inverted', _to_bool),  # used
-    ('DSPLowCutFilterEnabled', '', None),
-    ('DspLowCutFrequency', '', None),
-    ('DspLowCutNumTaps', '', None),
-    ('DspLowCutFilterType', '', None),
-    ('DSPHighCutFilterEnabled', '', None),
-    ('DspHighCutFrequency', '', None),
-    ('DspHighCutNumTaps', '', None),
-    ('DspHighCutFilterType', '', None),
-    ('DspDelayCompensation', '', None),
-    ('DspFilterDelay_µs', '', None),
-    ('DisabledSubChannels', '', None),
-    ('WaveformLength', '', int),
-    ('AlignmentPt', '', None),
-    ('ThreshVal', '', None),
-    ('MinRetriggerSamples', '', None),
-    ('SpikeRetriggerTime', '', None),
-    ('DualThresholding', '', None),
-    (r'Feature \w+ \d+', '', None),
-    ('SessionUUID', '', None),
-    ('FileUUID', '', None),
-    ('CheetahRev', '', None),  # only for older versions of Cheetah
-    ('ProbeName', '', None),
-    ('OriginalFileName', '', None),
-    ('TimeCreated', '', None),
-    ('TimeClosed', '', None),
-    ('ApplicationName', '', None),  # also include version number when present
-    ('AcquisitionSystem', '', None),
-    ('ReferenceChannel', '', None),
-]
+    startBlocks = []
+    endBlocks = []
+    sampFreqUsed = 0
+    microsPerSampUsed = 0
+
+class NlxHeader(OrderedDict):
+    """
+    Representation of basic information in all 16 kbytes Neuralynx file headers,
+    including dates opened and closed if given.
+    """
 
+    HEADER_SIZE = 2 ** 14  # Neuralynx files have a txt header of 16kB
 
-# Filename and datetime may appear in header lines starting with # at
-# beginning of header or in later versions as a property. The exact format
-# used depends on the application name and its version as well as the
-# -FileVersion property.
-#
-# There are 3 styles understood by this code and the patterns used for parsing
-# the items within each are stored in a dictionary. Each dictionary is then
-# stored in main dictionary keyed by an abbreviation for the style.
-header_pattern_dicts = {
-    # Cheetah before version 5 and BML
-    'bv5': dict(
-        datetime1_regex=r'## Time Opened: \(m/d/y\): (?P<date>\S+)  At Time: (?P<time>\S+)',
-        filename_regex = r'## File Name: (?P<filename>\S+)',
-        datetimeformat = '%m/%d/%Y %H:%M:%S.%f'),
-    # Cheetah version 5 before and including v 5.6.4
-    'bv5.6.4': dict(
-        datetime1_regex = r'## Time Opened \(m/d/y\): (?P<date>\S+)  \(h:m:s\.ms\) (?P<time>\S+)',
-        datetime2_regex = r'## Time Closed \(m/d/y\): (?P<date>\S+)  \(h:m:s\.ms\) (?P<time>\S+)',
-        filename_regex = r'## File Name (?P<filename>\S+)',
-        datetimeformat = '%m/%d/%Y %H:%M:%S.%f'),
-    # Cheetah after v 5.6.4 and default for others such as Pegasus
-    'def': dict(
-        datetime1_regex=r'-TimeCreated (?P<date>\S+) (?P<time>\S+)',
-        datetime2_regex = r'-TimeClosed (?P<date>\S+) (?P<time>\S+)',
-        filename_regex = r'-OriginalFileName "?(?P<filename>\S+)"?',
-        datetimeformat = '%Y/%m/%d %H:%M:%S')
+    # helper function to interpret boolean keys
+    def _to_bool(txt):
+        if txt == 'True':
+            return True
+        elif txt == 'False':
+            return False
+        else:
+            raise Exception('Can not convert %s to bool' % (txt))
+
+    # keys that may be present in header which we parse
+    txt_header_keys = [
+        ('AcqEntName', 'channel_names', None),  # used
+        ('FileType', '', None),
+        ('FileVersion', '', None),
+        ('RecordSize', '', None),
+        ('HardwareSubSystemName', '', None),
+        ('HardwareSubSystemType', '', None),
+        ('SamplingFrequency', 'sampling_rate', float),  # used
+        ('ADMaxValue', '', None),
+        ('ADBitVolts', 'bit_to_microVolt', None),  # used
+        ('NumADChannels', '', None),
+        ('ADChannel', 'channel_ids', None),  # used
+        ('InputRange', '', None),
+        ('InputInverted', 'input_inverted', _to_bool),  # used
+        ('DSPLowCutFilterEnabled', '', None),
+        ('DspLowCutFrequency', '', None),
+        ('DspLowCutNumTaps', '', None),
+        ('DspLowCutFilterType', '', None),
+        ('DSPHighCutFilterEnabled', '', None),
+        ('DspHighCutFrequency', '', None),
+        ('DspHighCutNumTaps', '', None),
+        ('DspHighCutFilterType', '', None),
+        ('DspDelayCompensation', '', None),
+        ('DspFilterDelay_µs', '', None),
+        ('DisabledSubChannels', '', None),
+        ('WaveformLength', '', int),
+        ('AlignmentPt', '', None),
+        ('ThreshVal', '', None),
+        ('MinRetriggerSamples', '', None),
+        ('SpikeRetriggerTime', '', None),
+        ('DualThresholding', '', None),
+        (r'Feature \w+ \d+', '', None),
+        ('SessionUUID', '', None),
+        ('FileUUID', '', None),
+        ('CheetahRev', '', None),  # only for older versions of Cheetah
+        ('ProbeName', '', None),
+        ('OriginalFileName', '', None),
+        ('TimeCreated', '', None),
+        ('TimeClosed', '', None),
+        ('ApplicationName', '', None),  # also include version number when present
+        ('AcquisitionSystem', '', None),
+        ('ReferenceChannel', '', None),
+    ]
+
+    # Filename and datetime may appear in header lines starting with # at
+    # beginning of header or in later versions as a property. The exact format
+    # used depends on the application name and its version as well as the
+    # -FileVersion property.
+    #
+    # There are 3 styles understood by this code and the patterns used for parsing
+    # the items within each are stored in a dictionary. Each dictionary is then
+    # stored in main dictionary keyed by an abbreviation for the style.
+    header_pattern_dicts = {
+        # Cheetah before version 5 and BML
+        'bv5': dict(
+            datetime1_regex=r'## Time Opened: \(m/d/y\): (?P<date>\S+)  At Time: (?P<time>\S+)',
+            filename_regex=r'## File Name: (?P<filename>\S+)',
+            datetimeformat='%m/%d/%Y %H:%M:%S.%f'),
+        # Cheetah version 5 before and including v 5.6.4
+        'bv5.6.4': dict(
+            datetime1_regex=r'## Time Opened \(m/d/y\): (?P<date>\S+)  \(h:m:s\.ms\) (?P<time>\S+)',
+            datetime2_regex=r'## Time Closed \(m/d/y\): (?P<date>\S+)  \(h:m:s\.ms\) (?P<time>\S+)',
+            filename_regex=r'## File Name (?P<filename>\S+)',
+            datetimeformat='%m/%d/%Y %H:%M:%S.%f'),
+        # Cheetah after v 5.6.4 and default for others such as Pegasus
+        'def': dict(
+            datetime1_regex=r'-TimeCreated (?P<date>\S+) (?P<time>\S+)',
+            datetime2_regex=r'-TimeClosed (?P<date>\S+) (?P<time>\S+)',
+            filename_regex=r'-OriginalFileName "?(?P<filename>\S+)"?',
+            datetimeformat='%Y/%m/%d %H:%M:%S')
     }
 
+    def buildForFile(filename):
+        """
+        Factory function to build NlxHeader for a given file.
+        """
 
-def read_txt_header(filename):
-    """
-    All file in neuralynx contains a 16kB header in txt format.
-    This function parses it to create info dict.
-    This includes datetime.
-    """
-    with open(filename, 'rb') as f:
-        txt_header = f.read(HEADER_SIZE)
-    txt_header = txt_header.strip(b'\x00').decode('latin-1')
-
-    # find keys
-    info = OrderedDict()
-    for k1, k2, type_ in txt_header_keys:
-        pattern = r'-(?P<name>' + k1 + r')\s+(?P<value>[\S ]*)'
-        matches = re.findall(pattern, txt_header)
-        for match in matches:
-            if k2 == '':
-                name = match[0]
-            else:
-                name = k2
-            value = match[1].rstrip(' ')
-            if type_ is not None:
-                value = type_(value)
-            info[name] = value
-
-    # if channel_ids or s not in info then the filename is used
-    name = os.path.splitext(os.path.basename(filename))[0]
-
-    # convert channel ids
-    if 'channel_ids' in info:
-        chid_entries = re.findall(r'\w+', info['channel_ids'])
-        info['channel_ids'] = [int(c) for c in chid_entries]
-    else:
-        info['channel_ids'] = [name]
-
-    # convert channel names
-    if 'channel_names' in info:
-        name_entries = re.findall(r'\w+', info['channel_names'])
-        if len(name_entries) == 1:
-            info['channel_names'] = name_entries * len(info['channel_ids'])
-        assert len(info['channel_names']) == len(info['channel_ids']), \
-            'Number of channel ids does not match channel names.'
-    else:
-        info['channel_names'] = [name] * len(info['channel_ids'])
-
-    # version and application name
-    # older Cheetah versions with CheetahRev property
-    if 'CheetahRev' in info:
-        assert 'ApplicationName' not in info
-        info['ApplicationName'] = 'Cheetah'
-        app_version = info['CheetahRev']
-    # new file version 3.4 does not contain CheetahRev property, but ApplicationName instead
-    elif 'ApplicationName' in info:
-        pattern = r'(\S*) "([\S ]*)"'
-        match = re.findall(pattern, info['ApplicationName'])
-        assert len(match) == 1, 'impossible to find application name and version'
-        info['ApplicationName'], app_version = match[0]
-    # BML Ncs file writing contained neither property, assume BML version 2
-    else:
-        info['ApplicationName'] = 'BML'
-        app_version = "2.0"
-
-    info['ApplicationVersion'] = distutils.version.LooseVersion(app_version)
-
-    # convert bit_to_microvolt
-    if 'bit_to_microVolt' in info:
-        btm_entries = re.findall(r'\S+', info['bit_to_microVolt'])
-        if len(btm_entries) == 1:
-            btm_entries = btm_entries * len(info['channel_ids'])
-        info['bit_to_microVolt'] = [float(e) * 1e6 for e in btm_entries]
-        assert len(info['bit_to_microVolt']) == len(info['channel_ids']), \
-            'Number of channel ids does not match bit_to_microVolt conversion factors.'
-
-    if 'InputRange' in info:
-        ir_entries = re.findall(r'\w+', info['InputRange'])
-        if len(ir_entries) == 1:
-            info['InputRange'] = [int(ir_entries[0])] * len(chid_entries)
+        with open(filename, 'rb') as f:
+            txt_header = f.read(NlxHeader.HEADER_SIZE)
+        txt_header = txt_header.strip(b'\x00').decode('latin-1')
+
+        # find keys
+        info = NlxHeader()
+        for k1, k2, type_ in NlxHeader.txt_header_keys:
+            pattern = r'-(?P<name>' + k1 + r')\s+(?P<value>[\S ]*)'
+            matches = re.findall(pattern, txt_header)
+            for match in matches:
+                if k2 == '':
+                    name = match[0]
+                else:
+                    name = k2
+                value = match[1].rstrip(' ')
+                if type_ is not None:
+                    value = type_(value)
+                info[name] = value
+
+        # if channel_ids or s not in info then the filename is used
+        name = os.path.splitext(os.path.basename(filename))[0]
+
+        # convert channel ids
+        if 'channel_ids' in info:
+            chid_entries = re.findall(r'\w+', info['channel_ids'])
+            info['channel_ids'] = [int(c) for c in chid_entries]
+        else:
+            info['channel_ids'] = [name]
+
+        # convert channel names
+        if 'channel_names' in info:
+            name_entries = re.findall(r'\w+', info['channel_names'])
+            if len(name_entries) == 1:
+                info['channel_names'] = name_entries * len(info['channel_ids'])
+            assert len(info['channel_names']) == len(info['channel_ids']), \
+                'Number of channel ids does not match channel names.'
         else:
-            info['InputRange'] = [int(e) for e in ir_entries]
-        assert len(info['InputRange']) == len(chid_entries), \
-            'Number of channel ids does not match input range values.'
-
-    # Filename and datetime depend on app name, app version, and -FileVersion
-    an = info['ApplicationName']
-    if an == 'Cheetah':
-        av = info['ApplicationVersion']
-        if av < '5':
-            hpd = header_pattern_dicts['bv5']
-        elif av <= '5.6.4':
-            hpd = header_pattern_dicts['bv5.6.4']
+            info['channel_names'] = [name] * len(info['channel_ids'])
+
+        # version and application name
+        # older Cheetah versions with CheetahRev property
+        if 'CheetahRev' in info:
+            assert 'ApplicationName' not in info
+            info['ApplicationName'] = 'Cheetah'
+            app_version = info['CheetahRev']
+        # new file version 3.4 does not contain CheetahRev property, but ApplicationName instead
+        elif 'ApplicationName' in info:
+            pattern = r'(\S*) "([\S ]*)"'
+            match = re.findall(pattern, info['ApplicationName'])
+            assert len(match) == 1, 'impossible to find application name and version'
+            info['ApplicationName'], app_version = match[0]
+        # BML Ncs file writing contained neither property, assume BML version 2
         else:
-            hpd = header_pattern_dicts['def']
-    elif an == 'BML':
-        hpd = header_pattern_dicts['bv5']
-    else:
-        hpd = header_pattern_dicts['def']
+            info['ApplicationName'] = 'BML'
+            app_version = "2.0"
+
+        info['ApplicationVersion'] = distutils.version.LooseVersion(app_version)
+
+        # convert bit_to_microvolt
+        if 'bit_to_microVolt' in info:
+            btm_entries = re.findall(r'\S+', info['bit_to_microVolt'])
+            if len(btm_entries) == 1:
+                btm_entries = btm_entries * len(info['channel_ids'])
+            info['bit_to_microVolt'] = [float(e) * 1e6 for e in btm_entries]
+            assert len(info['bit_to_microVolt']) == len(info['channel_ids']), \
+                'Number of channel ids does not match bit_to_microVolt conversion factors.'
+
+        if 'InputRange' in info:
+            ir_entries = re.findall(r'\w+', info['InputRange'])
+            if len(ir_entries) == 1:
+                info['InputRange'] = [int(ir_entries[0])] * len(chid_entries)
+            else:
+                info['InputRange'] = [int(e) for e in ir_entries]
+            assert len(info['InputRange']) == len(chid_entries), \
+                'Number of channel ids does not match input range values.'
+
+        # Filename and datetime depend on app name, app version, and -FileVersion
+        an = info['ApplicationName']
+        if an == 'Cheetah':
+            av = info['ApplicationVersion']
+            if av < '5':
+                hpd = NlxHeader.header_pattern_dicts['bv5']
+            elif av <= '5.6.4':
+                hpd = NlxHeader.header_pattern_dicts['bv5.6.4']
+            else:
+                hpd = NlxHeader.header_pattern_dicts['def']
+        elif an == 'BML':
+            hpd = NlxHeader.header_pattern_dicts['bv5']
+        else:
+            hpd = NlxHeader.header_pattern_dicts['def']
+
+        original_filename = re.search(hpd['filename_regex'], txt_header).groupdict()['filename']
 
-    original_filename = re.search(hpd['filename_regex'], txt_header).groupdict()['filename']
+        # opening time
+        dt1 = re.search(hpd['datetime1_regex'], txt_header).groupdict()
+        info['recording_opened'] = datetime.datetime.strptime(
+            dt1['date'] + ' ' + dt1['time'], hpd['datetimeformat'])
 
-    # opening time
-    dt1 = re.search(hpd['datetime1_regex'], txt_header).groupdict()
-    info['recording_opened'] = datetime.datetime.strptime(
-        dt1['date'] + ' ' + dt1['time'], hpd['datetimeformat'])
+        # close time, if available
+        if 'datetime2_regex' in hpd:
+            dt2 = re.search(hpd['datetime2_regex'], txt_header).groupdict()
+            info['recording_closed'] = datetime.datetime.strptime(
+                dt2['date'] + ' ' + dt2['time'], hpd['datetimeformat'])
 
-    # close time, if available
-    if 'datetime2_regex' in hpd:
-        dt2 = re.search(hpd['datetime2_regex'], txt_header).groupdict()
-        info['recording_closed'] = datetime.datetime.strptime(
-            dt2['date'] + ' ' + dt2['time'], hpd['datetimeformat'])
+        return info
 
-    return info
 
+class NcsHeader():
+    """
+    Representation of information in Ncs file headers, including exact
+    recording type.
+    """
 
 ncs_dtype = [('timestamp', 'uint64'), ('channel_id', 'uint32'), ('sample_rate', 'uint32'),
              ('nb_valid', 'uint32'), ('samples', 'int16', (BLOCK_SIZE,))]

From fecfed11dd96e8623f5b98cdf2b72eca6a4d2612 Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Thu, 20 Aug 2020 11:13:56 -0700
Subject: [PATCH 09/16] Add check for 8 hash marks at start of file.

---
 neo/rawio/neuralynxrawio.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index bc36949cd..4d8567a04 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -645,6 +645,9 @@ def buildForFile(filename):
             txt_header = f.read(NlxHeader.HEADER_SIZE)
         txt_header = txt_header.strip(b'\x00').decode('latin-1')
 
+        # must start with 8 # characters
+        assert txt_header.startswith("########"),'Neuralynx files must start with 8 # characters.'
+
         # find keys
         info = NlxHeader()
         for k1, k2, type_ in NlxHeader.txt_header_keys:

From 8ed68769dab60c80aa170b5a2541c3ec58d2b951 Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Wed, 2 Sep 2020 11:30:21 -0700
Subject: [PATCH 10/16] Ignore resources folder.

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index debc97c68..7c3113591 100644
--- a/.gitignore
+++ b/.gitignore
@@ -65,3 +65,4 @@ Thumbs.db
 neo/test/io/neurosharemergeio.py
 files_for_testing_neo
 /venv
+/neo/test/resources

From 300d89070e9f8728bac04960a906275924b8175b Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Thu, 3 Sep 2020 08:19:29 -0700
Subject: [PATCH 11/16] Begin restructure to header and block objects.

---
 neo/rawio/neuralynxrawio.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index 4d8567a04..684d60b9d 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -10,8 +10,8 @@
 
 
 NCS can contains gaps that can be detected in inregularity
-in timestamps of data blocks. Each gap lead to one new segment.
-NCS files need to be read entirely to detect those gaps.... too bad....
+in timestamps of data blocks. Each gap leads to one new segment.
+Some NCS files may need to be read entirely to detect those gaps which can be slow.
 
 
 Author: Julia Sprenger, Carlos Canova, Samuel Garcia
@@ -34,17 +34,17 @@
 
 class NeuralynxRawIO(BaseRawIO):
     """"
-    Class for reading dataset recorded by Neuralynx.
+    Class for reading datasets recorded by Neuralynx.
 
     Examples:
         >>> reader = NeuralynxRawIO(dirname='Cheetah_v5.5.1/original_data')
         >>> reader.parse_header()
 
-            Inspect all file in the directory.
+            Inspect all files in the directory.
 
         >>> print(reader)
 
-            Display all informations about signal channels, units, segment size....
+            Display all information about signal channels, units, segment size....
     """
     extensions = ['nse', 'ncs', 'nev', 'ntt']
     rawmode = 'one-dir'
@@ -75,8 +75,8 @@ def _parse_header(self):
         self._empty_nev = []
         self._empty_nse_ntt = []
 
-        # explore the directory looking for ncs, nev, nse and ntt
-        # And construct channels headers
+        # Explore the directory looking for ncs, nev, nse and ntt
+        # and construct channels headers.
         signal_annotations = []
         unit_annotations = []
         event_annotations = []
@@ -104,7 +104,7 @@ def _parse_header(self):
 
                 chan_uid = (chan_name, chan_id)
                 if ext == 'ncs':
-                    # a signal channels
+                    # a signal channel
                     units = 'uV'
                     gain = info['bit_to_microVolt'][idx]
                     if info.get('input_inverted', False):
@@ -207,7 +207,7 @@ def _parse_header(self):
             self._sigs_sampling_rate = sampling_rate[0]
 
         # read ncs files for gaps detection and nb_segment computation
-        self.read_ncs_files(self.ncs_filenames)
+        ncsBlocks = self.read_ncs_files(self.ncs_filenames)
 
         # timestamp limit in nev, nse
         # so need to scan all spike and event to
@@ -418,7 +418,13 @@ def _rescale_event_timestamp(self, event_timestamps, dtype):
 
     def read_ncs_files(self, ncs_filenames):
         """
-        Given a list of ncs files construct:
+<<<<<<< Updated upstream
+        Given a list of ncs files contrsuct:
+=======
+        Given a list of ncs files, return a dictionary of NcsBlocks indexed by channel uid.
+
+        :
+>>>>>>> Stashed changes
             * self._sigs_memmap = [ {} for seg_index in range(self._nb_segment) ]
             * self._sigs_t_start = []
             * self._sigs_t_stop = []
@@ -438,9 +444,9 @@ def read_ncs_files(self, ncs_filenames):
         """
         # :TODO: Needs to account for gaps being different in different blocks of channels.
         if len(ncs_filenames) == 0:
-            self._nb_segment = 1
-            self._timestamp_limits = None
-            return
+            return NcsBlocks()
+
+
 
         good_delta = int(BLOCK_SIZE * 1e6 / self._sigs_sampling_rate)
         chan_uid0 = list(ncs_filenames.keys())[0]

From ac0ae4dffa7348e8c7134ea1fbc6caf7c523e9cb Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Thu, 3 Sep 2020 10:27:13 -0700
Subject: [PATCH 12/16] Add bunch of explanatory comments.

---
 neo/rawio/neuralynxrawio.py | 92 ++++++++++++++++++++++++++-----------
 1 file changed, 65 insertions(+), 27 deletions(-)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index 684d60b9d..780b5fdb5 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -3,13 +3,13 @@
 This IO supports NCS, NEV, NSE and NTT file formats.
 
 
-NCS contains signals for one channel
+NCS contains sampled signal for one channel
 NEV contains events
 NSE contains spikes and waveforms for mono electrodes
 NTT contains spikes and waveforms for tetrodes
 
 
-NCS can contains gaps that can be detected in inregularity
+NCS can contains gaps that can be detected in irregularity
 in timestamps of data blocks. Each gap leads to one new segment.
 Some NCS files may need to be read entirely to detect those gaps which can be slow.
 
@@ -36,6 +36,8 @@ class NeuralynxRawIO(BaseRawIO):
     """"
     Class for reading datasets recorded by Neuralynx.
 
+    This version only works with rawmode of one-dir for a single directory.
+
     Examples:
         >>> reader = NeuralynxRawIO(dirname='Cheetah_v5.5.1/original_data')
         >>> reader.parse_header()
@@ -50,6 +52,15 @@ class NeuralynxRawIO(BaseRawIO):
     rawmode = 'one-dir'
 
     def __init__(self, dirname='', keep_original_times=False, **kargs):
+        """
+        Parameters
+        ----------
+        dirname: str
+            name of directory containing all files for dataset
+        keep_original_times:
+            if True, keep original start time as in files,
+            otherwise set 0 of time to first time in dataset
+        """
         self.dirname = dirname
         self.keep_original_times = keep_original_times
         BaseRawIO.__init__(self, **kargs)
@@ -71,7 +82,7 @@ def _parse_header(self):
         self._spike_memmap = {}
         self.internal_unit_ids = []  # channel_index > ((channel_name, channel_id), unit_id)
         self.internal_event_ids = []
-        self._empty_ncs = []  # this list contains filenames of empty records
+        self._empty_ncs = []  # this list contains filenames of empty files
         self._empty_nev = []
         self._empty_nse_ntt = []
 
@@ -90,7 +101,9 @@ def _parse_header(self):
             if ext not in self.extensions:
                 continue
 
-            if (os.path.getsize(filename) <= NlxHeader.HEADER_SIZE) and (ext in ['ncs']):
+            # Skip Ncs files with only header. Other empty file types
+            # will have an empty dataset constructed later.
+            if (os.path.getsize(filename) <= NlxHeader.HEADER_SIZE) and ext in ['ncs']:
                 self._empty_ncs.append(filename)
                 continue
 
@@ -104,7 +117,7 @@ def _parse_header(self):
 
                 chan_uid = (chan_name, chan_id)
                 if ext == 'ncs':
-                    # a signal channel
+                    # a sampled signal channel
                     units = 'uV'
                     gain = info['bit_to_microVolt'][idx]
                     if info.get('input_inverted', False):
@@ -140,8 +153,8 @@ def _parse_header(self):
                     signal_annotations.append(d)
 
                 elif ext in ('nse', 'ntt'):
-                    # nse and ntt are pretty similar except for the wavform shape
-                    # a file can contain several unit_id (so several unit channel)
+                    # nse and ntt are pretty similar except for the waveform shape.
+                    # A file can contain several unit_id (so several unit channel).
                     assert chan_id not in self.nse_ntt_filenames, \
                         'Several nse or ntt files have the same unit_id!!!'
                     self.nse_ntt_filenames[chan_uid] = filename
@@ -206,11 +219,13 @@ def _parse_header(self):
             assert sampling_rate.size == 1
             self._sigs_sampling_rate = sampling_rate[0]
 
-        # read ncs files for gaps detection and nb_segment computation
+        # Read ncs files for gaps detection and nb_segment computation.
+        # :TODO: current algorithm depends on side-effect of read_ncs_files on
+        #   self._sigs_memmap, self._sigs_t_start, self._sigs_t_stop,
+        #   self._sigs_length, self._nb_segment, self._timestamp_limits
         ncsBlocks = self.read_ncs_files(self.ncs_filenames)
 
-        # timestamp limit in nev, nse
-        # so need to scan all spike and event to
+        # Determine timestamp limits in nev, nse file by scanning them.
         ts0, ts1 = None, None
         for _data_memmap in (self._spike_memmap, self._nev_memmap):
             for _, data in _data_memmap.items():
@@ -221,7 +236,7 @@ def _parse_header(self):
                     ts0 = ts[0]
                     ts1 = ts[-1]
                 ts0 = min(ts0, ts[0])
-                ts1 = max(ts0, ts[-1])
+                ts1 = max(ts0, ts[-1]) # :FIXME: possible error, should be ts1
 
         if self._timestamp_limits is None:
             # case  NO ncs but HAVE nev or nse
@@ -249,7 +264,7 @@ def _parse_header(self):
             self.global_t_stop = self.global_t_stop - self.global_t_start
             self.global_t_start = 0
 
-        # fill into header dict
+        # fill header dictionary
         self.header = {}
         self.header['nb_block'] = 1
         self.header['nb_segment'] = [self._nb_segment]
@@ -285,6 +300,7 @@ def _parse_header(self):
                 # ~ ev_ann['digital_marker'] =
                 # ~ ev_ann['analog_marker'] =
 
+    # Accessors for segment times which are offset by appropriate global start time
     def _segment_t_start(self, block_index, seg_index):
         return self._seg_t_starts[seg_index] - self.global_t_start
 
@@ -297,7 +313,27 @@ def _get_signal_size(self, block_index, seg_index, channel_indexes):
     def _get_signal_t_start(self, block_index, seg_index, channel_indexes):
         return self._sigs_t_start[seg_index] - self.global_t_start
 
-    def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, channel_indexes):
+    def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, channel_indices):
+        """
+        Retrieve chunk of analog signal, a chunk being a set of contiguous samples.
+
+        PARAMETERS
+        ----------
+        block_index:
+            index of block in dataset, ignored as only 1 block in this implementation
+        seg_index:
+            index of segment to use
+        i_start:
+            sample index of first sample within segment to retrieve
+        i_stop:
+            sample index of last sample within segment to retrieve
+        channel_indices:
+            list of channel indices to return data for
+
+        RETURNS
+        -------
+            array of samples, with each requested channel in a column
+        """
         if i_start is None:
             i_start = 0
         if i_stop is None:
@@ -308,13 +344,15 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, chann
         sl0 = i_start % 512
         sl1 = sl0 + (i_stop - i_start)
 
-        if channel_indexes is None:
-            channel_indexes = slice(None)
+        if channel_indices is None:
+            channel_indices = slice(None)
 
-        channel_ids = self.header['signal_channels'][channel_indexes]['id']
-        channel_names = self.header['signal_channels'][channel_indexes]['name']
+        channel_ids = self.header['signal_channels'][channel_indices]['id']
+        channel_names = self.header['signal_channels'][channel_indices]['name']
 
+        # create buffer for samples
         sigs_chunk = np.zeros((i_stop - i_start, len(channel_ids)), dtype='int16')
+
         for i, chan_uid in enumerate(zip(channel_names, channel_ids)):
             data = self._sigs_memmap[seg_index][chan_uid]
             sub = data[block_start:block_stop]
@@ -329,6 +367,7 @@ def _spike_count(self, block_index, seg_index, unit_index):
 
         ts0, ts1 = self._timestamp_limits[seg_index]
 
+        # only count spikes inside the timestamp limits, inclusive, and for the specified unit
         keep = (ts >= ts0) & (ts <= ts1) & (unit_id == data['unit_id'])
         nb_spike = int(data[keep].size)
         return nb_spike
@@ -418,13 +457,9 @@ def _rescale_event_timestamp(self, event_timestamps, dtype):
 
     def read_ncs_files(self, ncs_filenames):
         """
-<<<<<<< Updated upstream
-        Given a list of ncs files contrsuct:
-=======
         Given a list of ncs files, return a dictionary of NcsBlocks indexed by channel uid.
 
-        :
->>>>>>> Stashed changes
+        :TODO: Current algorithm has side effects on following attributes:
             * self._sigs_memmap = [ {} for seg_index in range(self._nb_segment) ]
             * self._sigs_t_start = []
             * self._sigs_t_stop = []
@@ -442,11 +477,11 @@ def read_ncs_files(self, ncs_filenames):
         gap_indexes can be given (when cached) to avoid full read.
 
         """
-        # :TODO: Needs to account for gaps being different in different blocks of channels.
+        # :TODO: Needs to account for gaps and start and end times potentially
+        #    being different in different groups of channels. These groups typically
+        #    correspond to the channels collected by a single ADC card.
         if len(ncs_filenames) == 0:
-            return NcsBlocks()
-
-
+            return None
 
         good_delta = int(BLOCK_SIZE * 1e6 / self._sigs_sampling_rate)
         chan_uid0 = list(ncs_filenames.keys())[0]
@@ -469,7 +504,7 @@ def read_ncs_files(self, ncs_filenames):
             deltas0 = np.diff(timestamps0)
 
             # :TODO: This algorithm needs to account for older style files which had a rounded
-            # :TODO: off sampling rate in the header.
+            #   off sampling rate in the header.
             #
             # It should be that:
             # gap_indexes, = np.nonzero(deltas0!=good_delta)
@@ -487,6 +522,9 @@ def read_ncs_files(self, ncs_filenames):
             # update for lost_indexes
             # Sometimes NLX writes a faulty block, but it then validates how much samples it wrote
             # the validation field is in delta0['nb_valid'], it should be equal to BLOCK_SIZE
+            # :TODO: this algorithm ignores samples in partially filled blocks, which
+            #   is not strictly necessary as all channels might have same partially filled
+            #   blocks at the end.
 
             lost_indexes, = np.nonzero(data0['nb_valid'] < BLOCK_SIZE)
 

From 144c750d215dfae1ed785af74c5d56af99d9cab1 Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Thu, 3 Sep 2020 10:47:00 -0700
Subject: [PATCH 13/16] Fix up attributes needed for Pegasus. Comments.

---
 neo/rawio/neuralynxrawio.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index 780b5fdb5..1dc1e9f73 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -214,11 +214,17 @@ def _parse_header(self):
         unit_channels = np.array(unit_channels, dtype=_unit_channel_dtype)
         event_channels = np.array(event_channels, dtype=_event_channel_dtype)
 
+        # require all sampled signals, ncs files, to have same sampling rate
         if sig_channels.size > 0:
             sampling_rate = np.unique(sig_channels['sampling_rate'])
             assert sampling_rate.size == 1
             self._sigs_sampling_rate = sampling_rate[0]
 
+        # set 2 attributes needed later for header in case there are no ncs files in dataset,
+        #   e.g. Pegasus
+        self._timestamp_limits = None
+        self._nb_segment = 1
+
         # Read ncs files for gaps detection and nb_segment computation.
         # :TODO: current algorithm depends on side-effect of read_ncs_files on
         #   self._sigs_memmap, self._sigs_t_start, self._sigs_t_stop,
@@ -238,6 +244,7 @@ def _parse_header(self):
                 ts0 = min(ts0, ts[0])
                 ts1 = max(ts0, ts[-1]) # :FIXME: possible error, should be ts1
 
+        # decide on segment and global start and stop times based on files available
         if self._timestamp_limits is None:
             # case  NO ncs but HAVE nev or nse
             self._timestamp_limits = [(ts0, ts1)]

From 4d71c94169c3f18593887863af9aeb9e7d41add8 Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Thu, 3 Sep 2020 10:47:30 -0700
Subject: [PATCH 14/16] Correct comment.

---
 neo/test/iotest/test_neuralynxio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/neo/test/iotest/test_neuralynxio.py b/neo/test/iotest/test_neuralynxio.py
index 6a9bcd1e5..adc97ba6a 100644
--- a/neo/test/iotest/test_neuralynxio.py
+++ b/neo/test/iotest/test_neuralynxio.py
@@ -1,5 +1,5 @@
 """
-Tests of neo.io.blackrockio
+Tests of neo.io.neuralynxio.py
 """
 
 import time

From d84ab9c44e054b34692222d0437471e520729b47 Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Thu, 3 Sep 2020 11:12:14 -0700
Subject: [PATCH 15/16] Clean up style for PEP8.

---
 neo/rawio/neuralynxrawio.py | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index 1dc1e9f73..305049b18 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -97,7 +97,7 @@ def _parse_header(self):
 
             _, ext = os.path.splitext(filename)
             ext = ext[1:]  # remove dot
-            ext = ext.lower() # make lower case for comparisons
+            ext = ext.lower()  # make lower case for comparisons
             if ext not in self.extensions:
                 continue
 
@@ -165,7 +165,8 @@ def _parse_header(self):
                         self._empty_nse_ntt.append(filename)
                         data = np.zeros((0,), dtype=dtype)
                     else:
-                        data = np.memmap(filename, dtype=dtype, mode='r', offset=NlxHeader.HEADER_SIZE)
+                        data = np.memmap(filename, dtype=dtype, mode='r',
+                                         offset=NlxHeader.HEADER_SIZE)
 
                     self._spike_memmap[chan_uid] = data
 
@@ -178,7 +179,7 @@ def _parse_header(self):
                         unit_id = '{}'.format(unit_id)
                         wf_units = 'uV'
                         wf_gain = info['bit_to_microVolt'][idx]
-                        if info.get('input_inverted',False):
+                        if info.get('input_inverted', False):
                             wf_gain *= -1
                         wf_offset = 0.
                         wf_left_sweep = -1  # NOT KNOWN
@@ -198,7 +199,8 @@ def _parse_header(self):
                         data = np.zeros((0,), dtype=nev_dtype)
                         internal_ids = []
                     else:
-                        data = np.memmap(filename, dtype=nev_dtype, mode='r', offset=NlxHeader.HEADER_SIZE)
+                        data = np.memmap(filename, dtype=nev_dtype, mode='r',
+                                         offset=NlxHeader.HEADER_SIZE)
                         internal_ids = np.unique(data[['event_id', 'ttl_input']]).tolist()
                     for internal_event_id in internal_ids:
                         if internal_event_id not in self.internal_event_ids:
@@ -242,7 +244,7 @@ def _parse_header(self):
                     ts0 = ts[0]
                     ts1 = ts[-1]
                 ts0 = min(ts0, ts[0])
-                ts1 = max(ts0, ts[-1]) # :FIXME: possible error, should be ts1
+                ts1 = max(ts0, ts[-1])  # :FIXME: possible error, should be ts1
 
         # decide on segment and global start and stop times based on files available
         if self._timestamp_limits is None:
@@ -575,8 +577,8 @@ def read_ncs_files(self, ncs_filenames):
 
                 if chan_uid == chan_uid0:
                     ts0 = subdata[0]['timestamp']
-                    ts1 = subdata[-1]['timestamp'] \
-                            + np.uint64(BLOCK_SIZE / self._sigs_sampling_rate * 1e6)
+                    ts1 = subdata[-1]['timestamp'] +\
+                        np.uint64(BLOCK_SIZE / self._sigs_sampling_rate * 1e6)
                     self._timestamp_limits.append((ts0, ts1))
                     t_start = ts0 / 1e6
                     self._sigs_t_start.append(t_start)
@@ -585,6 +587,7 @@ def read_ncs_files(self, ncs_filenames):
                     length = subdata.size * BLOCK_SIZE
                     self._sigs_length.append(length)
 
+
 class NcsBlocks():
     """
     Contains information regarding the blocks of records in an Ncs file.
@@ -597,6 +600,7 @@ class NcsBlocks():
     sampFreqUsed = 0
     microsPerSampUsed = 0
 
+
 class NlxHeader(OrderedDict):
     """
     Representation of basic information in all 16 kbytes Neuralynx file headers,
@@ -670,13 +674,16 @@ def _to_bool(txt):
     header_pattern_dicts = {
         # Cheetah before version 5 and BML
         'bv5': dict(
-            datetime1_regex=r'## Time Opened: \(m/d/y\): (?P<date>\S+)  At Time: (?P<time>\S+)',
+            datetime1_regex=r'## Time Opened: \(m/d/y\): (?P<date>\S+)'
+                            r'  At Time: (?P<time>\S+)',
             filename_regex=r'## File Name: (?P<filename>\S+)',
             datetimeformat='%m/%d/%Y %H:%M:%S.%f'),
         # Cheetah version 5 before and including v 5.6.4
         'bv5.6.4': dict(
-            datetime1_regex=r'## Time Opened \(m/d/y\): (?P<date>\S+)  \(h:m:s\.ms\) (?P<time>\S+)',
-            datetime2_regex=r'## Time Closed \(m/d/y\): (?P<date>\S+)  \(h:m:s\.ms\) (?P<time>\S+)',
+            datetime1_regex=r'## Time Opened \(m/d/y\): (?P<date>\S+)'
+                            r'  \(h:m:s\.ms\) (?P<time>\S+)',
+            datetime2_regex=r'## Time Closed \(m/d/y\): (?P<date>\S+)'
+                            r'  \(h:m:s\.ms\) (?P<time>\S+)',
             filename_regex=r'## File Name (?P<filename>\S+)',
             datetimeformat='%m/%d/%Y %H:%M:%S.%f'),
         # Cheetah after v 5.6.4 and default for others such as Pegasus
@@ -697,7 +704,8 @@ def buildForFile(filename):
         txt_header = txt_header.strip(b'\x00').decode('latin-1')
 
         # must start with 8 # characters
-        assert txt_header.startswith("########"),'Neuralynx files must start with 8 # characters.'
+        assert txt_header.startswith("########"),\
+            'Neuralynx files must start with 8 # characters.'
 
         # find keys
         info = NlxHeader()
@@ -808,6 +816,7 @@ class NcsHeader():
     recording type.
     """
 
+
 ncs_dtype = [('timestamp', 'uint64'), ('channel_id', 'uint32'), ('sample_rate', 'uint32'),
              ('nb_valid', 'uint32'), ('samples', 'int16', (BLOCK_SIZE,))]
 

From e960e7e9565a71a645e703f299d83e3b66626216 Mon Sep 17 00:00:00 2001
From: "Peter N. Steinmetz" <PeterNSteinmetz@steinmetz.org>
Date: Wed, 28 Oct 2020 08:42:04 -0700
Subject: [PATCH 16/16] Revert to channel_indexes and fixed :FIXME:

The use of channel_indexes is a standard throughout neo at present.
---
 neo/rawio/neuralynxrawio.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/neo/rawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio.py
index 305049b18..0df584cb7 100644
--- a/neo/rawio/neuralynxrawio.py
+++ b/neo/rawio/neuralynxrawio.py
@@ -244,7 +244,7 @@ def _parse_header(self):
                     ts0 = ts[0]
                     ts1 = ts[-1]
                 ts0 = min(ts0, ts[0])
-                ts1 = max(ts0, ts[-1])  # :FIXME: possible error, should be ts1
+                ts1 = max(ts1, ts[-1])
 
         # decide on segment and global start and stop times based on files available
         if self._timestamp_limits is None:
@@ -322,7 +322,7 @@ def _get_signal_size(self, block_index, seg_index, channel_indexes):
     def _get_signal_t_start(self, block_index, seg_index, channel_indexes):
         return self._sigs_t_start[seg_index] - self.global_t_start
 
-    def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, channel_indices):
+    def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, channel_indexes):
         """
         Retrieve chunk of analog signal, a chunk being a set of contiguous samples.
 
@@ -336,7 +336,7 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, chann
             sample index of first sample within segment to retrieve
         i_stop:
             sample index of last sample within segment to retrieve
-        channel_indices:
+        channel_indexes:
             list of channel indices to return data for
 
         RETURNS
@@ -353,11 +353,11 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, chann
         sl0 = i_start % 512
         sl1 = sl0 + (i_stop - i_start)
 
-        if channel_indices is None:
-            channel_indices = slice(None)
+        if channel_indexes is None:
+            channel_indexes = slice(None)
 
-        channel_ids = self.header['signal_channels'][channel_indices]['id']
-        channel_names = self.header['signal_channels'][channel_indices]['name']
+        channel_ids = self.header['signal_channels'][channel_indexes]['id']
+        channel_names = self.header['signal_channels'][channel_indexes]['name']
 
         # create buffer for samples
         sigs_chunk = np.zeros((i_stop - i_start, len(channel_ids)), dtype='int16')