Update to use session based output files and handle clock signals

Author: dannyockilson

QUICK_START.md

@@ -127,7 +127,11 @@ This will:
 
 **What happens:** The tool finds all your recording files and extracts the brain signal data from each one. For each NDF file (like M1555404530.ndf), it creates a matching subdirectory (M1555404530/) with separate channel files (E0.txt, E1.txt, E2.txt, etc.).
 
-**Important note on channel zero:** The output channel 0 represents the clock messages channel and occur at 128 Hz.
+**Important note on channels:**
+- Channel 0: Clock signal (128 Hz) - used for timing synchronization
+- Channels 1-15: EEG data channels (512 Hz) - your brain activity recordings
+
+The tool automatically detects and uses the correct rate for each channel type.
 
 #### Performance Notes
 

README.md

@@ -137,7 +137,7 @@ python bulk_converter.py input_folder
 python bulk_converter.py input_folder output_folder
 
 # With custom settings
-python bulk_converter.py input_folder output_folder --sample-rate 512 --range 120
+python bulk_converter.py input_folder output_folder --range 120
 ```
 
 **Note:** The bulk converter now expects a directory structure with subdirectories containing E{channel}.txt files (e.g., E1.txt, E2.txt). Each subdirectory is converted into a single unified LabChart file with all channels as tab-separated columns.
@@ -148,7 +148,6 @@ python bulk_converter.py input_folder output_folder --sample-rate 512 --range 12
 python bulk_converter.py [input_dir] [output_dir] [options]
 
 Options:
-  --sample-rate, -sr    Sample rate in Hz (default: 512)
   --range, -r          Input dynamic range in mV (default: 120)
   --interval-length    Length of each interval in seconds (default: 1.0)
   --commas             Use European format (commas for decimals)
@@ -158,6 +157,8 @@ Options:
   --glitch-threshold   Glitch filter threshold (default: 500, 0 to disable)
   --verbose, -v        Verbose output
   --help, -h           Show help message
+
+Note: Sample rates are now auto-detected per channel (Ch0: 128Hz, others: 512Hz)
 ```
 
 #### Bulk Converter Examples
@@ -167,7 +168,7 @@ Options:
 python bulk_converter.py ndf_text_output output --range 120
 
 # AC transmitter with different settings
-python bulk_converter.py ndf_text_output output --range 30 --sample-rate 1024
+python bulk_converter.py ndf_text_output output --range 30
 
 # European format with microvolts
 python bulk_converter.py ndf_text_output output --commas --microvolts
@@ -206,10 +207,11 @@ Options:
   --output, -o          Output directory (default: input_path + '_text')
   --channels, -c        Specific channels to extract (default: all channels)
   --format, -f          Output format: simple, detailed, csv (default: simple)
-  --sample-rate, -sr    Sample rate in Hz (default: 512)
   --timestamps          Include timestamp information
   --no-metadata         Exclude metadata headers
   --verbose, -v         Verbose output
+
+Note: Sample rates are auto-detected per channel (Ch0: 128Hz, others: 512Hz)
 ```
 
 #### Output Formats
@@ -238,6 +240,9 @@ The NDF reader automatically handles:
 - **Message Parsing**: Decodes 8-byte telemetry messages
 - **Timing Reconstruction**: Converts timestamps to relative timing
 - **Data Validation**: Ensures 16-bit signal compatibility
+- **Per-Channel Sample Rates**: Auto-detects channel-specific rates
+  - Channel 0: 128 Hz (clock signal channel)
+  - Channels 1-15: 512 Hz (data channels)
 
 ### Manual Conversion (API Usage)
 
@@ -314,8 +319,8 @@ reader = NDFReader('data.ndf')
 print(f"Created: {reader.get_creation_date()}")
 print(f"Channels: {reader.get_available_channels()}")
 
-# Extract specific channel
-intervals = reader.read_channel_data(channel_num=1, sample_rate=512.0)
+# Extract specific channel (sample rate auto-detected: Ch0=128Hz, others=512Hz)
+intervals = reader.read_channel_data(channel_num=1)
 
 # Export to LabChart
 exporter = LabChartExporter(sample_rate=512.0, range_mV=120.0)
@@ -333,6 +338,7 @@ output_file = exporter.export_channel(
 from ndf_reader import SimpleBinarySignalReader
 
 # Read 16-bit binary data
+# Note: Binary files require explicit sample_rate since auto-detection only works for NDF files
 intervals = SimpleBinarySignalReader.read_signal(
     filepath="signal_data.bin",
     sample_rate=512.0,
@@ -353,6 +359,7 @@ exporter.export_channel(
 from ndf_reader import TextSignalReader
 
 # Read text file (one value per line)
+# Note: Text files require explicit sample_rate since auto-detection only works for NDF files
 intervals = TextSignalReader.read_signal(
     filepath="signal_data.txt",
     sample_rate=512.0,

bulk_converter.py

@@ -26,13 +26,16 @@
 
 def find_channel_directories(input_dir: str) -> List[str]:
     """
-    Find all subdirectories that contain E{channel}.txt files.
+    Find all session subdirectories that contain E{channel}.txt files.
+
+    Looks for directories named session_* or session_{timestamp} containing
+    channel data files from continuous recording sessions.
 
     Args:
         input_dir: Parent directory to search
 
     Returns:
-        List of subdirectory paths containing channel files
+        List of session directory paths containing channel files, sorted chronologically
     """
     subdirs = []
 
@@ -45,6 +48,7 @@ def find_channel_directories(input_dir: str) -> List[str]:
             if channel_files:
                 subdirs.append(full_path)
 
+    # Sort by directory name (which includes timestamp for session directories)
     return sorted(subdirs)
 
 
@@ -74,15 +78,15 @@ def find_channel_files(directory: str) -> Dict[int, str]:
 
 def load_channel_data(
     channel_files: Dict[int, str],
-    sample_rate: float,
     interval_length: float = 1.0,
 ) -> Dict[int, List[Tuple[float, List[int]]]]:
     """
-    Load data from all channel files.
+    Load data from all channel files with per-channel sample rates.
+
+    Channel 0 uses 128 Hz (clock signal), all other channels use 512 Hz.
 
     Args:
         channel_files: Dictionary mapping channel numbers to file paths
-        sample_rate: Sample rate in Hz
         interval_length: Length of each interval in seconds
 
     Returns:
@@ -92,14 +96,21 @@ def load_channel_data(
 
     for channel_num, file_path in channel_files.items():
         try:
+            # Auto-detect sample rate based on channel number
+            # Channel 0: 128 Hz (clock signal)
+            # Other channels: 512 Hz (default)
+            sample_rate = 128.0 if channel_num == 0 else 512.0
+
             intervals = TextSignalReader.read_signal(
                 filepath=file_path,
                 sample_rate=sample_rate,
                 interval_length=interval_length,
             )
             if intervals:
                 channel_data[channel_num] = intervals
-                print(f"  Channel {channel_num}: {len(intervals)} intervals loaded")
+                print(
+                    f"  Channel {channel_num}: {len(intervals)} intervals loaded ({sample_rate} Hz)"
+                )
             else:
                 print(f"  Channel {channel_num}: Warning - no data found")
         except Exception as e:
@@ -115,10 +126,13 @@ def convert_directory(
     interval_length: float = 1.0,
 ) -> Optional[str]:
     """
-    Convert all channel files in a directory to a single unified LabChart file.
+    Convert all channel files in a session directory to a single unified LabChart file.
+
+    Extracts session timestamp from directory name (session_{timestamp}) for both
+    creation date and output filename.
 
     Args:
-        input_dir: Directory containing E{channel}.txt files
+        input_dir: Session directory containing E{channel}.txt files
         output_dir: Output directory for LabChart file
         exporter: LabChartExporter instance
         interval_length: Length of each interval in seconds
@@ -138,21 +152,29 @@ def convert_directory(
 
     print(f"  Found {len(channel_files)} channel files: {sorted(channel_files.keys())}")
 
-    # Load all channel data
+    # Load all channel data with per-channel sample rates
     channel_data = load_channel_data(
         channel_files=channel_files,
-        sample_rate=exporter.sample_rate,
         interval_length=interval_length,
     )
 
     if not channel_data:
         print(f"  Warning: No valid data loaded from any channels")
         return None
 
-    # Get creation date from directory metadata or first file
-    first_file = list(channel_files.values())[0]
-    file_mtime = os.path.getmtime(first_file)
-    creation_date = datetime.fromtimestamp(file_mtime).strftime("%Y-%m-%d %H:%M:%S")
+    # Extract session timestamp from directory name (session_{timestamp})
+    # Fallback to file modification time if extraction fails
+    session_pattern = re.compile(r"session_(\d{10})")
+    match = session_pattern.match(dir_name)
+
+    if match:
+        timestamp = int(match.group(1))
+        creation_date = datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M:%S")
+    else:
+        # Fallback to first file modification time
+        first_file = list(channel_files.values())[0]
+        file_mtime = os.path.getmtime(first_file)
+        creation_date = datetime.fromtimestamp(file_mtime).strftime("%Y-%m-%d %H:%M:%S")
 
     # Create output filename based on directory name
     output_file = os.path.join(output_dir, f"{dir_name}.txt")
@@ -184,12 +206,16 @@ def bulk_convert(
     glitch_threshold: int = 500,
 ) -> List[str]:
     """
-    Bulk convert all channel directories to unified LabChart format.
+    Bulk convert all session directories to unified LabChart format.
+
+    Sample rates are auto-detected per channel:
+    - Channel 0: 128 Hz (clock signal)
+    - Other channels: 512 Hz
 
     Args:
-        input_dir: Input directory containing subdirectories with E{channel}.txt files
+        input_dir: Input directory containing session subdirectories with E{channel}.txt files
         output_dir: Output directory (default: input_dir + '_labchart')
-        sample_rate: Sample rate in Hz
+        sample_rate: DEPRECATED - Sample rates are now auto-detected per channel
         range_mV: Input dynamic range in millivolts
         interval_length: Length of each interval in seconds
         use_commas: Use European format (commas for decimals)
@@ -250,7 +276,9 @@ def bulk_convert(
             created_files.append(output_file)
 
     print(f"\nConversion complete!")
-    print(f"Successfully converted {len(created_files)} out of {len(channel_dirs)} directories")
+    print(
+        f"Successfully converted {len(created_files)} out of {len(channel_dirs)} directories"
+    )
     print(f"Output files: {output_dir}")
 
     return created_files
@@ -259,38 +287,44 @@ def bulk_convert(
 def main():
     """Main entry point for command line usage"""
     parser = argparse.ArgumentParser(
-        description="Bulk convert EEG channel directories to unified LabChart format",
+        description="Bulk convert EEG session directories to unified LabChart format",
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog="""
 Examples:
   # Basic conversion with default settings
-  python bulk_converter.py ndf_files_text
+  python bulk_converter.py test_sessions
 
-  # Custom output directory and sample rate
-  python bulk_converter.py ndf_files_text output --sample-rate 1024 --range 30
+  # Custom output directory with options
+  python bulk_converter.py test_sessions output --range 120 --commas
 
   # European format with microvolts
-  python bulk_converter.py data output --commas --microvolts
+  python bulk_converter.py test_sessions output --commas --microvolts
 
-  # High precision timing in milliseconds
-  python bulk_converter.py data output --milliseconds --absolute-time
+  # High precision timing with absolute UNIX timestamps
+  python bulk_converter.py test_sessions output --milliseconds --absolute-time
 
 Expected Input Structure:
   input_dir/
-  ├── M1555404530/
-  │   ├── E0.txt
-  │   ├── E1.txt
-  │   ├── E2.txt
-  │   └── E15.txt
-  └── M1555404531/
+  ├── session_1555404530/    (continuous recording session)
+  │   ├── E0.txt             (Channel 0: 128 Hz clock signal)
+  │   ├── E1.txt             (Channel 1: 512 Hz)
+  │   ├── E2.txt             (Channel 2: 512 Hz)
+  │   └── E15.txt            (Channel 15: 512 Hz)
+  └── session_1558948567/    (next session after gap)
       ├── E0.txt
       ├── E1.txt
       └── E2.txt
 
 Output:
   output_dir/
-  ├── M1555404530.txt  (contains all channels in unified format)
-  └── M1555404531.txt  (contains all channels in unified format)
+  ├── session_1555404530.txt  (all channels merged, tab-separated)
+  └── session_1558948567.txt  (all channels merged, tab-separated)
+
+Notes:
+  - Channel 0 is automatically detected as 128 Hz (clock signal)
+  - All other channels are automatically detected as 512 Hz
+  - The --sample-rate option is deprecated but retained for compatibility
+  - Session directories are typically created by ndf_to_text_converter.py
         """,
     )
 
@@ -310,7 +344,7 @@ def main():
         "-sr",
         type=float,
         default=512.0,
-        help="Sample rate in Hz (default: 512)",
+        help="DEPRECATED - Sample rates are auto-detected per channel (Ch0: 128Hz, others: 512Hz)",
     )
 
     parser.add_argument(

labchart_exporter.py

@@ -9,7 +9,7 @@
 import os
 import struct
 from datetime import datetime
-from typing import List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
 
 
 class LabChartExporter:
@@ -230,6 +230,10 @@ def export_multi_channel(
         """
         Export multiple channels to a single unified LabChart file.
 
+        Per-channel sample rates are auto-detected:
+        - Channel 0: 128 Hz (clock signal)
+        - Other channels: 512 Hz
+
         Args:
             output_file: Path to output file
             channel_data: Dictionary mapping channel numbers to their intervals
@@ -258,13 +262,17 @@ def export_multi_channel(
         all_samples = []
 
         for channel in channels:
+            # Determine sample rate for this channel
+            # Channel 0: 128 Hz (clock signal), others: 512 Hz
+            channel_sample_rate = 128.0 if channel == 0 else 512.0
+
             intervals = channel_data[channel]
             for start_time, signal_values in intervals:
                 # Apply glitch filter if enabled
                 filtered_values = self._apply_glitch_filter(signal_values)
 
-                # Calculate time for each sample
-                sample_period = 1.0 / self.sample_rate
+                # Calculate time for each sample using per-channel sample rate
+                sample_period = 1.0 / channel_sample_rate
                 for i, value in enumerate(filtered_values):
                     sample_time = start_time + (i * sample_period)
                     all_samples.append((sample_time, channel, value))
@@ -274,7 +282,7 @@ def export_multi_channel(
 
         # Group samples by timestamp using rounding for efficiency
         # Round to nearest nanosecond to handle floating point precision
-        timestamp_groups = {}
+        timestamp_groups: Dict[float, Dict[int, int]] = {}
 
         for sample_time, channel, value in all_samples:
             # Round timestamp to avoid floating point comparison issues
@@ -293,6 +301,10 @@ def export_multi_channel(
         if not self.absolute_time:
             if self.start_time is None:
                 self.start_time = sorted_times[0] if sorted_times else 0.0
+            # Ensure start_time is set for type checking
+            start_time_value = self.start_time
+        else:
+            start_time_value = 0.0  # Not used in absolute mode
 
         with open(output_file, "a") as f:
             for sample_time in sorted_times:
@@ -302,7 +314,7 @@ def export_multi_channel(
                 if self.absolute_time:
                     display_time = sample_time
                 else:
-                    display_time = sample_time - self.start_time
+                    display_time = sample_time - start_time_value
 
                 # Format time
                 time_str = self._format_value(display_time, is_time=True)