Fix digital line scaler data with non-zero offset (#212)

What is documented as a raw byte offset in the NI documentation is
actually a bit offset for digital channels.

nptdms/daqmx.py

@@ -65,10 +65,10 @@ def _read_data_chunk(self, file, data_objects, chunk_index):
                     scaler for scaler in obj.daqmx_metadata.scalers
                     if scaler.raw_buffer_index == raw_buffer_index]
                 for scaler in scalers_for_raw_buffer_index:
-                    offset = scaler.raw_byte_offset
+                    byte_offset = scaler.byte_offset()
                     scaler_size = scaler.data_type.size
                     byte_columns = tuple(
-                        range(offset, offset + scaler_size))
+                        range(byte_offset, byte_offset + scaler_size))
                     # Select columns for this scaler, so that number of values
                     # will be number of bytes per point * number of data
                     # points. Then use ravel to flatten the results into a
@@ -78,9 +78,8 @@ def _read_data_chunk(self, file, data_objects, chunk_index):
                     # should be correct
                     this_scaler_data.dtype = (
                         scaler.data_type.nptype.newbyteorder(self.endianness))
-                    if obj.daqmx_metadata.scaler_type == DIGITAL_LINE_SCALER:
-                        this_scaler_data = np.bitwise_and(this_scaler_data, 1)
-                    scaler_data[obj.path][scaler.scale_id] = this_scaler_data
+                    processed_data = scaler.postprocess_data(this_scaler_data)
+                    scaler_data[obj.path][scaler.scale_id] = processed_data
 
         return RawDataChunk.scaler_data(scaler_data)
 
@@ -163,8 +162,9 @@ def __init__(self, f, endianness, scaler_type):
 
         # size of vector of format changing scalers
         scaler_vector_length = types.Uint32.read(f, endianness)
+        scaler_class = _scaler_classes[scaler_type]
         self.scalers = [
-            DaqMxScaler(f, endianness, scaler_type)
+            scaler_class(f, endianness)
             for _ in range(scaler_vector_length)]
 
         # Read raw data widths.
@@ -200,19 +200,61 @@ class DaqMxScaler(object):
         'sample_format_bitmap',
         ]
 
-    def __init__(self, open_file, endianness, scaler_type):
+    def __init__(self, open_file, endianness):
         data_type_code = types.Uint32.read(open_file, endianness)
         self.data_type = DAQMX_TYPES[data_type_code]
 
         # more info for format changing scaler
         self.raw_buffer_index = types.Uint32.read(open_file, endianness)
         self.raw_byte_offset = types.Uint32.read(open_file, endianness)
-        if scaler_type == DIGITAL_LINE_SCALER:
-            self.sample_format_bitmap = types.Uint8.read(open_file, endianness)
-        else:
-            self.sample_format_bitmap = types.Uint32.read(open_file, endianness)
+        self.sample_format_bitmap = types.Uint32.read(open_file, endianness)
         self.scale_id = types.Uint32.read(open_file, endianness)
 
+    def byte_offset(self):
+        return self.raw_byte_offset
+
+    def postprocess_data(self, data):
+        return data
+
+    def __repr__(self):
+        properties = (
+            "%s=%s" % (name, _get_attr_repr(self, name))
+            for name in self.__slots__)
+
+        properties_list = ", ".join(properties)
+        return "%s(%s)" % (self.__class__.__name__, properties_list)
+
+
+class DigitalLineScaler(object):
+    """ Details of a DAQmx digital line scaler read from a TDMS file
+    """
+
+    __slots__ = [
+        'scale_id',
+        'data_type',
+        'raw_buffer_index',
+        'raw_bit_offset',
+        'sample_format_bitmap',
+        ]
+
+    def __init__(self, open_file, endianness):
+        data_type_code = types.Uint32.read(open_file, endianness)
+        self.data_type = DAQMX_TYPES[data_type_code]
+
+        # more info for format changing scaler
+        self.raw_buffer_index = types.Uint32.read(open_file, endianness)
+        self.raw_bit_offset = types.Uint32.read(open_file, endianness)
+        self.sample_format_bitmap = types.Uint8.read(open_file, endianness)
+        self.scale_id = types.Uint32.read(open_file, endianness)
+
+    def byte_offset(self):
+        return self.raw_bit_offset // 8
+
+    def postprocess_data(self, data):
+        bit_offset = self.raw_bit_offset % 8
+        bitmask = 1 << bit_offset
+        return np.right_shift(np.bitwise_and(data, bitmask), bit_offset)
+
     def __repr__(self):
         properties = (
             "%s=%s" % (name, _get_attr_repr(self, name))
@@ -238,3 +280,9 @@ def _get_attr_repr(obj, attr_name):
     4: types.Uint32,
     5: types.Int32,
 }
+
+
+_scaler_classes = {
+    FORMAT_CHANGING_SCALER: DaqMxScaler,
+    DIGITAL_LINE_SCALER: DigitalLineScaler,
+}

nptdms/test/test_daqmx.py

@@ -4,6 +4,7 @@
 from collections import defaultdict
 import logging
 import numpy as np
+import pytest
 
 from nptdms import TdmsFile
 from nptdms.log import log_manager
@@ -407,16 +408,16 @@ def test_digital_line_scaler_data():
     """ Test loading a DAQmx file with a single channel of U8 digital line scaler data
     """
 
-    scaler_metadata = daqmx_scaler_metadata(0, 0, 2, digital_line_scaler=True)
+    scaler_metadata = digital_scaler_metadata(0, 0, 0)
     metadata = segment_objects_metadata(
         root_metadata(),
         group_metadata(),
         daqmx_channel_metadata("Channel1", 4, [4], [scaler_metadata], digital_line_scaler=True))
     data = (
         "00 00 00 00"
-        "00 00 01 00"
+        "01 00 00 00"
         "00 00 00 00"
-        "00 00 01 00"
+        "01 00 00 00"
     )
 
     test_file = GeneratedFile()
@@ -429,30 +430,46 @@ def test_digital_line_scaler_data():
     np.testing.assert_array_equal(data, [0, 1, 0, 1])
 
 
-def test_digital_line_scaler_data_uses_first_bit_of_bytes():
-    """ Test DAQmx digital line scaler data only uses the first bit in each byte to represent a 1 or 0 value
+@pytest.mark.parametrize('byte_offset', [0, 1, 2, 3])
+def test_digital_line_scaler_with_multiple_channels(byte_offset):
+    """ Test DAQmx digital line scaler data with multiple channels
     """
 
-    scaler_metadata = daqmx_scaler_metadata(0, 0, 2, digital_line_scaler=True)
+    scaler_metadata_0 = digital_scaler_metadata(0, 0, byte_offset * 8 + 0)
+    scaler_metadata_1 = digital_scaler_metadata(0, 0, byte_offset * 8 + 1)
+    scaler_metadata_2 = digital_scaler_metadata(0, 0, byte_offset * 8 + 2)
     metadata = segment_objects_metadata(
         root_metadata(),
         group_metadata(),
-        daqmx_channel_metadata("Channel1", 4, [4], [scaler_metadata], digital_line_scaler=True))
-    data = (
-        "00 00 00 00"
-        "00 00 01 00"
-        "00 00 02 00"
-        "00 00 03 00"
+        daqmx_channel_metadata("Channel0", 4, [4], [scaler_metadata_0], digital_line_scaler=True),
+        daqmx_channel_metadata("Channel1", 4, [4], [scaler_metadata_1], digital_line_scaler=True),
+        daqmx_channel_metadata("Channel2", 4, [4], [scaler_metadata_2], digital_line_scaler=True),
     )
+    byte_values = [
+        "00",
+        "01",
+        "02",
+        "03",
+        "04",
+        "05",
+        "06",
+        "07",
+    ]
+    hex_data = " ".join("00" * byte_offset + b + "00" * (3 - byte_offset) for b in byte_values)
 
     test_file = GeneratedFile()
-    test_file.add_segment(segment_toc(), metadata, data)
+    test_file.add_segment(segment_toc(), metadata, hex_data)
     tdms_data = test_file.load()
 
-    data = tdms_data["Group"]["Channel1"].raw_data
+    for (channel_name, expected_data) in [
+        ("Channel0", [0, 1, 0, 1, 0, 1, 0, 1]),
+        ("Channel1", [0, 0, 1, 1, 0, 0, 1, 1]),
+        ("Channel2", [0, 0, 0, 0, 1, 1, 1, 1]),
+    ]:
+        data = tdms_data["Group"][channel_name].raw_data
 
-    assert data.dtype == np.uint8
-    np.testing.assert_array_equal(data, [0, 1, 0, 1])
+        assert data.dtype == np.uint8
+        np.testing.assert_array_equal(data, expected_data, "Incorrect data for channel '%s'" % channel_name)
 
 
 def test_lazily_reading_channel():
@@ -683,7 +700,7 @@ def group_metadata():
         "00 00 00 00")
 
 
-def daqmx_scaler_metadata(scale_id, type_id, byte_offset, raw_buffer_index=0, digital_line_scaler=False):
+def daqmx_scaler_metadata(scale_id, type_id, byte_offset, raw_buffer_index=0):
     return (
         # DAQmx data type (type ids don't match TDMS types)
         hexlify_value("<I", type_id) +
@@ -692,7 +709,21 @@ def daqmx_scaler_metadata(scale_id, type_id, byte_offset, raw_buffer_index=0, di
         # Raw byte offset
         hexlify_value("<I", byte_offset) +
         # Sample format bitmap (don't know what this is for...)
-        ("00" if digital_line_scaler else "00 00 00 00") +
+        "00 00 00 00" +
+        # Scale ID
+        hexlify_value("<I", scale_id))
+
+
+def digital_scaler_metadata(scale_id, type_id, bit_offset, raw_buffer_index=0):
+    return (
+        # DAQmx data type (type ids don't match TDMS types)
+        hexlify_value("<I", type_id) +
+        # Raw buffer index
+        hexlify_value("<I", raw_buffer_index) +
+        # Raw byte offset
+        hexlify_value("<I", bit_offset) +
+        # Sample format bitmap (don't know what this is for...)
+        "00" +
         # Scale ID
         hexlify_value("<I", scale_id))