fix: sl3 version 1 works again

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = sllib
-version = 0.2.2
+version = 0.2.3
 author = Peter Magnusson    
 author_email = me@kmpm.se
 description = Library for reading SLG, SL2 and SL3 files from Lowrance fishfinders

sllib/__init__.py

@@ -4,13 +4,14 @@
 from .frame import Frame
 from .header import Header
 
-__all__ = ['Frame', 'Reader', 'Header', 'create_reader']
+__all__ = ['Frame', 'Reader', 'Header', 'create_reader', '__version__']
 
+__version__ = '0.2.3'
 
 @contextmanager
-def create_reader(filename):
+def create_reader(filename, strict=False):
     f = open(filename, 'rb')
     try:
-        yield Reader(f)
+        yield Reader(f, strict=strict)
     finally:
         f.close()

sllib/frame.py

@@ -66,21 +66,22 @@ def to_dict(self, format=2, fields=None):
         return out
 
     @staticmethod
-    def read(stream: io.IOBase, format: List[int], blocksize: int = 0, strict: bool = False):
-        if not isinstance(format, list):
+    def read(stream: io.IOBase, formver: List[int], blocksize: int = 0, strict: bool = False):
+        if not isinstance(formver, list):
             raise TypeError('format must be a list')
 
-        if format[0] == 1:
+        if formver[0] == 1:
             # slg is a conditional format for each packet... yuck
             return _readSlg(stream, blocksize)
-        if format[0] == 2:
-            return _readSl2(stream, blocksize, 2)
-        if format[0] == 3:
-            return _readSlx(stream, blocksize, strict=strict, format=format)
+        if formver[0] == 2:
+            return _readSl2(stream, blocksize, formver)
+        if formver[0] == 3:
+            return _readSlx(stream, blocksize, strict=strict, formver=formver)
         raise Exception('unkown format')
 
 
-def _readSl2(stream: io.IOBase, blocksize: int, format: List[int]) -> Frame:
+def _readSl2(stream: io.IOBase, blocksize: int, formver: List[int]) -> Frame:
+    format = formver[0]
     f = FRAME_FORMATS[format]
     s = struct.calcsize(f)
     here = stream.tell()
@@ -126,10 +127,10 @@ def _readSl2(stream: io.IOBase, blocksize: int, format: List[int]) -> Frame:
     return b
 
 
-def _readSlx(stream: io.IOBase, blocksize: int, format: List[int], strict: bool) -> Frame:
-    fform = format[0]
-    # fver = format[1]
-    f = FRAME_FORMATS[fform]
+def _readSlx(stream: io.IOBase, blocksize: int, formver: List[int], strict: bool) -> Frame:
+    format = formver[0]
+    version = formver[1]
+    f = FRAME_FORMATS[format]
     s = struct.calcsize(f)
     here = stream.tell()
     bad = 0
@@ -149,7 +150,7 @@ def _readSlx(stream: io.IOBase, blocksize: int, format: List[int], strict: bool)
         elif here > 0:
             bad += 1
             if bad == 1:
-                logger.warn('unexpected offset at offset: %s. will try to find next frame', here)
+                logger.warn('unexpected offset %s at location: %s. will try to find next frame', data[0], here)
             if strict:
                 raise Exception('offset missmatch')
             # jump forward and try to catch next
@@ -160,23 +161,26 @@ def _readSlx(stream: io.IOBase, blocksize: int, format: List[int], strict: bool)
             raise Exception('location does not match expected offset')
 
     kv = {'headersize': s}
-    for i, d in enumerate(FRAME_DEFINITIONS[fform]):
+    for i, d in enumerate(FRAME_DEFINITIONS[format]):
         name = d['name']
         if not name == "-":
             kv[name] = data[i]
-            if name == 'flags' and FLAG_FORMATS[fform]:
+            if name == 'flags' and FLAG_FORMATS[format]:
                 if FLAG_AS_BINARY:
                     kv[name] = f'({kv[name]}) {kv[name]:016b}'
-                flagform = FLAG_FORMATS[fform]
+                flagform = FLAG_FORMATS[format]
                 flags = data[i]
                 for k, v in flagform.items():
                     kv[k] = flags & v == v
     b = Frame(**kv)
-    if b.channel <= 5:
+    packetsize = b.packetsize
+    if version == 1 and not b.has_tbd1:
+        packetsize = b.framesize - 168
+
+    if version == 1 or (version == 2 and b.channel <= 5):
         extra = 168-s
-        # logger.debug('low channel. reading extra %d bytes', extra)
         stream.read(extra)
-    b.packet = stream.read(b.packetsize)
+    b.packet = stream.read(packetsize)
 
     return b
 

tests/test_reader_sl3.py

@@ -27,7 +27,7 @@ def test_header_sl3(self):
             self.assertIn('channel', fields)
 
     def test_first(self):
-        with sllib.create_reader(self.path_sl3) as reader:
+        with sllib.create_reader(self.path_sl3, True) as reader:
             header = reader.header
             self.assertEqual(header.format, 3)
             x = next(reader)
@@ -53,7 +53,7 @@ def test_first(self):
             self.assertEqual(x.flags, 950)  # TODO: Validate !!!
 
     def test_read_all_v1(self):
-        with sllib.create_reader(self.path_sl3) as reader:
+        with sllib.create_reader(self.path_sl3, True) as reader:
             header = reader.header
             self.assertEqual(header.format, 3)
             self.assertEqual(header.version, 1)
@@ -67,7 +67,7 @@ def test_read_all_v1(self):
             }
             for frame in reader:
                 count += 1
-                if frame.channel <= 5:
+                if frame.has_tbd1:
                     goodCount += 1
                 # space
                 self.assertGreaterEqual(frame.longitude, box['lon']['min'])
@@ -81,15 +81,15 @@ def test_read_all_v1(self):
                 self.assertLessEqual(frame.channel, 9)
                 self.assertLessEqual(frame.frequency, 10)
 
-            self.assertEqual(count, 8691)
-            self.assertEqual(goodCount, 6228)
+            self.assertEqual(count, 10124, 'wrong number of frames')
+            self.assertEqual(goodCount, 7626, 'wrong number of frames with flag set')
 
             filesize = os.path.getsize(self.path_sl3)
             self.assertEqual(reader.tell(), filesize)
 
     def test_read_all_v2(self):
         filename = self.path_sl3_v2
-        with sllib.create_reader(filename) as reader:
+        with sllib.create_reader(filename, strict=True) as reader:
             header = reader.header
             self.assertEqual(header.format, 3)
             self.assertEqual(header.version, 2)

utils/offsets.py

@@ -1,3 +1,4 @@
+from typing import List
 import csv
 import os
 import struct
@@ -32,10 +33,11 @@ def create_csv_with_header(csvfile, fields) -> csv.DictWriter:
     return writer
 
 
-def readfile(stream: IOBase, writer: csv.DictWriter, format: int, version: int, maxcount: int = 20):
+def readfile(stream: IOBase, writer: csv.DictWriter, formver: List[int], maxcount: int = 20):
     count = 0
     last = 0
-    offset = 0
+    offset = 8
+    last_end = 8
     while True:
         stream.seek(offset)
         buf = stream.read(4)
@@ -47,17 +49,20 @@ def readfile(stream: IOBase, writer: csv.DictWriter, format: int, version: int,
 
         if data[0] == offset:  # yes, we have an equal
             stream.seek(offset)  # go back a bit
-            fr = Frame.read(stream, format)
-            dct = fr.to_dict(format)
+            fr = Frame.read(stream, formver)
+            told = stream.tell()
+            dct = fr.to_dict(formver[0])
             dct['start'] = offset
-            dct['end'] = stream.tell()
+            dct['end'] = told
+            dct['offby'] = offset - last_end
             dct['size'] = offset-last
-            dct['asdf'] = dct['packetsize'] - dct['framesize']
+            dct['asdf'] = [fr.channel, f'({fr.flags}) {fr.flags:016b}']
             writer.writerow(dct)
             # print(
             #     'match at', offset, 'now', dct['now'], 'size', offset - last, 'asd', now-offset-fr.headersize,
             #     fr.to_dict(format=3, fields=['offset', 'index', 'latitude', 'packetsize', 'headersize'])
             # )
+            last_end = told
             last = offset
             count += 1
 
@@ -74,10 +79,11 @@ def main(filename, maxcount):
     with open(filename, 'rb') as stream:
         with open(outfile, 'w', newline='') as csvfile:
             reader = Reader(stream)
+            formver = [reader.header.format, reader.header.version]
             print(reader.header)
-            fields = ['start', 'end', 'size', 'asdf'] + reader.header.fields
+            fields = ['start', 'end', 'offby', 'size', 'asdf'] + reader.fields
             writer = create_csv_with_header(csvfile, fields)
-            count = readfile(stream, writer, reader.header.format, reader.header.version, maxcount)
+            count = readfile(stream, writer, formver, maxcount)
             print(f'wrote {count} records to {outfile}')