fix: improve kaitai parser parser #119

src/hobbits-plugins/analyzers/KaitaiStruct/kaitaistruct.cpp

@@ -295,6 +295,7 @@ QSharedPointer<const AnalyzerResult> KaitaiStruct::analyzeBits(
     }
 
     QSharedPointer<BitInfo> bitInfo = BitInfo::copyFromContainer(container);
+    bitInfo->clearHighlightCategory(KAITAI_STRUCT_CATEGORY);
     bitInfo->addHighlights(highlights);
 
     if (!kscOutput.isEmpty()) {
@@ -310,11 +311,11 @@ QSharedPointer<const AnalyzerResult> KaitaiStruct::analyzeBits(
 RangeHighlight KaitaiStruct::makeHighlight(QString label, const QMap<QString, QPair<Range, QList<QString>>> &rangeData, int &colorIdx)
 {
     QList<QColor> colors = {
-        QColor(100, 220, 100, 85),
-        QColor(100, 0, 255, 50),
-        QColor(0, 150, 230, 100),
-        QColor(200, 140, 0, 100),
-        QColor(250, 50, 0, 100)
+        QColor(100, 220, 100, 200),
+        QColor(100, 0, 255, 200),
+        QColor(0, 150, 230, 200),
+        QColor(200, 140, 0, 200),
+        QColor(250, 50, 0, 200)
     };
     auto pair = rangeData.value(label);
     if (pair.second.isEmpty()) {

src/hobbits-plugins/analyzers/KaitaiStruct/scripts/runner.py

@@ -4,26 +4,28 @@
 import importlib
 import glob
 import os
+import binascii
+import traceback
 
 from kaitaistruct import KaitaiStruct
 
-#import pprint
-#pp = pprint.PrettyPrinter(indent=2)
+import pprint
+pp = pprint.PrettyPrinter(indent=2)
 
 def dump_struct(s, sections, prefix="", parent_offset = 0, root_io=None, root_offset=0):
     if isinstance(s, list):
-        #print(f"\n[BEGIN List Under '{prefix}']")
+        print(f"\n[BEGIN List Under '{prefix}']")
         for i, item in enumerate(s):
             label = prefix + "[" + str(i) + "]"
-            #print(label)
+            print(label)
             sections.append({
                 "label": label,
                 "parent": prefix
             })
             dump_struct(item, sections, label, parent_offset, root_io, root_offset)
-        #print(f"[END OF List Under '{prefix}']\n")
+        print(f"[END OF List Under '{prefix}']\n")
     elif isinstance(s, KaitaiStruct):
-        #pp.pprint(vars(s))
+        pp.pprint(vars(s))
         if hasattr(s, "_debug"):
 
             section_io = s._io
@@ -33,11 +35,11 @@ def dump_struct(s, sections, prefix="", parent_offset = 0, root_io=None, root_of
                 root_io = section_io
                 root_offset = root_offset + parent_offset
 
-            #print("\nITEMS {\n")
+            print("\nITEMS {\n")
             for name, descr in s._debug.items():
                 prop = getattr(s, name)
                 label = prefix + "." + name if prefix else name
-                #print(f"(\n  name: {label},\n  descr: {descr},\n  prop: {prop},\n  offset: {root_offset}\n)\n")
+                print(f"(\n  name: {label},\n  descr: {descr},\n  prop: {prop},\n  offset: {root_offset}\n)\n")
 
                 sections.append({
                     "start": descr["start"] + root_offset,
@@ -51,7 +53,119 @@ def dump_struct(s, sections, prefix="", parent_offset = 0, root_io=None, root_of
 
                 dump_struct(prop, sections, label, parent_offset, root_io, root_offset)
 
-            #print("}\n")
+            print("}\n")
+
+
+def process_value(value, section):
+
+    if isinstance(value, bytes):
+        if len(value) > 15:
+            value = f"0x{binascii.hexlify(value[:12]).decode()}..."
+        else:
+            value = f"0x{binascii.hexlify(value).decode()}"
+        section['value'] = value
+
+    elif isinstance(value, float):
+        section['value'] = value
+
+    elif isinstance(value, int):
+        section['value'] = value
+
+    elif isinstance(value, bool):
+        section['value'] = value
+
+    else:
+        return False
+
+    return True
+
+
+def parse_struct(struct, sections, prefix="", parent_offset = 0, root_io=None, root_offset=0):
+    if not isinstance(struct, KaitaiStruct):
+        return
+
+    print(f"Parsing {struct} at '{prefix}'")
+
+    # iterate through members in order to read lazy instances
+    for attr in vars(type(struct)):
+        #print(f"Checking attribute {attr}")
+        prop = getattr(type(struct), attr, None)
+        if isinstance(prop, property):
+            #print(f"Getting property {attr}")
+            try:
+                getattr(struct, attr)
+            except:
+                print(f"Failed when getting property {attr}: {traceback.format_exc()}")
+
+    if not hasattr(struct, "_debug"):
+        return
+
+
+
+    section_io = struct._io
+    if root_io is None:
+        root_io = section_io
+    elif section_io is not root_io:
+        root_io = section_io
+        root_offset = root_offset + parent_offset
+
+    for name, info in struct._debug.items():
+        try:
+            value = getattr(struct, name)
+        except:
+            print(f"Skipping {name}, not an attribute in struct")
+            continue
+
+        label = prefix + "." + name if prefix else name
+        parent_offset = info["start"] + root_offset
+
+        #print(f"{name}:")
+        #pp.pprint(info)
+
+        if not ("end" in info and "start" in info):
+            print(f"Skipping {name}, no start-end...")
+            continue
+
+        section = {
+            "start": info["start"] + root_offset,
+            "end": info["end"] + root_offset,
+            "label": label,
+            "parent": prefix
+        }
+
+
+        if isinstance(value, KaitaiStruct):
+            sections.append(section)
+            parse_struct(value, sections, label, parent_offset, root_io, root_offset)
+
+        elif isinstance(value, list) and len(value) > 0:
+            sections.append(section)
+            for idx, value_item in enumerate(value):
+                idx_label = f"{label}[{idx}]"
+                idx_section = {
+                    "label": idx_label,
+                    "parent": label
+                }
+
+                if isinstance(value_item, KaitaiStruct):
+                    sections.append(idx_section)
+                    parse_struct(value_item, sections, idx_label, parent_offset, root_io, root_offset)
+                else:
+                    if not struct._debug[name] or not struct._debug[name]['arr'] or idx >= len(struct._debug[name]['arr']):
+                        continue
+                    value_item_section = {
+                        "start": struct._debug[name]['arr'][idx]['start'] + root_offset,
+                        "end": struct._debug[name]['arr'][idx]['end'] + root_offset,
+                        "label": idx_section["label"],
+                        "parent": idx_section["parent"],
+                    }
+                    if process_value(value_item, value_item_section):
+                        sections.append(value_item_section)
+
+        elif process_value(value, section):
+            sections.append(section)
+
+
 
 def parse_data(input_filename, output_filename, action_progress):
     # locate the compiled struct module
@@ -69,16 +183,19 @@ def parse_data(input_filename, output_filename, action_progress):
 
     # parse the input data
     target = Struct.from_file(input_filename)
-    target._read()
+    try:
+        target._read()
 
-    action_progress.set_progress_percent(70)
+        action_progress.set_progress_percent(70)
 
-    # write the parser result to the output
-    sections = []
+        # write the parser result to the output
+        sections = []
 
-    dump_struct(target, sections)
+        parse_struct(target, sections)
+    finally:
+        target._io.close()
 
-    #pp.pprint(sections)
+    pp.pprint(sections)
 
     action_progress.set_progress_percent(80)
 

src/hobbits-widgets/displayhelper.cpp

@@ -213,47 +213,56 @@ QVector<QRectF> DisplayHelper::drawHighlightRects(
     }
 
     QVector<QRectF> rects;
-    if (container->info()->highlights(category).size() > 0) {
-        int lastHighlight = 0;
-        int rowOffset = -1;
-        for (qint64 i = frameOffset; i < container->frameCount(); i++) {
-            rowOffset++;
-            if (rowOffset >= rowCount) {
-                break;
-            }
-            Frame frame = container->frameAt(i);
-            Frame displayFrame =
-                Frame(
-                        container->bits(),
-                        frame.start() + bitOffset,
-                        qMin(frame.end(), frame.start() + bitOffset + colCount - 1));
-            QList<RangeHighlight> spots = getHighlightSpots(
-                    container->info()->highlights(category),
-                    lastHighlight,
-                    displayFrame);
-
-            while(!spots.isEmpty()) {
-                RangeHighlight spot = spots.takeFirst();
-                if (!spot.children().isEmpty()) {
-                    painter->setOpacity(0.35);
-                    int minIndex = 0;
-                    spots.append(getHighlightSpots(spot.children(), minIndex, displayFrame));
-                }
-                else {
-                    painter->setOpacity(1);
-                }
-                qint64 displayStart = (spot.range().start() - displayFrame.start());
-                double hx = getGroupedOffset(displayStart, colWidth, colGroupSize, bitOffset, colGroupMargin);
-                double hy = (i - frameOffset) * rowHeight;
-                qint64 displayEnd = (spot.range().end() - displayFrame.start());
-                double hw =
-                    getGroupedOffset(displayEnd, colWidth, colGroupSize, bitOffset, colGroupMargin) + colWidth - hx;
-                double hh = rowHeight;
-                painter->setBrush(QColor::fromRgba(spot.color()));
-                painter->drawRect(QRectF(hx, hy, hw, hh));
+
+    if (container->info()->highlights(category).size() < 1) {
+        return rects;
+    }
+
+    int bufW = getGroupedOffset(colCount + 1, colWidth, colGroupSize, bitOffset, colGroupMargin);
+    int bufH = rowHeight * (rowCount + 1);
+    QImage buffer(bufW, bufH, QImage::Format_RGBA8888);
+    QPainter bufPainter(&buffer);
+    bufPainter.setPen(Qt::transparent);
+    buffer.fill(Qt::transparent);
+
+    int lastHighlight = 0;
+    int rowOffset = -1;
+    for (qint64 i = frameOffset; i < container->frameCount(); i++) {
+        rowOffset++;
+        if (rowOffset >= rowCount) {
+            break;
+        }
+        Frame frame = container->frameAt(i);
+        Frame displayFrame =
+            Frame(
+                    container->bits(),
+                    frame.start() + bitOffset,
+                    qMin(frame.end(), frame.start() + bitOffset + colCount - 1));
+        QList<RangeHighlight> spots = getHighlightSpots(
+                container->info()->highlights(category),
+                lastHighlight,
+                displayFrame);
+
+        while(!spots.isEmpty()) {
+            RangeHighlight spot = spots.takeFirst();
+            if (!spot.children().isEmpty()) {
+                int minIndex = 0;
+                spots.append(getHighlightSpots(spot.children(), minIndex, displayFrame));
             }
+            qint64 displayStart = (spot.range().start() - displayFrame.start());
+            double hx = getGroupedOffset(displayStart, colWidth, colGroupSize, bitOffset, colGroupMargin);
+            double hy = (i - frameOffset) * rowHeight;
+            qint64 displayEnd = (spot.range().end() - displayFrame.start());
+            double hw =
+                getGroupedOffset(displayEnd, colWidth, colGroupSize, bitOffset, colGroupMargin) + colWidth - hx;
+            double hh = rowHeight;
+            bufPainter.setBrush(QColor::fromRgb(spot.color()));
+            bufPainter.drawRect(QRectF(hx, hy, hw, hh));
         }
     }
+    painter->setOpacity(0.25);
+    painter->drawImage(0, 0, buffer);
+
     return rects;
 }
 
@@ -585,7 +594,7 @@ int DisplayHelper::drawTextRaster(QPainter *painter,
                 handle,
                 painter,
                 QSizeF(double(textSize.width()) / double(bitsPerChar), rowH),
-                textSectionSize,
+                QSize(charsPerRow * bitsPerChar, rows),
                 handle->bitOffset(),
                 handle->frameOffset(),
                 columnGrouping,

src/hobbits-widgets/widgetssettings.cpp

@@ -9,7 +9,7 @@ WidgetsSettings::WidgetsSettings()
     SettingsManager::instance().m_data.setUiSetting(SettingsManager::ONE_COLOR_KEY, QColor(Qt::black));
     SettingsManager::instance().m_data.setUiSetting(SettingsManager::ZERO_COLOR_KEY, QColor(253, 254, 229));
     SettingsManager::instance().m_data.setUiSetting(SettingsManager::BYTE_HUE_SAT_KEY, QColor::fromHsl(120, 200, 128));
-    SettingsManager::instance().m_data.setUiSetting(SettingsManager::FOCUS_COLOR_KEY, QColor(50, 190, 0, 85));
+    SettingsManager::instance().m_data.setUiSetting(SettingsManager::FOCUS_COLOR_KEY, QColor(255, 255, 255, 85));
     SettingsManager::instance().m_data.setUiSetting(SettingsManager::HIGHLIGHT_1_COLOR_KEY, QColor(100, 220, 100, 85));
     SettingsManager::instance().m_data.setUiSetting(SettingsManager::HIGHLIGHT_2_COLOR_KEY, QColor(100, 0, 255, 50));
     SettingsManager::instance().m_data.setUiSetting(SettingsManager::HIGHLIGHT_3_COLOR_KEY, QColor(0, 150, 230, 100));