perf: improve json parse speed by jsony

CHANGELOG-Japanese.md

@@ -5,6 +5,7 @@
 **改善:**
 
 - 重複するコードを削除するためのリファクタリング。 (#99) (@fukusuket)
+- JSONのパースを`jsony`に変更することで、処理速度が2倍以上速くなった。 (#122) (@fukusuket)
 
 ## 2.4.0 [2024/02/22] - Ninja Day Release
 

CHANGELOG.md

@@ -5,6 +5,7 @@
 **Enhancements:**
 
 - Refactoring to remove duplicate code. (#99) (@fukusuket)
+- Processing speed is more than twice as fast by changing the JSON parsing to `jsony`. (#122) (@fukusuket)
 
 ## 2.4.0 [2024/02/22] - Ninja Day Release
 

src/takajo.nim

@@ -15,10 +15,12 @@ import threadpool
 import uri
 import os
 import std/enumerate
+import std/options
 import std/xmlparser
 import std/xmltree
 import suru
 import takajopkg/general
+import takajopkg/hayabusaJson
 import takajopkg/stackUtil
 import takajopkg/takajoTerminal
 import takajopkg/ttpResult

src/takajopkg/extractScriptblocks.nim

@@ -73,6 +73,15 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo
         currentIndex  = 0
         stackedRecords = newTable[string, Script]()
         summaryRecords = newOrderedTable[string, array[7, string]]()
+        timestamp: string
+        computerName: string
+        eventLevel: string
+        ruleTitle: string
+        scriptBlock: string
+        scriptBlockId: string
+        messageNumber: int
+        messageTotal: int
+        path: string
 
     bar[0].total = totalLines
     bar.setup()
@@ -81,30 +90,23 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo
         inc currentIndex
         inc bar
         bar.update(1000000000) # refresh every second
-        let jsonLine = parseJson(line)
-        var
-            timestamp: string
-            computerName: string
-            eventLevel: string
-            ruleTitle: string
-            scriptBlock: string
-            scriptBlockId: string
-            messageNumber: int
-            messageTotal: int
-            path: string
-
-        if jsonLine["EventID"].getInt(0) != 4104 or isMinLevel(jsonLine["Level"].getStr(), level) == false:
+        let jsonLineOpt = parseLine(line)
+        if jsonLineOpt.isNone:
+            continue
+        let jsonLine:HayabusaJson = jsonLineOpt.get()
+
+        if jsonLine.EventID != 4104 or isMinLevel(jsonLine.Level, level) == false:
             continue
         try:
-            timestamp = jsonLine["Timestamp"].getStr()
-            computerName = jsonLine["Computer"].getStr()
-            eventLevel = jsonLine["Level"].getStr()
-            ruleTitle = jsonLine["RuleTitle"].getStr()
-            scriptBlock = jsonLine["Details"]["ScriptBlock"].getStr()
-            scriptBlockId = jsonLine["ExtraFieldInfo"]["ScriptBlockId"].getStr()
-            messageNumber = jsonLine["ExtraFieldInfo"]["MessageNumber"].getInt()
-            messageTotal = jsonLine["ExtraFieldInfo"]["MessageTotal"].getInt()
-            path = jsonLine["ExtraFieldInfo"].getOrDefault("Path").getStr()
+            timestamp = jsonLine.Timestamp
+            computerName = jsonLine.Computer
+            eventLevel = jsonLine.Level
+            ruleTitle = jsonLine.RuleTitle
+            scriptBlock = jsonLine.Details["ScriptBlock"].getStr()
+            scriptBlockId = jsonLine.ExtraFieldInfo["ScriptBlockId"].getStr()
+            messageNumber = jsonLine.ExtraFieldInfo["MessageNumber"].getInt()
+            messageTotal = jsonLine.ExtraFieldInfo["MessageTotal"].getInt()
+            path = jsonLine.ExtraFieldInfo.getOrDefault("Path").getStr()
             if path == "":
                 path = "no-path"
         except CatchableError:

src/takajopkg/general.nim

@@ -9,6 +9,7 @@ import std/tables
 import terminal
 import times
 import takajoTerminal
+
 from std/streams import newFileStream
 
 
@@ -19,12 +20,6 @@ proc getJsonValue*(jsonResponse: JsonNode, keys: seq[string], default: string =
             value = value[key]
         else:
             return default
-        #[
-        try:
-            value = value[key]
-        except KeyError:
-            return default]#
-    # Check if the value is an integer or a string
     if value.kind == JInt:
         return $value.getInt()  # Convert to string
     elif value.kind == JString:

src/takajopkg/hayabusaJson.nim

@@ -0,0 +1,30 @@
+import json
+import jsony
+import std/options
+
+type HayabusaJson* = ref object
+    Timestamp*: string
+    RuleTitle*: string
+    Computer*: string
+    Channel*: string
+    Level*: string
+    EventID*: int
+    RuleAuthor*: string
+    RuleModifiedDate*: string
+    Status*: string
+    RecordID*: int
+    Details*: JsonNode
+    ExtraFieldInfo*: JsonNode
+    Provider*: string
+    RuleCreationDate*: string
+    RuleFile*: string
+    EvtxFile*: string
+    MitreTags*: seq[string]
+    MitreTactics*: seq[string]
+    OtherTags*: seq[string]
+
+proc parseLine*(line:string): Option[HayabusaJson] =
+  try:
+      return some(line.fromJson(HayabusaJson))
+  except CatchableError:
+      return none(HayabusaJson)

src/takajopkg/listDomains.nim

@@ -21,10 +21,7 @@ proc listDomains(includeSubdomains: bool = false, includeWorkstations: bool = fa
     echo ""
 
     var
-        channel, domain = ""
-        eventId = 0
         domainHashSet = initHashSet[string]()
-        jsonLine: JsonNode
         bar: SuruBar = initSuruBar()
 
     bar[0].total = totalLines
@@ -33,13 +30,16 @@ proc listDomains(includeSubdomains: bool = false, includeWorkstations: bool = fa
     for line in lines(timeline):
         inc bar
         bar.update(1000000000)
-        jsonLine = parseJson(line)
-        channel = jsonLine["Channel"].getStr()
-        eventId = jsonLine["EventID"].getInt()
-        let details = jsonLine["Details"]
+        let jsonLineOpt = parseLine(line)
+        if jsonLineOpt.isNone:
+            continue
+        let jsonLine:HayabusaJson = jsonLineOpt.get()
+        let eventId = jsonLine.EventID
+        let channel = jsonLine.Channel
+        let details = jsonLine.Details
         # Found a Sysmon 22 DNS Query event
         if channel == "Sysmon" and eventId == 22:
-            domain = details.extractStr("Query")
+            var domain = details.extractStr("Query")
 
             # If includeWorkstations is false, only add domain if it contains a period
             # Filter out ".", "*.lan" and "*.LAN"

src/takajopkg/listHashes.nim

@@ -26,9 +26,8 @@ proc listHashes(level: string = "high", output: string, quiet: bool = false, tim
 
     var
         md5hashes, sha1hashes, sha256hashes, impHashes = initHashSet[string]()
-        channel, eventLevel, hashes = ""
-        eventId, md5hashCount, sha1hashCount, sha256hashCount, impHashCount = 0
-        jsonLine: JsonNode
+        hashes = ""
+        md5hashCount, sha1hashCount, sha256hashCount, impHashCount = 0
         bar: SuruBar = initSuruBar()
 
     bar[0].total = totalLines
@@ -37,15 +36,18 @@ proc listHashes(level: string = "high", output: string, quiet: bool = false, tim
     for line in lines(timeline):
         inc bar
         bar.update(1000000000) # refresh every second
-        jsonLine = parseJson(line)
-        channel = jsonLine["Channel"].getStr()
-        eventId = jsonLine["EventID"].getInt()
-        eventLevel = jsonLine["Level"].getStr()
+        let jsonLineOpt = parseLine(line)
+        if jsonLineOpt.isNone:
+            continue
+        let jsonLine:HayabusaJson = jsonLineOpt.get()
+        let eventId = jsonLine.EventID
+        let channel = jsonLine.Channel
+        let eventLevel = jsonLine.Level
 
         # Found a Sysmon 1 process creation event
         if channel == "Sysmon" and eventId == 1 and isMinLevel(eventLevel, level) == true:
             try:
-                hashes = jsonLine["Details"]["Hashes"].getStr() # Hashes are not enabled by default so this field may not exist.
+                hashes = jsonLine.Details["Hashes"].getStr() # Hashes are not enabled by default so this field may not exist.
                 let pairs = hashes.split(",")  # Split the string into key-value pairs. Ex: MD5=DE9C75F34F47B60A71BBA03760F0579E,SHA256=12F06D3B1601004DB3F7F1A07E7D3AF4CC838E890E0FF50C51E4A0C9366719ED,IMPHASH=336674CB3C8337BDE2C22255345BFF43
                 for pair in pairs:
                     let keyVal = pair.split("=")

src/takajopkg/listIpAddresses.nim

@@ -25,10 +25,8 @@ proc listIpAddresses(inbound: bool = true, outbound: bool = true, output: string
     echo ""
 
     var
-        channel, ipAddress = ""
-        eventId = 0
+        ipAddress = ""
         ipHashSet = initHashSet[string]()
-        jsonLine: JsonNode
         bar: SuruBar = initSuruBar()
 
     bar[0].total = totalLines
@@ -37,20 +35,23 @@ proc listIpAddresses(inbound: bool = true, outbound: bool = true, output: string
     for line in lines(timeline):
         inc bar
         bar.update(1000000000)
-        jsonLine = parseJson(line)
-        channel = jsonLine["Channel"].getStr()
-        eventId = jsonLine["EventID"].getInt()
+        let jsonLineOpt = parseLine(line)
+        if jsonLineOpt.isNone:
+            continue
+        let jsonLine:HayabusaJson = jsonLineOpt.get()
+        let eventId = jsonLine.EventID
+        let channel = jsonLine.Channel
 
         # Search for events with a SrcIP field if inbound == true
         if inbound == true:
-            ipAddress = getJsonValue(jsonLine, @["Details", "SrcIP"])
+            ipAddress = getJsonValue(jsonLine.Details, @["SrcIP"])
             if (not isPrivateIP(ipAddress) or privateIp) and
                 isMulticast(ipAddress) == false and isLoopback(ipAddress) == false and ipAddress != "Unknown" and ipAddress != "-":
                 ipHashSet.incl(ipAddress)
 
         # Search for events with a TgtIP field if outbound == true
         if outbound == true:
-                ipAddress = getJsonValue(jsonLine, @["Details", "TgtIP"])
+                ipAddress = getJsonValue(jsonLine.Details, @["TgtIP"])
                 if (not isPrivateIP(ipAddress) or privateIp) and
                     isMulticast(ipAddress) == false and isLoopback(ipAddress) == false and ipAddress != "Unknown" and ipAddress != "-":
                     ipHashSet.incl(ipAddress)

src/takajopkg/splitJsonTimeline.nim

@@ -33,8 +33,11 @@ proc splitJsonTimeline(output: string = "output", quiet: bool = false, timeline:
         inc bar
         bar.update(1000000000) # refresh every second
 
-        let jsonLine = parseJson(line)
-        let computerName = jsonLine["Computer"].getStr()
+        let jsonLineOpt = parseLine(line)
+        if jsonLineOpt.isNone:
+            continue
+        let jsonLine:HayabusaJson = jsonLineOpt.get()
+        let computerName = jsonLine.Computer
 
         if not filesTable.hasKey(computerName):
             let filename = output & "/" & computerName & "-HayabusaResults.jsonl"

src/takajopkg/stackCmdlines.nim

@@ -12,12 +12,15 @@ proc stackCmdlines(level: string = "low", ignoreSysmon: bool = false, ignoreSecu
     for line in lines(timeline):
         inc bar
         bar.update(1000000000) # refresh every second
-        let jsonLine = parseJson(line)
-        let eventId = jsonLine["EventID"].getInt(0)
-        let channel = jsonLine["Channel"].getStr("N/A")
+        let jsonLineOpt = parseLine(line)
+        if jsonLineOpt.isNone:
+            continue
+        let jsonLine:HayabusaJson = jsonLineOpt.get()
+        let eventId = jsonLine.EventID
+        let channel = jsonLine.Channel
         if (eventId == 1 and not ignoreSysmon and channel == "Sysmon") or
            (eventId == 4688 and not ignoreSecurity and channel == "Sec"):
-            let stackKey = jsonLine["Details"]["Cmdline"].getStr("N/A")
+            let stackKey = jsonLine.Details["Cmdline"].getStr("N/A")
             stackResult(stackKey, stack, level, jsonLine)
     bar.finish()
     outputResult(output, "Cmdline", stack)

src/takajopkg/stackDNS.nim

@@ -12,13 +12,16 @@ proc stackDNS(level: string = "informational", output: string = "", quiet: bool
     for line in lines(timeline):
         inc bar
         bar.update(1000000000) # refresh every second
-        let jsonLine = parseJson(line)
-        let eventId = jsonLine["EventID"].getInt(0)
-        let channel = jsonLine["Channel"].getStr("N/A")
+        let jsonLineOpt = parseLine(line)
+        if jsonLineOpt.isNone:
+            continue
+        let jsonLine:HayabusaJson = jsonLineOpt.get()
+        let eventId = jsonLine.EventID
+        let channel = jsonLine.Channel
         if (eventId == 22 and channel == "Sysmon"):
-            let prog = jsonLine["Details"]["Proc"].getStr("N/A")
-            let query = jsonLine["Details"]["Query"].getStr("N/A")
-            let res = jsonLine["Details"]["Result"].getStr("N/A")
+            let prog = jsonLine.Details["Proc"].getStr("N/A")
+            let query = jsonLine.Details["Query"].getStr("N/A")
+            let res = jsonLine.Details["Result"].getStr("N/A")
             let stackKey = prog & " -> " & query & " -> " & res
             stackResult(stackKey, stack, level, jsonLine, @[prog, query, res])
     bar.finish()

src/takajopkg/stackLogons.nim

@@ -35,18 +35,21 @@ proc stackLogons(localSrcIpAddresses = false, output: string = "", quiet: bool =
     for line in lines(timeline):
         inc bar
         bar.update(1000000000) # refresh every second
-        let jsonLine = parseJson(line)
-        let ruleTitle = jsonLine["RuleTitle"].getStr()
+        let jsonLineOpt = parseLine(line)
+        if jsonLineOpt.isNone:
+            continue
+        let jsonLine:HayabusaJson = jsonLineOpt.get()
+        let ruleTitle = jsonLine.RuleTitle
 
         # EID 4624 Successful Logon
         if isEID_4624(ruleTitle) == true:
             inc EID_4624_count
 
-            tgtUser = getJsonValue(jsonLine, @["Details", "TgtUser"])
-            tgtComp = getJsonValue(jsonLine, @["Computer"])
-            logonType = getJsonValue(jsonLine, @["Details", "Type"])
-            srcIP = getJsonValue(jsonLine, @["Details", "SrcIP"])
-            srcComp = getJsonValue(jsonLine, @["Details", "SrcComp"])
+            tgtUser = getJsonValue(jsonLine.Details, @["TgtUser"])
+            tgtComp = jsonLine.Computer
+            logonType = getJsonValue(jsonLine.Details, @["Type"])
+            srcIP = getJsonValue(jsonLine.Details, @["SrcIP"])
+            srcComp = getJsonValue(jsonLine.Details, @["SrcComp"])
 
             if not localSrcIpAddresses and isLocalIP(srcIP):
                 discard

src/takajopkg/stackProcesses.nim

@@ -11,12 +11,15 @@ proc stackProcesses(level: string = "low", ignoreSysmon: bool = false, ignoreSec
     for line in lines(timeline):
         inc bar
         bar.update(1000000000) # refresh every second
-        let jsonLine = parseJson(line)
-        let eventId = jsonLine["EventID"].getInt(0)
-        let channel = jsonLine["Channel"].getStr("N/A")
+        let jsonLineOpt = parseLine(line)
+        if jsonLineOpt.isNone:
+            continue
+        let jsonLine:HayabusaJson = jsonLineOpt.get()
+        let eventId = jsonLine.EventID
+        let channel = jsonLine.Channel
         if (eventId == 1 and not ignoreSysmon and channel == "Sysmon") or
            (eventId == 4688 and not ignoreSecurity and channel == "Sec"):
-            let stackKey = jsonLine["Details"]["Proc"].getStr("N/A")
+            let stackKey = jsonLine.Details["Proc"].getStr("N/A")
             stackResult(stackKey, stack, level, jsonLine)
     bar.finish()
     outputResult(output, "Process", stack)

src/takajopkg/stackServices.nim

@@ -13,13 +13,16 @@ proc stackServices(level: string = "informational", ignoreSystem: bool = false,
     for line in lines(timeline):
         inc bar
         bar.update(1000000000) # refresh every second
-        let jsonLine = parseJson(line)
-        let eventId = jsonLine["EventID"].getInt(0)
-        let channel = jsonLine["Channel"].getStr("N/A")
+        let jsonLineOpt = parseLine(line)
+        if jsonLineOpt.isNone:
+            continue
+        let jsonLine:HayabusaJson = jsonLineOpt.get()
+        let eventId = jsonLine.EventID
+        let channel = jsonLine.Channel
         if (eventId == 7045 and not ignoreSystem and channel == "Sys") or
            (eventId == 4697 and not ignoreSecurity and channel == "Sec"):
-            let svc = jsonLine["Details"]["Svc"].getStr("N/A")
-            let path = jsonLine["Details"]["Path"].getStr("N/A")
+            let svc = jsonLine.Details["Svc"].getStr("N/A")
+            let path = jsonLine.Details["Path"].getStr("N/A")
             let stackKey = svc & " -> " & path
             stackResult(stackKey, stack, level, jsonLine, @[svc, path])
     bar.finish()