From 11f3fef26c7f8d5ff9f07a0d5a60dd579b4f6b32 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Wed, 28 Feb 2024 22:11:29 +0900 Subject: [PATCH 1/4] perf: use jsony for json parsing --- src/takajo.nim | 2 + src/takajopkg/extractScriptblocks.nim | 43 ++++++----- src/takajopkg/general.nim | 7 +- src/takajopkg/hayabusaJson.nim | 22 ++++++ src/takajopkg/listDomains.nim | 13 ++-- src/takajopkg/listHashes.nim | 15 ++-- src/takajopkg/listIpAddresses.nim | 14 ++-- src/takajopkg/splitJsonTimeline.nim | 4 +- src/takajopkg/stackCmdlines.nim | 8 +-- src/takajopkg/stackDNS.nim | 12 ++-- src/takajopkg/stackLogons.nim | 14 ++-- src/takajopkg/stackProcesses.nim | 8 +-- src/takajopkg/stackServices.nim | 10 +-- src/takajopkg/stackTasks.nim | 12 ++-- src/takajopkg/stackUtil.nim | 17 +++++ src/takajopkg/timelineLogon.nim | 72 +++++++++---------- src/takajopkg/timelinePartitionDiagnostic.nim | 39 +++++----- src/takajopkg/timelineSuspiciousProcesses.nim | 64 ++++++++--------- src/takajopkg/timelineTasks.nim | 20 +++--- src/takajopkg/ttpSummary.nim | 12 ++-- src/takajopkg/ttpVisualize.nim | 8 +-- takajo.nimble | 3 +- 22 files changed, 225 insertions(+), 194 deletions(-) create mode 100644 src/takajopkg/hayabusaJson.nim diff --git a/src/takajo.nim b/src/takajo.nim index b553682..b4e45ee 100644 --- a/src/takajo.nim +++ b/src/takajo.nim @@ -1,6 +1,7 @@ import algorithm import cligen import json +import jsony import nancy import puppy import re @@ -19,6 +20,7 @@ import std/xmlparser import std/xmltree import suru import takajopkg/general +import takajopkg/hayabusaJson import takajopkg/stackUtil import takajopkg/takajoTerminal import takajopkg/ttpResult diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index bf56bbe..c3b3ec6 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -73,6 +73,15 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo currentIndex = 0 stackedRecords = newTable[string, Script]() summaryRecords = newOrderedTable[string, array[7, string]]() + timestamp: string + computerName: string + eventLevel: string + ruleTitle: string + scriptBlock: string + scriptBlockId: string + messageNumber: int + messageTotal: int + path: string bar[0].total = totalLines bar.setup() @@ -81,30 +90,20 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo inc currentIndex inc bar bar.update(1000000000) # refresh every second - let jsonLine = parseJson(line) - var - timestamp: string - computerName: string - eventLevel: string - ruleTitle: string - scriptBlock: string - scriptBlockId: string - messageNumber: int - messageTotal: int - path: string - - if jsonLine["EventID"].getInt(0) != 4104 or isMinLevel(jsonLine["Level"].getStr(), level) == false: + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + + if jsonLine.EventID != 4104 or isMinLevel(jsonLine.Level, level) == false: continue try: - timestamp = jsonLine["Timestamp"].getStr() - computerName = jsonLine["Computer"].getStr() - eventLevel = jsonLine["Level"].getStr() - ruleTitle = jsonLine["RuleTitle"].getStr() - scriptBlock = jsonLine["Details"]["ScriptBlock"].getStr() - scriptBlockId = jsonLine["ExtraFieldInfo"]["ScriptBlockId"].getStr() - messageNumber = jsonLine["ExtraFieldInfo"]["MessageNumber"].getInt() - messageTotal = jsonLine["ExtraFieldInfo"]["MessageTotal"].getInt() - path = jsonLine["ExtraFieldInfo"].getOrDefault("Path").getStr() + timestamp = jsonLine.Timestamp + computerName = jsonLine.Computer + eventLevel = jsonLine.Level + ruleTitle = jsonLine.RuleTitle + scriptBlock = jsonLine.Details["ScriptBlock"].getStr() + scriptBlockId = jsonLine.ExtraFieldInfo["ScriptBlockId"].getStr() + messageNumber = jsonLine.ExtraFieldInfo["MessageNumber"].getInt() + messageTotal = jsonLine.ExtraFieldInfo["MessageTotal"].getInt() + path = jsonLine.ExtraFieldInfo.getOrDefault("Path").getStr() if path == "": path = "no-path" except CatchableError: diff --git a/src/takajopkg/general.nim b/src/takajopkg/general.nim index d7cee25..0a66286 100644 --- a/src/takajopkg/general.nim +++ b/src/takajopkg/general.nim @@ -9,6 +9,7 @@ import std/tables import terminal import times import takajoTerminal + from std/streams import newFileStream @@ -19,12 +20,6 @@ proc getJsonValue*(jsonResponse: JsonNode, keys: seq[string], default: string = value = value[key] else: return default - #[ - try: - value = value[key] - except KeyError: - return default]# - # Check if the value is an integer or a string if value.kind == JInt: return $value.getInt() # Convert to string elif value.kind == JString: diff --git a/src/takajopkg/hayabusaJson.nim b/src/takajopkg/hayabusaJson.nim new file mode 100644 index 0000000..39f1f93 --- /dev/null +++ b/src/takajopkg/hayabusaJson.nim @@ -0,0 +1,22 @@ +import json + +type HayabusaJson* = ref object + Timestamp*: string + RuleTitle*: string + Computer*: string + Channel*: string + Level*: string + EventID*: int + RuleAuthor*: string + RuleModifiedDate*: string + Status*: string + RecordID*: int + Details*: JsonNode + ExtraFieldInfo*: JsonNode + Provider*: string + RuleCreationDate*: string + RuleFile*: string + EvtxFile*: string + MitreTags*: seq[string] + MitreTactics*: seq[string] + OtherTags*: seq[string] \ No newline at end of file diff --git a/src/takajopkg/listDomains.nim b/src/takajopkg/listDomains.nim index 6161d67..1c47c74 100644 --- a/src/takajopkg/listDomains.nim +++ b/src/takajopkg/listDomains.nim @@ -21,10 +21,7 @@ proc listDomains(includeSubdomains: bool = false, includeWorkstations: bool = fa echo "" var - channel, domain = "" - eventId = 0 domainHashSet = initHashSet[string]() - jsonLine: JsonNode bar: SuruBar = initSuruBar() bar[0].total = totalLines @@ -33,13 +30,13 @@ proc listDomains(includeSubdomains: bool = false, includeWorkstations: bool = fa for line in lines(timeline): inc bar bar.update(1000000000) - jsonLine = parseJson(line) - channel = jsonLine["Channel"].getStr() - eventId = jsonLine["EventID"].getInt() - let details = jsonLine["Details"] + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let eventId = jsonLine.EventID + let channel = jsonLine.Channel + let details = jsonLine.Details # Found a Sysmon 22 DNS Query event if channel == "Sysmon" and eventId == 22: - domain = details.extractStr("Query") + var domain = details.extractStr("Query") # If includeWorkstations is false, only add domain if it contains a period # Filter out ".", "*.lan" and "*.LAN" diff --git a/src/takajopkg/listHashes.nim b/src/takajopkg/listHashes.nim index 5072c28..288151f 100644 --- a/src/takajopkg/listHashes.nim +++ b/src/takajopkg/listHashes.nim @@ -26,9 +26,8 @@ proc listHashes(level: string = "high", output: string, quiet: bool = false, tim var md5hashes, sha1hashes, sha256hashes, impHashes = initHashSet[string]() - channel, eventLevel, hashes = "" - eventId, md5hashCount, sha1hashCount, sha256hashCount, impHashCount = 0 - jsonLine: JsonNode + hashes = "" + md5hashCount, sha1hashCount, sha256hashCount, impHashCount = 0 bar: SuruBar = initSuruBar() bar[0].total = totalLines @@ -37,15 +36,15 @@ proc listHashes(level: string = "high", output: string, quiet: bool = false, tim for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - jsonLine = parseJson(line) - channel = jsonLine["Channel"].getStr() - eventId = jsonLine["EventID"].getInt() - eventLevel = jsonLine["Level"].getStr() + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let eventId = jsonLine.EventID + let channel = jsonLine.Channel + let eventLevel = jsonLine.Level # Found a Sysmon 1 process creation event if channel == "Sysmon" and eventId == 1 and isMinLevel(eventLevel, level) == true: try: - hashes = jsonLine["Details"]["Hashes"].getStr() # Hashes are not enabled by default so this field may not exist. + hashes = jsonLine.Details["Hashes"].getStr() # Hashes are not enabled by default so this field may not exist. let pairs = hashes.split(",") # Split the string into key-value pairs. Ex: MD5=DE9C75F34F47B60A71BBA03760F0579E,SHA256=12F06D3B1601004DB3F7F1A07E7D3AF4CC838E890E0FF50C51E4A0C9366719ED,IMPHASH=336674CB3C8337BDE2C22255345BFF43 for pair in pairs: let keyVal = pair.split("=") diff --git a/src/takajopkg/listIpAddresses.nim b/src/takajopkg/listIpAddresses.nim index 5ad8cd4..38b5093 100644 --- a/src/takajopkg/listIpAddresses.nim +++ b/src/takajopkg/listIpAddresses.nim @@ -25,10 +25,8 @@ proc listIpAddresses(inbound: bool = true, outbound: bool = true, output: string echo "" var - channel, ipAddress = "" - eventId = 0 + ipAddress = "" ipHashSet = initHashSet[string]() - jsonLine: JsonNode bar: SuruBar = initSuruBar() bar[0].total = totalLines @@ -37,20 +35,20 @@ proc listIpAddresses(inbound: bool = true, outbound: bool = true, output: string for line in lines(timeline): inc bar bar.update(1000000000) - jsonLine = parseJson(line) - channel = jsonLine["Channel"].getStr() - eventId = jsonLine["EventID"].getInt() + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let eventId = jsonLine.EventID + let channel = jsonLine.Channel # Search for events with a SrcIP field if inbound == true if inbound == true: - ipAddress = getJsonValue(jsonLine, @["Details", "SrcIP"]) + ipAddress = getJsonValue(jsonLine.Details, @["SrcIP"]) if (not isPrivateIP(ipAddress) or privateIp) and isMulticast(ipAddress) == false and isLoopback(ipAddress) == false and ipAddress != "Unknown" and ipAddress != "-": ipHashSet.incl(ipAddress) # Search for events with a TgtIP field if outbound == true if outbound == true: - ipAddress = getJsonValue(jsonLine, @["Details", "TgtIP"]) + ipAddress = getJsonValue(jsonLine.Details, @["TgtIP"]) if (not isPrivateIP(ipAddress) or privateIp) and isMulticast(ipAddress) == false and isLoopback(ipAddress) == false and ipAddress != "Unknown" and ipAddress != "-": ipHashSet.incl(ipAddress) diff --git a/src/takajopkg/splitJsonTimeline.nim b/src/takajopkg/splitJsonTimeline.nim index 0f114fc..a87bb31 100644 --- a/src/takajopkg/splitJsonTimeline.nim +++ b/src/takajopkg/splitJsonTimeline.nim @@ -33,8 +33,8 @@ proc splitJsonTimeline(output: string = "output", quiet: bool = false, timeline: inc bar bar.update(1000000000) # refresh every second - let jsonLine = parseJson(line) - let computerName = jsonLine["Computer"].getStr() + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let computerName = jsonLine.Computer if not filesTable.hasKey(computerName): let filename = output & "/" & computerName & "-HayabusaResults.jsonl" diff --git a/src/takajopkg/stackCmdlines.nim b/src/takajopkg/stackCmdlines.nim index d518a04..5f90959 100644 --- a/src/takajopkg/stackCmdlines.nim +++ b/src/takajopkg/stackCmdlines.nim @@ -12,12 +12,12 @@ proc stackCmdlines(level: string = "low", ignoreSysmon: bool = false, ignoreSecu for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine = parseJson(line) - let eventId = jsonLine["EventID"].getInt(0) - let channel = jsonLine["Channel"].getStr("N/A") + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let eventId = jsonLine.EventID + let channel = jsonLine.Channel if (eventId == 1 and not ignoreSysmon and channel == "Sysmon") or (eventId == 4688 and not ignoreSecurity and channel == "Sec"): - let stackKey = jsonLine["Details"]["Cmdline"].getStr("N/A") + let stackKey = jsonLine.Details["Cmdline"].getStr("N/A") stackResult(stackKey, stack, level, jsonLine) bar.finish() outputResult(output, "Cmdline", stack) diff --git a/src/takajopkg/stackDNS.nim b/src/takajopkg/stackDNS.nim index 7116c67..dcf6efd 100644 --- a/src/takajopkg/stackDNS.nim +++ b/src/takajopkg/stackDNS.nim @@ -12,13 +12,13 @@ proc stackDNS(level: string = "informational", output: string = "", quiet: bool for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine = parseJson(line) - let eventId = jsonLine["EventID"].getInt(0) - let channel = jsonLine["Channel"].getStr("N/A") + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let eventId = jsonLine.EventID + let channel = jsonLine.Channel if (eventId == 22 and channel == "Sysmon"): - let prog = jsonLine["Details"]["Proc"].getStr("N/A") - let query = jsonLine["Details"]["Query"].getStr("N/A") - let res = jsonLine["Details"]["Result"].getStr("N/A") + let prog = jsonLine.Details["Proc"].getStr("N/A") + let query = jsonLine.Details["Query"].getStr("N/A") + let res = jsonLine.Details["Result"].getStr("N/A") let stackKey = prog & " -> " & query & " -> " & res stackResult(stackKey, stack, level, jsonLine, @[prog, query, res]) bar.finish() diff --git a/src/takajopkg/stackLogons.nim b/src/takajopkg/stackLogons.nim index 5826a53..be7a88e 100644 --- a/src/takajopkg/stackLogons.nim +++ b/src/takajopkg/stackLogons.nim @@ -35,18 +35,18 @@ proc stackLogons(localSrcIpAddresses = false, output: string = "", quiet: bool = for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine = parseJson(line) - let ruleTitle = jsonLine["RuleTitle"].getStr() + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let ruleTitle = jsonLine.RuleTitle # EID 4624 Successful Logon if isEID_4624(ruleTitle) == true: inc EID_4624_count - tgtUser = getJsonValue(jsonLine, @["Details", "TgtUser"]) - tgtComp = getJsonValue(jsonLine, @["Computer"]) - logonType = getJsonValue(jsonLine, @["Details", "Type"]) - srcIP = getJsonValue(jsonLine, @["Details", "SrcIP"]) - srcComp = getJsonValue(jsonLine, @["Details", "SrcComp"]) + tgtUser = getJsonValue(jsonLine.Details, @["TgtUser"]) + tgtComp = jsonLine.Computer + logonType = getJsonValue(jsonLine.Details, @["Type"]) + srcIP = getJsonValue(jsonLine.Details, @["SrcIP"]) + srcComp = getJsonValue(jsonLine.Details, @["SrcComp"]) if not localSrcIpAddresses and isLocalIP(srcIP): discard diff --git a/src/takajopkg/stackProcesses.nim b/src/takajopkg/stackProcesses.nim index a10b2ce..e100a02 100644 --- a/src/takajopkg/stackProcesses.nim +++ b/src/takajopkg/stackProcesses.nim @@ -11,12 +11,12 @@ proc stackProcesses(level: string = "low", ignoreSysmon: bool = false, ignoreSec for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine = parseJson(line) - let eventId = jsonLine["EventID"].getInt(0) - let channel = jsonLine["Channel"].getStr("N/A") + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let eventId = jsonLine.EventID + let channel = jsonLine.Channel if (eventId == 1 and not ignoreSysmon and channel == "Sysmon") or (eventId == 4688 and not ignoreSecurity and channel == "Sec"): - let stackKey = jsonLine["Details"]["Proc"].getStr("N/A") + let stackKey = jsonLine.Details["Proc"].getStr("N/A") stackResult(stackKey, stack, level, jsonLine) bar.finish() outputResult(output, "Process", stack) diff --git a/src/takajopkg/stackServices.nim b/src/takajopkg/stackServices.nim index 57b4395..fa2fdc3 100644 --- a/src/takajopkg/stackServices.nim +++ b/src/takajopkg/stackServices.nim @@ -13,13 +13,13 @@ proc stackServices(level: string = "informational", ignoreSystem: bool = false, for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine = parseJson(line) - let eventId = jsonLine["EventID"].getInt(0) - let channel = jsonLine["Channel"].getStr("N/A") + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let eventId = jsonLine.EventID + let channel = jsonLine.Channel if (eventId == 7045 and not ignoreSystem and channel == "Sys") or (eventId == 4697 and not ignoreSecurity and channel == "Sec"): - let svc = jsonLine["Details"]["Svc"].getStr("N/A") - let path = jsonLine["Details"]["Path"].getStr("N/A") + let svc = jsonLine.Details["Svc"].getStr("N/A") + let path = jsonLine.Details["Path"].getStr("N/A") let stackKey = svc & " -> " & path stackResult(stackKey, stack, level, jsonLine, @[svc, path]) bar.finish() diff --git a/src/takajopkg/stackTasks.nim b/src/takajopkg/stackTasks.nim index a98fe78..b0f8c65 100644 --- a/src/takajopkg/stackTasks.nim +++ b/src/takajopkg/stackTasks.nim @@ -15,13 +15,13 @@ proc stackTasks(level: string = "informational", ignoreSysmon: bool = false, ign for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine = parseJson(line) - let eventId = jsonLine["EventID"].getInt(0) - let channel = jsonLine["Channel"].getStr("N/A") + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let eventId = jsonLine.EventID + let channel = jsonLine.Channel if eventId == 4698 and channel == "Sec": - let user = jsonLine["Details"]["User"].getStr("N/A") - let name = jsonLine["Details"]["Name"].getStr("N/A") - let content = jsonLine["Details"]["Content"].getStr("N/A").replace("\\r\\n", "") + let user = jsonLine.Details["User"].getStr("N/A") + let name = jsonLine.Details["Name"].getStr("N/A") + let content = jsonLine.Details["Content"].getStr("N/A").replace("\\r\\n", "") let node = parseXml(content) let commands = node.findAll("Command") var command = "" diff --git a/src/takajopkg/stackUtil.nim b/src/takajopkg/stackUtil.nim index 16cd166..603bc77 100644 --- a/src/takajopkg/stackUtil.nim +++ b/src/takajopkg/stackUtil.nim @@ -2,6 +2,7 @@ import general import json import nancy import takajoTerminal +import hayabusaJson import std/algorithm import std/enumerate import std/sets @@ -71,6 +72,22 @@ proc stackResult*(key:string, stack: var Table[string, StackRecord], minLevel:st val.otherColumn = otherColumn stack[key] = val +proc stackResult*(key:string, stack: var Table[string, StackRecord], minLevel:string, jsonLine:HayabusaJson, otherColumn:seq[string] = @[]) = + let level = jsonLine.Level + if not isMinLevel(level, minLevel): + return + var val: StackRecord + if key notin stack: + val = StackRecord(key: key, eid: intToStr(jsonLine.EventID), channel: jsonLine.Channel) + else: + val = stack[key] + val.count += 1 + val.levels.incl(level) + val.levelsOrder = val.calcLevelOrder() + val.ruleTitles.incl(jsonLine.RuleTitle) + val.otherColumn = otherColumn + stack[key] = val + proc outputResult*(output:string, culumnName: string, stack: Table[string, StackRecord], otherHeader:seq[string] = newSeq[string]()) = echo "" if stack.len == 0: diff --git a/src/takajopkg/timelineLogon.nim b/src/takajopkg/timelineLogon.nim index 200446f..74f5f5e 100644 --- a/src/takajopkg/timelineLogon.nim +++ b/src/takajopkg/timelineLogon.nim @@ -39,23 +39,23 @@ proc timelineLogon(calculateElapsedTime: bool = true, output: string, outputLogo for line in lines(timeline): inc bar bar.update(1000000000) - let jsonLine = parseJson(line) - let ruleTitle = jsonLine["RuleTitle"].getStr() + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let ruleTitle = jsonLine.RuleTitle #EID 4624 Successful Logon if isEID_4624(ruleTitle) == true: inc EID_4624_count var singleResultTable = newTable[string, string]() singleResultTable["Event"] = "Successful Logon" - singleResultTable["Timestamp"] = jsonLine["Timestamp"].getStr() - singleResultTable["Channel"] = jsonLine["Channel"].getStr() + singleResultTable["Timestamp"] = jsonLine.Timestamp + singleResultTable["Channel"] = jsonLine.Channel singleResultTable["EventID"] = "4624" - let details = jsonLine["Details"] - let extraFieldInfo = jsonLine["ExtraFieldInfo"] + let details = jsonLine.Details + let extraFieldInfo = jsonLine.ExtraFieldInfo let logonType = details.extractStr("Type") # Will be Int if field mapping is turned off singleResultTable["Type"] = logonType # Needs to be logonNumberToString(logonType) if data field mapping is turned off. TODO singleResultTable["Auth"] = extraFieldInfo.extractStr("AuthenticationPackageName") - singleResultTable["TargetComputer"] = jsonLine.extractStr("Computer") + singleResultTable["TargetComputer"] = jsonLine.Computer singleResultTable["TargetUser"] = details.extractStr("TgtUser") let impersonationLevel = extraFieldInfo.extractStr("ImpersonationLevel") singleResultTable["Impersonation"] = impersonationLevelIdToName(impersonationLevel) @@ -80,15 +80,15 @@ proc timelineLogon(calculateElapsedTime: bool = true, output: string, outputLogo inc EID_4625_count var singleResultTable = newTable[string, string]() singleResultTable["Event"] = "Failed Logon" - singleResultTable["Timestamp"] = jsonLine["Timestamp"].getStr() - singleResultTable["Channel"] = jsonLine["Channel"].getStr() + singleResultTable["Timestamp"] = jsonLine.Timestamp + singleResultTable["Channel"] = jsonLine.Channel singleResultTable["EventID"] = "4625" - let details = jsonLine["Details"] - let extraFieldInfo = jsonLine["ExtraFieldInfo"] + let details = jsonLine.Details + let extraFieldInfo = jsonLine.ExtraFieldInfo let logonType = details.extractStr("Type") singleResultTable["Type"] = logonType # Needs to be logonNumberToString(logonType) if data field mapping is turned off. TODO singleResultTable["Auth"] = details.extractStr("AuthPkg") - singleResultTable["TargetComputer"] = jsonLine.extractStr("Computer") + singleResultTable["TargetComputer"] = jsonLine.Computer singleResultTable["TargetUser"] = details.extractStr("TgtUser") singleResultTable["SourceIP"] = details.extractStr("SrcIP") singleResultTable["Process"] = details.extractStr("Proc") @@ -105,17 +105,17 @@ proc timelineLogon(calculateElapsedTime: bool = true, output: string, outputLogo # If we want to calculate ElapsedTime if calculateElapsedTime == true: # Create the key in the format of LID:Computer:User with a value of the timestamp - let key = jsonLine["Details"]["LID"].getStr() & ":" & jsonLine["Computer"].getStr() & ":" & jsonLine["Details"]["User"].getStr() - let logoffTime = jsonLine["Timestamp"].getStr() + let key = jsonLine.Details["LID"].getStr() & ":" & jsonLine.Computer & ":" & jsonLine.Details["User"].getStr() + let logoffTime = jsonLine.Timestamp logoffEvents[key] = logoffTime if outputLogoffEvents == true: var singleResultTable = newTable[string, string]() singleResultTable["Event"] = "User Initiated Logoff" - singleResultTable["Timestamp"] = jsonLine["Timestamp"].getStr() - singleResultTable["Channel"] = jsonLine["Channel"].getStr() - singleResultTable["TargetComputer"] = jsonLine.extractStr("Computer") + singleResultTable["Timestamp"] = jsonLine.Timestamp + singleResultTable["Channel"] = jsonLine.Channel + singleResultTable["TargetComputer"] = jsonLine.Computer singleResultTable["EventID"] = "4634" - let details = jsonLine["Details"] + let details = jsonLine.Details singleResultTable["TargetUser"] = details.extractStr("User") singleResultTable["LID"] = details.extractStr("LID") seqOfResultsTables.add(singleResultTable) @@ -126,17 +126,17 @@ proc timelineLogon(calculateElapsedTime: bool = true, output: string, outputLogo # If we want to calculate ElapsedTime if calculateElapsedTime == true: # Create the key in the format of LID:Computer:User with a value of the timestamp - let key = jsonLine["Details"]["LID"].getStr() & ":" & jsonLine["Computer"].getStr() & ":" & jsonLine["Details"]["User"].getStr() - let logoffTime = jsonLine["Timestamp"].getStr() + let key = jsonLine.Details["LID"].getStr() & ":" & jsonLine.Computer & ":" & jsonLine.Details["User"].getStr() + let logoffTime = jsonLine.Timestamp logoffEvents[key] = logoffTime if outputLogoffEvents == true: var singleResultTable = newTable[string, string]() singleResultTable["Event"] = "User Initiated Logoff" - singleResultTable["Timestamp"] = jsonLine["Timestamp"].getStr() - singleResultTable["Channel"] = jsonLine["Channel"].getStr() - singleResultTable["TargetComputer"] = jsonLine.extractStr("Computer") + singleResultTable["Timestamp"] = jsonLine.Timestamp + singleResultTable["Channel"] = jsonLine.Channel + singleResultTable["TargetComputer"] = jsonLine.Computer singleResultTable["EventID"] = "4647" - let details = jsonLine["Details"] + let details = jsonLine.Details singleResultTable["TargetUser"] = details.extractStr("User") singleResultTable["LID"] = details.extractStr("LID") seqOfResultsTables.add(singleResultTable) @@ -146,11 +146,11 @@ proc timelineLogon(calculateElapsedTime: bool = true, output: string, outputLogo inc EID_4648_count var singleResultTable = newTable[string, string]() singleResultTable["Event"] = "Explicit Logon" - singleResultTable["Timestamp"] = jsonLine["Timestamp"].getStr() - singleResultTable["Channel"] = jsonLine["Channel"].getStr() + singleResultTable["Timestamp"] = jsonLine.Timestamp + singleResultTable["Channel"] = jsonLine.Channel singleResultTable["EventID"] = "4648" - let details = jsonLine["Details"] - let extraFieldInfo = jsonLine["ExtraFieldInfo"] + let details = jsonLine.Details + let extraFieldInfo = jsonLine.ExtraFieldInfo singleResultTable["TargetComputer"] = details.extractStr("TgtSvr") singleResultTable["TargetUser"] = details.extractStr("TgtUser") singleResultTable["SourceUser"] = details.extractStr("SrcUser") @@ -158,7 +158,7 @@ proc timelineLogon(calculateElapsedTime: bool = true, output: string, outputLogo singleResultTable["Process"] = details.extractStr("Proc") singleResultTable["LID"] = details.extractStr("LID") singleResultTable["LGUID"] = extraFieldInfo.extractStr("LogonGuid") - singleResultTable["SourceComputer"] = jsonLine.extractStr("Computer") # 4648 is a little different in that the log is saved on the source computer so Computer will be the source. + singleResultTable["SourceComputer"] = jsonLine.Computer # 4648 is a little different in that the log is saved on the source computer so Computer will be the source. seqOfResultsTables.add(singleResultTable) # EID 4672 Admin Logon @@ -167,18 +167,18 @@ proc timelineLogon(calculateElapsedTime: bool = true, output: string, outputLogo # The timing will be very close to the 4624 log so I am checking if the Computer, LID and TgtUser are the same and then if the two events happened within 10 seconds. if ruleTitle == "Admin Logon": inc EID_4672_count - let key = jsonLine["Details"]["LID"].getStr() & ":" & jsonLine["Computer"].getStr() & ":" & jsonLine["Details"]["TgtUser"].getStr() - let adminLogonTime = jsonLine["Timestamp"].getStr() + let key = jsonLine.Details["LID"].getStr() & ":" & jsonLine.Computer & ":" & jsonLine.Details["TgtUser"].getStr() + let adminLogonTime = jsonLine.Timestamp adminLogonEvents[key] = adminLogonTime if outputAdminLogonEvents == true: var singleResultTable = newTable[string, string]() singleResultTable["Event"] = "Admin Logon" - singleResultTable["Timestamp"] = jsonLine["Timestamp"].getStr() - singleResultTable["Channel"] = jsonLine["Channel"].getStr() + singleResultTable["Timestamp"] = jsonLine.Timestamp + singleResultTable["Channel"] = jsonLine.Channel singleResultTable["EventID"] = "4672" - singleResultTable["TargetComputer"] = jsonLine.extractStr("Computer") - let eventId = jsonLine["EventID"].getInt() - let details = jsonLine["Details"] + singleResultTable["TargetComputer"] = jsonLine.Computer + let eventId = jsonLine.EventID + let details = jsonLine.Details singleResultTable["TargetUser"] = details.extractStr("TgtUser") singleResultTable["LID"] = details.extractStr("LID") seqOfResultsTables.add(singleResultTable) diff --git a/src/takajopkg/timelinePartitionDiagnostic.nim b/src/takajopkg/timelinePartitionDiagnostic.nim index e42e3c1..0417473 100644 --- a/src/takajopkg/timelinePartitionDiagnostic.nim +++ b/src/takajopkg/timelinePartitionDiagnostic.nim @@ -5,25 +5,25 @@ proc changeByteOrder(vsnStr:string): string = res.add(vsnStr[i - 1] & vsnStr[i]) return join(res.reversed, "") -proc extractVSN(jsonLine: JsonNode) : array[4, string] = +proc extractVSN(jsonLine: HayabusaJson) : array[4, string] = var res:array[4, string] = ["n/a", "n/a", "n/a", "n/a"] - if "PartitionStyle" notin jsonLine["ExtraFieldInfo"] or jsonLine["ExtraFieldInfo"]["PartitionStyle"].getInt() != 0: + if "PartitionStyle" notin jsonLine.ExtraFieldInfo or jsonLine.ExtraFieldInfo["PartitionStyle"].getInt() != 0: return res for i, vbrNo in ["Vbr0", "Vbr1", "Vbr2", "Vbr3"]: - if vbrNo notin jsonLine["ExtraFieldInfo"]: + if vbrNo notin jsonLine.ExtraFieldInfo: continue - let vbr = jsonLine["ExtraFieldInfo"][vbrNo].getStr() + let vbr = jsonLine.ExtraFieldInfo[vbrNo].getStr() if vbr.len < 18: res[i] = "" continue let sig = vbr[6..17] var vsnLittleEndian = "" - if sig == "4D53444F5335" and jsonLine["ExtraFieldInfo"][vbrNo].getStr().len > 141: # FAT32 - vsnLittleEndian = jsonLine["ExtraFieldInfo"][vbrNo].getStr()[134..141] - elif sig == "455846415420" and jsonLine["ExtraFieldInfo"][vbrNo].getStr().len > 207: #ExFAT - vsnLittleEndian = jsonLine["ExtraFieldInfo"][vbrNo].getStr()[200..207] - elif sig == "4E5446532020" and jsonLine["ExtraFieldInfo"][vbrNo].getStr().len > 151: # NTFS - vsnLittleEndian = jsonLine["ExtraFieldInfo"][vbrNo].getStr()[144..151] + if sig == "4D53444F5335" and jsonLine.ExtraFieldInfo[vbrNo].getStr().len > 141: # FAT32 + vsnLittleEndian = jsonLine.ExtraFieldInfo[vbrNo].getStr()[134..141] + elif sig == "455846415420" and jsonLine.ExtraFieldInfo[vbrNo].getStr().len > 207: #ExFAT + vsnLittleEndian = jsonLine.ExtraFieldInfo[vbrNo].getStr()[200..207] + elif sig == "4E5446532020" and jsonLine.ExtraFieldInfo[vbrNo].getStr().len > 151: # NTFS + vsnLittleEndian = jsonLine.ExtraFieldInfo[vbrNo].getStr()[144..151] if vsnLittleEndian != "": res[i] = changeByteOrder(vsnLittleEndian) return res @@ -44,7 +44,6 @@ proc timelinePartitionDiagnostic(output: string = "", quiet: bool = false, timel echo "" var - jsonLine: JsonNode bar: SuruBar = initSuruBar() seqOfResultsTables: seq[Table[string, string]] @@ -54,17 +53,19 @@ proc timelinePartitionDiagnostic(output: string = "", quiet: bool = false, timel for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - jsonLine = parseJson(line) + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let eventId = jsonLine.EventID + let channel = jsonLine.Channel - if jsonLine["EventID"].getInt() != 1006 or jsonLine["Channel"].getStr() != "MS-Win-Partition/Diagnostic": + if eventId != 1006 or channel != "MS-Win-Partition/Diagnostic": continue var singleResultTable = initTable[string, string]() - singleResultTable["Timestamp"] = jsonLine["Timestamp"].getStr() - singleResultTable["Computer"] = jsonLine["Computer"].getStr() - singleResultTable["Manufacturer"] = jsonLine["Details"]["Manufacturer"].getStr() - singleResultTable["Model"] = jsonLine["Details"]["Model"].getStr() - singleResultTable["Revision"] = jsonLine["Details"]["Revision"].getStr() - singleResultTable["SerialNumber"] = jsonLine["Details"]["SerialNumber"].getStr() + singleResultTable["Timestamp"] = jsonLine.Timestamp + singleResultTable["Computer"] = jsonLine.Computer + singleResultTable["Manufacturer"] = jsonLine.Details["Manufacturer"].getStr() + singleResultTable["Model"] = jsonLine.Details["Model"].getStr() + singleResultTable["Revision"] = jsonLine.Details["Revision"].getStr() + singleResultTable["SerialNumber"] = jsonLine.Details["SerialNumber"].getStr() for i, vsn in extractVSN(jsonLine): var val = "n/a" if vsn != "": diff --git a/src/takajopkg/timelineSuspiciousProcesses.nim b/src/takajopkg/timelineSuspiciousProcesses.nim index 1a19a15..1599438 100644 --- a/src/takajopkg/timelineSuspiciousProcesses.nim +++ b/src/takajopkg/timelineSuspiciousProcesses.nim @@ -36,34 +36,34 @@ proc timelineSuspiciousProcesses(level: string = "high", output: string = "", qu for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - jsonLine = parseJson(line) - channel = jsonLine["Channel"].getStr() - eventId = jsonLine["EventID"].getInt() - eventLevel = jsonLine["Level"].getStr() + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let eventId = jsonLine.EventID + let channel = jsonLine.Channel + let eventLevel = jsonLine.Level # Found a Security 4688 process creation event if channel == "Sec" and eventId == 4688 and isMinLevel(eventLevel, level) == true: inc suspicousProcessCount_Sec_4688 try: - cmdLine = jsonLine["Details"]["Cmdline"].getStr() + cmdLine = jsonLine.Details["Cmdline"].getStr() except KeyError: cmdLine = "" eventType = "Security 4688" - timestamp = jsonLine["Timestamp"].getStr() - ruleTitle = jsonLine["RuleTitle"].getStr() - computer = jsonLine["Computer"].getStr() - process = jsonLine["Details"]["Proc"].getStr() - pidStr = $jsonLine["Details"]["PID"].getInt() - user = jsonLine["Details"]["User"].getStr() - lid = jsonLine["Details"]["LID"].getStr() + timestamp = jsonLine.Timestamp + ruleTitle = jsonLine.RuleTitle + computer = jsonLine.Computer + process = jsonLine.Details["Proc"].getStr() + pidStr = $jsonLine.Details["PID"].getInt() + user = jsonLine.Details["User"].getStr() + lid = jsonLine.Details["LID"].getStr() try: if pidStr == "0": # -F, --no-field-data-mapping JSONL requires hex conversion - pidStr = intToStr(fromHex[int](jsonLine["Details"]["PID"].getStr())) + pidStr = intToStr(fromHex[int](jsonLine.Details["PID"].getStr())) except ValueError: discard try: - ruleAuthor = jsonLine["RuleAuthor"].getStr() + ruleAuthor = jsonLine.RuleAuthor except KeyError: ruleAuthor = "" @@ -98,32 +98,32 @@ proc timelineSuspiciousProcesses(level: string = "high", output: string = "", qu # Found a Sysmon 1 process creation event if channel == "Sysmon" and eventId == 1 and isMinLevel(eventLevel, level) == true: inc suspicousProcessCount_Sysmon_1 - cmdLine = jsonLine["Details"]["Cmdline"].getStr() + cmdLine = jsonLine.Details["Cmdline"].getStr() eventType = "Sysmon 1" - timestamp = jsonLine["Timestamp"].getStr() - ruleTitle = jsonLine["RuleTitle"].getStr() - computer = jsonLine["Computer"].getStr() - process = jsonLine["Details"]["Proc"].getStr() - pidStr = $jsonLine["Details"]["PID"].getInt() - user = jsonLine["Details"]["User"].getStr() - lid = jsonLine["Details"]["LID"].getStr() - lguid = jsonLine["Details"]["LGUID"].getStr() - processGuid = jsonLine["Details"]["PGUID"].getStr() - parentCmdline = jsonLine["Details"]["ParentCmdline"].getStr() - parentPid = $jsonLine["Details"]["ParentPID"].getInt() - parentGuid = jsonLine["Details"]["ParentPGUID"].getStr() - description = jsonLine["Details"]["Description"].getStr() - product = jsonLine["Details"]["Product"].getStr() + timestamp = jsonLine.Timestamp + ruleTitle = jsonLine.RuleTitle + computer = jsonLine.Computer + process = jsonLine.Details["Proc"].getStr() + pidStr = $jsonLine.Details["PID"].getInt() + user = jsonLine.Details["User"].getStr() + lid = jsonLine.Details["LID"].getStr() + lguid = jsonLine.Details["LGUID"].getStr() + processGuid = jsonLine.Details["PGUID"].getStr() + parentCmdline = jsonLine.Details["ParentCmdline"].getStr() + parentPid = $jsonLine.Details["ParentPID"].getInt() + parentGuid = jsonLine.Details["ParentPGUID"].getStr() + description = jsonLine.Details["Description"].getStr() + product = jsonLine.Details["Product"].getStr() try: - company = jsonLine["Details"]["Company"].getStr() + company = jsonLine.Details["Company"].getStr() except KeyError: company = "" try: - ruleAuthor = jsonLine["RuleAuthor"].getStr() + ruleAuthor = jsonLine.RuleAuthor except KeyError: ruleAuthor = "" try: - hashes = jsonLine["Details"]["Hashes"].getStr() # Hashes are not enabled by default so this field may not exist. + hashes = jsonLine.Details["Hashes"].getStr() # Hashes are not enabled by default so this field may not exist. let pairs = hashes.split(",") # Split the string into key-value pairs. Ex: MD5=DE9C75F34F47B60A71BBA03760F0579E,SHA256=12F06D3B1601004DB3F7F1A07E7D3AF4CC838E890E0FF50C51E4A0C9366719ED,IMPHASH=336674CB3C8337BDE2C22255345BFF43 for pair in pairs: let keyVal = pair.split("=") diff --git a/src/takajopkg/timelineTasks.nim b/src/takajopkg/timelineTasks.nim index 4b90e2b..8bed7c3 100644 --- a/src/takajopkg/timelineTasks.nim +++ b/src/takajopkg/timelineTasks.nim @@ -43,17 +43,17 @@ proc timelineTasks(output: string, outputLogoffEvents: bool = false, quiet: bool for line in lines(timeline): inc bar bar.update(1000000000) - let jsonLine = parseJson(line) - let eventId = jsonLine["EventID"].getInt(0) - let channel = jsonLine["Channel"].getStr("N/A") + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let eventId = jsonLine.EventID + let channel = jsonLine.Channel if eventId == 4698 and channel == "Sec": - let ts = jsonLine["Timestamp"].getStr("N/A") - let taskName = jsonLine["Details"]["Name"].getStr("N/A") - let subjectUserName = jsonLine["Details"]["User"].getStr("N/A") - let subjectUserSid = jsonLine["ExtraFieldInfo"]["SubjectUserSid"].getStr("N/A") - let subjectDomainName = jsonLine["ExtraFieldInfo"]["SubjectDomainName"].getStr("N/A") - let subjectLogonId = jsonLine["Details"]["LID"].getStr("N/A") - let content = jsonLine["Details"]["Content"].getStr("N/A").replace("\\r\\n", "") + let ts = jsonLine.Timestamp + let taskName = jsonLine.Details["Name"].getStr("N/A") + let subjectUserName = jsonLine.Details["User"].getStr("N/A") + let subjectUserSid = jsonLine.ExtraFieldInfo["SubjectUserSid"].getStr("N/A") + let subjectDomainName = jsonLine.ExtraFieldInfo["SubjectDomainName"].getStr("N/A") + let subjectLogonId = jsonLine.Details["LID"].getStr("N/A") + let content = jsonLine.Details["Content"].getStr("N/A").replace("\\r\\n", "") var basicTable = initOrderedTable[string, string]() basicTable["Timestamp"] = ts basicTable["TaskName"] = taskName diff --git a/src/takajopkg/ttpSummary.nim b/src/takajopkg/ttpSummary.nim index 0cf9cf7..8469a17 100644 --- a/src/takajopkg/ttpSummary.nim +++ b/src/takajopkg/ttpSummary.nim @@ -58,18 +58,18 @@ proc ttpSummary(output: string = "", quiet: bool = false, timeline: string) = for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine = parseJson(line) + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) try: - let com = jsonLine["Computer"].getStr() - for tag in jsonLine["MitreTags"]: - if tag.getStr() notin attack: + let com = jsonLine.Computer + for tag in jsonLine.MitreTags: + if tag notin attack: continue - let res = attack[tag.getStr()] + let res = attack[tag] let dat = res["Tactic"].getStr() let tac = tac_no[dat] & dat let tec = res["Technique"].getStr() let sub = res["Sub-Technique"].getStr() - let rul = jsonLine["RuleTitle"].getStr() + let rul = jsonLine.RuleTitle seqOfResultsTables.add([com, tac, tec, sub, rul]) except CatchableError: continue diff --git a/src/takajopkg/ttpVisualize.nim b/src/takajopkg/ttpVisualize.nim index cf01408..34d6286 100644 --- a/src/takajopkg/ttpVisualize.nim +++ b/src/takajopkg/ttpVisualize.nim @@ -22,11 +22,11 @@ proc ttpVisualize(output: string = "mitre-ttp-heatmap.json", quiet: bool = false for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine = parseJson(line) + let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) try: - for tag in jsonLine["MitreTags"]: - let techniqueID = tag.getStr() - let ruleTitle = strip(jsonLine["RuleTitle"].getStr()) + for tag in jsonLine.MitreTags: + let techniqueID = tag + let ruleTitle = strip(jsonLine.RuleTitle) if stackedMitreTags.hasKey(techniqueID) and ruleTitle notin stackedMitreTags[techniqueID]: stackedMitreTags[techniqueID] = stackedMitreTags[techniqueID] & "," & ruleTitle stackedMitreTagsCount[techniqueID] += 1 diff --git a/takajo.nimble b/takajo.nimble index 793fea6..ee5c0b5 100644 --- a/takajo.nimble +++ b/takajo.nimble @@ -16,4 +16,5 @@ requires "cligen >= 1.5" requires "suru#f6f1e607c585b2bc2f71309996643f0555ff6349" requires "puppy >= 2.1.0" requires "termstyle" -requires "nancy" \ No newline at end of file +requires "nancy" +requires "jsony >= 1.1.5" \ No newline at end of file From b3ecb80fe4401264ccc9bd31a8817a863e2a0850 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Wed, 28 Feb 2024 22:43:40 +0900 Subject: [PATCH 2/4] perf: use jsony for json parsing --- src/takajo.nim | 1 + src/takajopkg/extractScriptblocks.nim | 5 ++++- src/takajopkg/hayabusaJson.nim | 10 +++++++++- src/takajopkg/listDomains.nim | 5 ++++- src/takajopkg/listHashes.nim | 5 ++++- src/takajopkg/listIpAddresses.nim | 5 ++++- src/takajopkg/splitJsonTimeline.nim | 5 ++++- src/takajopkg/stackCmdlines.nim | 5 ++++- src/takajopkg/stackDNS.nim | 5 ++++- src/takajopkg/stackLogons.nim | 5 ++++- src/takajopkg/stackProcesses.nim | 5 ++++- src/takajopkg/stackServices.nim | 5 ++++- src/takajopkg/stackTasks.nim | 5 ++++- src/takajopkg/timelineLogon.nim | 5 ++++- src/takajopkg/timelinePartitionDiagnostic.nim | 5 ++++- src/takajopkg/timelineSuspiciousProcesses.nim | 5 ++++- src/takajopkg/timelineTasks.nim | 5 ++++- src/takajopkg/ttpSummary.nim | 5 ++++- src/takajopkg/ttpVisualize.nim | 5 ++++- 19 files changed, 78 insertions(+), 18 deletions(-) diff --git a/src/takajo.nim b/src/takajo.nim index b4e45ee..70cfd42 100644 --- a/src/takajo.nim +++ b/src/takajo.nim @@ -16,6 +16,7 @@ import threadpool import uri import os import std/enumerate +import std/options import std/xmlparser import std/xmltree import suru diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index c3b3ec6..9abdf68 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -90,7 +90,10 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo inc currentIndex inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() if jsonLine.EventID != 4104 or isMinLevel(jsonLine.Level, level) == false: continue diff --git a/src/takajopkg/hayabusaJson.nim b/src/takajopkg/hayabusaJson.nim index 39f1f93..dd818b8 100644 --- a/src/takajopkg/hayabusaJson.nim +++ b/src/takajopkg/hayabusaJson.nim @@ -1,4 +1,6 @@ import json +import jsony +import std/options type HayabusaJson* = ref object Timestamp*: string @@ -19,4 +21,10 @@ type HayabusaJson* = ref object EvtxFile*: string MitreTags*: seq[string] MitreTactics*: seq[string] - OtherTags*: seq[string] \ No newline at end of file + OtherTags*: seq[string] + +proc parseLine*(line:string): Option[HayabusaJson] = + try: + return some(line.fromJson(HayabusaJson)) + except CatchableError: + return none(HayabusaJson) \ No newline at end of file diff --git a/src/takajopkg/listDomains.nim b/src/takajopkg/listDomains.nim index 1c47c74..322da3d 100644 --- a/src/takajopkg/listDomains.nim +++ b/src/takajopkg/listDomains.nim @@ -30,7 +30,10 @@ proc listDomains(includeSubdomains: bool = false, includeWorkstations: bool = fa for line in lines(timeline): inc bar bar.update(1000000000) - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let eventId = jsonLine.EventID let channel = jsonLine.Channel let details = jsonLine.Details diff --git a/src/takajopkg/listHashes.nim b/src/takajopkg/listHashes.nim index 288151f..c1502d5 100644 --- a/src/takajopkg/listHashes.nim +++ b/src/takajopkg/listHashes.nim @@ -36,7 +36,10 @@ proc listHashes(level: string = "high", output: string, quiet: bool = false, tim for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let eventId = jsonLine.EventID let channel = jsonLine.Channel let eventLevel = jsonLine.Level diff --git a/src/takajopkg/listIpAddresses.nim b/src/takajopkg/listIpAddresses.nim index 38b5093..2f9b7da 100644 --- a/src/takajopkg/listIpAddresses.nim +++ b/src/takajopkg/listIpAddresses.nim @@ -35,7 +35,10 @@ proc listIpAddresses(inbound: bool = true, outbound: bool = true, output: string for line in lines(timeline): inc bar bar.update(1000000000) - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let eventId = jsonLine.EventID let channel = jsonLine.Channel diff --git a/src/takajopkg/splitJsonTimeline.nim b/src/takajopkg/splitJsonTimeline.nim index a87bb31..4b25ba1 100644 --- a/src/takajopkg/splitJsonTimeline.nim +++ b/src/takajopkg/splitJsonTimeline.nim @@ -33,7 +33,10 @@ proc splitJsonTimeline(output: string = "output", quiet: bool = false, timeline: inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let computerName = jsonLine.Computer if not filesTable.hasKey(computerName): diff --git a/src/takajopkg/stackCmdlines.nim b/src/takajopkg/stackCmdlines.nim index 5f90959..b6dc500 100644 --- a/src/takajopkg/stackCmdlines.nim +++ b/src/takajopkg/stackCmdlines.nim @@ -12,7 +12,10 @@ proc stackCmdlines(level: string = "low", ignoreSysmon: bool = false, ignoreSecu for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let eventId = jsonLine.EventID let channel = jsonLine.Channel if (eventId == 1 and not ignoreSysmon and channel == "Sysmon") or diff --git a/src/takajopkg/stackDNS.nim b/src/takajopkg/stackDNS.nim index dcf6efd..2a7b579 100644 --- a/src/takajopkg/stackDNS.nim +++ b/src/takajopkg/stackDNS.nim @@ -12,7 +12,10 @@ proc stackDNS(level: string = "informational", output: string = "", quiet: bool for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let eventId = jsonLine.EventID let channel = jsonLine.Channel if (eventId == 22 and channel == "Sysmon"): diff --git a/src/takajopkg/stackLogons.nim b/src/takajopkg/stackLogons.nim index be7a88e..258dec7 100644 --- a/src/takajopkg/stackLogons.nim +++ b/src/takajopkg/stackLogons.nim @@ -35,7 +35,10 @@ proc stackLogons(localSrcIpAddresses = false, output: string = "", quiet: bool = for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let ruleTitle = jsonLine.RuleTitle # EID 4624 Successful Logon diff --git a/src/takajopkg/stackProcesses.nim b/src/takajopkg/stackProcesses.nim index e100a02..760528c 100644 --- a/src/takajopkg/stackProcesses.nim +++ b/src/takajopkg/stackProcesses.nim @@ -11,7 +11,10 @@ proc stackProcesses(level: string = "low", ignoreSysmon: bool = false, ignoreSec for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let eventId = jsonLine.EventID let channel = jsonLine.Channel if (eventId == 1 and not ignoreSysmon and channel == "Sysmon") or diff --git a/src/takajopkg/stackServices.nim b/src/takajopkg/stackServices.nim index fa2fdc3..24d69f9 100644 --- a/src/takajopkg/stackServices.nim +++ b/src/takajopkg/stackServices.nim @@ -13,7 +13,10 @@ proc stackServices(level: string = "informational", ignoreSystem: bool = false, for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let eventId = jsonLine.EventID let channel = jsonLine.Channel if (eventId == 7045 and not ignoreSystem and channel == "Sys") or diff --git a/src/takajopkg/stackTasks.nim b/src/takajopkg/stackTasks.nim index b0f8c65..50b5e84 100644 --- a/src/takajopkg/stackTasks.nim +++ b/src/takajopkg/stackTasks.nim @@ -15,7 +15,10 @@ proc stackTasks(level: string = "informational", ignoreSysmon: bool = false, ign for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let eventId = jsonLine.EventID let channel = jsonLine.Channel if eventId == 4698 and channel == "Sec": diff --git a/src/takajopkg/timelineLogon.nim b/src/takajopkg/timelineLogon.nim index 74f5f5e..b124979 100644 --- a/src/takajopkg/timelineLogon.nim +++ b/src/takajopkg/timelineLogon.nim @@ -39,7 +39,10 @@ proc timelineLogon(calculateElapsedTime: bool = true, output: string, outputLogo for line in lines(timeline): inc bar bar.update(1000000000) - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let ruleTitle = jsonLine.RuleTitle #EID 4624 Successful Logon diff --git a/src/takajopkg/timelinePartitionDiagnostic.nim b/src/takajopkg/timelinePartitionDiagnostic.nim index 0417473..77ced5d 100644 --- a/src/takajopkg/timelinePartitionDiagnostic.nim +++ b/src/takajopkg/timelinePartitionDiagnostic.nim @@ -53,7 +53,10 @@ proc timelinePartitionDiagnostic(output: string = "", quiet: bool = false, timel for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let eventId = jsonLine.EventID let channel = jsonLine.Channel diff --git a/src/takajopkg/timelineSuspiciousProcesses.nim b/src/takajopkg/timelineSuspiciousProcesses.nim index 1599438..01b0a81 100644 --- a/src/takajopkg/timelineSuspiciousProcesses.nim +++ b/src/takajopkg/timelineSuspiciousProcesses.nim @@ -36,7 +36,10 @@ proc timelineSuspiciousProcesses(level: string = "high", output: string = "", qu for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let eventId = jsonLine.EventID let channel = jsonLine.Channel let eventLevel = jsonLine.Level diff --git a/src/takajopkg/timelineTasks.nim b/src/takajopkg/timelineTasks.nim index 8bed7c3..7d6d1c5 100644 --- a/src/takajopkg/timelineTasks.nim +++ b/src/takajopkg/timelineTasks.nim @@ -43,7 +43,10 @@ proc timelineTasks(output: string, outputLogoffEvents: bool = false, quiet: bool for line in lines(timeline): inc bar bar.update(1000000000) - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() let eventId = jsonLine.EventID let channel = jsonLine.Channel if eventId == 4698 and channel == "Sec": diff --git a/src/takajopkg/ttpSummary.nim b/src/takajopkg/ttpSummary.nim index 8469a17..3433810 100644 --- a/src/takajopkg/ttpSummary.nim +++ b/src/takajopkg/ttpSummary.nim @@ -58,7 +58,10 @@ proc ttpSummary(output: string = "", quiet: bool = false, timeline: string) = for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() try: let com = jsonLine.Computer for tag in jsonLine.MitreTags: diff --git a/src/takajopkg/ttpVisualize.nim b/src/takajopkg/ttpVisualize.nim index 34d6286..ff56e75 100644 --- a/src/takajopkg/ttpVisualize.nim +++ b/src/takajopkg/ttpVisualize.nim @@ -22,7 +22,10 @@ proc ttpVisualize(output: string = "mitre-ttp-heatmap.json", quiet: bool = false for line in lines(timeline): inc bar bar.update(1000000000) # refresh every second - let jsonLine:HayabusaJson = line.fromJson(HayabusaJson) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine:HayabusaJson = jsonLineOpt.get() try: for tag in jsonLine.MitreTags: let techniqueID = tag From be774f6c278fa356205f6b33c572dd1d9a5baa68 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Wed, 28 Feb 2024 22:54:30 +0900 Subject: [PATCH 3/4] fix: remove unused import --- src/takajo.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/src/takajo.nim b/src/takajo.nim index 70cfd42..79e2c84 100644 --- a/src/takajo.nim +++ b/src/takajo.nim @@ -1,7 +1,6 @@ import algorithm import cligen import json -import jsony import nancy import puppy import re From 3af3cfda8fbb8cf5321f9720d6930e2654680413 Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Thu, 29 Feb 2024 08:50:52 +0900 Subject: [PATCH 4/4] update changelog --- CHANGELOG-Japanese.md | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 5a0c403..0f8b655 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -5,6 +5,7 @@ **改善:** - 重複するコードを削除するためのリファクタリング。 (#99) (@fukusuket) +- JSONのパースを`jsony`に変更することで、処理速度が2倍以上速くなった。 (#122) (@fukusuket) ## 2.4.0 [2024/02/22] - Ninja Day Release diff --git a/CHANGELOG.md b/CHANGELOG.md index f034d60..d5c36a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ **Enhancements:** - Refactoring to remove duplicate code. (#99) (@fukusuket) +- Processing speed is more than twice as fast by changing the JSON parsing to `jsony`. (#122) (@fukusuket) ## 2.4.0 [2024/02/22] - Ninja Day Release