From 54d878fd57a9250b44965429750a4d20e7850b3e Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Tue, 1 Jun 2021 10:59:25 +1000 Subject: [PATCH] Artifacts can now import/export other artifacts. (#1087) This allows developing shared code which can be easily used by different artifacts. Also included a initial implementation of a Shellbags artifact. --- api/notebooks.go | 7 +- .../definitions/Windows/Forensics/Lnk.yaml | 37 +++- .../Windows/Forensics/Shellbags.yaml | 68 ++++++++ .../definitions/Windows/Sys/Drivers.yaml | 19 ++- .../server/testcases/binary_parsers.out.yaml | 17 +- artifacts/testdata/windows/evtx.out.yaml | 38 ++--- artifacts/testdata/windows/registry.out.yaml | 6 - artifacts/testdata/windows/vss.out.yaml | 38 ++--- datastore/filebased.go | 7 +- go.mod | 7 +- go.sum | 9 +- gui/velociraptor/package-lock.json | 4 +- .../src/components/core/paged-table.js | 10 ++ gui/velociraptor/src/components/utils/hex.js | 159 ++++++++++++++++++ .../src/components/vfs/file-hex-view.css | 9 + gui/velociraptor/src/dark-mode.css | 13 +- reporting/cells.go | 11 ++ services/launcher/compiler.go | 46 +++++ utils/path.go | 13 +- vql/functions/unhex.go | 43 +++++ 20 files changed, 477 insertions(+), 84 deletions(-) create mode 100644 artifacts/definitions/Windows/Forensics/Shellbags.yaml create mode 100644 gui/velociraptor/src/components/utils/hex.js create mode 100644 vql/functions/unhex.go diff --git a/api/notebooks.go b/api/notebooks.go index ed73474a1e..966fc4c0ba 100644 --- a/api/notebooks.go +++ b/api/notebooks.go @@ -1077,7 +1077,12 @@ func updateCellContents( // markdown fragments. cell_content, err := reporting.ConvertVQLCellToContent(input) if err != nil { - return nil, err + // Ignore errors and just treat the whole + // thing as VQL - this will fail to render the + // comment and just ignore it - it is probably + // malformed. + cell_content = &reporting.Content{} + cell_content.PushVQL(input) } for _, fragment := range cell_content.Fragments { diff --git a/artifacts/definitions/Windows/Forensics/Lnk.yaml b/artifacts/definitions/Windows/Forensics/Lnk.yaml index c42e50ab41..1706a37e47 100644 --- a/artifacts/definitions/Windows/Forensics/Lnk.yaml +++ b/artifacts/definitions/Windows/Forensics/Lnk.yaml @@ -20,8 +20,7 @@ parameters: description: Also upload the link files themselves. type: bool -sources: - - query: | +export: | LET Profile = ''' [ ["ShellLinkHeader", 0, [ @@ -134,10 +133,17 @@ sources: "end_bit": 7, }], + ["Subtype", 2, "BitField", { + "type": "uint8", + "start_bit": 0, + "end_bit": 1, + }], + # For now only support some common shell bags ["ShellBag", 0, "Union", { "selector": "x=>x.Type", "choices": { + "64": "ShellBag0x40", "48": "ShellBag0x30", "16": "ShellBag0x1f", "32": "ShellBag0x20", @@ -145,6 +151,18 @@ sources: }] ]], + ["ShellBag0x40", 0, [ + ["Name", 5, "String", { + encoding: "utf8", + }], + ["Description", 0, "Value", { + "value": 'x=>dict( + Type="NetworkLocation", + ShortName=x.Name + )' + }] + ]], + # A LinkInfo stores information about the destination of the link. ["LinkInfo", "x=>x.LinkInfoSize", [ ["Offset", 0, "Value", {"value": "x=>x.StartOf"}], @@ -279,11 +297,16 @@ sources: # Volume name ["ShellBag0x20", 0, [ - ["Name", 3, "String"], - ["Description", 0, "Value", { + ["__Name", 3, "String"], + # Name is only valid if the first bit is set. + ["Name", 3, "Value", { + "value": "x=>if(condition=x.ParentOf.Subtype, then=x.__Name, else='')", + }], + ["Description", 0, "Value", { "value": 'x=>dict( - ShortName=x.Name, - Type="Volume" + LongName=x.Name, + ShortName=x.Name, + Type="Volume" )' }] ]], @@ -376,6 +399,8 @@ sources: ] ''' +sources: + - query: | LET link_files = SELECT parse_binary( filename=FullPath, profile=Profile, struct="ShellLinkHeader") AS Parsed, diff --git a/artifacts/definitions/Windows/Forensics/Shellbags.yaml b/artifacts/definitions/Windows/Forensics/Shellbags.yaml new file mode 100644 index 0000000000..6c80e278a0 --- /dev/null +++ b/artifacts/definitions/Windows/Forensics/Shellbags.yaml @@ -0,0 +1,68 @@ +name: Windows.Forensics.Shellbags +description: | + Windows uses the Shellbag keys to store user preferences for GUI + folder display within Windows Explorer. + +reference: + - https://www.sans.org/blog/computer-forensic-artifacts-windows-7-shellbags/ + +parameters: + - name: MRUGlob + type: csv + default: | + Glob + HKEY_CURRENT_USER\Software\Microsoft\Windows\Shell\BagMRU\** + HKEY_CURRENT_USER\Software\Classes\Local Settings\Software\Microsoft\Windows\Shell\BagMRU\** + +imports: + # Link files use the same internal format as shellbags so we import + # the profile here. + - Windows.Forensics.Lnk + +sources: + - query: | + LET X = SELECT FullPath, + parse_binary(profile=Profile, filename=Data.value, accessor="data", struct="ItemIDList") as _Parsed, + base64encode(string=Data.value) AS _RawData, ModTime + FROM glob(globs=MRUGlob.Glob, accessor="registry") + WHERE Data.type = "BINARY" AND Name =~ "[0-9]+" + + LET AllResults <= SELECT FullPath, + _Parsed.ShellBag.Description AS Description, + _Parsed, _RawData, ModTime + FROM X + + // Recursive function to join path components together. + LET FormPath(MRUPath, Description) = SELECT * FROM chain( + b={ + SELECT MRUPath AS FullPath, Description, + -- Signify unknown component as ? + Description.LongName || Description.ShortName || "?" AS Name + FROM scope() + }, + c={ + SELECT * FROM foreach(row={ + SELECT FullPath, Description + FROM AllResults + WHERE FullPath = dirname(path=MRUPath, sep="\\") + LIMIT 1 + }, query={ + SELECT * FROM FormPath(MRUPath=FullPath, Description=Description) + }) + }) + ORDER BY FullPath + + // Now display all hits and their reconstructed path + LET ReconstructedPath = SELECT ModTime, FullPath, Description, { + SELECT * FROM FormPath(MRUPath=FullPath, Description=Description) + } AS Chain, _RawData, _Parsed + FROM AllResults + + SELECT ModTime, FullPath, Description, + join(array=Chain.Name, sep=" -> ") AS Path, + _RawData, _Parsed + FROM ReconstructedPath + +column_types: + - name: _RawData + type: base64 diff --git a/artifacts/definitions/Windows/Sys/Drivers.yaml b/artifacts/definitions/Windows/Sys/Drivers.yaml index 777ab50e14..45bd473b8d 100644 --- a/artifacts/definitions/Windows/Sys/Drivers.yaml +++ b/artifacts/definitions/Windows/Sys/Drivers.yaml @@ -1,10 +1,17 @@ name: Windows.Sys.Drivers -description: Details for in-use Windows device drivers. This does not display installed but unused drivers. +description: | + Details for in-use Windows device drivers. This does not display installed but unused drivers. + sources: - precondition: SELECT OS From info() where OS = 'windows' - queries: - - | - SELECT * from wmi( - query="select * from Win32_PnPSignedDriver", - namespace="ROOT\\CIMV2") + query: | + SELECT * from wmi( + query="select * from Win32_PnPSignedDriver", + namespace="ROOT\\CIMV2") + + - name: RunningDrivers + query: | + SELECT * from wmi( + query="select * from Win32_SystemDriver", + namespace="ROOT\\CIMV2") diff --git a/artifacts/testdata/server/testcases/binary_parsers.out.yaml b/artifacts/testdata/server/testcases/binary_parsers.out.yaml index b9759cbc66..fa92bfe4b9 100644 --- a/artifacts/testdata/server/testcases/binary_parsers.out.yaml +++ b/artifacts/testdata/server/testcases/binary_parsers.out.yaml @@ -78,6 +78,7 @@ SELECT ModuleName, Timestamp, Functions[1] FROM Artifact.Windows.System.Powershe "ItemIDSize": 20, "Offset": 78, "Type": 16, + "Subtype": 1, "ShellBag": { "Description": { "ShortName": "My Computer", @@ -89,9 +90,11 @@ SELECT ModuleName, Timestamp, Functions[1] FROM Artifact.Windows.System.Powershe "ItemIDSize": 25, "Offset": 98, "Type": 32, + "Subtype": 1, "ShellBag": { "Name": "F:\\", "Description": { + "LongName": "F:\\", "ShortName": "F:\\", "Type": "Volume" } @@ -101,6 +104,7 @@ SELECT ModuleName, Timestamp, Functions[1] FROM Artifact.Windows.System.Powershe "ItemIDSize": 74, "Offset": 123, "Type": 48, + "Subtype": 1, "ShellBag": { "Size": 74, "Type": 49, @@ -108,14 +112,14 @@ SELECT ModuleName, Timestamp, Functions[1] FROM Artifact.Windows.System.Powershe "Directory", "Unicode" ], - "LastModificationTime": "2020-12-14T02:28:18Z", + "LastModificationTime": "2020-11-12T01:24:36Z", "ShortName": "tmp", "Extension": { "Size": 56, "Version": 9, "Signature": "0xbeef0004", - "CreateDate": "2020-12-14T02:28:18Z", - "LastAccessed": "2020-12-14T02:28:18Z", + "CreateDate": "2020-11-12T01:24:36Z", + "LastAccessed": "2020-11-12T01:24:36Z", "MFTReference": { "MFTID": 8651087, "SequenceNumber": 0 @@ -127,9 +131,9 @@ SELECT ModuleName, Timestamp, Functions[1] FROM Artifact.Windows.System.Powershe "Directory", "Unicode" ], - "Modified": "2020-12-14T02:28:18Z", - "LastAccessed": "2020-12-14T02:28:18Z", - "CreateDate": "2020-12-14T02:28:18Z", + "Modified": "2020-11-12T01:24:36Z", + "LastAccessed": "2020-11-12T01:24:36Z", + "CreateDate": "2020-11-12T01:24:36Z", "ShortName": "tmp", "LongName": "tmp", "MFTID": 8651087, @@ -141,6 +145,7 @@ SELECT ModuleName, Timestamp, Functions[1] FROM Artifact.Windows.System.Powershe "ItemIDSize": 84, "Offset": 197, "Type": 48, + "Subtype": 0, "ShellBag": { "Size": 84, "Type": 50, diff --git a/artifacts/testdata/windows/evtx.out.yaml b/artifacts/testdata/windows/evtx.out.yaml index 1f5c1000f7..06d7817e49 100644 --- a/artifacts/testdata/windows/evtx.out.yaml +++ b/artifacts/testdata/windows/evtx.out.yaml @@ -275,12 +275,12 @@ SELECT * FROM parse_evtx(filename=srcDir + '/artifacts/testdata/files/Security_1 } ]SELECT EventID, DomainName, UserName, LogonType, SourceIP, Description FROM Artifact.Windows.EventLogs.RDPAuth( Security=srcDir + '/artifacts/testdata/files/RDPAuth_Security.evtx', System= srcDir + '/artifacts/testdata/files/RDPAuth_System.evtx', LocalSessionManager= srcDir + '/artifacts/testdata/files/RDPAuth_LocalSessionManager.evtx', RemoteConnectionManager= srcDir + '/artifacts/testdata/files/RDPAuth_RemoteConnectionManager.evtx') WHERE NOT LogonType = 3 GROUP BY EventID[ { - "EventID": 1149, - "DomainName": "null", + "EventID": 23, + "DomainName": "WINDOMAIN", "UserName": "vagrant", "LogonType": "null", - "SourceIP": "192.168.38.1", - "Description": "RDP_INITIATION_SUCCESSFUL" + "SourceIP": "null", + "Description": "RDP_SESSION_LOGOFF" }, { "EventID": 21, @@ -299,12 +299,12 @@ SELECT * FROM parse_evtx(filename=srcDir + '/artifacts/testdata/files/Security_1 "Description": "RDP_REMOTE_CONNECTED" }, { - "EventID": 23, - "DomainName": "WINDOMAIN", - "UserName": "vagrant", + "EventID": 40, + "DomainName": "null", + "UserName": "null", "LogonType": "null", "SourceIP": "null", - "Description": "RDP_SESSION_LOGOFF" + "Description": "RDP_REMOTE_DISCONNECTED_REASON" }, { "EventID": 24, @@ -323,12 +323,20 @@ SELECT * FROM parse_evtx(filename=srcDir + '/artifacts/testdata/files/Security_1 "Description": "RDP_REMOTE_RECONNECTION" }, { - "EventID": 40, + "EventID": 1149, "DomainName": "null", - "UserName": "null", + "UserName": "vagrant", + "LogonType": "null", + "SourceIP": "192.168.38.1", + "Description": "RDP_INITIATION_SUCCESSFUL" + }, + { + "EventID": 4647, + "DomainName": "WINDOMAIN", + "UserName": "vagrant", "LogonType": "null", "SourceIP": "null", - "Description": "RDP_REMOTE_DISCONNECTED_REASON" + "Description": "USER_INITIATED_LOGOFF" }, { "EventID": 4624, @@ -345,13 +353,5 @@ SELECT * FROM parse_evtx(filename=srcDir + '/artifacts/testdata/files/Security_1 "LogonType": 10, "SourceIP": "null", "Description": "LOGOFF_DISCONNECT" - }, - { - "EventID": 4647, - "DomainName": "WINDOMAIN", - "UserName": "vagrant", - "LogonType": "null", - "SourceIP": "null", - "Description": "USER_INITIATED_LOGOFF" } ] \ No newline at end of file diff --git a/artifacts/testdata/windows/registry.out.yaml b/artifacts/testdata/windows/registry.out.yaml index f7d3463984..95b4b80131 100644 --- a/artifacts/testdata/windows/registry.out.yaml +++ b/artifacts/testdata/windows/registry.out.yaml @@ -40,9 +40,6 @@ SELECT FullPath FROM glob(globs="/*", accessor="reg")[ { "FullPath": "\\HKEY_LOCAL_MACHINE\\BCD00000000" }, - { - "FullPath": "\\HKEY_LOCAL_MACHINE\\DRIVERS" - }, { "FullPath": "\\HKEY_LOCAL_MACHINE\\HARDWARE" }, @@ -59,9 +56,6 @@ SELECT FullPath FROM glob(globs="/*", accessor="reg")[ { "FullPath": "\\HKEY_LOCAL_MACHINE\\BCD00000000" }, - { - "FullPath": "\\HKEY_LOCAL_MACHINE\\DRIVERS" - }, { "FullPath": "\\HKEY_LOCAL_MACHINE\\HARDWARE" }, diff --git a/artifacts/testdata/windows/vss.out.yaml b/artifacts/testdata/windows/vss.out.yaml index 40b8e1a2c1..d2c75a5c2a 100644 --- a/artifacts/testdata/windows/vss.out.yaml +++ b/artifacts/testdata/windows/vss.out.yaml @@ -6,41 +6,41 @@ SELECT FullPath, SHA1, Source, Deduped FROM Artifact.Windows.Search.VSS(SearchFi "Deduped": true } ]SELECT FullPath, SHA1, Source, Deduped FROM Artifact.Windows.Search.VSS(SearchFilesGlob='c:\\Users\\test2.txt')[ - { - "FullPath": "\\\\?\\GLOBALROOT\\Device\\HarddiskVolumeShadowCopy1\\Users\\test2.txt", - "SHA1": "1896ab50888c9c3d4150f5044b2568cea317a8e8", - "Source": "_HarddiskVolumeShadowCopy1", - "Deduped": false - }, { "FullPath": "\\\\.\\C:\\Users\\test2.txt", "SHA1": "de23e5237d8b6c785cf2cbde1924e081aa66b674", "Source": "C:", "Deduped": true + }, + { + "FullPath": "\\\\?\\GLOBALROOT\\Device\\HarddiskVolumeShadowCopy1\\Users\\test2.txt", + "SHA1": "1896ab50888c9c3d4150f5044b2568cea317a8e8", + "Source": "_HarddiskVolumeShadowCopy1", + "Deduped": true } ]SELECT EventID,ServiceName,Source FROM Artifact.Windows.EventLogs.ServiceCreationComspec( EventLog="C:\\Windows\\system32\\winevt\\logs\\System.evtx", SearchVSS=True)[ { "EventID": 7045, - "ServiceName": "TestingDetection1", - "Source": "_HarddiskVolumeShadowCopy1" + "ServiceName": "TestingDetection2", + "Source": "C:" }, { "EventID": 7045, - "ServiceName": "TestingDetection2", - "Source": "C:" + "ServiceName": "TestingDetection1", + "Source": "_HarddiskVolumeShadowCopy1" } ]SELECT EventID, Channel, Message=~ 'Clear',Source FROM Artifact.Windows.EventLogs.Cleared( SearchVSS=True)[ { "EventID": 104, "Channel": "System", "Message =~ 'Clear'": true, - "Source": "_HarddiskVolumeShadowCopy1" + "Source": "C:" }, { - "EventID": 104, - "Channel": "Application", + "EventID": 1102, + "Channel": "Security", "Message =~ 'Clear'": true, - "Source": "_HarddiskVolumeShadowCopy1" + "Source": "C:" }, { "EventID": 104, @@ -50,14 +50,14 @@ SELECT FullPath, SHA1, Source, Deduped FROM Artifact.Windows.Search.VSS(SearchFi }, { "EventID": 104, - "Channel": "System", + "Channel": "Application", "Message =~ 'Clear'": true, - "Source": "C:" + "Source": "_HarddiskVolumeShadowCopy1" }, { - "EventID": 1102, - "Channel": "Security", + "EventID": 104, + "Channel": "System", "Message =~ 'Clear'": true, - "Source": "C:" + "Source": "_HarddiskVolumeShadowCopy1" } ] \ No newline at end of file diff --git a/datastore/filebased.go b/datastore/filebased.go index 7335a2c7da..ae6c49a5a1 100644 --- a/datastore/filebased.go +++ b/datastore/filebased.go @@ -504,7 +504,12 @@ func SanitizeString(component string) []rune { component += "_" } - result := make([]rune, len(component)*4) + length := len(component) + if length > 1024 { + length = 1024 + } + + result := make([]rune, length*4) result_idx := 0 for _, c := range []byte(component) { diff --git a/go.mod b/go.mod index 468f4a7617..de88f990ab 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/Showmax/go-fqdn v1.0.0 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/Velocidex/ahocorasick v0.0.0-20180712114356-e1c353eeaaee - github.com/Velocidex/amsi v0.0.0-20200608120838-e5d93b76f119 // indirect + github.com/Velocidex/amsi v0.0.0-20200608120838-e5d93b76f119 github.com/Velocidex/cgofuse v1.1.2 github.com/Velocidex/go-elasticsearch/v7 v7.3.1-0.20191001125819-fee0ef9cac6b github.com/Velocidex/go-yara v1.1.10-0.20210423154840-dace8239c158 @@ -25,7 +25,6 @@ require ( github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 github.com/alecthomas/chroma v0.7.2 github.com/alecthomas/participle v0.7.1 - github.com/alecthomas/repr v0.0.0-20210301060118-828286944d6a // indirect github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0 github.com/aws/aws-sdk-go v1.26.7 github.com/bi-zone/etw v0.0.0-20200916105032-b215904fae4f @@ -121,8 +120,8 @@ require ( www.velocidex.com/golang/go-prefetch v0.0.0-20200722101157-37e4751dd5ca www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196 www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 - www.velocidex.com/golang/vfilter v0.0.0-20210515085940-25d96b94dafb - www.velocidex.com/golang/vtypes v0.0.0-20210323032031-b61f37170666 + www.velocidex.com/golang/vfilter v0.0.0-20210531163909-d0a0745e41b7 + www.velocidex.com/golang/vtypes v0.0.0-20210531054645-7ec8b6e24d88 ) // replace www.velocidex.com/golang/go-pe => /home/mic/projects/go-pe diff --git a/go.sum b/go.sum index a8db8a94aa..f60f09e15f 100644 --- a/go.sum +++ b/go.sum @@ -728,7 +728,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -938,8 +937,8 @@ www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196 h1:3oYZ7hPN www.velocidex.com/golang/oleparse v0.0.0-20190327031422-34195d413196/go.mod h1:i7M+d4Vxir8nmDACh+c6CsUU1r1Wcj00aRgNp8mXcPQ= www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500 h1:XqZddiAbjPIsTZcEPbqqqABS/ZV5SB7j33eczNsqD60= www.velocidex.com/golang/regparser v0.0.0-20190625082115-b02dc43c2500/go.mod h1:DVzloLH8L+oF3zma1Jisaat5bGF+4VLggDcYlIp00ns= -www.velocidex.com/golang/vfilter v0.0.0-20210108051106-c18c13c24eff/go.mod h1:EdP5LDT3l9khZVjDVD2YKoDvECi3AW0e0074quDPNpA= -www.velocidex.com/golang/vfilter v0.0.0-20210515085940-25d96b94dafb h1:S8UGW4jtwOEqibeKsErTyTr/0EqV3Fn4Ns8S4/0tNKA= www.velocidex.com/golang/vfilter v0.0.0-20210515085940-25d96b94dafb/go.mod h1:KB724xBNYh4lgipyGwsvx0/5hXRqsKjmrMrkSjGESvU= -www.velocidex.com/golang/vtypes v0.0.0-20210323032031-b61f37170666 h1:4VjIpQYv3WWXizcMMwAHi8hp5B6pbWPD1jsNxygWhRE= -www.velocidex.com/golang/vtypes v0.0.0-20210323032031-b61f37170666/go.mod h1:34AZRfhNvJ1QAwPpYrDxjCyOFys+NbSmH6LLVSjsAEg= +www.velocidex.com/golang/vfilter v0.0.0-20210531163909-d0a0745e41b7 h1:3xbomFv89/F/RlXE5jHDFz2aQHL9iFc+dtB/VRVaJs0= +www.velocidex.com/golang/vfilter v0.0.0-20210531163909-d0a0745e41b7/go.mod h1:KB724xBNYh4lgipyGwsvx0/5hXRqsKjmrMrkSjGESvU= +www.velocidex.com/golang/vtypes v0.0.0-20210531054645-7ec8b6e24d88 h1:Fb7nzCAXBVzXbSUXO5sq5msYieB2GMfG7BGnZRDjKMM= +www.velocidex.com/golang/vtypes v0.0.0-20210531054645-7ec8b6e24d88/go.mod h1:PIG8uSY330pJd620KPksZpTaAsX3sIMiiNJQihZph6c= diff --git a/gui/velociraptor/package-lock.json b/gui/velociraptor/package-lock.json index c3391660d9..b9dd9a5e2f 100644 --- a/gui/velociraptor/package-lock.json +++ b/gui/velociraptor/package-lock.json @@ -1,11 +1,11 @@ { "name": "velociraptor", - "version": "0.5.7", + "version": "0.6.0", "lockfileVersion": 2, "requires": true, "packages": { "": { - "version": "0.5.7", + "version": "0.6.0", "dependencies": { "@fortawesome/fontawesome-svg-core": "^1.2.30", "@fortawesome/free-solid-svg-icons": "^5.14.0", diff --git a/gui/velociraptor/src/components/core/paged-table.js b/gui/velociraptor/src/components/core/paged-table.js index 152db31797..3efdf03085 100644 --- a/gui/velociraptor/src/components/core/paged-table.js +++ b/gui/velociraptor/src/components/core/paged-table.js @@ -23,6 +23,7 @@ import Spinner from '../utils/spinner.js'; import api from '../core/api-service.js'; import VeloTimestamp from "../utils/time.js"; import ClientLink from '../clients/client-link.js'; +import HexView from '../utils/hex.js'; import { InspectRawJson, ColumnToggleList, sizePerPageRenderer, PrepareData } from './table.js'; @@ -147,6 +148,15 @@ class VeloPagedTable extends Component { if (column === column_types[i].name) { let type = column_types[i].type; switch (type) { + case "base64": + return (cell, row, rowIndex)=>{ + let decoded = cell.slice(0,1000); + try { + decoded = atob(cell); + } catch(e) {}; + + return ; + }; case "timestamp": return (cell, row, rowIndex)=>; diff --git a/gui/velociraptor/src/components/utils/hex.js b/gui/velociraptor/src/components/utils/hex.js new file mode 100644 index 0000000000..bbdd81f93f --- /dev/null +++ b/gui/velociraptor/src/components/utils/hex.js @@ -0,0 +1,159 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import _ from 'lodash'; +import Button from 'react-bootstrap/Button'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; + + +// A hex viewer suitable for small amountfs of text - No paging. +export default class HexView extends React.Component { + static propTypes = { + data: PropTypes.string, + height: PropTypes.number, + }; + + state = { + rows: 25, + columns: 0x10, + hexDataRows: [], + parsed_data: "", + expanded: false, + } + + componentDidMount = () => { + this.parseFileContentToHexRepresentation_(this.props.data); + this.setState({parsed_data: this.props.data}); + } + + componentDidUpdate = (prevProps, prevState, rootNode) => { + return this.props.data !== this.state.parsed_data; + } + + parseFileContentToHexRepresentation_ = (fileContent) => { + if (!fileContent) { + fileContent = ""; + } + let hexDataRows = []; + for(var i = 0; i < this.state.rows; i++){ + let offset = 0; + var rowOffset = offset + (i * this.state.columns); + var data = fileContent.substr(i * this.state.columns, this.state.columns); + var data_row = []; + for (var j = 0; j < data.length; j++) { + var char = data.charCodeAt(j).toString(16); + data_row.push(('0' + char).substr(-2)); // add leading zero if necessary + }; + + if (data_row.length === 0) { + break; + }; + + hexDataRows.push({ + offset: rowOffset, + data_row: data_row, + data: data, + safe_data: data.replace(/[^\x20-\x7f]/g, '.'), + }); + + } + + this.setState({hexDataRows: hexDataRows, loading: false}); + }; + + + render() { + let height = this.props.height || 5; + let more = this.state.hexDataRows.length > height; + let hexArea = + + + { _.map(this.state.hexDataRows, (row, idx)=>{ + if (idx >= height && !this.state.expanded) { + return <>; + } + return + + ; }) + } + +
+ { _.map(row.data_row, (x, idx)=>{ + return { x } ; + })} +
; + + let contextArea = + + + { _.map(this.state.hexDataRows, (row, idx)=>{ + if (idx >= height && !this.state.expanded) { + return <>; + } + return ; + })} + +
{ row.safe_data }
; + + return ( +
+
+
+
+ + + + + + + + + + + + + + + { more && (this.state.expanded ? + + + + : + + ) } + +
Offset00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f
+ + + { _.map(this.state.hexDataRows, (row, idx)=>{ + if (idx >= height && !this.state.expanded) { + return <>; + } + return + + ; })} + +
+ { row.offset } +
+ 		+ { hexArea } + + { contextArea } +
+ +
+ +
+
+
+
+
+ ); + } +}; diff --git a/gui/velociraptor/src/components/vfs/file-hex-view.css b/gui/velociraptor/src/components/vfs/file-hex-view.css index 23c437c5ee..2720e681a4 100644 --- a/gui/velociraptor/src/components/vfs/file-hex-view.css +++ b/gui/velociraptor/src/components/vfs/file-hex-view.css @@ -39,3 +39,12 @@ font-size: 2.0em; color: var(--color-no-content-color); } + + +.file-hex-view td, .file-hex-view th { + border: 0px; +} + +.file-hex-view td, .file-hex-view button { + width: 100%; +} diff --git a/gui/velociraptor/src/dark-mode.css b/gui/velociraptor/src/dark-mode.css index 6ff1210b8c..ff8307f4a7 100644 --- a/gui/velociraptor/src/dark-mode.css +++ b/gui/velociraptor/src/dark-mode.css @@ -54,7 +54,7 @@ --card-heading-background-color: #444444; --table-heading-background-color: #444444; --color-table-row-selected: #8f8f8f50; - + --color-monospace-color: #eeeeee; --scrollbar-track: #121212; --app-footer-background-color: #444444; } @@ -253,12 +253,6 @@ background-color: var(--background-color); } -/* hexview */ -.dark-mode .hexdump { - filter: invert(1); -} - - /* width */ .dark-mode ::-webkit-scrollbar { width: 10px; @@ -357,3 +351,8 @@ white-space: nowrap; display: block; } + + +.dark-mode .file-hex-view td, .file-hex-view th { + border: 0px; +} diff --git a/reporting/cells.go b/reporting/cells.go index ccf9533edc..41ea475c98 100644 --- a/reporting/cells.go +++ b/reporting/cells.go @@ -6,6 +6,17 @@ import ( "github.com/alecthomas/participle/lexer/stateful" ) +/* + Notebooks cells of VQL type may have markdown interspersed + throughout. This allows users to document their VQL queries directly + in the cell itself rather than having to add another cell. Markdown + is allowed in multiline comments + + This module parses out the VQL into a series of fragments, each are + either VQL or Comment sections. The VQL is evaluated together with + the comment is rendered as a markdown block. +*/ + var ( def = lexer.Must(stateful.New(stateful.Rules{ "Root": { diff --git a/services/launcher/compiler.go b/services/launcher/compiler.go index 6fdf618b8a..7409bd8dbf 100644 --- a/services/launcher/compiler.go +++ b/services/launcher/compiler.go @@ -118,9 +118,55 @@ LET %v <= if( result.OpsPerSecond = artifact.Resources.OpsPerSecond } + err := resolveImports(config_obj, artifact, result) + if err != nil { + return err + } + return mergeSources(config_obj, options, artifact, result) } +func resolveImports(config_obj *config_proto.Config, + artifact *artifacts_proto.Artifact, + result *actions_proto.VQLCollectorArgs) error { + // Resolve imports if needed. First check if the artifact + // itself declares exports for itself (by default each + // artifact imports its own exports). + if artifact.Export != "" { + result.Query = append(result.Query, &actions_proto.VQLRequest{ + VQL: artifact.Export, + }) + } + + if artifact.Imports == nil { + return nil + } + + manager, err := services.GetRepositoryManager() + if err != nil { + return err + } + global_repo, err := manager.GetGlobalRepository(config_obj) + if err != nil { + return err + } + + // These are a list of names to be imported. + for _, imported := range artifact.Imports { + dependent_artifact, pres := global_repo.Get(config_obj, imported) + if !pres { + return fmt.Errorf("Artifact %v imports %v which is not known.", + artifact.Name, imported) + } + if dependent_artifact.Export != "" { + result.Query = append(result.Query, &actions_proto.VQLRequest{ + VQL: dependent_artifact.Export, + }) + } + } + return nil +} + func mergeSources( config_obj *config_proto.Config, options services.CompilerOptions, diff --git a/utils/path.go b/utils/path.go index c90c0bc810..79a41de727 100644 --- a/utils/path.go +++ b/utils/path.go @@ -154,8 +154,13 @@ func SplitPlainComponents(path string) []string { } func escapeComponent(component string) string { + length := len(component) + if length > 1024 { + length = 1024 + } + hasQuotes := false - result := make([]byte, 0, len(component)*2) + result := make([]byte, 0, length*2) for i := 0; i < len(component); i++ { result = append(result, component[i]) @@ -266,7 +271,11 @@ func shouldEscape(c byte) bool { var hexTable = []byte("0123456789ABCDEF") func SanitizeString(component string) string { - result := make([]byte, len(component)*4) + length := len(component) + if length > 1024 { + length = 1024 + } + result := make([]byte, length*4) result_idx := 0 for _, c := range []byte(component) { diff --git a/vql/functions/unhex.go b/vql/functions/unhex.go new file mode 100644 index 0000000000..72326af021 --- /dev/null +++ b/vql/functions/unhex.go @@ -0,0 +1,43 @@ +package functions + +import ( + "context" + "encoding/hex" + + "github.com/Velocidex/ordereddict" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" + "www.velocidex.com/golang/vfilter" + "www.velocidex.com/golang/vfilter/arg_parser" +) + +type UnhexFunctionArgs struct { + String string `vfilter:"optional,field=string,doc=Hex string to decode"` +} + +type UnhexFunction struct{} + +func (self *UnhexFunction) Call(ctx context.Context, + scope vfilter.Scope, + args *ordereddict.Dict) vfilter.Any { + arg := &UnhexFunctionArgs{} + err := arg_parser.ExtractArgsWithContext(ctx, scope, args, arg) + if err != nil { + scope.Log("unhex: %s", err.Error()) + return false + } + + res, _ := hex.DecodeString(arg.String) + return string(res) +} + +func (self UnhexFunction) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo { + return &vfilter.FunctionInfo{ + Name: "unhex", + Doc: "Apply hex decoding to the string.", + ArgType: type_map.AddType(scope, &UnhexFunctionArgs{}), + } +} + +func init() { + vql_subsystem.RegisterFunction(&UnhexFunction{}) +}