This repository has been archived by the owner on Nov 16, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 514
/
Performance, Json and dynamics operator, external data.txt
366 lines (284 loc) · 15.2 KB
/
Performance, Json and dynamics operator, external data.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
print Topic = "l33tSpeak: Advanced hunting in Microsoft 365 Defender", Presenters = "Michael Melone, Tali Ash", Company = "Microsoft", Date = todatetime("17 NOV 2020")
// -------------------------
// Topic 1: Advanced Hunting, KQL, and performance
// Advanced hunting query best practices in Microsoft Threat Protection - Microsoft 365 security | Microsoft Docs
// -------------------------
// Advanced Hunting is built on Azure Data Explorer, which is a Write Once Read Many (WORM)
// technology. When you write a query against Advanced Hunting:
// - Data is based on recent activity (usually delayed just a few minutes)
// - There is never any impact to the endpoint
// - Queries may be throttled or limited based on how they're written to limit impact to other sessions
// Recommended documentation https://docs.microsoft.com/en-us/microsoft-365/security/mtp/advanced-hunting-best-practices?view=o365-worldwide
// #1 best way to improve query performance: reduce timeframe
// Visualized filter at the top right
IdentityInfo
| where Department == "Finance"
| distinct AccountObjectId
| join IdentityLogonEvents on AccountObjectId
| where Application == "Office 365"
// Filter timestamp in the query using “ago”
IdentityInfo
| where Department == "Finance"
| distinct AccountObjectId
| join (IdentityLogonEvents | where Timestamp > ago(10d)) on AccountObjectId
| where Application == "Office 365"
// Filter timestamp in the query using “between”
let selectedTimestamp = datetime(2020-11-12T19:35:03.9859771Z);
IdentityInfo
| where Department == "Finance"
| distinct AccountObjectId
| join (IdentityLogonEvents | where Timestamp between ((selectedTimestamp - 2h) .. (selectedTimestamp + 2h))) on AccountObjectId
| where Application == "Office 365"
AlertInfo
| where AlertId == "caC27D7C90-E9E7-3207-9FF8-94335F0E27D3"
| project AlertTime = Timestamp , Title, Severity, AlertId
| join AlertEvidence on AlertId
| where EntityType == "User"
| project AccountObjectId , AlertTime
| join IdentityLogonEvents on AccountObjectId
| where Timestamp between ((AlertTime - 1h) ..(AlertTime + 1h))
// Has beats contains: When looking for full tokens, has works better, since it doesn't look for substrings.
DeviceNetworkEvents
| where RemoteUrl contains "team"
| take 50
DeviceNetworkEvents
| where RemoteUrl has "team" //only teams will work as it is the full token
| take 50
DeviceNetworkEvents
| where RemoteUrl contains "microsoft.com"
| take 50
DeviceNetworkEvents
| where RemoteUrl has "microsoft.com"
| take 50
// Use case-sensitive operators when possible.
// Names of case-sensitive string operators, such as has_cs and contains_cs, generally end with _cs.
DeviceNetworkEvents
| where RemoteUrl has_cs "microsoft.com"
| take 50
// Use == and not =~, Use in and not in~
// Get latest information on user/device
DeviceInfo
| where DeviceName == "alexw-pc" and isnotempty(OSPlatform)
| summarize arg_max(Timestamp, *) by DeviceId
IdentityInfo
| where Department == "Finance"
| extend ingestionTime = ingestion_time()
| summarize arg_max(ingestionTime, *) by AccountObjectId
// Optimize the join operator
// - If you are using a join, try to reduce the dataset before joining to limit the join size
// - If you use too many resources you may be put in 'time out' for a bit
EmailEvents
| project NetworkMessageId, Subject, Timestamp, SenderFromAddress , SenderIPv4 , RecipientEmailAddress , AttachmentCount
| join kind=leftouter(EmailAttachmentInfo
| project NetworkMessageId,FileName, FileType, ThreatTypes, SHA256, RecipientEmailAddress )
on NetworkMessageId
// filter the left table as much as you can
// Key for the join should be accurate as possible
EmailEvents
| where AttachmentCount > 0
|project NetworkMessageId, Subject, Timestamp, SenderFromAddress , SenderIPv4 , RecipientEmailAddress , AttachmentCount
| join kind=inner (EmailAttachmentInfo
| project NetworkMessageId,FileName, FileType, ThreatTypes, SHA256, RecipientEmailAddress )
on NetworkMessageId, RecipientEmailAddress
// Smaller table on the left side, with kind = inner, as default join (innerunique)
// will remove left side duplications, so if a single email has more than one attachments we will miss it
EmailAttachmentInfo
| project NetworkMessageId, FileName, FileType, ThreatTypes, SHA256, RecipientEmailAddress
| join kind=inner
(EmailEvents
| where AttachmentCount > 0
|project NetworkMessageId, Subject, Timestamp, SenderFromAddress , SenderIPv4 , RecipientEmailAddress , AttachmentCount)
on NetworkMessageId, RecipientEmailAddress
// Check for specific alerts
AlertInfo
| join AlertEvidence on AlertId
| where EntityType == "Machine"
// Attempts to clear security event logs.
| where Title in("Event log was cleared",
// List alerts flagging attempts to delete backup files.
"File backups were deleted",
// Potential Cobalt Strike activity - Note that other threat activity can also
// trigger alerts for suspicious decoded content
"Suspicious decoded content",
// Cobalt Strike activity
"\'Atosev\' malware was detected",
"\'Ploty\' malware was detected",
"\'Bynoco\' malware was detected")
| extend AlertTime = Timestamp
| distinct DeviceName, AlertTime, AlertId, Title
| join DeviceLogonEvents on $left.DeviceName == $right.DeviceName
// Creating 10 day Window surrounding alert activity
| where Timestamp < AlertTime +5d and Timestamp > AlertTime - 5d
// Projecting specific columns
| project Title, DeviceName, DeviceId, Timestamp, LogonType, AccountDomain,
AccountName, AccountSid, AlertTime, AlertId, RemoteIP, RemoteDeviceName
// Check for specific alerts
AlertInfo
// Attempts to clear security event logs.
| where Title in("Event log was cleared",
// List alerts flagging attempts to delete backup files.
"File backups were deleted",
// Potential Cobalt Strike activity - Note that other threat activity can also
// trigger alerts for suspicious decoded content
"Suspicious decoded content",
// Cobalt Strike activity
"\'Atosev\' malware was detected",
"\'Ploty\' malware was detected",
"\'Bynoco\' malware was detected")
| extend AlertTime = Timestamp
| join AlertEvidence on AlertId
| where EntityType == "Machine"
| distinct DeviceName, AlertTime, AlertId, Title
| join DeviceLogonEvents on $left.DeviceName == $right.DeviceName
// Creating 10 day Window surrounding alert activity
| where Timestamp < AlertTime +5d and Timestamp > AlertTime - 5d
// Projecting specific columns
| project Title, DeviceName, DeviceId, Timestamp, LogonType, AccountDomain,
AccountName, AccountSid, AlertTime, AlertId, RemoteIP, RemoteDeviceName
// - Queries are limited to 10k results through the web UI, 100k results via API
// -----------------------------
// Topic 2: Ransomware tips / recommendations (3 examples)
// Ransomware is a very real challenge in today’s enterprise.
// -----------------------------
// Defender for Endpoint provides a bunch of different types of alerts for
// known ransomware and ransomware-like behavior
// To accomplish these tests, I explicitly excluded the folder where I placed the malware,
// disable automatic response by AutoIR, and ensured EDR in block mode was disabled.
AlertEvidence
| where DeviceId == 'eb610cc67fe0b99300a076324f5f4cd409324872'
| join kind=rightsemi AlertInfo on AlertId
| order by Timestamp asc
// Much of the ransomware that makes the news is what we refer to as human operated
// ransomware. This differs from other forms of ransomware in that it typically begins
// with an attacker compromising a vulnerability, performing credential theft until they
// attain enough authorization to deploy ransomware broadly, then doing so.
// …in other words, it is a targeted attack and should be treated as such. To track and
// eliminate this activity you will want to use the ABC method discussed in Tracking the
// Adversary episode 4. Its not about the malware, but rather the persistence mechanisms
// implemented by the attacker and any credentials they control.
// On the ransomware front, we can try to detect ransomware-like behaviors, for example:
// src: https://github.com/microsoft/Microsoft-365-Defender-Hunting-Queries/blob/master/Execution/Possible%20Ransomware%20Related%20Destruction%20Activity.md
DeviceProcessEvents
| where Timestamp > ago(7d)
| where (FileName =~ 'vssadmin.exe' and ProcessCommandLine has "delete shadows" and ProcessCommandLine has "/all" and ProcessCommandLine has "/quiet" ) // Clearing shadow copies
or (FileName =~ 'cipher.exe' and ProcessCommandLine contains "/w") // Wiping drive free space
or (FileName =~ 'schtasks.exe' and ProcessCommandLine has "/change" and ProcessCommandLine has @"\Microsoft\Windows\SystemRestore\SR" and ProcessCommandLine has "/disable") // Disabling system restore task
or (FileName =~ 'fsutil.exe' and ProcessCommandLine has "usn" and ProcessCommandLine has "deletejournal" and ProcessCommandLine has "/d") // Deleting USN journal
or (FileName =~ 'icacls.exe' and ProcessCommandLine has @'"C:\*"' and ProcessCommandLine contains '/grant Everyone:F') // Attempts to re-ACL all files on the C drive to give everyone full control
or (FileName =~ 'powershell.exe' and (
ProcessCommandLine matches regex @'\s+-((?i)encod?e?d?c?o?m?m?a?n?d?|e|en|enc|ec)\s+' and replace(@'\x00','', base64_decode_tostring(extract("[A-Za-z0-9+/]{50,}[=]{0,2}",0 , ProcessCommandLine))) matches regex @".*(Win32_Shadowcopy).*(.Delete\(\)).*"
) or ProcessCommandLine matches regex @".*(Win32_Shadowcopy).*(.Delete\(\)).*"
) // This query looks for PowerShell-based commands used to delete shadow copies
// -----------------------------
// Topic 3: Handling JSON and the dynamic type
// -----------------------------
// Many tables in Advanced Hunting are JSON strings. While you can parse these as strings,
// turning them into dynamic type columns is usually a lot more effective.
DeviceEvents
| summarize arg_max(Timestamp, *) by ActionType
| project-reorder ActionType, AdditionalFields
// Let's look at the AdditionalFields column of a PnpDeviceConnected event
DeviceEvents
| where ActionType == 'PnpDeviceConnected'
| take 10
| project-reorder AdditionalFields
// Imagine we wanted to audit which plug and play devices were used on the network. To
// accomplish this we need to turn this string into (a much more useful) dynamic.
// From there we can access individual properties of the object using dotted notation.
DeviceEvents
| where ActionType == 'PnpDeviceConnected'
| project ATDynamic = parse_json(AdditionalFields)
| extend ClassName = ATDynamic.ClassName, ClassId = ATDynamic.ClassId, DeviceId = ATDynamic.DeviceId, DeviceDescription = ATDynamic.DeviceDescription
| project-reorder ClassName, ClassId, DeviceDescription
| take 10
// ...but that takes forever to type, right? Enter bag_unpack()!
DeviceEvents
| where ActionType == 'PnpDeviceConnected'
| take 100
| project ATDynamic = parse_json(AdditionalFields)
| evaluate bag_unpack(ATDynamic)
// Lists can be indexed too. For example, let's take a look at signature versions reported
// by Threat and Vulnerability Management
DeviceTvmSecureConfigurationAssessment
| where ConfigurationId == 'scid-2011'
| take 100
| project-reorder Context
// Here we have a list within a list. The inner list has three values (for machines running
// Defender Antimalware):
// - Signature version
// - Engine version
// - Update date
// To get them out:
DeviceTvmSecureConfigurationAssessment
| where ConfigurationId == 'scid-2011'
| take 100
| extend x = todynamic(Context)
| project DeviceId, DeviceName, SignatureVersion = x[0][0], EngineVerision = x[0][1], UpdateDate = x[0][2]
// Now let's say you wanted to generate a report that has a count of the number of different
// plug and play devices added per machine, but also lists out all of the ClassId's from
// each of the plug and play devices.
// First, we need to change that project to an extend so that we can get the DeviceId back.
DeviceEvents
| where ActionType == 'PnpDeviceConnected'
| take 100
| extend ATDynamic = parse_json(AdditionalFields)
| evaluate bag_unpack(ATDynamic)
// Error! It looks like we have a field in the Additional Fields column called DeviceId already.
// To work around this we can use the second parameter of bag_unpack() to assign a prefix.
DeviceEvents
| where ActionType == 'PnpDeviceConnected'
| take 100
| extend ATDynamic = parse_json(AdditionalFields)
| evaluate bag_unpack(ATDynamic, 'AdditionalFields_')
// Perfect. Now we need to turn the ClassId into a string so that we can use it in summary
// operations, such as dcount(). After that, we will use a new operator, make_set(), to create
// a new deduplicated list of ClassIds associated with the device.
DeviceEvents
| where ActionType == 'PnpDeviceConnected'
| take 100
| extend ATDynamic = parse_json(AdditionalFields)
| evaluate bag_unpack(ATDynamic, 'AdditionalFields_')
| extend ClassId = tostring(AdditionalFields_ClassId)
| summarize PnPEvents = count(), DifferentPnPDevices = dcount(ClassId), makeset(ClassId), (LastPnPEvent, DeviceName) = arg_max(Timestamp, DeviceName) by DeviceId
// Now what if we want to make our own JSON objects with properties values? That's where the pack*() series of
// functions come in.
// pack
// https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/packfunction
// The pack function takes specific keys and values and packs them into a JSON object. Parameters are
// provided in pairs with the first pair being the key and the second being the value.
print x = pack('foo','bar','wibble','wobble')
print x = pack('foo','bar','wibble','wobble')
| evaluate bag_unpack(x)
// ..but let's say instead you wanted to package your query results as a JSON object.
// pack_all() is quite handy for this.
EmailEvents
| take 100
| extend packed = pack_all()
| project-reorder packed
// This is all well and good, but you might need to aggregate all of these individual rows
// into a single row, right? This is where our aggrigation functions come in.
// makelist() is like makeset(), but without deduplication. Let's use makelist() to create
// a list of e-mails in JSON format based on senders and their domain.
EmailEvents
| take 100
| project SenderFromDomain, SenderFromAddress, packed = pack_all()
| summarize makelist(packed) by SenderFromDomain, SenderFromAddress
// -----------------------------
// Topic 4: externaldata operator
// externaldata operator allows importing data from externally stored files and use it inside a query.
// externaldata operator - https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/externaldata-operator
// In our docs: https://docs.microsoft.com/en-us/microsoft-365/security/mtp/advanced-hunting-best-practices?view=o365-worldwide#ingest-data-from-external-sources
// -----------------------------
EmailAttachmentInfo
| where SHA256 in (externaldata(TimeGenerated:datetime, SHA256:string)
[@"https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Sample%20Data/Feeds/Microsoft.Covid19.Indicators.csv"]
with (format="csv"))
// See if any process created a file matching a hash on the list
let covidIndicators = (externaldata(TimeGenerated:datetime, FileHashValue:string, FileHashType: string )
[@"https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Sample%20Data/Feeds/Microsoft.Covid19.Indicators.csv"]
with (format="csv"))
| where FileHashType == 'sha256'; //and TimeGenerated > ago(1d);
covidIndicators
| join (DeviceFileEvents
| where ActionType == 'FileCreated'
| take 100) on $left.FileHashValue == $right.SHA256