-
Notifications
You must be signed in to change notification settings - Fork 4.9k
/
pipeline.yml
395 lines (378 loc) · 12.7 KB
/
pipeline.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
---
description: Pipeline for parsing MISP Threat Intel
processors:
####################
# Event ECS fields #
####################
- set:
field: event.ingested
value: "{{_ingest.timestamp}}"
- set:
field: event.kind
value: enrichment
- set:
field: event.category
value: threat
- set:
field: event.type
value: indicator
######################
# General ECS fields #
######################
- rename:
field: message
target_field: event.original
ignore_missing: true
- json:
field: event.original
target_field: json
- fingerprint:
fields:
- json.Event.Attribute.uuid
- json.Event.Object.Attribute.uuid
target_field: "_id"
ignore_missing: true
- rename:
field: json.Event
target_field: misp
ignore_missing: true
- set:
field: threat.indicator.provider
value: misp
if: ctx.misp?.Orgc?.local != 'false'
- set:
field: threat.indicator.provider
value: "{{misp.Orgc.name}}"
if: ctx.misp?.Orgc?.local == 'false'
ignore_empty_value: true
# Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event
- remove:
field:
- misp.ShadowAttribute
- misp.RelatedEvent
- misp.Galaxy
- misp.Attribute.Galaxy
- misp.Attribute.ShadowAttribute
- misp.EventReport
- misp.Object.Attribute.Galaxy
- misp.Object.Attribute.ShadowAttribute
ignore_missing: true
- remove:
field:
- misp.Attribute
ignore_missing: true
if: ctx.misp?.Attribute.size() == 0
- remove:
field:
- misp.Object
ignore_missing: true
if: ctx.misp?.Object.size() == 0
- date:
field: misp.timestamp
formats:
- UNIX
ignore_failure: true
- rename:
field: misp.Attribute
target_field: misp.attribute
ignore_missing: true
- rename:
field: misp.Object
target_field: misp.object
ignore_missing: true
- rename:
field: misp.object.Attribute
target_field: misp.object.attribute
ignore_missing: true
- rename:
field: misp.Orgc
target_field: misp.orgc
ignore_missing: true
- rename:
field: misp.Org
target_field: misp.org
ignore_missing: true
- rename:
field: misp.Tag
target_field: misp.tag
ignore_missing: true
# # Dance around issue of not being able to split the document into two.
# # Make the Object.Attribute field primary if it exists, but keep the
# # outer Attribute as context.
- rename:
field: misp.attribute
target_field: misp.context.attribute
ignore_missing: true
if: ctx.misp?.object != null
- rename:
field: misp.object.attribute
target_field: misp.attribute
ignore_missing: true
if: ctx.misp?.object != null
#####################
# Threat ECS Fields #
#####################
- set:
field: threat.feed.name
value: "[Filebeat] MISP"
- set:
field: threat.feed.dashboard_id
value: "ad9c7430-72de-11eb-a3e3-b3cc7c78a70f"
- rename:
field: misp.attribute.first_seen
target_field: threat.indicator.first_seen
ignore_missing: true
- rename:
field: misp.attribute.last_seen
target_field: threat.indicator.last_seen
ignore_missing: true
- convert:
field: misp.analysis
type: long
target_field: threat.indicator.scanner_stats
ignore_missing: true
- convert:
field: misp.threat_level_id
type: long
ignore_missing: true
## File/Hash indicator operations
- set:
field: threat.indicator.type
value: file
if: "ctx.misp?.attribute?.type != null && (['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type.startsWith('filename'))"
- rename:
field: misp.attribute.value
target_field: "threat.indicator.file.hash.{{misp.attribute.type}}"
ignore_missing: true
if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp?.attribute?.type.startsWith('filename')"
- rename:
field: misp.attribute.value
target_field: threat.indicator.file.name
ignore_missing: true
if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type == 'filename'"
- grok:
field: misp.attribute.type
patterns:
- "%{WORD}\\|%{WORD:_tmp.hashtype}"
ignore_missing: true
if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
- grok:
field: misp.attribute.value
patterns:
- "%{DATA:threat.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}"
ignore_missing: true
if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
- set:
field: threat.indicator.file.hash.{{_tmp.hashtype}}
value: "{{_tmp.hashvalue}}"
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"
## URL/URI indicator operations
- set:
field: threat.indicator.type
value: url
if: "ctx.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)"
- uri_parts:
field: misp.attribute.value
target_field: threat.indicator.url
keep_original: true
remove_if_successful: true
if: ctx.threat?.indicator?.type == 'url' && ctx.misp?.attribute?.type != 'uri'
- set:
field: threat.indicator.url.full
value: "{{{threat.indicator.url.original}}}"
ignore_empty_value: true
if: "ctx.threat?.indicator?.type == 'url' && ctx.misp?.attribute?.type != 'uri'"
## Regkey indicator operations
- set:
field: threat.indicator.type
value: windows-registry-key
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('regkey')"
- rename:
field: misp.attribute.value
target_field: threat.indicator.registry.key
ignore_missing: true
if: "ctx.threat?.indicator?.type == 'windows-registry-key' && ctx.misp?.attribute?.type == 'regkey'"
- grok:
field: misp.attribute.value
patterns:
- "%{DATA:threat.indicator.registry.key}\\|%{DATA:threat.indicator.registry.value}"
ignore_missing: true
if: "ctx.misp?.attribute?.type == 'regkey|value'"
## AS indicator operations
- set:
field: threat.indicator.type
value: autonomous-system
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type == 'AS'"
- convert:
field: misp.attribute.value
type: long
target_field: threat.indicator.as.number
ignore_missing: true
if: ctx.threat?.indicator?.type == 'autonomous-system'
## Domain/IP/Port indicator operations
- set:
field: threat.indicator.type
value: domain-name
if: "ctx.misp?.attribute?.type != null && (ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type.startsWith('domain'))"
- set:
field: threat.indicator.type
value: ipv4-addr
if: "ctx.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
- rename:
field: misp.attribute.value
target_field: threat.indicator.url.domain
ignore_missing: true
if: "ctx.threat?.indicator?.type == 'domain-name' && ctx.misp?.attribute?.type != 'domain|ip' && ctx.threat?.indicator?.url?.domain == null"
- rename:
field: misp.attribute.value
target_field: threat.indicator.ip
ignore_missing: true
if: "ctx.threat?.indicator?.type == 'ipv4-addr' && ctx.misp?.attribute?.type != 'domain|ip' && !['ip-src|port', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
- grok:
field: misp.attribute.value
patterns:
- "%{DATA:threat.indicator.url.domain}\\|%{IP:threat.indicator.ip}"
ignore_missing: true
if: ctx.misp?.attribute?.type == 'domain|ip' && ctx.threat?.indicator?.url?.domain == null
- grok:
field: misp.attribute.value
patterns:
- "%{IP:threat.indicator.ip}\\|%{NUMBER:threat.indicator.port}"
ignore_missing: true
if: "['ip-src|port', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
# MISP may send a CIDR such as 1.2.3.0/22, which is not a valid Elasticsearch IP data type
ignore_failure: true
## Email indicator operations
# Currently this ignores email-message, except setting the type it will leave the rest of the fields under misp.
- set:
field: threat.indicator.type
value: email-addr
if: "ctx.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
- set:
field: threat.indicator.type
value: email-message
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
- rename:
field: misp.attribute.value
target_field: threat.indicator.email.address
ignore_missing: true
if: ctx.threat?.indicator?.type == 'email-addr'
- rename:
field: misp.event_creator_email
target_field: user.email
ignore_missing: true
- append:
field: user.roles
value: "reporting_user"
if: ctx?.user?.email != null
## MAC Address indicator operations
- set:
field: threat.indicator.type
value: mac-addr
if: "ctx.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)"
- rename:
field: misp.attribute.value
target_field: threat.indicator.mac
ignore_missing: true
if: ctx.threat?.indicator?.type == 'mac-addr'
###################
# Tags ECS fields #
###################
# Stripping special characters from tags
- script:
lang: painless
if: ctx.misp?.tag != null
source: |
def tags = ctx.misp.tag.stream()
.map(t -> t.name.replace('\\', '').replace('"', ''))
.collect(Collectors.toList());
def tlpTags = tags.stream()
.filter(t -> t.startsWith('tlp:'))
.map(t -> t.replace('tlp:', ''))
.collect(Collectors.toList());
ctx.tags = tags;
ctx.threat.indicator.marking = [ 'tlp': tlpTags ];
# Setting indicator type to unknown if it does not match anything
- set:
field: threat.indicator.type
value: unknown
if: ctx.threat?.indicator?.type == null
#################
# Convert types #
#################
- convert:
field: misp.attribute.distribution
type: long
ignore_missing: true
- convert:
field: misp.context.attribute.distribution
type: long
ignore_missing: true
- convert:
field: threat.indicator.port
type: long
ignore_missing: true
- convert:
field: misp.attribute_count
type: long
ignore_missing: true
######################
# Cleanup processors #
######################
- remove:
field: event.original
if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
ignore_failure: true
ignore_missing: true
- script:
lang: painless
if: ctx?.misp != null
source: |
void handleMap(Map map) {
for (def x : map.values()) {
if (x instanceof Map) {
handleMap(x);
} else if (x instanceof List) {
handleList(x);
}
}
map.values().removeIf(v -> v == null);
}
void handleList(List list) {
for (def x : list) {
if (x instanceof Map) {
handleMap(x);
} else if (x instanceof List) {
handleList(x);
}
}
}
handleMap(ctx);
# Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event
- remove:
field:
- misp.attribute.value
ignore_missing: true
if: ctx.threat?.indicator?.type != 'unknown'
- remove:
field:
# This removes a number of fields that may be wanted in the future when
# misp.attribute and misp.object.attribute can
# be separated. At the root of .object are fields that mirror fields at
# the root of misp.
- misp.object
ignore_missing: true
- remove:
field:
- misp.Attribute.timestamp
- misp.timestamp
- misp.tag
- misp.org
- misp.analysis
- _tmp
- json
ignore_missing: true
on_failure:
- set:
field: error.message
value: "{{ _ingest.on_failure_message }}"