Skip to content

Commit

Permalink
Reaqta: implemented grater/less fields translation, fixed from_stix f…
Browse files Browse the repository at this point in the history 
…ields sorting, fixed unittests (#938)
  • Loading branch information
Arthur Muradyan committed May 24, 2022
1 parent 2b41434 commit 5872672
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 36 deletions.
42 changes: 21 additions & 21 deletions stix_shifter_modules/reaqta/stix_translation/json/from_stix_map.json
View file
@@ -1,7 +1,7 @@
{
"directory": {
"fields": {
"path": ["path", "accessor.path", "consumer.workingDirectory", "__etwHomePath"]
"path": ["__etwHomePath", "accessor.path", "consumer.workingDirectory", "path"]
}
},
"file": {
Expand All @@ -14,14 +14,14 @@
"hashes.'MD5'": ["md5"],
"hashes.'SHA-1'": ["sha1"],
"hashes.'SHA-256'": ["sha256"],
"name": ["filename", "consumer.script.filename"],
"name": ["consumer.script.filename", "filename"],
"parent_directory_ref.path": ["path"],
"size": ["eventdata.size.gte", "eventdata.size.lte"]
"size": ["eventdata.size"]
}
},
"ipv4-addr": {
"fields": {
"value": ["login.ip", "ip"]
"value": ["ip", "login.ip"]
}
},
"ipv6-addr": {
Expand All @@ -46,9 +46,9 @@
"extensions.'x-reaqta-process'.logon_id": ["service.login.id"],
"extensions.'x-reaqta-process'.no_gui": ["service.hasGui"],
"extensions.'x-reaqta-process'.privilege_level": ["service.privilege"],
"name": ["login.processName", "__etwCallerProcessName", "__etwProcessName", "__etwLogonProcessName"],
"name": ["__etwCallerProcessName", "__etwLogonProcessName", "__etwProcessName", "login.processName"],
"parent_ref.binary_ref.name": ["service.ppid"],
"pid": ["wmi.clientPid", "__etwProcessId", "__etwCallerProcessId", "engine.ppid", "host.pid", "accessor.pid", "eventdata.targetProcessId", "pid", "service.pid", "service.ppid", "wmiHost.pid", "engine.pid", "allocator.ppid", "ppid", "allocator.pid", "accessor.ppid"]
"pid": ["__etwCallerProcessId", "__etwProcessId", "accessor.pid", "accessor.ppid", "allocator.pid", "allocator.ppid", "engine.pid", "engine.ppid", "eventdata.targetProcessId", "host.pid", "pid", "ppid", "service.pid", "service.ppid", "wmi.clientPid", "wmiHost.pid"]
}
},
"url": {
Expand All @@ -58,12 +58,12 @@
},
"user-account": {
"fields": {
"user_id": ["login.src.username", "__etwNewTargetUserName", "__etwTargetOutboundUserName", "wmi.user", "accessor.user", "engine.user", "service.user", "__etwOldTargetUserName", "allocator.user", "login.dst.username", "user"]
"user_id": ["__etwNewTargetUserName", "__etwOldTargetUserName", "__etwTargetOutboundUserName", "accessor.user", "allocator.user", "engine.user", "login.dst.username", "login.src.username", "service.user", "user", "wmi.user"]
}
},
"windows-process-ext": {
"fields": {
"owner_sid": ["engine.user.sid", "accessor.user.sid", "login.src.sid", "service.user.sid", "allocator.user.sid", "user.sid"]
"owner_sid": ["accessor.user.sid", "allocator.user.sid", "engine.user.sid", "login.src.sid", "service.user.sid", "user.sid"]
}
},
"x-ibm-finding": {
Expand All @@ -89,7 +89,7 @@
"extensions.'x-reaqta-consumer'.showWindowCommand": ["consumer.showWindowCmd"],
"extensions.'x-wmi-event'.client_machine_fqn": ["wmi.clientMachineFqn"],
"host_id": ["__etwWorkstation"],
"hostname": ["wmi.clientMachine", "wmi.machineName", "__etwWorkstationName"],
"hostname": ["__etwWorkstationName", "wmi.clientMachine", "wmi.machineName"],
"ip_refs[*].value": ["ip"]
}
},
Expand Down Expand Up @@ -126,9 +126,9 @@
},
"x-reaqta-cert": {
"fields": {
"expired": ["allocator.expired", "service.expired", "eventdata.cert.expired", "expired", "engine.expired", "accessor.expired"],
"signer": ["engine.signer", "allocator.signer", "service.signer", "signer", "accessor.signer", "eventdata.cert.signer"],
"trusted": ["eventdata.cert.trusted", "service.trusted", "allocator.trusted", "engine.trusted", "accessor.trusted", "trusted"]
"expired": ["accessor.expired", "allocator.expired", "engine.expired", "eventdata.cert.expired", "expired", "service.expired"],
"signer": ["accessor.signer", "allocator.signer", "engine.signer", "eventdata.cert.signer", "service.signer", "signer"],
"trusted": ["accessor.trusted", "allocator.trusted", "engine.trusted", "eventdata.cert.trusted", "service.trusted", "trusted"]
}
},
"x-reaqta-consumer": {
Expand Down Expand Up @@ -167,7 +167,7 @@
"etwTargetLinkedLogonId": ["__etwTargetLinkedLogonId"],
"etwTargetLogonGuid": ["__etwTargetLogonGuid"],
"etwTargetLogonId": ["login.targetLogonId"],
"etwTargetOutboundDomainName": ["__etwTargetServerName", "__etwTargetOutboundDomainName"],
"etwTargetOutboundDomainName": ["__etwTargetOutboundDomainName", "__etwTargetServerName"],
"etwTargetSid": ["__etwTargetSid"],
"etwTargetUserSid": ["login.dst.sid"],
"etwTask": ["eventdata.etwTask"],
Expand All @@ -185,7 +185,7 @@
"etw_display_name": ["__etwDisplayName"],
"etw_dummy": ["__etwDummy"],
"etw_elevated_token": ["__etwElevatedToken"],
"etw_event_record_id": ["eventdata.etwEventVersion", "__etwEventRecordId"],
"etw_event_record_id": ["__etwEventRecordId", "eventdata.etwEventVersion"],
"etw_failure_reason": ["__etwFailureReason"],
"etw_home_directory": ["__etwHomeDirectory"],
"etw_impersonation_level": ["__etwImpersonationLevel"],
Expand Down Expand Up @@ -225,8 +225,8 @@
"query": ["wmi.query"],
"queryLanguage": ["wmi.queryLanguage"],
"queryName": ["eventdata.dns"],
"region_size": ["eventdata.regionSize.gte", "eventdata.regionSize.lte"],
"relevance": ["eventdata.relevance.lte", "eventdata.relevance.gte", "eventdata.relevance"],
"region_size": ["eventdata.regionSize"],
"relevance": ["eventdata.relevance"],
"return_code": ["eventdata.returnCode"],
"root_object": ["path"],
"service_name": ["service.name"],
Expand All @@ -246,14 +246,14 @@
},
"x-reaqta-process": {
"fields": {
"logon_id": ["accessor.login.id", "engine.login.id", "allocator.login.id", "service.login.id", "login.id"],
"no_gui": ["hasGui", "service.hasGui", "engine.hasGui", "allocator.hasGui", "accessor.hasGui"],
"privilege_level": ["privilege", "allocator.privilege", "service.privilege", "accessor.privilege", "engine.privilege"]
"logon_id": ["accessor.login.id", "allocator.login.id", "engine.login.id", "login.id", "service.login.id"],
"no_gui": ["accessor.hasGui", "allocator.hasGui", "engine.hasGui", "hasGui", "service.hasGui"],
"privilege_level": ["accessor.privilege", "allocator.privilege", "engine.privilege", "privilege", "service.privilege"]
}
},
"x-reaqta-program": {
"fields": {
"arch": ["accessor.arch", "allocator.arch", "service.arch", "engine.arch", "arch"],
"arch": ["accessor.arch", "allocator.arch", "arch", "engine.arch", "service.arch"],
"fsname": ["filename"]
}
},
Expand All @@ -268,7 +268,7 @@
"extensions.'x-reaqta-cert'.signer": ["signer"],
"extensions.'x-reaqta-cert'.trusted": ["trusted"],
"extensions.'x-reaqta-etw'.etw_cert_thumbprint": ["__etwCertThumbprint"],
"issuer": ["accessor.issuer", "eventdata.cert.issuer", "allocator.issuer", "__etwCertIssuerName", "issuer", "engine.issuer", "service.issuer"],
"issuer": ["__etwCertIssuerName", "accessor.issuer", "allocator.issuer", "engine.issuer", "eventdata.cert.issuer", "issuer", "service.issuer"],
"serial_number": ["__etwCertSerialNumber"]
}
}
Expand Down
13 changes: 12 additions & 1 deletion stix_shifter_modules/reaqta/stix_translation/query_constructor.py
View file
Expand Up @@ -57,6 +57,15 @@ def _format_universal_field(field) -> str:
return '${}'.format(field)
return field

@staticmethod
def _format_range_field(field, comparator) -> str:
if field in GREATER_LESS_FIELDS:
if comparator == ComparisonComparators.LessThanOrEqual:
return '{}.lte'.format(field)
else:
return '{}.gte'.format(field)
return field

@staticmethod
def _check_value_type(value):
value = str(value)
Expand Down Expand Up @@ -86,7 +95,7 @@ def _format_qualifier(self, qualifier, time_range) -> str:

return formated_qualifier

def _parse_mapped_fields(self, value, comparator, mapped_fields_array) -> str:
def _parse_mapped_fields(self, value, comparator, expression_comparator, mapped_fields_array) -> str:
"""Convert a list of mapped fields into a query string."""
comparison_strings = []
str_ = None
Expand All @@ -97,6 +106,7 @@ def _parse_mapped_fields(self, value, comparator, mapped_fields_array) -> str:
for val in value:
for mapped_field in mapped_fields_array:
mapped_field = self._format_universal_field(mapped_field)
mapped_field = self._format_range_field(mapped_field, expression_comparator)
comparison_strings.append(f'{mapped_field} {comparator} {val}')

# Only wrap in () if there's more than one comparison string
Expand Down Expand Up @@ -135,6 +145,7 @@ def _parse_expression(self, expression, qualifier=None) -> str:
comparison_string = self._parse_mapped_fields(
value=value,
comparator=comparator,
expression_comparator=expression.comparator,
mapped_fields_array=mapped_fields_array
)

Expand Down
37 changes: 23 additions & 14 deletions stix_shifter_modules/reaqta/test/stix_translation/test_stix_to_hunt_query.py
View file
Expand Up @@ -46,7 +46,7 @@ def test_timeinterval(self):
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries']

test_string = ['((login.ip = "172.16.60.184" OR $ip = "172.16.60.184")) {}'.format(TEST_START_STOP_TRANSLATED1)]
test_string = ['(($ip = "172.16.60.184" OR login.ip = "172.16.60.184")) {}'.format(TEST_START_STOP_TRANSLATED1)]

self.assertQuery(query, test_string, stix_pattern)

Expand All @@ -58,7 +58,7 @@ def test_no_timeinterval(self):
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries'][0]

assert '((login.ip = "192.168.1.2" OR $ip = "192.168.1.2")) AND happenedAfter = "' in query
assert '(($ip = "192.168.1.2" OR login.ip = "192.168.1.2")) AND happenedAfter = "' in query

found = re.findall(TEST_DATE_PATTERN, query)
assert len(found) == 2
Expand All @@ -68,7 +68,7 @@ def test_one_observation_expression_with_timeinterval(self):
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries']

test_string = ['(eventdata.url = "www.example.com" OR (login.ip = "192.168.1.2" OR $ip = "192.168.1.2")) {}'.format(TEST_START_STOP_TRANSLATED1, TEST_START_STOP_TRANSLATED1)]
test_string = ['(eventdata.url = "www.example.com" OR ($ip = "192.168.1.2" OR login.ip = "192.168.1.2")) {}'.format(TEST_START_STOP_TRANSLATED1, TEST_START_STOP_TRANSLATED1)]

self.assertQuery(query, test_string, stix_pattern)

Expand All @@ -77,7 +77,7 @@ def test_two_observation_expressions_with_two_timeintervals(self):
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries']

test_string = ['((login.ip = "192.168.1.2" OR $ip = "192.168.1.2")) {} OR (eventdata.url = "www.example.com") {}'.format(TEST_START_STOP_TRANSLATED1, TEST_START_STOP_TRANSLATED2)]
test_string = ['(($ip = "192.168.1.2" OR login.ip = "192.168.1.2")) {} OR (eventdata.url = "www.example.com") {}'.format(TEST_START_STOP_TRANSLATED1, TEST_START_STOP_TRANSLATED2)]

self.assertQuery(query, test_string, stix_pattern)

Expand All @@ -93,7 +93,7 @@ def test_two_observation_expressions_with_one_timeinterval(self):
test_string = ['(($ip = "192.168.1.2") {}) OR ((eventdata.url = "www.example.com") {})'.format(TEST_START_STOP_TRANSLATED1, TEST_START_STOP_TRANSLATED1)]
self.assertNotEqual(query, test_string)

assert '((login.ip = "192.168.1.2" OR $ip = "192.168.1.2")) AND happenedAfter = "' in query[0]
assert '(($ip = "192.168.1.2" OR login.ip = "192.168.1.2")) AND happenedAfter = "' in query[0]
assert 'OR (eventdata.url = "www.example.com") AND happenedAfter = "' in query[0]

found = re.findall(TEST_DATE_PATTERN, query[0])
Expand All @@ -105,7 +105,7 @@ def test_combined_observation_expressions_with_timeintervals(self):
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries']

test_string = ['((login.ip = "192.168.1.2" OR $ip = "192.168.1.2")) {} OR (eventdata.url = "www.example.com") {}'.format(TEST_START_STOP_TRANSLATED1, TEST_START_STOP_TRANSLATED2)]
test_string = ['(($ip = "192.168.1.2" OR login.ip = "192.168.1.2")) {} OR (eventdata.url = "www.example.com") {}'.format(TEST_START_STOP_TRANSLATED1, TEST_START_STOP_TRANSLATED2)]

self.assertQuery(query, test_string, stix_pattern)

Expand All @@ -116,7 +116,7 @@ def test_not_operator(self):
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries']

test_string = ['($ip != "172.31.60.104" OR (NOT login.ip = "172.31.60.104" OR NOT $ip = "172.31.60.104"))' \
test_string = ['($ip != "172.31.60.104" OR (NOT $ip = "172.31.60.104" OR NOT login.ip = "172.31.60.104"))' \
' AND happenedAfter = "2022-03-24T20:21:35.519Z" AND happenedBefore = "2022-03-24T20:21:35.619Z"']

self.assertQuery(query, test_string, stix_pattern)
Expand All @@ -127,7 +127,7 @@ def test_in_operator(self):
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries']

test_string = ['(eventdata.localPort = "443" OR eventdata.localPort = "446") {} OR ((login.ip = "127.0.0.1" OR $ip = "127.0.0.1" OR login.ip = "127.0.0.2" OR $ip = "127.0.0.2")) {}'.format(TEST_START_STOP_TRANSLATED1, TEST_START_STOP_TRANSLATED2)]
test_string = ['(eventdata.localPort = "443" OR eventdata.localPort = "446") {} OR (($ip = "127.0.0.1" OR login.ip = "127.0.0.1" OR $ip = "127.0.0.2" OR login.ip = "127.0.0.2")) {}'.format(TEST_START_STOP_TRANSLATED1, TEST_START_STOP_TRANSLATED2)]

self.assertQuery(query, test_string, stix_pattern)

Expand All @@ -137,7 +137,7 @@ def test_match_operator(self):
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries']

test_string = ['(($filename = "serv" OR consumer.script.filename = "serv")) {}'.format(TEST_START_STOP_TRANSLATED1)]
test_string = ['((consumer.script.filename = "serv" OR $filename = "serv")) {}'.format(TEST_START_STOP_TRANSLATED1)]

self.assertQuery(query, test_string, stix_pattern)

Expand All @@ -147,7 +147,7 @@ def test_like_operator(self):
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries']

test_string = ['(($filename = "svc" OR consumer.script.filename = "svc")) {}'.format(TEST_START_STOP_TRANSLATED1)]
test_string = ['((consumer.script.filename = "svc" OR $filename = "svc")) {}'.format(TEST_START_STOP_TRANSLATED1)]

self.assertQuery(query, test_string, stix_pattern)

Expand All @@ -160,7 +160,7 @@ def test_combined(self):
query = queries['queries']

test_string = ['($sha256 = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" OR $md5 != "23db6982caef9e9152f1a5b2589e6ca3" AND $ip = "127.0.0.1") {} ' \
'AND ((login.ip = "12.0.0.2" OR $ip = "12.0.0.2") OR (login.ip = "12.0.0.1" OR $ip = "12.0.0.1") OR (login.ip = "10.0.0.1" OR $ip = "10.0.0.1")) {} ' \
'AND (($ip = "12.0.0.2" OR login.ip = "12.0.0.2") OR ($ip = "12.0.0.1" OR login.ip = "12.0.0.1") OR ($ip = "10.0.0.1" OR login.ip = "10.0.0.1")) {} ' \
'AND (eventdata.url = "http://ccc.ddd" OR eventdata.url = "http://aaa.bbb") {}'
.format(TEST_START_STOP_TRANSLATED1, TEST_START_STOP_TRANSLATED1, TEST_START_STOP_TRANSLATED1)]

Expand All @@ -175,7 +175,7 @@ def test_ipv4_addr(self):
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries']

test_string = ['((login.ip = "192.168.122.84" OR $ip = "192.168.122.84") OR (login.ip = "192.168.122.83" OR $ip = "192.168.122.83")) {}'.format(TEST_START_STOP_TRANSLATED1)]
test_string = ['(($ip = "192.168.122.84" OR login.ip = "192.168.122.84") OR ($ip = "192.168.122.83" OR login.ip = "192.168.122.83")) {}'.format(TEST_START_STOP_TRANSLATED1)]

self.assertQuery(query, test_string, stix_pattern)

Expand All @@ -202,7 +202,7 @@ def test_file_name(self):
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries']

test_string = ['(($filename = "winword.exe" OR consumer.script.filename = "winword.exe")) {}'.format(TEST_START_STOP_TRANSLATED1)]
test_string = ['((consumer.script.filename = "winword.exe" OR $filename = "winword.exe")) {}'.format(TEST_START_STOP_TRANSLATED1)]

self.assertQuery(query, test_string, stix_pattern)

Expand Down Expand Up @@ -240,7 +240,7 @@ def test_directory_file_path(self):

path = "c:\\program files\\microsoft office\\root\\office16\\winword.exe"

test_string = ['(($path = "{}" OR accessor.path = "{}" OR consumer.workingDirectory = "{}" OR __etwHomePath = "{}") OR $path = "{}") {}'.format(path, path, path, path, path, TEST_START_STOP_TRANSLATED1)]
test_string = ['((__etwHomePath = "{}" OR accessor.path = "{}" OR consumer.workingDirectory = "{}" OR $path = "{}") OR $path = "{}") {}'.format(path, path, path, path, path, TEST_START_STOP_TRANSLATED1)]

self.assertQuery(query, test_string, stix_pattern)

Expand All @@ -262,6 +262,15 @@ def test_network_traffic_ip_port(self):

self.assertQuery(query, test_string, stix_pattern)

def test_file_size_greater_less_or_equal_than(self):
stix_pattern = "[file:size <= '10' AND file:size >= '50'] {}".format(TEST_START_STOP_STIX_VALUE1)
queries = translation.translate('reaqta', 'query', '{}', stix_pattern)
query = queries['queries']

test_string = ['(eventdata.size.gte = 50 AND eventdata.size.lte = 10) {}'.format(TEST_START_STOP_TRANSLATED1)]

self.assertQuery(query, test_string, stix_pattern)

###############################
## custom stix objects check ##
###############################
Expand Down

0 comments on commit 5872672

Please sign in to comment.