diff --git a/siem-converter/app/converter/backends/graylog/mappings/graylog_cti.py b/siem-converter/app/converter/backends/graylog/mappings/graylog_cti.py index f2f32690..eb66315b 100644 --- a/siem-converter/app/converter/backends/graylog/mappings/graylog_cti.py +++ b/siem-converter/app/converter/backends/graylog/mappings/graylog_cti.py @@ -1,12 +1,12 @@ DEFAULT_GRAYLOG_MAPPING = { - "SourceIP": "sourceAddress", - "DestinationIP": "destinationAddress", - "Domain": "destinationDnsDomain", - "URL": "requestUrl", - "HashMd5": "fileHash", - "HashSha1": "fileHash", - "HashSha256": "fileHash", - "HashSha512": "fileHash", + "SourceIP": "source.ip", + "DestinationIP": "destination.ip", + "Domain": "destination.domain", + "URL": "url.original", + "HashMd5": "file.hash.md5", + "HashSha1": "file.hash.sha1", + "HashSha256": "file.hash.sha256", + "HashSha512": "file.hash.sha512", "Emails": "emails", "Files": "filePath" } diff --git a/siem-converter/app/converter/backends/microsoft/mappings/microsoft_sentinel_cti.py b/siem-converter/app/converter/backends/microsoft/mappings/microsoft_sentinel_cti.py index 8f9ac3ea..f2cf3a9d 100644 --- a/siem-converter/app/converter/backends/microsoft/mappings/microsoft_sentinel_cti.py +++ b/siem-converter/app/converter/backends/microsoft/mappings/microsoft_sentinel_cti.py @@ -1,5 +1,5 @@ DEFAULT_MICROSOFT_SENTINEL_MAPPING = { - "DestinationIP": "DestinationIP", + "DestinationIP": "DestinationIp", "SourceIP": "SourceIp", "HashSha512": "FileHashSha512", "HashSha256": "FileHashSha256", @@ -7,6 +7,6 @@ "Emails": "SenderFromAddress", "Domain": "DestinationHostname", "HashSha1": "FileHashSha1", - "Files": "FileName", + "Files": "TargetFileName", "URL": "URL" } diff --git a/siem-converter/app/converter/core/parser_cti.py b/siem-converter/app/converter/core/parser_cti.py index ff173df1..4d33fe31 100644 --- a/siem-converter/app/converter/core/parser_cti.py +++ b/siem-converter/app/converter/core/parser_cti.py @@ -5,23 +5,29 @@ from pydantic import BaseModel from app.converter.core.exceptions.iocs import IocsLimitExceededException, EmptyIOCSException -from app.converter.tools.const import IP_IOC_REGEXP_PATTERN, DOMAIN_IOC_REGEXP_PATTERN, URL_IOC_REGEXP_PATTERN,\ - hash_regexes, IOCType, HashType, IocParsingRule +from app.converter.tools.const import IP_IOC_REGEXP_PATTERN, DOMAIN_IOC_REGEXP_PATTERN, URL_IOC_REGEXP_PATTERN, \ + hash_regexes, IOCType, HashType, IocParsingRule, HASH_MAP class Iocs(BaseModel): ip: list[str] = [] url: list[str] = [] domain: list[str] = [] - hash: list[str] = [] + hash_dict: dict = {} def get_total_count(self) -> int: - return len(self.ip) + len(self.url) + len(self.domain) + len(self.hash) + hash_len = 0 + for value in self.hash_dict.values(): + hash_len += len(value) + return len(self.ip) + len(self.url) + len(self.domain) + hash_len def return_iocs(self) -> dict: - if all(not value for value in [self.ip, self.url, self.domain, self.hash]): + if all(not value for value in [self.ip, self.url, self.domain, self.hash_dict]): raise EmptyIOCSException() - return {"ip": self.ip, "url": self.url, "domain": self.domain, "hash": self.hash} + result = {"DestinationIP": self.ip, "URL": self.url, "Domain": self.domain} + for key, value in self.hash_dict.items(): + result[HASH_MAP[key]] = value + return result class CTIParser: @@ -47,7 +53,7 @@ def get_iocs_from_string( if not include_hash_types: include_hash_types = list(hash_regexes.keys()) for hash_type in include_hash_types: - iocs.hash.extend(self._find_all_str_by_regex(string, hash_regexes[hash_type])) + iocs.hash_dict[hash_type] = self._find_all_str_by_regex(string, hash_regexes[hash_type]) iocs = self.remove_duplicates(iocs) iocs = self.remove_exceptions(iocs, exceptions) if ioc_parsing_rules is None or "remove_private_and_reserved_ips" in ioc_parsing_rules: @@ -69,14 +75,16 @@ def remove_duplicates(self, iocs): iocs.ip = self._remove_duplicates_from_list(iocs.ip) iocs.domain = self._remove_duplicates_from_list(iocs.domain) iocs.url = self._remove_duplicates_from_list(iocs.url) - iocs.hash = self._remove_duplicates_from_list(iocs.hash) + for key, value in iocs.hash_dict.items(): + iocs.hash_dict[key] = self._remove_duplicates_from_list(value) return iocs def remove_exceptions(self, iocs, exceptions=None): iocs.ip = self._remove_exceptions(iocs.ip, exceptions) iocs.domain = self._remove_exceptions(iocs.domain, exceptions) iocs.url = self._remove_exceptions(iocs.url, exceptions) - iocs.hash = self._remove_exceptions(iocs.hash, exceptions) + for key, value in iocs.hash_dict.items(): + iocs.hash_dict[key] = self._remove_exceptions(value, exceptions) return iocs @staticmethod diff --git a/siem-converter/app/converter/core/render_cti.py b/siem-converter/app/converter/core/render_cti.py index daf18a51..bf8d2c68 100644 --- a/siem-converter/app/converter/core/render_cti.py +++ b/siem-converter/app/converter/core/render_cti.py @@ -34,9 +34,6 @@ class RenderCTI: final_result_for_one: str = "union * | where {result}\n" default_mapping = None - def get_default_mapping(self, include_source_ip=False): - return self.prepare_mapping(self.default_mapping, include_source_ip) - def create_field_value(self, field: str, value: str, generic_field: str): return self.data_map.format(key=field, value=value) @@ -53,35 +50,17 @@ def render(self, data: List[List[IocsChunkValue]]) -> list[str]: def collect_data_values(self, chunk): data_values = [] key_chunk = [] - processing_key = chunk[0].generic_field + processing_key = chunk[0].platform_field for value in chunk: - if processing_key != value.generic_field: + if processing_key != value.platform_field: data_values.append(self.or_group.format(or_group=self.or_operator.join(key_chunk), processing_key=processing_key)) key_chunk = [] - processing_key = value.generic_field - key_chunk.append(self.create_field_value(field=value.generic_field, + processing_key = value.platform_field + key_chunk.append(self.create_field_value(field=value.platform_field, value=value.value, generic_field=value.generic_field)) if key_chunk: data_values.append( self.or_group.format(or_group=self.or_operator.join(key_chunk), processing_key=processing_key)) return data_values - - def prepare_mapping(self, mapping: dict, include_source_ip: bool = False) -> dict: - m = { - "DestinationIP": "ip", - "Domain": "domain", - "URL": "url", - "HashMd5": "hash", - "HashSha1": "hash", - "HashSha256": "hash", - "HashSha512": "hash", - } - if include_source_ip: - m["SourceIP"] = "ip" - res = {} - for key, new_field_name in mapping.items(): - if key in m: - res[new_field_name] = m[key] - return res diff --git a/siem-converter/app/converter/cti_converter.py b/siem-converter/app/converter/cti_converter.py index 7a9a4ed6..a0070b1f 100644 --- a/siem-converter/app/converter/cti_converter.py +++ b/siem-converter/app/converter/cti_converter.py @@ -17,14 +17,8 @@ def __init__(self): self.logger = logging.getLogger("cti_converter") self.parser = CTIParser() - def _reverse_mapping(self, platform: CTIPlatform, data: Dict[str, List[str]], - include_source_ip: bool = False) -> Dict[str, str]: - mapping: Dict = self.renders.get(platform.name).get_default_mapping(include_source_ip) - reverse_mapping = {} - for platform_field, generic_field in mapping.items(): - if data.get(generic_field): - reverse_mapping.update({generic_field: platform_field}) - return reverse_mapping + def _get_render_mapping(self, platform: CTIPlatform, include_source_ip: bool = False) -> Dict[str, str]: + return self.renders.get(platform.name).default_mapping @handle_translation_exceptions def __parse_iocs_from_string(self, text: str, include_ioc_types: list = None, include_hash_types: list = None, @@ -39,11 +33,10 @@ def __parse_iocs_from_string(self, text: str, include_ioc_types: list = None, in @handle_translation_exceptions def __render_translation(self, parsed_data: dict, platform_data: CTIPlatform, iocs_per_query: int, include_source_ip: bool = False) -> List[str]: - reverse_mapping = self._reverse_mapping(data=parsed_data, - include_source_ip=include_source_ip, platform=platform_data) + mapping = self._get_render_mapping(platform=platform_data, include_source_ip=include_source_ip) platform = self.renders.get(platform_data.name) platform_generation = self.generate(data=parsed_data, platform=platform, iocs_per_query=iocs_per_query, - mapping=reverse_mapping) + mapping=mapping) return platform_generation def convert(self, text: str, @@ -73,9 +66,10 @@ def _get_iocs_chunk(chunks_size: int, data: Dict[str, List[str]], result = [] for generic_field, iocs_list in data.items(): for ioc in iocs_list: - result.append(IocsChunkValue(generic_field=generic_field, - platform_field=mapping.get(generic_field, generic_field), - value=ioc)) + if mapping.get(generic_field): + result.append(IocsChunkValue(generic_field=generic_field, + platform_field=mapping[generic_field], + value=ioc)) return [result[i:i + chunks_size] for i in range(0, len(result), chunks_size)] def generate(self, platform: RenderCTI, iocs_per_query, data: Dict[str, List[str]], diff --git a/siem-converter/app/converter/tools/const.py b/siem-converter/app/converter/tools/const.py index 611083d9..1616189a 100644 --- a/siem-converter/app/converter/tools/const.py +++ b/siem-converter/app/converter/tools/const.py @@ -10,6 +10,13 @@ "replace_dots", "remove_private_and_reserved_ips", "replace_hxxp" ] +HASH_MAP = { + "md5": "HashMd5", + "sha1": "HashSha1", + "sha256": "HashSha256", + "sha512": "HashSha512" +} + hash_regexes = { "md5": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{32})(?=[\s)\]\"',;\n<|]|$)", "sha1": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{40})(?=[\s)\]\"',;\n<|]|$)",