diff --git a/uncoder-core/app/translator/const.py b/uncoder-core/app/translator/const.py index 6db1167e..767fe882 100644 --- a/uncoder-core/app/translator/const.py +++ b/uncoder-core/app/translator/const.py @@ -9,4 +9,4 @@ CTI_IOCS_PER_QUERY_LIMIT = 25 -DEFAULT_VALUE_TYPE = Union[int, str, StrValue, list[Union[int, str, StrValue]]] +DEFAULT_VALUE_TYPE = Union[bool, int, str, StrValue, list[Union[int, str, StrValue]]] diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 8e9f8373..778fbfb2 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -90,7 +90,7 @@ def _map_bool_value(value: bool) -> str: def _pre_process_value( self, field: str, - value: Union[int, str, StrValue], + value: Union[bool, int, str, StrValue], value_type: str = ValueType.value, wrap_str: bool = False, wrap_int: bool = False, diff --git a/uncoder-core/app/translator/core/str_value_manager.py b/uncoder-core/app/translator/core/str_value_manager.py index b5718e3a..5ee02312 100644 --- a/uncoder-core/app/translator/core/str_value_manager.py +++ b/uncoder-core/app/translator/core/str_value_manager.py @@ -130,6 +130,25 @@ def has_spec_symbols(self) -> bool: return any(isinstance(el, BaseSpecSymbol) for el in self.split_value) +RE_STR_SPEC_SYMBOLS_MAP = { + "?": ReZeroOrOneQuantifier, + "*": ReZeroOrMoreQuantifier, + "+": ReOneOrMoreQuantifier, + "^": ReCaretSymbol, + "$": ReEndOfStrSymbol, + ".": ReAnySymbol, + "[": ReLeftSquareBracket, + "]": ReRightSquareBracket, + "(": ReLeftParenthesis, + ")": ReRightParenthesis, + "{": ReLeftCurlyBracket, + "}": ReRightCurlyBracket, + "|": ReOrOperator, + ",": ReCommaSymbol, + "-": ReHyphenSymbol, +} + + CONTAINER_SPEC_SYMBOLS_MAP = { SingleSymbolWildCard: "?", UnboundLenWildCard: "*", diff --git a/uncoder-core/app/translator/mappings/platforms/anomali/common.yml b/uncoder-core/app/translator/mappings/platforms/anomali/common.yml new file mode 100644 index 00000000..7b069f4c --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/anomali/common.yml @@ -0,0 +1,31 @@ +platform: Anomali +description: Common field mapping + +field_mapping: + c-uri-query: url + c-useragent: user_agent + CommandLine: command_line + DestinationHostname: dest + DestinationIp: dest_ip + DestinationPort: dest_port + Details: reg_value_data + dst_ip: dest_ip + dst_port: dest_port + EventID: event_id + EventName: event_name + FileName: file_name + FilePath: file_path + Image: image + NewProcessName: image + OriginalFileName: original_file_name + ParentCommandLine: parent_command_line + ParentImage: parent_image + ParentProcessID: parent_process_id + Platform: platform + ProcessCommandLine: command_line + ProcessID: process_id + SourceImage: parent_image + SourcePort: src_port + TargetFilename: file_name + TargetObject: reg_key + UserAgent: user_agent diff --git a/uncoder-core/app/translator/mappings/platforms/anomali/default.yml b/uncoder-core/app/translator/mappings/platforms/anomali/default.yml new file mode 100644 index 00000000..fed2954e --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/anomali/default.yml @@ -0,0 +1,5 @@ +platform: Anomali +source: default + + +default_log_source: {} diff --git a/uncoder-core/app/translator/platforms/anomali/__init__.py b/uncoder-core/app/translator/platforms/anomali/__init__.py new file mode 100644 index 00000000..5cd64d01 --- /dev/null +++ b/uncoder-core/app/translator/platforms/anomali/__init__.py @@ -0,0 +1 @@ +from app.translator.platforms.anomali.renders.anomali import AnomaliQueryRender # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/anomali/const.py b/uncoder-core/app/translator/platforms/anomali/const.py new file mode 100644 index 00000000..3d14733d --- /dev/null +++ b/uncoder-core/app/translator/platforms/anomali/const.py @@ -0,0 +1,11 @@ +from app.translator.core.models.platform_details import PlatformDetails + +ANOMALI_QUERY_DETAILS = { + "platform_id": "anomali-aql-query", + "name": "Anomali Security Analytics Query", + "group_name": "Anomali Security Analytics", + "platform_name": "Query", + "group_id": "anomali", +} + +anomali_query_details = PlatformDetails(**ANOMALI_QUERY_DETAILS) diff --git a/uncoder-core/app/translator/platforms/anomali/mapping.py b/uncoder-core/app/translator/platforms/anomali/mapping.py new file mode 100644 index 00000000..5c7e13a3 --- /dev/null +++ b/uncoder-core/app/translator/platforms/anomali/mapping.py @@ -0,0 +1,18 @@ +from app.translator.core.mapping import BaseCommonPlatformMappings, LogSourceSignature +from app.translator.platforms.anomali.const import anomali_query_details + + +class AnomaliLogSourceSignature(LogSourceSignature): + def is_suitable(self) -> bool: + return True + + def __str__(self) -> str: + return "" + + +class AnomaliMappings(BaseCommonPlatformMappings): + def prepare_log_source_signature(self, mapping: dict) -> AnomaliLogSourceSignature: # noqa: ARG002 + return AnomaliLogSourceSignature() + + +anomali_query_mappings = AnomaliMappings(platform_dir="anomali", platform_details=anomali_query_details) diff --git a/uncoder-core/app/translator/platforms/anomali/renders/__init__.py b/uncoder-core/app/translator/platforms/anomali/renders/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/uncoder-core/app/translator/platforms/anomali/renders/anomali.py b/uncoder-core/app/translator/platforms/anomali/renders/anomali.py new file mode 100644 index 00000000..1da26ab7 --- /dev/null +++ b/uncoder-core/app/translator/platforms/anomali/renders/anomali.py @@ -0,0 +1,100 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.custom_types.values import ValueType +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.managers import render_manager +from app.translator.platforms.anomali.const import anomali_query_details +from app.translator.platforms.anomali.mapping import AnomaliMappings, anomali_query_mappings +from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager + + +class AnomaliFieldValueRender(BaseFieldValueRender): + details: PlatformDetails = anomali_query_details + str_value_manager = sql_str_value_manager + + @staticmethod + def _wrap_str_value(value: str) -> str: + return f"'{value}'" + + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" + return f"{field} = {self._pre_process_value(field, value, wrap_str=True)}" + + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" + return f"{field} != {self._pre_process_value(field, value, wrap_str=True)}" + + def less_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} < {self._pre_process_value(field, value, wrap_str=True)}" + + def less_or_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} <= {self._pre_process_value(field, value, wrap_str=True)}" + + def greater_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} > {self._pre_process_value(field, value, wrap_str=True)}" + + def greater_or_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} >= {self._pre_process_value(field, value, wrap_str=True)}" + + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" + return f"{field} like '%{self._pre_process_value(field, value)}%'" + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" + return f"{field} like '%{self._pre_process_value(field, value)}'" + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" + return f"{field} like '{self._pre_process_value(field, value)}%'" + + def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" + regex_str = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=True) + return f"regexp_like({field}, {regex_str})" + + def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f'message contains "{self._pre_process_value(field, value)}"' + + +@render_manager.register +class AnomaliQueryRender(PlatformQueryRender): + details: PlatformDetails = anomali_query_details + mappings: AnomaliMappings = anomali_query_mappings + + or_token = "OR" + and_token = "AND" + not_token = "NOT" + + comment_symbol = "--" + is_single_line_comment = True + + field_value_render = AnomaliFieldValueRender(or_token=or_token) + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"| where {query}" if query else "" diff --git a/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py b/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py index 2e189db0..6c2a071b 100644 --- a/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py @@ -23,26 +23,12 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.str_value_manager import ( CONTAINER_SPEC_SYMBOLS_MAP, + RE_STR_SPEC_SYMBOLS_MAP, BaseSpecSymbol, - ReAnySymbol, - ReCaretSymbol, - ReCommaSymbol, ReDigitalSymbol, - ReEndOfStrSymbol, - ReHyphenSymbol, - ReLeftCurlyBracket, - ReLeftParenthesis, - ReLeftSquareBracket, - ReOneOrMoreQuantifier, - ReOrOperator, - ReRightCurlyBracket, - ReRightParenthesis, - ReRightSquareBracket, ReWhiteSpaceSymbol, ReWordBoundarySymbol, ReWordSymbol, - ReZeroOrMoreQuantifier, - ReZeroOrOneQuantifier, SingleSymbolWildCard, StrValue, StrValueManager, @@ -50,23 +36,6 @@ ) from app.translator.platforms.base.aql.escape_manager import aql_escape_manager -RE_STR_SPEC_SYMBOLS_MAP = { - "?": ReZeroOrOneQuantifier, - "*": ReZeroOrMoreQuantifier, - "+": ReOneOrMoreQuantifier, - "^": ReCaretSymbol, - "$": ReEndOfStrSymbol, - ".": ReAnySymbol, - "[": ReLeftSquareBracket, - "]": ReRightSquareBracket, - "(": ReLeftParenthesis, - ")": ReRightParenthesis, - "{": ReLeftCurlyBracket, - "}": ReRightCurlyBracket, - "|": ReOrOperator, - ",": ReCommaSymbol, - "-": ReHyphenSymbol, -} AQL_CONTAINER_SPEC_SYMBOLS_MAP = copy.copy(CONTAINER_SPEC_SYMBOLS_MAP) AQL_CONTAINER_SPEC_SYMBOLS_MAP.update({SingleSymbolWildCard: "_", UnboundLenWildCard: "%"}) diff --git a/uncoder-core/app/translator/platforms/base/sql/escape_manager.py b/uncoder-core/app/translator/platforms/base/sql/escape_manager.py new file mode 100644 index 00000000..8e5be92a --- /dev/null +++ b/uncoder-core/app/translator/platforms/base/sql/escape_manager.py @@ -0,0 +1,18 @@ +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.escape_manager import EscapeManager +from app.translator.core.models.escape_details import EscapeDetails + + +class SQLEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [EscapeDetails(pattern=r"(')", escape_symbols=r"'\1")], + ValueType.regex_value: [ + EscapeDetails(pattern=r"([$^*+()\[\]{}|.?\-\\])", escape_symbols=r"\\\1"), + EscapeDetails(pattern=r"(')", escape_symbols=r"'\1"), + ], + } + + +sql_escape_manager = SQLEscapeManager() diff --git a/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py b/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py new file mode 100644 index 00000000..0c2f03ba --- /dev/null +++ b/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py @@ -0,0 +1,100 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" + +import copy +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.str_value_manager import ( + CONTAINER_SPEC_SYMBOLS_MAP, + RE_STR_SPEC_SYMBOLS_MAP, + BaseSpecSymbol, + ReDigitalSymbol, + ReWhiteSpaceSymbol, + ReWordBoundarySymbol, + ReWordSymbol, + SingleSymbolWildCard, + StrValue, + StrValueManager, + UnboundLenWildCard, +) +from app.translator.platforms.base.sql.escape_manager import sql_escape_manager + +SQL_CONTAINER_SPEC_SYMBOLS_MAP = copy.copy(CONTAINER_SPEC_SYMBOLS_MAP) +SQL_CONTAINER_SPEC_SYMBOLS_MAP.update({SingleSymbolWildCard: "_", UnboundLenWildCard: "%"}) + + +class SQLStrValueManager(StrValueManager): + escape_manager = sql_escape_manager + container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = SQL_CONTAINER_SPEC_SYMBOLS_MAP + re_str_alpha_num_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { + "b": ReWordBoundarySymbol, + "w": ReWordSymbol, + "d": ReDigitalSymbol, + "s": ReWhiteSpaceSymbol, + } + re_str_spec_symbols_map = RE_STR_SPEC_SYMBOLS_MAP + str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { + "_": SingleSymbolWildCard, + "%": UnboundLenWildCard, + } + + def from_str_to_container(self, value: str) -> StrValue: + split = [] + prev_char = None + for char in value: + if char in self.str_spec_symbols_map: + split.append(self.str_spec_symbols_map[char]()) + else: + if char == "'": + if prev_char == "'": + split.append(char) + prev_char = char + continue + split.append(char) + + prev_char = char + + return StrValue(value, self._concat(split)) + + def from_re_str_to_container(self, value: str) -> StrValue: + value = value.replace("''", "'") + return super().from_re_str_to_container(value) + + def from_container_to_str(self, container: StrValue, value_type: str = ValueType.value) -> str: + result = "" + for el in container.split_value: + if isinstance(el, str): + result += self.escape_manager.escape(el, value_type) + elif isinstance(el, BaseSpecSymbol): + if value_type == ValueType.regex_value: + if isinstance(el, SingleSymbolWildCard): + result += "." + continue + if isinstance(el, UnboundLenWildCard): + result += ".*" + continue + + if pattern := self.container_spec_symbols_map.get(type(el)): + result += pattern + + return result + + +sql_str_value_manager = SQLStrValueManager() diff --git a/uncoder-core/app/translator/platforms/sigma/str_value_manager.py b/uncoder-core/app/translator/platforms/sigma/str_value_manager.py index ae5120df..751db716 100644 --- a/uncoder-core/app/translator/platforms/sigma/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/sigma/str_value_manager.py @@ -18,50 +18,18 @@ """ from app.translator.core.str_value_manager import ( - ReAnySymbol, - ReCaretSymbol, - ReCommaSymbol, + RE_STR_SPEC_SYMBOLS_MAP, ReDigitalSymbol, - ReEndOfStrSymbol, - ReHyphenSymbol, - ReLeftCurlyBracket, - ReLeftParenthesis, - ReLeftSquareBracket, - ReOneOrMoreQuantifier, - ReOrOperator, - ReRightCurlyBracket, - ReRightParenthesis, - ReRightSquareBracket, ReWhiteSpaceSymbol, ReWordBoundarySymbol, ReWordSymbol, - ReZeroOrMoreQuantifier, - ReZeroOrOneQuantifier, SingleSymbolWildCard, StrValue, StrValueManager, - UnboundLenWildCard, + UnboundLenWildCard ) from app.translator.platforms.sigma.escape_manager import sigma_escape_manager -RE_STR_SPEC_SYMBOLS_MAP = { - "?": ReZeroOrOneQuantifier, - "*": ReZeroOrMoreQuantifier, - "+": ReOneOrMoreQuantifier, - "^": ReCaretSymbol, - "$": ReEndOfStrSymbol, - ".": ReAnySymbol, - "[": ReLeftSquareBracket, - "]": ReRightSquareBracket, - "(": ReLeftParenthesis, - ")": ReRightParenthesis, - "{": ReLeftCurlyBracket, - "}": ReRightCurlyBracket, - "|": ReOrOperator, - ",": ReCommaSymbol, - "-": ReHyphenSymbol, -} - class SigmaStrValueManager(StrValueManager): escape_manager = sigma_escape_manager diff --git a/uncoder-core/app/translator/translator.py b/uncoder-core/app/translator/translator.py index a62f870d..15cf428d 100644 --- a/uncoder-core/app/translator/translator.py +++ b/uncoder-core/app/translator/translator.py @@ -1,14 +1,12 @@ import logging -from typing import Optional, Union +from typing import Optional from app.translator.core.exceptions.core import UnsupportedPlatform from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer -from app.translator.core.parser import PlatformQueryParser +from app.translator.core.parser import QueryParser from app.translator.core.render import QueryRender from app.translator.managers import ParserManager, RenderManager, parser_manager, render_manager from app.translator.platforms.elasticsearch.const import ELASTIC_QUERY_TYPES -from app.translator.platforms.roota.parsers.roota import RootAParser -from app.translator.platforms.sigma.parsers.sigma import SigmaParser from app.translator.tools.decorators import handle_translation_exceptions @@ -19,7 +17,7 @@ class Translator: def __init__(self): self.logger = logging.getLogger("translator") - def __get_parser(self, source: str) -> Union[PlatformQueryParser, RootAParser, SigmaParser]: + def __get_parser(self, source: str) -> QueryParser: parser = self.parser_manager.get(source) if not parser: raise UnsupportedPlatform(platform=source, is_parser=True) @@ -41,13 +39,16 @@ def __is_one_vendor_translation(source: str, target: str) -> bool: return False + def parse_raw_query(self, text: str, source: str) -> tuple[QueryParser, RawQueryContainer]: + parser = self.__get_parser(source) + text = parser.remove_comments(text) + return parser, parser.parse_raw_query(text, language=source) + @handle_translation_exceptions def __parse_incoming_data( self, text: str, source: str, target: Optional[str] = None ) -> tuple[RawQueryContainer, Optional[TokenizedQueryContainer]]: - parser = self.__get_parser(source) - text = parser.remove_comments(text) - raw_query_container = parser.parse_raw_query(text, language=source) + parser, raw_query_container = self.parse_raw_query(text=text, source=source) tokenized_query_container = None if not (target and self.__is_one_vendor_translation(raw_query_container.language, target)): tokenized_query_container = parser.parse(raw_query_container) @@ -117,3 +118,6 @@ def get_parsers(self) -> list: def get_renders(self) -> list: return self.render_manager.get_platforms_details + + +app_translator = Translator()