From 1cf80f3de4193c86b91b51171cbb95fba9dcbd40 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Thu, 4 Jul 2024 17:23:57 +0300 Subject: [PATCH 1/5] function value --- uncoder-core/app/translator/core/const.py | 3 +- .../translator/core/models/function_value.py | 39 +++++++++++ .../translator/core/models/query_container.py | 4 +- uncoder-core/app/translator/core/parser.py | 48 +++++++------ uncoder-core/app/translator/core/render.py | 63 +++++++++++------ uncoder-core/app/translator/core/tokenizer.py | 27 +++++-- .../platforms/base/aql/parsers/aql.py | 14 ++-- .../platforms/base/aql/tokenizer.py | 48 ++++++++++++- .../platforms/base/lucene/parsers/lucene.py | 9 +-- .../platforms/base/spl/parsers/spl.py | 10 +-- .../platforms/base/sql/parsers/sql.py | 9 +-- .../platforms/chronicle/parsers/chronicle.py | 9 +-- .../forti_siem/renders/forti_siem_rule.py | 70 ++++++++----------- .../platforms/logscale/parsers/logscale.py | 10 +-- .../microsoft/parsers/microsoft_sentinel.py | 10 +-- 15 files changed, 245 insertions(+), 128 deletions(-) create mode 100644 uncoder-core/app/translator/core/models/function_value.py diff --git a/uncoder-core/app/translator/core/const.py b/uncoder-core/app/translator/core/const.py index a8788ada..3802ea63 100644 --- a/uncoder-core/app/translator/core/const.py +++ b/uncoder-core/app/translator/core/const.py @@ -1,6 +1,7 @@ from typing import Union from app.translator.core.models.field import Alias, Field, FieldValue, Keyword +from app.translator.core.models.function_value import FunctionValue from app.translator.core.models.identifier import Identifier -TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias] +QUERY_TOKEN_TYPE = Union[FieldValue, FunctionValue, Keyword, Identifier, Field, Alias] diff --git a/uncoder-core/app/translator/core/models/function_value.py b/uncoder-core/app/translator/core/models/function_value.py new file mode 100644 index 00000000..bbcea219 --- /dev/null +++ b/uncoder-core/app/translator/core/models/function_value.py @@ -0,0 +1,39 @@ +from typing import Optional, Union + +from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS +from app.translator.core.models.functions.base import Function +from app.translator.core.models.identifier import Identifier +from app.translator.core.str_value_manager import StrValue + + +class FunctionValue: + def __init__(self, function: Function, operator: Identifier, value: Union[int, str, StrValue, list, tuple]): + self.function = function + self.operator = operator + self.values = [] + self.__add_value(value) + + @property + def value(self) -> Union[int, str, StrValue, list[Union[int, str, StrValue]]]: + if isinstance(self.values, list) and len(self.values) == 1: + return self.values[0] + return self.values + + @value.setter + def value(self, new_value: Union[int, str, StrValue, list[Union[int, str, StrValue]]]) -> None: + self.values = [] + self.__add_value(new_value) + + def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -> None: + if value and isinstance(value, (list, tuple)): + for v in value: + self.__add_value(v) + elif ( + value + and isinstance(value, str) + and value.isnumeric() + and self.operator.token_type not in STR_SEARCH_OPERATORS + ): + self.values.append(int(value)) + elif value is not None and isinstance(value, (int, str)): + self.values.append(value) diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index 0d90f237..4ca79b48 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import Optional -from app.translator.core.const import TOKEN_TYPE +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import DEFAULT_MAPPING_NAME from app.translator.core.models.field import Field @@ -65,6 +65,6 @@ class RawQueryDictContainer: @dataclass class TokenizedQueryContainer: - tokens: list[TOKEN_TYPE] + tokens: list[QUERY_TOKEN_TYPE] meta_info: MetaInfoContainer functions: ParsedFunctions = field(default_factory=ParsedFunctions) diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index 18b50739..77a41464 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -18,15 +18,15 @@ import re from abc import ABC, abstractmethod -from typing import Union +from typing import Optional, Union -from app.translator.core.const import TOKEN_TYPE +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword -from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.field import Field, FieldValue +from app.translator.core.models.function_value import FunctionValue +from app.translator.core.models.functions.base import Function from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.tokenizer import QueryTokenizer @@ -55,24 +55,30 @@ class PlatformQueryParser(QueryParser, ABC): tokenizer: QueryTokenizer = None platform_functions: PlatformFunctions = None - def get_fields_tokens(self, tokens: list[Union[FieldValue, Keyword, Identifier]]) -> list[Field]: - return [token.field for token in self.tokenizer.filter_tokens(tokens, FieldValue)] - - def get_tokens_and_source_mappings( - self, query: str, log_sources: dict[str, Union[str, list[str]]] - ) -> tuple[list[TOKEN_TYPE], list[SourceMapping]]: + def get_query_tokens(self, query: str) -> list[QUERY_TOKEN_TYPE]: if not query: raise TokenizerGeneralException("Can't translate empty query. Please provide more details") - tokens = self.tokenizer.tokenize(query=query) - field_tokens = self.get_fields_tokens(tokens=tokens) + return self.tokenizer.tokenize(query=query) + + def get_field_tokens( + self, query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None + ) -> list[Field]: + field_tokens = [] + for token in query_tokens: + if isinstance(token, FieldValue): + field_tokens.append(token.field) + elif isinstance(token, FunctionValue): + field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args([token.function])) + + if functions: + field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args(functions)) + + return field_tokens + + def get_source_mappings( + self, field_tokens: list[Field], log_sources: dict[str, Union[str, list[str]]] + ) -> list[SourceMapping]: field_names = [field.source_name for field in field_tokens] source_mappings = self.mappings.get_suitable_source_mappings(field_names=field_names, **log_sources) self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping) - - return tokens, source_mappings - - def set_functions_fields_generic_names( - self, functions: ParsedFunctions, source_mappings: list[SourceMapping] - ) -> None: - field_tokens = self.tokenizer.get_field_tokens_from_func_args(args=functions.functions) - self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping) + return source_mappings diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index b4b1ccc5..b002dbec 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -22,7 +22,7 @@ from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.const import TOKEN_TYPE +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.context_vars import return_only_first_query_ctx_var, wrap_query_with_meta_info_ctx_var from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType @@ -32,6 +32,7 @@ from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword, PredefinedField +from app.translator.core.models.function_value import FunctionValue from app.translator.core.models.functions.base import Function, RenderedFunctions from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails @@ -258,7 +259,9 @@ def map_predefined_field(self, predefined_field: PredefinedField) -> str: return mapped_predefined_field_name - def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + def apply_token( # noqa: PLR0911 + self, token: Union[FieldValue, Function, Keyword, Identifier], source_mapping: SourceMapping + ) -> str: if isinstance(token, FieldValue): if token.alias: mapped_fields = [token.alias.name] @@ -286,6 +289,12 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp ] ) return self.group_token % joined if len(cross_paired_fields) > 1 else joined + if isinstance(token, FunctionValue): + func_render = self.platform_functions.manager.get_in_query_render(token.function.name) + rendered_func = func_render.render(token.function, source_mapping) + return self.field_value_render.apply_field_value( + field=rendered_func, operator=token.operator, value=token.value + ) if isinstance(token, Function): func_render = self.platform_functions.manager.get_in_query_render(token.name) return func_render.render(token, source_mapping) @@ -296,7 +305,7 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp return token.token_type - def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str: + def generate_query(self, tokens: list[QUERY_TOKEN_TYPE], source_mapping: SourceMapping) -> str: result_values = [] unmapped_fields = set() for token in tokens: @@ -412,37 +421,45 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap defined_raw_log_fields.append(prefix) return "\n".join(defined_raw_log_fields) + def _generate_from_tokenized_query_container_by_source_mapping( + self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping + ) -> str: + rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) + prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix) + + if source_mapping.raw_log_fields: + defined_raw_log_fields = self.generate_raw_log_fields( + fields=query_container.meta_info.query_fields, source_mapping=source_mapping + ) + prefix += f"\n{defined_raw_log_fields}" + query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) + not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported + return self.finalize_query( + prefix=prefix, + query=query, + functions=rendered_functions.rendered, + not_supported_functions=not_supported_functions, + meta_info=query_container.meta_info, + source_mapping=source_mapping, + ) + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} errors = [] source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) for source_mapping in source_mappings: - rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) - prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix) try: - if source_mapping.raw_log_fields: - defined_raw_log_fields = self.generate_raw_log_fields( - fields=query_container.meta_info.query_fields, source_mapping=source_mapping - ) - prefix += f"\n{defined_raw_log_fields}" - result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) - except StrictPlatformException as err: - errors.append(err) - continue - else: - not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported - finalized_query = self.finalize_query( - prefix=prefix, - query=result, - functions=rendered_functions.rendered, - not_supported_functions=not_supported_functions, - meta_info=query_container.meta_info, - source_mapping=source_mapping, + finalized_query = self._generate_from_tokenized_query_container_by_source_mapping( + query_container, source_mapping ) if return_only_first_query_ctx_var.get() is True: return finalized_query queries_map[source_mapping.source_id] = finalized_query + except StrictPlatformException as err: + errors.append(err) + continue + if not queries_map and errors: raise errors[0] return self.finalize(queries_map) diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index ff9385ba..a967cd74 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -20,17 +20,20 @@ from abc import ABC, abstractmethod from typing import Any, ClassVar, Optional, Union -from app.translator.core.const import TOKEN_TYPE +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager +from app.translator.core.exceptions.functions import NotSupportedFunctionException from app.translator.core.exceptions.parser import ( QueryParenthesesException, TokenizerGeneralException, UnsupportedOperatorException, ) +from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import SourceMapping from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword +from app.translator.core.models.function_value import FunctionValue from app.translator.core.models.functions.base import Function from app.translator.core.models.functions.eval import EvalArg from app.translator.core.models.functions.group_by import GroupByFunction @@ -64,6 +67,7 @@ class QueryTokenizer(BaseTokenizer): # do not modify, use subclasses to define this attribute field_pattern: str = None + function_pattern: str = None _value_pattern: str = None value_pattern: str = None multi_value_pattern: str = None @@ -73,6 +77,7 @@ class QueryTokenizer(BaseTokenizer): wildcard_symbol = None escape_manager: EscapeManager = None str_value_manager: StrValueManager = None + platform_functions: PlatformFunctions = None def __init_subclass__(cls, **kwargs): cls._validate_re_patterns() @@ -268,9 +273,16 @@ def _check_field_value_match(self, query: str, white_space_pattern: str = r"\s+" return False + def search_function_value(self, query: str) -> tuple[FunctionValue, str]: # noqa: ARG002 + raise NotSupportedFunctionException + + @staticmethod + def _check_function_value_match(query: str) -> bool: # noqa: ARG004 + return False + def _get_next_token( self, query: str - ) -> tuple[Union[FieldValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]: + ) -> tuple[Union[FieldValue, FunctionValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]: query = query.strip("\n").strip(" ").strip("\n") if query.startswith(GroupType.L_PAREN): return Identifier(token_type=GroupType.L_PAREN), query[1:] @@ -280,6 +292,8 @@ def _get_next_token( logical_operator = logical_operator_search.group("logical_operator") pos = logical_operator_search.end() return Identifier(token_type=logical_operator.lower()), query[pos:] + if self.platform_functions and self._check_function_value_match(query): + return self.search_function_value(query) if self._check_field_value_match(query): return self.search_field_value(query) if self.keyword_pattern and re.match(self.keyword_pattern, query): @@ -288,7 +302,7 @@ def _get_next_token( raise TokenizerGeneralException("Unsupported query entry") @staticmethod - def _validate_parentheses(tokens: list[TOKEN_TYPE]) -> None: + def _validate_parentheses(tokens: list[QUERY_TOKEN_TYPE]) -> None: parentheses = [] for token in tokens: if isinstance(token, Identifier) and token.token_type in (GroupType.L_PAREN, GroupType.R_PAREN): @@ -320,8 +334,9 @@ def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: @staticmethod def filter_tokens( - tokens: list[TOKEN_TYPE], token_type: Union[type[FieldValue], type[Field], type[Keyword], type[Identifier]] - ) -> list[TOKEN_TYPE]: + tokens: list[QUERY_TOKEN_TYPE], + token_type: Union[type[FieldValue], type[Field], type[Keyword], type[Identifier]], + ) -> list[QUERY_TOKEN_TYPE]: return [token for token in tokens if isinstance(token, token_type)] def get_field_tokens_from_func_args( # noqa: PLR0912 @@ -339,6 +354,8 @@ def get_field_tokens_from_func_args( # noqa: PLR0912 elif isinstance(arg, FieldValue): if arg.field: result.append(arg.field) + elif isinstance(arg, FunctionValue): + result.extend(self.get_field_tokens_from_func_args(args=[arg.function])) elif isinstance(arg, GroupByFunction): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses)) diff --git a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py index f911ea27..4bc3f46a 100644 --- a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py @@ -27,12 +27,12 @@ from app.translator.platforms.base.aql.functions import AQLFunctions, aql_functions from app.translator.platforms.base.aql.log_source_map import LOG_SOURCE_FUNCTIONS_MAP from app.translator.platforms.base.aql.mapping import AQLMappings, aql_mappings -from app.translator.platforms.base.aql.tokenizer import AQLTokenizer, aql_tokenizer +from app.translator.platforms.base.aql.tokenizer import AQLTokenizer from app.translator.tools.utils import get_match_group class AQLQueryParser(PlatformQueryParser): - tokenizer: AQLTokenizer = aql_tokenizer + tokenizer: AQLTokenizer = AQLTokenizer(aql_functions) mappings: AQLMappings = aql_mappings platform_functions: AQLFunctions = aql_functions @@ -116,10 +116,10 @@ def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, log_sources, functions = self._parse_query(raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - fields_tokens = self.get_fields_tokens(tokens=tokens) - self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens, functions.functions) + source_mappings = self.get_source_mappings(field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) diff --git a/uncoder-core/app/translator/platforms/base/aql/tokenizer.py b/uncoder-core/app/translator/platforms/base/aql/tokenizer.py index 54a797eb..d584e16d 100644 --- a/uncoder-core/app/translator/platforms/base/aql/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/aql/tokenizer.py @@ -21,11 +21,15 @@ from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType +from app.translator.core.functions import PlatformFunctions from app.translator.core.models.field import FieldValue, Keyword +from app.translator.core.models.function_value import FunctionValue +from app.translator.core.models.functions.base import Function from app.translator.core.models.identifier import Identifier from app.translator.core.str_value_manager import StrValue from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.base.aql.const import NUM_VALUE_PATTERN, SINGLE_QUOTES_VALUE_PATTERN, UTF8_PAYLOAD_PATTERN +from app.translator.platforms.base.aql.functions.const import AQLFunctionGroupType from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager from app.translator.tools.utils import get_match_group @@ -46,6 +50,7 @@ class AQLTokenizer(QueryTokenizer): multi_value_operators_map: ClassVar[dict[str, str]] = {"in": OperatorType.EQ} field_pattern = r'(?P"[a-zA-Z\._\-\s]+"|[a-zA-Z\._\-]+)' + function_pattern = r'(?P[a-zA-Z_]+)\((?:(?:[a-zA-Z\._\-\s]+)|(?:"[a-zA-Z\._\-]+"))\)' bool_value_pattern = rf"(?P<{ValueType.bool_value}>true|false)\s*" _value_pattern = rf"{NUM_VALUE_PATTERN}|{bool_value_pattern}|{SINGLE_QUOTES_VALUE_PATTERN}" multi_value_pattern = rf"""\((?P<{ValueType.multi_value}>[:a-zA-Z\"\*0-9=+%#\-_\/\\'\,.&^@!\(\s]*)\)""" @@ -54,6 +59,9 @@ class AQLTokenizer(QueryTokenizer): wildcard_symbol = "%" str_value_manager = aql_str_value_manager + def __init__(self, platform_functions: PlatformFunctions = None): + self.platform_functions = platform_functions + @staticmethod def should_process_value_wildcards(operator: Optional[str]) -> bool: return operator and operator.lower() in ("like", "ilike") @@ -79,7 +87,7 @@ def get_operator_and_value( return super().get_operator_and_value(match, mapped_operator, operator) def escape_field_name(self, field_name: str) -> str: - return field_name.replace('"', r"\"").replace(" ", r"\ ") + return field_name.replace('"', r"\"").replace(" ", r"\ ").replace("(", "\(").replace(")", "\)") @staticmethod def create_field_value(field_name: str, operator: Identifier, value: Union[str, list]) -> FieldValue: @@ -93,5 +101,39 @@ def search_keyword(self, query: str) -> tuple[Keyword, str]: pos = keyword_search.end() return keyword, query[pos:] - -aql_tokenizer = AQLTokenizer() + def _search_function_value(self, function: Function, query: str) -> tuple[FunctionValue, str]: + operator = self.search_operator(query, function.raw) + if self.is_multi_value_flow(function.raw, operator, query): + query, grouped_values = self.search_multi_value(query=query, operator=operator, field_name=function.raw) + tokens = [ # always consists of 1 element + FunctionValue(function=function, operator=Identifier(token_type=op), value=values) + for op, values in grouped_values.items() + ] + return tokens[0], query + + query, operator, value = self.search_single_value(query=query, operator=operator, field_name=function.raw) + operator_token = Identifier(token_type=operator) + return FunctionValue(function=function, operator=operator_token, value=value), query + + def search_function_value(self, query: str) -> tuple[FunctionValue, str]: + str_conversion_func_parser = self.platform_functions.manager.get_parser(AQLFunctionGroupType.str_conversion) + if str_conversion_func_parser and (func_match := str_conversion_func_parser.get_func_match(query)): + function = str_conversion_func_parser.parse(func_match.name, func_match.match) + return self._search_function_value(function, query) + + return super().search_function_value(query) + + def _check_function_value_match(self, query: str, white_space_pattern: str = r"\s+") -> bool: + single_value_operator_group = rf"(?:{'|'.join(self.single_value_operators_map)})" + single_value_pattern = rf"""{self.function_pattern}\s*{single_value_operator_group}\s*{self.value_pattern}\s*""" + if re.match(single_value_pattern, query, re.IGNORECASE): + return True + + if self.multi_value_operators_map: + multi_value_operator_group = rf"(?:{'|'.join(self.multi_value_operators_map)})" + pattern = f"{self.function_pattern}{white_space_pattern}{multi_value_operator_group}{white_space_pattern}" + multi_value_pattern = rf"{pattern}{self.multi_value_pattern}" + if re.match(multi_value_pattern, query, re.IGNORECASE): + return True + + return False diff --git a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py index c748c1e4..5fb57284 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py @@ -47,9 +47,10 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, log_sources = self._parse_query(raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - fields_tokens = self.get_fields_tokens(tokens=tokens) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens) + source_mappings = self.get_source_mappings(field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info) diff --git a/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py b/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py index 92ba415d..27a1559d 100644 --- a/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py @@ -67,10 +67,10 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain return self.platform_functions.parse_tstats_func(raw_query_container) query, log_sources, functions = self._parse_query(raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - fields_tokens = self.get_fields_tokens(tokens=tokens) - self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens, functions.functions) + source_mappings = self.get_source_mappings(field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) diff --git a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py index d324d4ba..4a882467 100644 --- a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py @@ -43,9 +43,10 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, Optional[str]]]: def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, log_sources = self._parse_query(raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - fields_tokens = self.get_fields_tokens(tokens=tokens) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens) + source_mappings = self.get_source_mappings(field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info) diff --git a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py index 8c0e8431..b36d1197 100644 --- a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py @@ -34,9 +34,10 @@ class ChronicleQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*//.*(?:\n|$)" def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: - tokens, source_mappings = self.get_tokens_and_source_mappings(raw_query_container.query, {}) - fields_tokens = self.get_fields_tokens(tokens=tokens) + query_tokens = self.get_query_tokens(raw_query_container.query) + field_tokens = self.get_field_tokens(query_tokens) + source_mappings = self.get_source_mappings(field_tokens, {}) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info) diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index dfbc2ee6..1b0a3008 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -18,8 +18,7 @@ from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.const import TOKEN_TYPE -from app.translator.core.context_vars import return_only_first_query_ctx_var +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType @@ -197,7 +196,7 @@ class FortiSiemRuleRender(PlatformQueryRender): field_value_render = FortiSiemFieldValueRender(or_token=or_token) @staticmethod - def __is_negated_token(prev_token: TOKEN_TYPE) -> bool: + def __is_negated_token(prev_token: QUERY_TOKEN_TYPE) -> bool: return isinstance(prev_token, Identifier) and prev_token.token_type == LogicalOperatorType.NOT @staticmethod @@ -208,7 +207,7 @@ def __should_negate(is_negated_token: bool = False, negation_ctx: bool = False) return is_negated_token or negation_ctx @staticmethod - def __negate_token(token: TOKEN_TYPE) -> None: + def __negate_token(token: QUERY_TOKEN_TYPE) -> None: if isinstance(token, Identifier): if token.token_type == LogicalOperatorType.AND: token.token_type = LogicalOperatorType.OR @@ -218,7 +217,7 @@ def __negate_token(token: TOKEN_TYPE) -> None: token_type = token.operator.token_type token.operator.token_type = _NOT_OPERATORS_MAP_SUBSTITUTES.get(token_type) or token_type - def __replace_not_tokens(self, tokens: list[TOKEN_TYPE]) -> list[TOKEN_TYPE]: + def __replace_not_tokens(self, tokens: list[QUERY_TOKEN_TYPE]) -> list[QUERY_TOKEN_TYPE]: not_token_indices = [] negation_ctx_stack = [] for index, token in enumerate(tokens[1:], start=1): @@ -244,40 +243,33 @@ def __replace_not_tokens(self, tokens: list[TOKEN_TYPE]) -> list[TOKEN_TYPE]: return tokens - def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: - queries_map = {} - source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) - - for source_mapping in source_mappings: - is_event_type_set = False - field_values = [token for token in query_container.tokens if isinstance(token, FieldValue)] - mapped_fields_set = set() - for field_value in field_values: - mapped_fields = self.map_field(field_value.field, source_mapping) - mapped_fields_set = mapped_fields_set.union(set(mapped_fields)) - if _EVENT_TYPE_FIELD in mapped_fields: - is_event_type_set = True - self.__update_event_type_values(field_value, source_mapping.source_id) - - tokens = self.__replace_not_tokens(query_container.tokens) - result = self.generate_query(tokens=tokens, source_mapping=source_mapping) - prefix = "" if is_event_type_set else self.generate_prefix(source_mapping.log_source_signature) - rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) - not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported - finalized_query = self.finalize_query( - prefix=prefix, - query=result, - functions=rendered_functions.rendered, - not_supported_functions=not_supported_functions, - meta_info=query_container.meta_info, - source_mapping=source_mapping, - fields=mapped_fields_set, - ) - if return_only_first_query_ctx_var.get() is True: - return finalized_query - queries_map[source_mapping.source_id] = finalized_query - - return self.finalize(queries_map) + def _generate_from_tokenized_query_container_by_source_mapping( + self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping + ) -> str: + is_event_type_set = False + field_values = [token for token in query_container.tokens if isinstance(token, FieldValue)] + mapped_fields_set = set() + for field_value in field_values: + mapped_fields = self.map_field(field_value.field, source_mapping) + mapped_fields_set = mapped_fields_set.union(set(mapped_fields)) + if _EVENT_TYPE_FIELD in mapped_fields: + is_event_type_set = True + self.__update_event_type_values(field_value, source_mapping.source_id) + + tokens = self.__replace_not_tokens(query_container.tokens) + result = self.generate_query(tokens=tokens, source_mapping=source_mapping) + prefix = "" if is_event_type_set else self.generate_prefix(source_mapping.log_source_signature) + rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) + not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported + return self.finalize_query( + prefix=prefix, + query=result, + functions=rendered_functions.rendered, + not_supported_functions=not_supported_functions, + meta_info=query_container.meta_info, + source_mapping=source_mapping, + fields=mapped_fields_set, + ) @staticmethod def __update_event_type_values(field_value: FieldValue, source_id: str) -> None: diff --git a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py index e1015ff2..668796ae 100644 --- a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py @@ -42,10 +42,10 @@ def _parse_query(self, query: str) -> tuple[str, ParsedFunctions]: def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, functions = self._parse_query(query=raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, {}) - fields_tokens = self.get_fields_tokens(tokens=tokens) - self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens, functions.functions) + source_mappings = self.get_source_mappings(field_tokens, {}) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py index 507c8c17..2325367f 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py @@ -43,10 +43,10 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]], ParsedFun def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, log_sources, functions = self._parse_query(query=raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - fields_tokens = self.get_fields_tokens(tokens=tokens) - self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens, functions.functions) + source_mappings = self.get_source_mappings(field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) From c773bc30bc3518aeed5a8d5782c03b857b509459 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Mon, 8 Jul 2024 16:00:19 +0300 Subject: [PATCH 2/5] query tokens refactoring --- uncoder-core/app/translator/core/const.py | 11 +- .../app/translator/core/mixins/logic.py | 18 +-- .../app/translator/core/mixins/operator.py | 2 +- .../app/translator/core/models/field.py | 136 ------------------ .../translator/core/models/function_value.py | 39 ----- .../translator/core/models/query_container.py | 2 +- .../core/models/query_tokens/__init__.py | 0 .../core/models/query_tokens/field.py | 39 +++++ .../core/models/query_tokens/field_field.py | 18 +++ .../core/models/query_tokens/field_value.py | 35 +++++ .../models/query_tokens/function_value.py | 14 ++ .../models/{ => query_tokens}/identifier.py | 0 .../core/models/query_tokens/keyword.py | 27 ++++ .../core/models/query_tokens/value.py | 30 ++++ uncoder-core/app/translator/core/parser.py | 5 +- uncoder-core/app/translator/core/render.py | 24 ++-- uncoder-core/app/translator/core/tokenizer.py | 13 +- .../platforms/base/aql/tokenizer.py | 7 +- .../platforms/base/lucene/tokenizer.py | 8 +- .../platforms/base/spl/tokenizer.py | 7 +- .../platforms/base/sql/tokenizer.py | 4 +- .../platforms/chronicle/tokenizer.py | 4 +- .../forti_siem/renders/forti_siem_rule.py | 4 +- .../renders/logrhythm_axon_query.py | 54 +++---- .../platforms/logscale/tokenizer.py | 8 +- .../opensearch/renders/opensearch_rule.py | 6 +- .../palo_alto/renders/cortex_xsiam.py | 43 +++++- .../platforms/sigma/models/compiler.py | 5 +- .../platforms/sigma/models/modifiers.py | 4 +- .../platforms/sigma/parsers/sigma.py | 3 +- .../platforms/sigma/renders/sigma.py | 3 +- .../translator/platforms/sigma/tokenizer.py | 5 +- 32 files changed, 303 insertions(+), 275 deletions(-) delete mode 100644 uncoder-core/app/translator/core/models/field.py delete mode 100644 uncoder-core/app/translator/core/models/function_value.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/__init__.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/field.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/field_field.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/field_value.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/function_value.py rename uncoder-core/app/translator/core/models/{ => query_tokens}/identifier.py (100%) create mode 100644 uncoder-core/app/translator/core/models/query_tokens/keyword.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/value.py diff --git a/uncoder-core/app/translator/core/const.py b/uncoder-core/app/translator/core/const.py index 3802ea63..c8fd16ce 100644 --- a/uncoder-core/app/translator/core/const.py +++ b/uncoder-core/app/translator/core/const.py @@ -1,7 +1,10 @@ from typing import Union -from app.translator.core.models.field import Alias, Field, FieldValue, Keyword -from app.translator.core.models.function_value import FunctionValue -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field import Alias, Field +from app.translator.core.models.query_tokens.field_field import FieldField +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword -QUERY_TOKEN_TYPE = Union[FieldValue, FunctionValue, Keyword, Identifier, Field, Alias] +QUERY_TOKEN_TYPE = Union[FieldField, FieldValue, FunctionValue, Keyword, Identifier, Field, Alias] diff --git a/uncoder-core/app/translator/core/mixins/logic.py b/uncoder-core/app/translator/core/mixins/logic.py index b24a1c99..7002e847 100644 --- a/uncoder-core/app/translator/core/mixins/logic.py +++ b/uncoder-core/app/translator/core/mixins/logic.py @@ -1,19 +1,21 @@ -from typing import Union - +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_field import FieldField +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword class ANDLogicOperatorMixin: @staticmethod - def get_missed_and_token_indices(tokens: list[Union[FieldValue, Keyword, Identifier]]) -> list[int]: + def get_missed_and_token_indices(tokens: list[QUERY_TOKEN_TYPE]) -> list[int]: missed_and_indices = [] for index in range(len(tokens) - 1): token = tokens[index] next_token = tokens[index + 1] if ( - isinstance(token, (FieldValue, Keyword)) + isinstance(token, (FieldField, FieldValue, FunctionValue, Keyword)) or isinstance(token, Identifier) and token.token_type == GroupType.R_PAREN ) and not ( @@ -23,9 +25,7 @@ def get_missed_and_token_indices(tokens: list[Union[FieldValue, Keyword, Identif missed_and_indices.append(index + 1) return list(reversed(missed_and_indices)) - def add_and_token_if_missed( - self, tokens: list[Union[FieldValue, Keyword, Identifier]] - ) -> list[Union[FieldValue, Keyword, Identifier]]: + def add_and_token_if_missed(self, tokens: list[QUERY_TOKEN_TYPE]) -> list[QUERY_TOKEN_TYPE]: indices = self.get_missed_and_token_indices(tokens=tokens) for index in indices: tokens.insert(index, Identifier(token_type=LogicalOperatorType.AND)) diff --git a/uncoder-core/app/translator/core/mixins/operator.py b/uncoder-core/app/translator/core/mixins/operator.py index dee82395..dec9e3f4 100644 --- a/uncoder-core/app/translator/core/mixins/operator.py +++ b/uncoder-core/app/translator/core/mixins/operator.py @@ -19,7 +19,7 @@ from typing import Optional, Union from app.translator.core.custom_types.tokens import OperatorType -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.identifier import Identifier class WildCardMixin: diff --git a/uncoder-core/app/translator/core/models/field.py b/uncoder-core/app/translator/core/models/field.py deleted file mode 100644 index d9facb77..00000000 --- a/uncoder-core/app/translator/core/models/field.py +++ /dev/null @@ -1,136 +0,0 @@ -from typing import Optional, Union - -from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS, OperatorType -from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping -from app.translator.core.models.identifier import Identifier -from app.translator.core.str_value_manager import StrValue - - -class Alias: - def __init__(self, name: str): - self.name = name - - -class Field: - def __init__(self, source_name: str): - self.source_name = source_name - self.__generic_names_map = {} - - def get_generic_field_name(self, source_id: str) -> Optional[str]: - return self.__generic_names_map.get(source_id) - - def add_generic_names_map(self, generic_names_map: dict) -> None: - self.__generic_names_map = generic_names_map - - def set_generic_names_map(self, source_mappings: list[SourceMapping], default_mapping: SourceMapping) -> None: - generic_names_map = { - source_mapping.source_id: source_mapping.fields_mapping.get_generic_field_name(self.source_name) - or self.source_name - for source_mapping in source_mappings - } - if DEFAULT_MAPPING_NAME not in generic_names_map: - fields_mapping = default_mapping.fields_mapping - generic_names_map[DEFAULT_MAPPING_NAME] = ( - fields_mapping.get_generic_field_name(self.source_name) or self.source_name - ) - - self.__generic_names_map = generic_names_map - - -class PredefinedField: - def __init__(self, name: str): - self.name = name - - -class FieldField: - def __init__( - self, - source_name_left: str, - operator: Identifier, - source_name_right: str, - is_alias_left: bool = False, - is_alias_right: bool = False, - ): - self.field_left = Field(source_name=source_name_left) if not is_alias_left else None - self.alias_left = Alias(name=source_name_left) if is_alias_left else None - self.operator = operator - self.field_right = Field(source_name=source_name_right) if not is_alias_right else None - self.alias_right = Alias(name=source_name_right) if is_alias_right else None - - -class FieldValue: - def __init__( - self, - source_name: str, - operator: Identifier, - value: Union[int, str, StrValue, list, tuple], - is_alias: bool = False, - is_predefined_field: bool = False, - ): - # mapped by platform fields mapping - self.field = Field(source_name=source_name) if not (is_alias or is_predefined_field) else None - # not mapped - self.alias = Alias(name=source_name) if is_alias else None - # mapped by platform predefined fields mapping - self.predefined_field = PredefinedField(name=source_name) if is_predefined_field else None - - self.operator = operator - self.values = [] - self.__add_value(value) - - @property - def value(self) -> Union[int, str, StrValue, list[Union[int, str, StrValue]]]: - if isinstance(self.values, list) and len(self.values) == 1: - return self.values[0] - return self.values - - @value.setter - def value(self, new_value: Union[int, str, StrValue, list[Union[int, str, StrValue]]]) -> None: - self.values = [] - self.__add_value(new_value) - - def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -> None: - if value and isinstance(value, (list, tuple)): - for v in value: - self.__add_value(v) - elif ( - value - and isinstance(value, str) - and value.isnumeric() - and self.operator.token_type not in STR_SEARCH_OPERATORS - ): - self.values.append(int(value)) - elif value is not None and isinstance(value, (int, str)): - self.values.append(value) - - def __repr__(self): - if self.alias: - return f"{self.alias.name} {self.operator.token_type} {self.values}" - - if self.predefined_field: - return f"{self.predefined_field.name} {self.operator.token_type} {self.values}" - - return f"{self.field.source_name} {self.operator.token_type} {self.values}" - - -class Keyword: - def __init__(self, value: Union[str, list[str]]): - self.operator: Identifier = Identifier(token_type=OperatorType.KEYWORD) - self.name = "keyword" - self.values = [] - self.__add_value(value=value) - - @property - def value(self) -> Union[str, list[str]]: - if isinstance(self.values, list) and len(self.values) == 1: - return self.values[0] - return self.values - - def __add_value(self, value: Union[str, list[str]]) -> None: - if value and isinstance(value, (list, tuple)): - self.values.extend(value) - elif value and isinstance(value, str): - self.values.append(value) - - def __repr__(self): - return f"{self.name} {self.operator.token_type} {self.values}" diff --git a/uncoder-core/app/translator/core/models/function_value.py b/uncoder-core/app/translator/core/models/function_value.py deleted file mode 100644 index bbcea219..00000000 --- a/uncoder-core/app/translator/core/models/function_value.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Optional, Union - -from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS -from app.translator.core.models.functions.base import Function -from app.translator.core.models.identifier import Identifier -from app.translator.core.str_value_manager import StrValue - - -class FunctionValue: - def __init__(self, function: Function, operator: Identifier, value: Union[int, str, StrValue, list, tuple]): - self.function = function - self.operator = operator - self.values = [] - self.__add_value(value) - - @property - def value(self) -> Union[int, str, StrValue, list[Union[int, str, StrValue]]]: - if isinstance(self.values, list) and len(self.values) == 1: - return self.values[0] - return self.values - - @value.setter - def value(self, new_value: Union[int, str, StrValue, list[Union[int, str, StrValue]]]) -> None: - self.values = [] - self.__add_value(new_value) - - def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -> None: - if value and isinstance(value, (list, tuple)): - for v in value: - self.__add_value(v) - elif ( - value - and isinstance(value, str) - and value.isnumeric() - and self.operator.token_type not in STR_SEARCH_OPERATORS - ): - self.values.append(int(value)) - elif value is not None and isinstance(value, (int, str)): - self.values.append(value) diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index 4ca79b48..7c56c71a 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -6,8 +6,8 @@ from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import DEFAULT_MAPPING_NAME -from app.translator.core.models.field import Field from app.translator.core.models.functions.base import ParsedFunctions +from app.translator.core.models.query_tokens.field import Field class MetaInfoContainer: diff --git a/uncoder-core/app/translator/core/models/query_tokens/__init__.py b/uncoder-core/app/translator/core/models/query_tokens/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/uncoder-core/app/translator/core/models/query_tokens/field.py b/uncoder-core/app/translator/core/models/query_tokens/field.py new file mode 100644 index 00000000..84d07e4e --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/field.py @@ -0,0 +1,39 @@ +from typing import Optional + +from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping + + +class Alias: + def __init__(self, name: str): + self.name = name + + +class Field: + def __init__(self, source_name: str): + self.source_name = source_name + self.__generic_names_map = {} + + def get_generic_field_name(self, source_id: str) -> Optional[str]: + return self.__generic_names_map.get(source_id) + + def add_generic_names_map(self, generic_names_map: dict) -> None: + self.__generic_names_map = generic_names_map + + def set_generic_names_map(self, source_mappings: list[SourceMapping], default_mapping: SourceMapping) -> None: + generic_names_map = { + source_mapping.source_id: source_mapping.fields_mapping.get_generic_field_name(self.source_name) + or self.source_name + for source_mapping in source_mappings + } + if DEFAULT_MAPPING_NAME not in generic_names_map: + fields_mapping = default_mapping.fields_mapping + generic_names_map[DEFAULT_MAPPING_NAME] = ( + fields_mapping.get_generic_field_name(self.source_name) or self.source_name + ) + + self.__generic_names_map = generic_names_map + + +class PredefinedField: + def __init__(self, name: str): + self.name = name diff --git a/uncoder-core/app/translator/core/models/query_tokens/field_field.py b/uncoder-core/app/translator/core/models/query_tokens/field_field.py new file mode 100644 index 00000000..86099f08 --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/field_field.py @@ -0,0 +1,18 @@ +from app.translator.core.models.query_tokens.field import Alias, Field +from app.translator.core.models.query_tokens.identifier import Identifier + + +class FieldField: + def __init__( + self, + source_name_left: str, + operator: Identifier, + source_name_right: str, + is_alias_left: bool = False, + is_alias_right: bool = False, + ): + self.field_left = Field(source_name=source_name_left) if not is_alias_left else None + self.alias_left = Alias(name=source_name_left) if is_alias_left else None + self.operator = operator + self.field_right = Field(source_name=source_name_right) if not is_alias_right else None + self.alias_right = Alias(name=source_name_right) if is_alias_right else None diff --git a/uncoder-core/app/translator/core/models/query_tokens/field_value.py b/uncoder-core/app/translator/core/models/query_tokens/field_value.py new file mode 100644 index 00000000..cf491da4 --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/field_value.py @@ -0,0 +1,35 @@ +from typing import Union + +from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS +from app.translator.core.models.query_tokens.field import Alias, Field, PredefinedField +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.value import Value +from app.translator.core.str_value_manager import StrValue + + +class FieldValue(Value): + def __init__( + self, + source_name: str, + operator: Identifier, + value: Union[bool, int, str, StrValue, list, tuple], + is_alias: bool = False, + is_predefined_field: bool = False, + ): + super().__init__(value, cast_to_int=operator.token_type not in STR_SEARCH_OPERATORS) + # mapped by platform fields mapping + self.field = Field(source_name=source_name) if not (is_alias or is_predefined_field) else None + # not mapped + self.alias = Alias(name=source_name) if is_alias else None + # mapped by platform predefined fields mapping + self.predefined_field = PredefinedField(name=source_name) if is_predefined_field else None + self.operator = operator + + def __repr__(self): + if self.alias: + return f"{self.alias.name} {self.operator.token_type} {self.values}" + + if self.predefined_field: + return f"{self.predefined_field.name} {self.operator.token_type} {self.values}" + + return f"{self.field.source_name} {self.operator.token_type} {self.values}" diff --git a/uncoder-core/app/translator/core/models/query_tokens/function_value.py b/uncoder-core/app/translator/core/models/query_tokens/function_value.py new file mode 100644 index 00000000..6ffd49bc --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/function_value.py @@ -0,0 +1,14 @@ +from typing import Union + +from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS +from app.translator.core.models.functions.base import Function +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.value import Value +from app.translator.core.str_value_manager import StrValue + + +class FunctionValue(Value): + def __init__(self, function: Function, operator: Identifier, value: Union[int, str, StrValue, list, tuple]): + super().__init__(value, cast_to_int=operator.token_type not in STR_SEARCH_OPERATORS) + self.function = function + self.operator = operator diff --git a/uncoder-core/app/translator/core/models/identifier.py b/uncoder-core/app/translator/core/models/query_tokens/identifier.py similarity index 100% rename from uncoder-core/app/translator/core/models/identifier.py rename to uncoder-core/app/translator/core/models/query_tokens/identifier.py diff --git a/uncoder-core/app/translator/core/models/query_tokens/keyword.py b/uncoder-core/app/translator/core/models/query_tokens/keyword.py new file mode 100644 index 00000000..4e753c51 --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/keyword.py @@ -0,0 +1,27 @@ +from typing import Union + +from app.translator.core.custom_types.tokens import OperatorType +from app.translator.core.models.query_tokens.identifier import Identifier + + +class Keyword: + def __init__(self, value: Union[str, list[str]]): + self.operator: Identifier = Identifier(token_type=OperatorType.KEYWORD) + self.name = "keyword" + self.values = [] + self.__add_value(value=value) + + @property + def value(self) -> Union[str, list[str]]: + if isinstance(self.values, list) and len(self.values) == 1: + return self.values[0] + return self.values + + def __add_value(self, value: Union[str, list[str]]) -> None: + if value and isinstance(value, (list, tuple)): + self.values.extend(value) + elif value and isinstance(value, str): + self.values.append(value) + + def __repr__(self): + return f"{self.name} {self.operator.token_type} {self.values}" diff --git a/uncoder-core/app/translator/core/models/query_tokens/value.py b/uncoder-core/app/translator/core/models/query_tokens/value.py new file mode 100644 index 00000000..82b37167 --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/value.py @@ -0,0 +1,30 @@ +from typing import Optional, Union + +from app.translator.core.str_value_manager import StrValue + + +class Value: + def __init__(self, value: Union[bool, int, str, StrValue, list, tuple], cast_to_int: bool = False): + self.values = [] + self.__cast_to_int = cast_to_int + self.__add_value(value) + + @property + def value(self) -> Union[bool, int, str, StrValue, list[Union[int, str, StrValue]]]: + if isinstance(self.values, list) and len(self.values) == 1: + return self.values[0] + return self.values + + @value.setter + def value(self, new_value: Union[bool, int, str, StrValue, list[Union[int, str, StrValue]]]) -> None: + self.values = [] + self.__add_value(new_value) + + def __add_value(self, value: Optional[Union[bool, int, str, StrValue, list, tuple]]) -> None: + if value and isinstance(value, (list, tuple)): + for v in value: + self.__add_value(v) + elif value and isinstance(value, str) and value.isnumeric() and self.__cast_to_int: + self.values.append(int(value)) + elif value is not None and isinstance(value, (bool, int, str)): + self.values.append(value) diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index 77a41464..c51ada8c 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -24,11 +24,12 @@ from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping -from app.translator.core.models.field import Field, FieldValue -from app.translator.core.models.function_value import FunctionValue from app.translator.core.models.functions.base import Function from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.models.query_tokens.field import Field +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue from app.translator.core.tokenizer import QueryTokenizer diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index b002dbec..618f2d37 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -31,12 +31,15 @@ from app.translator.core.exceptions.parser import UnsupportedOperatorException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping -from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword, PredefinedField -from app.translator.core.models.function_value import FunctionValue from app.translator.core.models.functions.base import Function, RenderedFunctions -from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer +from app.translator.core.models.query_tokens.field import Field, PredefinedField +from app.translator.core.models.query_tokens.field_field import FieldField +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.str_value_manager import StrValue, StrValueManager @@ -75,6 +78,10 @@ def _get_value_type(field_name: str, value: Union[int, str, StrValue], value_typ def _wrap_str_value(value: str) -> str: return value + @staticmethod + def _map_bool_value(value: bool) -> str: + return "true" if value else "false" + def _pre_process_value( self, field: str, value: Union[int, str, StrValue], value_type: str = ValueType.value, wrap_str: bool = False ) -> Union[int, str]: @@ -85,6 +92,8 @@ def _pre_process_value( if isinstance(value, str): value = self.str_value_manager.escape_manager.escape(value, value_type) return self._wrap_str_value(value) if wrap_str else value + if isinstance(value, bool): + return self._map_bool_value(value) return value def _pre_process_values_list( @@ -259,9 +268,7 @@ def map_predefined_field(self, predefined_field: PredefinedField) -> str: return mapped_predefined_field_name - def apply_token( # noqa: PLR0911 - self, token: Union[FieldValue, Function, Keyword, Identifier], source_mapping: SourceMapping - ) -> str: + def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): if token.alias: mapped_fields = [token.alias.name] @@ -290,14 +297,11 @@ def apply_token( # noqa: PLR0911 ) return self.group_token % joined if len(cross_paired_fields) > 1 else joined if isinstance(token, FunctionValue): - func_render = self.platform_functions.manager.get_in_query_render(token.function.name) + func_render = self.platform_functions.manager.get_render(token.function.name) rendered_func = func_render.render(token.function, source_mapping) return self.field_value_render.apply_field_value( field=rendered_func, operator=token.operator, value=token.value ) - if isinstance(token, Function): - func_render = self.platform_functions.manager.get_in_query_render(token.name) - return func_render.render(token, source_mapping) if isinstance(token, Keyword): return self.field_value_render.apply_field_value(field="", operator=token.operator, value=token.value) if token.token_type in LogicalOperatorType: diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index a967cd74..08295917 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -32,8 +32,6 @@ ) from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword -from app.translator.core.models.function_value import FunctionValue from app.translator.core.models.functions.base import Function from app.translator.core.models.functions.eval import EvalArg from app.translator.core.models.functions.group_by import GroupByFunction @@ -41,14 +39,19 @@ from app.translator.core.models.functions.rename import RenameArg from app.translator.core.models.functions.sort import SortArg from app.translator.core.models.functions.union import UnionFunction -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field import Field +from app.translator.core.models.query_tokens.field_field import FieldField +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.str_value_manager import StrValue, StrValueManager from app.translator.tools.utils import get_match_group class BaseTokenizer(ABC): @abstractmethod - def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: + def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]: raise NotImplementedError @@ -315,7 +318,7 @@ def _validate_parentheses(tokens: list[QUERY_TOKEN_TYPE]) -> None: if parentheses: raise QueryParenthesesException - def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: + def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]: tokenized = [] while query: next_token, sliced_query = self._get_next_token(query=query) diff --git a/uncoder-core/app/translator/platforms/base/aql/tokenizer.py b/uncoder-core/app/translator/platforms/base/aql/tokenizer.py index d584e16d..16aa96fe 100644 --- a/uncoder-core/app/translator/platforms/base/aql/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/aql/tokenizer.py @@ -22,10 +22,11 @@ from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.functions import PlatformFunctions -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.function_value import FunctionValue from app.translator.core.models.functions.base import Function -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.str_value_manager import StrValue from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.base.aql.const import NUM_VALUE_PATTERN, SINGLE_QUOTES_VALUE_PATTERN, UTF8_PAYLOAD_PATTERN diff --git a/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py b/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py index eb54b7ea..b56f5bee 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py @@ -19,11 +19,13 @@ import re from typing import ClassVar, Optional, Union +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.mixins.logic import ANDLogicOperatorMixin -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.str_value_manager import StrValue from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.base.lucene.escape_manager import lucene_escape_manager @@ -135,6 +137,6 @@ def _check_field_value_match(self, query: str, white_space_pattern: str = r"\s*" return super()._check_field_value_match(query, white_space_pattern=white_space_pattern) - def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: + def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]: tokens = super().tokenize(query=query) return self.add_and_token_if_missed(tokens=tokens) diff --git a/uncoder-core/app/translator/platforms/base/spl/tokenizer.py b/uncoder-core/app/translator/platforms/base/spl/tokenizer.py index 8a030519..57a5a695 100644 --- a/uncoder-core/app/translator/platforms/base/spl/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/spl/tokenizer.py @@ -17,13 +17,12 @@ """ import re -from typing import Any, ClassVar, Optional, Union +from typing import Any, ClassVar, Optional +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.mixins.logic import ANDLogicOperatorMixin -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.base.spl.const import DOUBLE_QUOTES_VALUE_PATTERN as D_Q_V_PATTERN from app.translator.platforms.base.spl.const import FIELD_PATTERN @@ -77,6 +76,6 @@ def get_operator_and_value( return super().get_operator_and_value(match, mapped_operator, operator) - def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: + def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]: tokens = super().tokenize(query=query) return self.add_and_token_if_missed(tokens=tokens) diff --git a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py index 944d3c9b..8292ca14 100644 --- a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py @@ -21,8 +21,8 @@ from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType -from app.translator.core.models.field import FieldValue -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.tokenizer import QueryTokenizer from app.translator.tools.utils import get_match_group diff --git a/uncoder-core/app/translator/platforms/chronicle/tokenizer.py b/uncoder-core/app/translator/platforms/chronicle/tokenizer.py index 5278da4a..a0943952 100644 --- a/uncoder-core/app/translator/platforms/chronicle/tokenizer.py +++ b/uncoder-core/app/translator/platforms/chronicle/tokenizer.py @@ -21,8 +21,8 @@ from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType -from app.translator.core.models.field import FieldValue -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.chronicle.escape_manager import chronicle_escape_manager from app.translator.tools.utils import get_match_group diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index 1b0a3008..0696e2ba 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -24,10 +24,10 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import FieldValue -from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index 95dbc40a..a38b8a64 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -20,16 +20,15 @@ from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.context_vars import return_only_first_query_ctx_var +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import LogicalOperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.exceptions.core import StrictPlatformException from app.translator.core.exceptions.render import BaseRenderException from app.translator.core.mapping import LogSourceSignature, SourceMapping -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logrhythm_axon.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_axon_query_details @@ -218,7 +217,7 @@ def _finalize_search_query(query: str) -> str: def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 return str(log_source_signature) - def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue) and token.field: try: mapped_fields = self.map_field(token.field, source_mapping) @@ -242,30 +241,23 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp return super().apply_token(token, source_mapping) - def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: - queries_map = {} - source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) - - for source_mapping in source_mappings: - prefix = self.generate_prefix(source_mapping.log_source_signature) - if "product" in query_container.meta_info.parsed_logsources: - prefix = f"{prefix} CONTAINS {query_container.meta_info.parsed_logsources['product'][0]}" - else: - prefix = f"{prefix} CONTAINS anything" - - result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) - rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) - not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported - finalized_query = self.finalize_query( - prefix=prefix, - query=result, - functions=rendered_functions.rendered, - not_supported_functions=not_supported_functions, - meta_info=query_container.meta_info, - source_mapping=source_mapping, - ) - if return_only_first_query_ctx_var.get() is True: - return finalized_query - queries_map[source_mapping.source_id] = finalized_query - - return self.finalize(queries_map) + def _generate_from_tokenized_query_container_by_source_mapping( + self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping + ) -> str: + prefix = self.generate_prefix(source_mapping.log_source_signature) + if "product" in query_container.meta_info.parsed_logsources: + prefix = f"{prefix} CONTAINS {query_container.meta_info.parsed_logsources['product'][0]}" + else: + prefix = f"{prefix} CONTAINS anything" + + result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) + rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) + not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported + return self.finalize_query( + prefix=prefix, + query=result, + functions=rendered_functions.rendered, + not_supported_functions=not_supported_functions, + meta_info=query_container.meta_info, + source_mapping=source_mapping, + ) diff --git a/uncoder-core/app/translator/platforms/logscale/tokenizer.py b/uncoder-core/app/translator/platforms/logscale/tokenizer.py index c765c8a9..9c7c33e5 100644 --- a/uncoder-core/app/translator/platforms/logscale/tokenizer.py +++ b/uncoder-core/app/translator/platforms/logscale/tokenizer.py @@ -17,13 +17,13 @@ """ import re -from typing import Any, ClassVar, Optional, Union +from typing import Any, ClassVar, Optional +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.mixins.logic import ANDLogicOperatorMixin -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.logscale.escape_manager import logscale_escape_manager from app.translator.tools.utils import get_match_group @@ -71,6 +71,6 @@ def _get_next_token(self, query: str) -> (list, str): return super()._get_next_token(query) - def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: + def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]: tokens = super().tokenize(query=query) return self.add_and_token_if_missed(tokens=tokens) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py index 09cd5b62..c5c67ed4 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py @@ -21,12 +21,12 @@ import json from typing import Optional, Union +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer +from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.managers import render_manager from app.translator.platforms.opensearch.const import OPENSEARCH_RULE, opensearch_rule_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings @@ -78,7 +78,7 @@ def finalize_query( rule_str = json.dumps(rule, indent=4, sort_keys=False) return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) - def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue) and token.field: for field in self.map_field(token.field, source_mapping): self.fields.update({field: f"{{ctx.results.0.hits.hits.0._source.{field}}}"}) diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index 31bdd1e0..14dc6498 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -16,17 +16,19 @@ limitations under the License. ----------------------------------------------------------------- """ - +from contextlib import suppress from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.context_vars import preset_log_source_str_ctx_var +from app.translator.core.const import QUERY_TOKEN_TYPE +from app.translator.core.context_vars import preset_log_source_str_ctx_var, return_only_first_query_ctx_var from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType -from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.exceptions.core import StrictPlatformException +from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager @@ -207,7 +209,7 @@ def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, fun log_source_str = preset_log_source_str_ctx_var.get() or str(log_source_signature) return f"{functions_prefix}{log_source_str}" - def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue) and token.field: field_name = token.field.source_name if values_map := SOURCE_MAPPING_TO_FIELD_VALUE_MAP.get(source_mapping.source_id, {}).get(field_name): @@ -223,3 +225,32 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp @staticmethod def _finalize_search_query(query: str) -> str: return f"| filter {query}" if query else "" + + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + queries_map = {} + errors = [] + source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) + + last_mapping_index = len(source_mappings) - 1 + for index, source_mapping in enumerate(source_mappings): + try: + finalized_query = self._generate_from_tokenized_query_container_by_source_mapping( + query_container, source_mapping + ) + if return_only_first_query_ctx_var.get() is True: + return finalized_query + queries_map[source_mapping.source_id] = finalized_query + except StrictPlatformException as err: + errors.append(err) + if index != last_mapping_index or source_mapping.source_id == DEFAULT_MAPPING_NAME or queries_map: + continue + + with suppress(StrictPlatformException): + finalized_query = self._generate_from_tokenized_query_container_by_source_mapping( + query_container, self.mappings.get_source_mapping(DEFAULT_MAPPING_NAME) + ) + queries_map[source_mapping.source_id] = finalized_query + + if not queries_map and errors: + raise errors[0] + return self.finalize(queries_map) diff --git a/uncoder-core/app/translator/platforms/sigma/models/compiler.py b/uncoder-core/app/translator/platforms/sigma/models/compiler.py index 2c0b6472..c6092498 100644 --- a/uncoder-core/app/translator/platforms/sigma/models/compiler.py +++ b/uncoder-core/app/translator/platforms/sigma/models/compiler.py @@ -19,8 +19,9 @@ from typing import Union from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.keyword import Keyword +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.platforms.sigma.models.group import Group from app.translator.platforms.sigma.models.operator import NOT, Operator diff --git a/uncoder-core/app/translator/platforms/sigma/models/modifiers.py b/uncoder-core/app/translator/platforms/sigma/models/modifiers.py index 446eb310..fa98c8ce 100644 --- a/uncoder-core/app/translator/platforms/sigma/models/modifiers.py +++ b/uncoder-core/app/translator/platforms/sigma/models/modifiers.py @@ -1,8 +1,8 @@ from typing import ClassVar, Optional, Union from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType -from app.translator.core.models.field import FieldValue -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.str_value_manager import StrValue from app.translator.platforms.sigma.str_value_manager import sigma_str_value_manager diff --git a/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py b/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py index 9f2fd7ab..5dd16651 100644 --- a/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py @@ -21,7 +21,8 @@ from app.translator.core.exceptions.core import SigmaRuleValidationException from app.translator.core.mixins.rule import YamlRuleMixin -from app.translator.core.models.field import Field, FieldValue +from app.translator.core.models.query_tokens.field import Field +from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import QueryParser diff --git a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py index 856fd4a3..9eaae45c 100644 --- a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py @@ -24,7 +24,8 @@ from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping -from app.translator.core.models.field import FieldValue, Keyword +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.render import QueryRender diff --git a/uncoder-core/app/translator/platforms/sigma/tokenizer.py b/uncoder-core/app/translator/platforms/sigma/tokenizer.py index 0893588f..faa0970a 100644 --- a/uncoder-core/app/translator/platforms/sigma/tokenizer.py +++ b/uncoder-core/app/translator/platforms/sigma/tokenizer.py @@ -21,8 +21,9 @@ from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType from app.translator.core.exceptions.parser import TokenizerGeneralException -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.keyword import Keyword +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.platforms.sigma.models.modifiers import ModifierManager From 0b092b4e8a1d623348f8b523e970953379c42bac Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Mon, 8 Jul 2024 16:12:14 +0300 Subject: [PATCH 3/5] fixes --- .../translator/core/models/query_tokens/keyword.py | 14 ++++---------- .../translator/core/models/query_tokens/value.py | 8 ++++---- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/uncoder-core/app/translator/core/models/query_tokens/keyword.py b/uncoder-core/app/translator/core/models/query_tokens/keyword.py index 4e753c51..09382791 100644 --- a/uncoder-core/app/translator/core/models/query_tokens/keyword.py +++ b/uncoder-core/app/translator/core/models/query_tokens/keyword.py @@ -2,22 +2,16 @@ from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.value import Value -class Keyword: +class Keyword(Value): def __init__(self, value: Union[str, list[str]]): + super().__init__(value) self.operator: Identifier = Identifier(token_type=OperatorType.KEYWORD) self.name = "keyword" - self.values = [] - self.__add_value(value=value) - @property - def value(self) -> Union[str, list[str]]: - if isinstance(self.values, list) and len(self.values) == 1: - return self.values[0] - return self.values - - def __add_value(self, value: Union[str, list[str]]) -> None: + def _add_value(self, value: Union[str, list[str]]) -> None: if value and isinstance(value, (list, tuple)): self.values.extend(value) elif value and isinstance(value, str): diff --git a/uncoder-core/app/translator/core/models/query_tokens/value.py b/uncoder-core/app/translator/core/models/query_tokens/value.py index 82b37167..d3d77eb0 100644 --- a/uncoder-core/app/translator/core/models/query_tokens/value.py +++ b/uncoder-core/app/translator/core/models/query_tokens/value.py @@ -7,7 +7,7 @@ class Value: def __init__(self, value: Union[bool, int, str, StrValue, list, tuple], cast_to_int: bool = False): self.values = [] self.__cast_to_int = cast_to_int - self.__add_value(value) + self._add_value(value) @property def value(self) -> Union[bool, int, str, StrValue, list[Union[int, str, StrValue]]]: @@ -18,12 +18,12 @@ def value(self) -> Union[bool, int, str, StrValue, list[Union[int, str, StrValue @value.setter def value(self, new_value: Union[bool, int, str, StrValue, list[Union[int, str, StrValue]]]) -> None: self.values = [] - self.__add_value(new_value) + self._add_value(new_value) - def __add_value(self, value: Optional[Union[bool, int, str, StrValue, list, tuple]]) -> None: + def _add_value(self, value: Optional[Union[bool, int, str, StrValue, list, tuple]]) -> None: if value and isinstance(value, (list, tuple)): for v in value: - self.__add_value(v) + self._add_value(v) elif value and isinstance(value, str) and value.isnumeric() and self.__cast_to_int: self.values.append(int(value)) elif value is not None and isinstance(value, (bool, int, str)): From 7e8169e942aae7660a25d9126b43a71e380242bd Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Tue, 9 Jul 2024 11:53:23 +0300 Subject: [PATCH 4/5] resolve conflicts --- .../translator/mappings/platforms/palo_alto_cortex/default.yml | 1 + .../app/translator/mappings/platforms/qradar/default.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml index fa904aaf..ac3f8c9c 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml @@ -126,3 +126,4 @@ field_mapping: DestinationOS: xdm.target.host.os url_category: xdm.network.http.url_category EventSeverity: xdm.alert.severity + duration: xdm.event.duration diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml index 00dcef55..1e098a77 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml @@ -77,4 +77,5 @@ field_mapping: EventSeverity: EventSeverity Source: - Source - - source \ No newline at end of file + - source + duration: duration \ No newline at end of file From f0d2b7614b904e089251860c79b3a17f23ee783f Mon Sep 17 00:00:00 2001 From: "oleksandr.volha" Date: Wed, 10 Jul 2024 11:52:49 +0300 Subject: [PATCH 5/5] fixes --- .../translator/core/custom_types/functions.py | 1 - uncoder-core/app/translator/core/functions.py | 16 ++-------------- .../app/translator/core/models/functions/base.py | 16 ++++++++++++---- .../app/translator/core/models/functions/bin.py | 12 +++--------- .../app/translator/core/models/functions/eval.py | 4 ++-- .../translator/core/models/functions/group_by.py | 2 +- .../app/translator/core/models/functions/join.py | 4 ++-- .../translator/core/models/functions/rename.py | 2 +- .../app/translator/core/models/functions/sort.py | 2 +- .../models/functions/{timeframe.py => time.py} | 8 +------- .../platforms/base/aql/functions/__init__.py | 2 +- .../platforms/base/aql/functions/const.py | 1 + .../platforms/palo_alto/functions/const.py | 2 -- 13 files changed, 27 insertions(+), 45 deletions(-) rename uncoder-core/app/translator/core/models/functions/{timeframe.py => time.py} (68%) diff --git a/uncoder-core/app/translator/core/custom_types/functions.py b/uncoder-core/app/translator/core/custom_types/functions.py index 17452c5b..8cecc010 100644 --- a/uncoder-core/app/translator/core/custom_types/functions.py +++ b/uncoder-core/app/translator/core/custom_types/functions.py @@ -22,7 +22,6 @@ class FunctionType(CustomEnum): upper = "upper" array_length = "array_length" - compare = "compare" extract_time = "extract_time" ipv4_is_in_range = "ipv4_is_in_range" diff --git a/uncoder-core/app/translator/core/functions.py b/uncoder-core/app/translator/core/functions.py index 924b8d53..2517129b 100644 --- a/uncoder-core/app/translator/core/functions.py +++ b/uncoder-core/app/translator/core/functions.py @@ -25,8 +25,8 @@ from app.translator.core.exceptions.functions import NotSupportedFunctionException from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import Alias, Field from app.translator.core.models.functions.base import Function, ParsedFunctions, RenderedFunctions +from app.translator.core.models.query_tokens.field import Alias, Field from app.translator.tools.utils import execute_module from settings import INIT_FUNCTIONS @@ -83,7 +83,6 @@ def parse(self, func_body: str, raw: str) -> Function: class FunctionRender(ABC): function_names_map: ClassVar[dict[str, str]] = {} order_to_render: int = 0 - in_query_render: bool = False render_to_prefix: bool = False manager: PlatformFunctionsManager = None @@ -117,7 +116,6 @@ def __init__(self): self._parsers_map: dict[str, FunctionParser] = {} # {platform_func_name: FunctionParser} self._renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender} - self._in_query_renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender} self._order_to_render: dict[str, int] = {} # {generic_func_name: int} def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRender]: @@ -126,8 +124,6 @@ def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRe for generic_function_name in render.function_names_map: self._renders_map[generic_function_name] = render self._order_to_render[generic_function_name] = render.order_to_render - if render.in_query_render: - self._in_query_renders_map[generic_function_name] = render return render_class @@ -149,24 +145,16 @@ def get_hof_parser(self, platform_func_name: str) -> HigherOrderFunctionParser: raise NotSupportedFunctionException - def get_parser(self, platform_func_name: str) -> FunctionParser: + def get_parser(self, platform_func_name: str) -> Optional[FunctionParser]: if INIT_FUNCTIONS and (parser := self._parsers_map.get(platform_func_name)): return parser - raise NotSupportedFunctionException - def get_render(self, generic_func_name: str) -> FunctionRender: if INIT_FUNCTIONS and (render := self._renders_map.get(generic_func_name)): return render raise NotSupportedFunctionException - def get_in_query_render(self, generic_func_name: str) -> FunctionRender: - if INIT_FUNCTIONS and (render := self._in_query_renders_map.get(generic_func_name)): - return render - - raise NotSupportedFunctionException - @property def order_to_render(self) -> dict[str, int]: if INIT_FUNCTIONS: diff --git a/uncoder-core/app/translator/core/models/functions/base.py b/uncoder-core/app/translator/core/models/functions/base.py index 187a92c2..05fe7535 100644 --- a/uncoder-core/app/translator/core/models/functions/base.py +++ b/uncoder-core/app/translator/core/models/functions/base.py @@ -1,16 +1,24 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Optional, Union +from typing import TYPE_CHECKING, Optional, Union -from app.translator.core.models.field import Alias, Field, FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field import Alias, Field +from app.translator.core.models.query_tokens.field_field import FieldField +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword + +if TYPE_CHECKING: + from app.translator.core.models.query_tokens.function_value import FunctionValue @dataclass class Function: name: str = None - args: list[Union[Alias, Field, FieldValue, Keyword, Function, Identifier, str, bool]] = field(default_factory=list) + args: list[ + Union[Alias, Field, FieldField, FieldValue, FunctionValue, Keyword, Function, Identifier, int, str, bool] + ] = field(default_factory=list) alias: Optional[Alias] = None raw: str = "" diff --git a/uncoder-core/app/translator/core/models/functions/bin.py b/uncoder-core/app/translator/core/models/functions/bin.py index a54884e6..828fb891 100644 --- a/uncoder-core/app/translator/core/models/functions/bin.py +++ b/uncoder-core/app/translator/core/models/functions/bin.py @@ -2,21 +2,15 @@ from typing import Optional from app.translator.core.custom_types.functions import FunctionType -from app.translator.core.models.field import Field +from app.translator.core.custom_types.time import TimeFrameType from app.translator.core.models.functions.base import Function -from app.translator.tools.custom_enum import CustomEnum - - -class SpanType(CustomEnum): - days = "days" - hours = "hours" - minutes = "minutes" +from app.translator.core.models.query_tokens.field import Field @dataclass class Span: value: str = "1" - type_: str = SpanType.days + type_: str = TimeFrameType.days @dataclass diff --git a/uncoder-core/app/translator/core/models/functions/eval.py b/uncoder-core/app/translator/core/models/functions/eval.py index 6e32449f..5632870e 100644 --- a/uncoder-core/app/translator/core/models/functions/eval.py +++ b/uncoder-core/app/translator/core/models/functions/eval.py @@ -2,9 +2,9 @@ from typing import Union from app.translator.core.custom_types.functions import FunctionType -from app.translator.core.models.field import Alias, Field from app.translator.core.models.functions.base import Function -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field import Alias, Field +from app.translator.core.models.query_tokens.identifier import Identifier @dataclass diff --git a/uncoder-core/app/translator/core/models/functions/group_by.py b/uncoder-core/app/translator/core/models/functions/group_by.py index 04b3d4e6..ef1fa745 100644 --- a/uncoder-core/app/translator/core/models/functions/group_by.py +++ b/uncoder-core/app/translator/core/models/functions/group_by.py @@ -2,8 +2,8 @@ from typing import Union from app.translator.core.custom_types.functions import FunctionType -from app.translator.core.models.field import Alias from app.translator.core.models.functions.base import Function +from app.translator.core.models.query_tokens.field import Alias @dataclass diff --git a/uncoder-core/app/translator/core/models/functions/join.py b/uncoder-core/app/translator/core/models/functions/join.py index 0f44da68..cd1ed4db 100644 --- a/uncoder-core/app/translator/core/models/functions/join.py +++ b/uncoder-core/app/translator/core/models/functions/join.py @@ -2,10 +2,10 @@ from typing import Union from app.translator.core.custom_types.functions import FunctionType -from app.translator.core.models.field import Alias, Field from app.translator.core.models.functions.base import Function -from app.translator.core.models.identifier import Identifier from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.core.models.query_tokens.field import Alias, Field +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.tools.custom_enum import CustomEnum diff --git a/uncoder-core/app/translator/core/models/functions/rename.py b/uncoder-core/app/translator/core/models/functions/rename.py index 06455e05..5a9dba6a 100644 --- a/uncoder-core/app/translator/core/models/functions/rename.py +++ b/uncoder-core/app/translator/core/models/functions/rename.py @@ -1,8 +1,8 @@ from dataclasses import dataclass from app.translator.core.custom_types.functions import FunctionType -from app.translator.core.models.field import Alias, Field from app.translator.core.models.functions.base import Function +from app.translator.core.models.query_tokens.field import Alias, Field @dataclass diff --git a/uncoder-core/app/translator/core/models/functions/sort.py b/uncoder-core/app/translator/core/models/functions/sort.py index e35646dc..63993401 100644 --- a/uncoder-core/app/translator/core/models/functions/sort.py +++ b/uncoder-core/app/translator/core/models/functions/sort.py @@ -2,8 +2,8 @@ from typing import Union from app.translator.core.custom_types.functions import FunctionType -from app.translator.core.models.field import Alias, Field from app.translator.core.models.functions.base import Function +from app.translator.core.models.query_tokens.field import Alias, Field from app.translator.tools.custom_enum import CustomEnum diff --git a/uncoder-core/app/translator/core/models/functions/timeframe.py b/uncoder-core/app/translator/core/models/functions/time.py similarity index 68% rename from uncoder-core/app/translator/core/models/functions/timeframe.py rename to uncoder-core/app/translator/core/models/functions/time.py index b9fedc82..eb6a1229 100644 --- a/uncoder-core/app/translator/core/models/functions/timeframe.py +++ b/uncoder-core/app/translator/core/models/functions/time.py @@ -1,14 +1,8 @@ from dataclasses import dataclass from app.translator.core.custom_types.functions import FunctionType +from app.translator.core.custom_types.time import TimeFrameType from app.translator.core.models.functions.base import Function -from app.translator.tools.custom_enum import CustomEnum - - -class TimeFrameType(CustomEnum): - days = "days" - hours = "hours" - minutes = "minutes" @dataclass diff --git a/uncoder-core/app/translator/platforms/base/aql/functions/__init__.py b/uncoder-core/app/translator/platforms/base/aql/functions/__init__.py index 3aed1306..813ec885 100644 --- a/uncoder-core/app/translator/platforms/base/aql/functions/__init__.py +++ b/uncoder-core/app/translator/platforms/base/aql/functions/__init__.py @@ -23,9 +23,9 @@ from app.translator.core.custom_types.functions import FunctionType from app.translator.core.exceptions.functions import InvalidFunctionSignature, NotSupportedFunctionException from app.translator.core.functions import PlatformFunctions -from app.translator.core.models.field import Field from app.translator.core.models.functions.base import Function, ParsedFunctions from app.translator.core.models.functions.sort import SortLimitFunction +from app.translator.core.models.query_tokens.field import Field from app.translator.platforms.base.aql.const import TABLE_PATTERN from app.translator.platforms.base.aql.functions.const import ( AGGREGATION_FUNCTIONS_MAP, diff --git a/uncoder-core/app/translator/platforms/base/aql/functions/const.py b/uncoder-core/app/translator/platforms/base/aql/functions/const.py index 141820b4..e9481cc0 100644 --- a/uncoder-core/app/translator/platforms/base/aql/functions/const.py +++ b/uncoder-core/app/translator/platforms/base/aql/functions/const.py @@ -24,6 +24,7 @@ class AQLFunctionType(CustomEnum): class AQLFunctionGroupType(CustomEnum): agg = "agg" + str_conversion = "str_conversion" class AQLSortOrderType(CustomEnum): diff --git a/uncoder-core/app/translator/platforms/palo_alto/functions/const.py b/uncoder-core/app/translator/platforms/palo_alto/functions/const.py index 8009a40e..95bb3982 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/functions/const.py +++ b/uncoder-core/app/translator/platforms/palo_alto/functions/const.py @@ -31,8 +31,6 @@ class CortexXQLFunctionType(CustomEnum): timeframe = "timeframe" union = "union" - compare = "compare" - class XqlSortOrderType(CustomEnum): asc = "asc"