fix conflicts

Author: dtsprm

uncoder-core/app/translator/core/custom_types/functions.py

@@ -9,25 +9,33 @@ class FunctionType(CustomEnum):
     min = "min"
     sum = "sum"
 
-    divide = "divide"
+    values = "values"
 
     earliest = "earliest"
     latest = "latest"
 
+    divide = "divide"
+    multiply = "multiply"
+
     lower = "lower"
+    split = "split"
     upper = "upper"
 
-    compare_boolean = "compare_boolean"
-
+    array_length = "array_length"
+    extract_time = "extract_time"
     ipv4_is_in_range = "ipv4_is_in_range"
 
     bin = "bin"
     eval = "eval"
     fields = "fields"
+    iploc = "iploc"
+    join = "join"
     rename = "rename"
     search = "search"
     sort_limit = "sort_limit"
     stats = "stats"
     table = "table"
     timeframe = "timeframe"
-    values = "values"
+    union = "union"
+
+    aggregation_data_function = "aggregation_data_function"

uncoder-core/app/translator/core/custom_types/predefined_fields.py

@@ -10,3 +10,7 @@ class IPLocationType(CustomEnum):
     lat_lon = "ip_loc_lat_lon"
     region = "ip_loc_region"
     timezone = "ip_loc_timezone"
+
+
+class TimeType(CustomEnum):
+    timestamp = "timestamp"

uncoder-core/app/translator/core/exceptions/core.py

@@ -1,10 +1,6 @@
 from typing import Optional
 
 
-class NotImplementedException(BaseException):
-    ...
-
-
 class BasePlatformException(BaseException):
     ...
 

uncoder-core/app/translator/core/exceptions/render.py

@@ -14,5 +14,5 @@ class FunctionRenderException(BaseRenderException):
 
 class UnsupportedRenderMethod(BaseRenderException):
     def __init__(self, platform_name: str, method: str):
-        message = f"Cannot translate. {platform_name} backend does not support {method}."
+        message = f'Cannot translate. {platform_name} backend does not support "{method}".'
         super().__init__(message)

uncoder-core/app/translator/core/functions.py

@@ -21,13 +21,13 @@
 import re
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from functools import cached_property
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
 
-from app.translator.core.exceptions.functions import InvalidFunctionSignature, NotSupportedFunctionException
+from app.translator.core.exceptions.functions import NotSupportedFunctionException
 from app.translator.core.mapping import SourceMapping
-from app.translator.core.models.field import Field
 from app.translator.core.models.functions.base import Function, ParsedFunctions, RenderedFunctions
+from app.translator.core.models.query_tokens.field import Alias, Field, PredefinedField
+from app.translator.tools.utils import execute_module
 from settings import INIT_FUNCTIONS
 
 if TYPE_CHECKING:
@@ -41,6 +41,14 @@ class FunctionMatchContainer:
 
 
 class BaseFunctionParser(ABC):
+    function_names_map: ClassVar[dict[str, str]] = {}
+    functions_group_name: str = None
+    manager: PlatformFunctionsManager = None
+
+    def set_functions_manager(self, manager: PlatformFunctionsManager) -> BaseFunctionParser:
+        self.manager = manager
+        return self
+
     @abstractmethod
     def parse(self, *args, **kwargs) -> Function:
         raise NotImplementedError
@@ -73,72 +81,81 @@ def parse(self, func_body: str, raw: str) -> Function:
 
 
 class FunctionRender(ABC):
+    function_names_map: ClassVar[dict[str, str]] = {}
+    order_to_render: int = 0
+    render_to_prefix: bool = False
+    manager: PlatformFunctionsManager = None
+
+    def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRender:
+        self.manager = manager
+        return self
+
     @abstractmethod
     def render(self, function: Function, source_mapping: SourceMapping) -> str:
         raise NotImplementedError
 
-    @staticmethod
-    def concat_kwargs(kwargs: dict[str, str]) -> str:
-        result = ""
-        for key, value in kwargs.items():
-            if value:
-                result = f"{result}, {key}={value}" if result else f"{key}={value}"
+    def map_field(self, field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
+        if isinstance(field, Alias):
+            return field.name
 
-        return result
+        if isinstance(field, Field):
+            mappings = self.manager.platform_functions.platform_query_render.mappings
+            mapped_fields = mappings.map_field(field, source_mapping)
+            return mapped_fields[0]
 
-    @staticmethod
-    def map_field(field: Field, source_mapping: SourceMapping) -> str:
-        generic_field_name = field.get_generic_field_name(source_mapping.source_id)
-        mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
-        if isinstance(mapped_field, list):
-            mapped_field = mapped_field[0]
+        if isinstance(field, PredefinedField):
+            return self.manager.platform_functions.platform_query_render.map_predefined_field(field)
 
-        return mapped_field if mapped_field else field.source_name
+        raise NotSupportedFunctionException
 
 
 class PlatformFunctionsManager:
+    platform_functions: PlatformFunctions = None
+
     def __init__(self):
-        self._parsers_map: dict[str, HigherOrderFunctionParser] = {}
-        self._renders_map: dict[str, FunctionRender] = {}
-        self._in_query_renders_map: dict[str, FunctionRender] = {}
-        self._names_map: dict[str, str] = {}
-        self._order_to_render: dict[str, int] = {}
-        self._render_to_prefix_functions: list[str] = []
-
-    def post_init_configure(self, platform_render: PlatformQueryRender) -> None:
-        raise NotImplementedError
+        # {platform_func_name: HigherOrderFunctionParser}
+        self._hof_parsers_map: dict[str, HigherOrderFunctionParser] = {}
+        self._parsers_map: dict[str, FunctionParser] = {}  # {platform_func_name: FunctionParser}
 
-    @cached_property
-    def _inverted_names_map(self) -> dict[str, str]:
-        return {value: key for key, value in self._names_map.items()}
+        self._renders_map: dict[str, FunctionRender] = {}  # {generic_func_name: FunctionRender}
+        self._order_to_render: dict[str, int] = {}  # {generic_func_name: int}
 
-    def get_parser(self, generic_func_name: str) -> HigherOrderFunctionParser:
-        if INIT_FUNCTIONS and (parser := self._parsers_map.get(generic_func_name)):
-            return parser
+    def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRender]:
+        render = render_class()
+        render.manager = self
+        for generic_function_name in render.function_names_map:
+            self._renders_map[generic_function_name] = render
+            self._order_to_render[generic_function_name] = render.order_to_render
 
-        raise NotSupportedFunctionException
+        return render_class
 
-    def get_render(self, generic_func_name: str) -> FunctionRender:
-        if INIT_FUNCTIONS and (render := self._renders_map.get(generic_func_name)):
-            return render
+    def register_parser(self, parser_class: type[BaseFunctionParser]) -> type[BaseFunctionParser]:
+        parser = parser_class()
+        parser.manager = self
+        parsers_map = self._hof_parsers_map if isinstance(parser, HigherOrderFunctionParser) else self._parsers_map
+        for platform_function_name in parser.function_names_map:
+            parsers_map[platform_function_name] = parser
 
-        raise NotSupportedFunctionException
+        if parser.functions_group_name:
+            parsers_map[parser.functions_group_name] = parser
 
-    def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
-        if INIT_FUNCTIONS and (render := self._in_query_renders_map.get(generic_func_name)):
-            return render
+        return parser_class
+
+    def get_hof_parser(self, platform_func_name: str) -> HigherOrderFunctionParser:
+        if INIT_FUNCTIONS and (parser := self._hof_parsers_map.get(platform_func_name)):
+            return parser
 
         raise NotSupportedFunctionException
 
-    def get_generic_func_name(self, platform_func_name: str) -> Optional[str]:
-        if INIT_FUNCTIONS and (generic_func_name := self._names_map.get(platform_func_name)):
-            return generic_func_name
+    def get_parser(self, platform_func_name: str) -> Optional[FunctionParser]:
+        if INIT_FUNCTIONS and (parser := self._parsers_map.get(platform_func_name)):
+            return parser
 
-        raise NotSupportedFunctionException
+    def get_render(self, generic_func_name: str) -> FunctionRender:
+        if INIT_FUNCTIONS and (render := self._renders_map.get(generic_func_name)):
+            return render
 
-    def get_platform_func_name(self, generic_func_name: str) -> Optional[str]:
-        if INIT_FUNCTIONS:
-            return self._inverted_names_map.get(generic_func_name)
+        raise NotSupportedFunctionException
 
     @property
     def order_to_render(self) -> dict[str, int]:
@@ -147,39 +164,22 @@ def order_to_render(self) -> dict[str, int]:
 
         return {}
 
-    @property
-    def render_to_prefix_functions(self) -> list[str]:
-        if INIT_FUNCTIONS:
-            return self._render_to_prefix_functions
-
-        return []
-
 
 class PlatformFunctions:
+    dir_path: str = None
+    platform_query_render: PlatformQueryRender = None
     manager: PlatformFunctionsManager = PlatformFunctionsManager()
+
     function_delimiter = "|"
 
-    def parse(self, query: str) -> ParsedFunctions:
-        parsed = []
-        not_supported = []
-        invalid = []
-        functions = query.split(self.function_delimiter)
-        for func in functions:
-            split_func = func.strip().split(" ")
-            func_name, func_body = split_func[0], " ".join(split_func[1:])
-            try:
-                func_parser = self.manager.get_parser(self.manager.get_generic_func_name(func_name))
-                parsed.append(func_parser.parse(func_body, func))
-            except NotSupportedFunctionException:
-                not_supported.append(func)
-            except InvalidFunctionSignature:
-                invalid.append(func)
+    def __init__(self):
+        self.manager.platform_functions = self
+        if self.dir_path:
+            execute_module(f"{self.dir_path}/parsers/__init__.py")
+            execute_module(f"{self.dir_path}/renders/__init__.py")
 
-        return ParsedFunctions(
-            functions=parsed,
-            not_supported=[self.wrap_function_with_delimiter(func) for func in not_supported],
-            invalid=invalid,
-        )
+    def parse(self, query: str) -> ParsedFunctions:  # noqa: ARG002
+        return ParsedFunctions()
 
     def _sort_functions_to_render(self, functions: list[Function]) -> list[Function]:
         return sorted(functions, key=lambda func: self.manager.order_to_render.get(func.name, 0))
@@ -193,7 +193,7 @@ def render(self, functions: list[Function], source_mapping: SourceMapping) -> Re
             try:
                 func_render = self.manager.get_render(func.name)
                 _rendered = func_render.render(func, source_mapping)
-                if func.name in self.manager.render_to_prefix_functions:
+                if func_render.render_to_prefix:
                     rendered_prefix += _rendered
                 else:
                     rendered += self.wrap_function_with_delimiter(_rendered)

uncoder-core/app/translator/core/mapping.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Optional, TypeVar
+from typing import TYPE_CHECKING, Optional, TypeVar, Union
 
 from app.translator.core.exceptions.core import StrictPlatformException
 from app.translator.core.models.platform_details import PlatformDetails
@@ -19,9 +19,14 @@ class LogSourceSignature(ABC):
     wildcard_symbol = "*"
 
     @abstractmethod
-    def is_suitable(self, *args, **kwargs) -> bool:
+    def is_suitable(self, **kwargs) -> bool:
         raise NotImplementedError("Abstract method")
 
+    @staticmethod
+    def _check_conditions(conditions: list[Union[bool, None]]) -> bool:
+        conditions = [condition for condition in conditions if condition is not None]
+        return bool(conditions) and all(conditions)
+
     @abstractmethod
     def __str__(self) -> str:
         raise NotImplementedError("Abstract method")
@@ -70,7 +75,7 @@ def update(self, fields_mapping: FieldsMapping) -> None:
         self.__render_mapping.update(fields_mapping.__render_mapping)
 
     def is_suitable(self, field_names: list[str]) -> bool:
-        return set(field_names).issubset(set(self.__parser_mapping.keys()))
+        return bool(field_names) and set(field_names).issubset(set(self.__parser_mapping.keys()))
 
 
 _LogSourceSignatureType = TypeVar("_LogSourceSignatureType", bound=LogSourceSignature)
@@ -147,9 +152,23 @@ def prepare_fields_mapping(field_mapping: dict) -> FieldsMapping:
     def prepare_log_source_signature(self, mapping: dict) -> LogSourceSignature:
         raise NotImplementedError("Abstract method")
 
-    @abstractmethod
-    def get_suitable_source_mappings(self, *args, **kwargs) -> list[SourceMapping]:
-        raise NotImplementedError("Abstract method")
+    def get_suitable_source_mappings(
+        self, field_names: list[str], log_sources: dict[str, list[Union[int, str]]]
+    ) -> list[SourceMapping]:
+        by_log_sources_and_fields = []
+        by_fields = []
+        for source_mapping in self._source_mappings.values():
+            if source_mapping.source_id == DEFAULT_MAPPING_NAME:
+                continue
+
+            if source_mapping.fields_mapping.is_suitable(field_names):
+                by_fields.append(source_mapping)
+
+                log_source_signature: LogSourceSignature = source_mapping.log_source_signature
+                if log_source_signature and log_source_signature.is_suitable(**log_sources):
+                    by_log_sources_and_fields.append(source_mapping)
+
+        return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]]
 
     def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
         return self._source_mappings.get(source_id)