diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index 0ecccbc1..97726ae9 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -72,7 +72,7 @@ def __init__( source_id: str, log_source_signature: _LogSourceSignatureType = None, fields_mapping: Optional[FieldsMapping] = None, - raw_log_fields: Optional[list] = None, + raw_log_fields: Optional[dict] = None, ): self.source_id = source_id self.log_source_signature = log_source_signature @@ -103,7 +103,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]: continue field_mappings_dict = mapping_dict.get("field_mapping", {}) - raw_log_fields = mapping_dict.get("raw_log_fields", []) + raw_log_fields = mapping_dict.get("raw_log_fields", {}) field_mappings_dict.update({field: field for field in raw_log_fields}) fields_mapping = self.prepare_fields_mapping(field_mapping=field_mappings_dict) self.update_default_source_mapping(default_mapping=default_mapping, fields_mapping=fields_mapping) diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 81dec670..06aba5c0 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -21,6 +21,7 @@ from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -192,7 +193,7 @@ class PlatformQueryRender(QueryRender): field_value_map = BaseQueryFieldValue(or_token=or_token) query_pattern = "{table} {query} {functions}" - raw_log_field_pattern: str = None + raw_log_field_pattern_map: dict = None def __init__(self): self.operator_map = { @@ -283,6 +284,7 @@ def finalize_query( **kwargs, # noqa: ARG002 ) -> str: query = self.query_pattern.format(prefix=prefix, query=query, functions=functions).strip() + query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) if not_supported_functions: rendered_not_supported = self.render_not_supported_functions(not_supported_functions) @@ -323,6 +325,16 @@ def _generate_from_raw_query_container(self, query_container: RawQueryContainer) prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info ) + def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]: + if raw_log_field_pattern := self.raw_log_field_pattern_map.get(field_type): + return raw_log_field_pattern.pattern.format(field=field) + + def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping) -> Optional[str]: + if self.raw_log_field_pattern_map is None: + return + if raw_log_field_type := source_mapping.raw_log_fields.get(field): + return self.process_raw_log_field(field=field, field_type=raw_log_field_type) + def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMapping) -> str: defined_raw_log_fields = [] for field in fields: @@ -334,10 +346,8 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap ) if not mapped_field and self.is_strict_mapping: raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name) - if mapped_field not in source_mapping.raw_log_fields: - continue - field_prefix = self.raw_log_field_pattern.format(field=mapped_field) - defined_raw_log_fields.append(field_prefix) + if field_prefix := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping): + defined_raw_log_fields.append(field_prefix) return "\n".join(set(defined_raw_log_fields)) def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: @@ -368,6 +378,8 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue meta_info=query_container.meta_info, source_mapping=source_mapping, ) + if return_only_first_query_ctx_var.get() is True: + return finalized_query queries_map[source_mapping.source_id] = finalized_query if not queries_map and errors: raise errors[0] diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_cloudtrail.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_cloudtrail.yml new file mode 100644 index 00000000..f8327a54 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_cloudtrail.yml @@ -0,0 +1,35 @@ +platform: Palo Alto XSIAM +source: aws_cloudtrail + + +default_log_source: + dataset: amazon_aws_raw + +field_mapping: + eventSource: eventSource + eventName: eventName + errorCode: errorCode + errorMessage: errorMessage + eventType: eventType + requestParameters: requestParameters + responseElements: responseElements + status: status + terminatingRuleId: terminatingRuleId + userAgent: userAgent + AdditionalEventData.MFAUsed: additionalEventData.MFAUsed + + +raw_log_fields: + additionalEventData.MFAUsed: object + requestParameters.ipPermissions.items.ipRanges.items.cidrIP: object + requestParameters.ipPermissions.items.ipRanges.items.fromPort: object + requestParameters.attribute: object + requestParameters.userData: list + responseElements.ConsoleLogin: object + responseElements.pendingModifiedValues.masterUserPassword: object + responseElements.publiclyAccessible: object + userIdentity.arn: object + userIdentity.principalId: object + userIdentity.sessionContext.sessionIssuer.type: object + userIdentity.type: object + userIdentity.userName: object \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_eks.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_eks.yml new file mode 100644 index 00000000..e7ba2c05 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_eks.yml @@ -0,0 +1,25 @@ +platform: Palo Alto XSIAM +source: aws_eks + + +default_log_source: + dataset: amazon_aws_raw + +field_mapping: + aws_node_type: aws_node_type + requestURI: requestURI + stage: stage + verb: verb + + +raw_log_fields: + annotations.authorization.k8s.io\/decision: object + annotations.podsecuritypolicy.policy.k8s.io\/admit-policy: object + objectRef.namespace: object + objectRef.resource: object + objectRef.subresource: object + requestObject.rules.resources: object + requestObject.rules.verbs: object + requestObject.spec.containers.image: object + user.groups: object + user.username: object \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_aadnoninteractiveusersigninlogs.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_aadnoninteractiveusersigninlogs.yml new file mode 100644 index 00000000..f3e64cc0 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_aadnoninteractiveusersigninlogs.yml @@ -0,0 +1,16 @@ +platform: Palo Alto XSIAM +source: azure_aadnoninteractiveusersigninlogs + + +default_log_source: + dataset: msft_azure_raw + +field_mapping: + UserAgent: properties.userAgent + Type: properties.type + AuthenticationProcessingDetails: properties.authenticationProcessingDetails + +raw_log_fields: + properties.userAgent: object + properties.type: object + properties.authenticationProcessingDetails: object \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_application.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_application.yml index 71143f9c..d40073fd 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_application.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_application.yml @@ -9,16 +9,16 @@ field_mapping: Provider_Name: provider_name raw_log_fields: - - src_ip - - source - - additional_information - - EventData - - Channel - - statement - - Faulting application path - - object_name - - class_type - - action_id - - Data - - Message - - Level + src_ip: regex + source: regex + additional_information: regex + EventData: regex + Channel: regex + statement: regex + Faulting application path: regex + object_name: regex + class_type: regex + action_id: regex + Data: regex + Message: regex + Level: regex diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_network_connection.yml index 6c19d8f2..9c535767 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_network_connection.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_network_connection.yml @@ -55,4 +55,4 @@ field_mapping: raw_log_fields: - - Initiated \ No newline at end of file + Initiated: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_pipe_created.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_pipe_created.yml index 2e7ea732..8deb0974 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_pipe_created.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_pipe_created.yml @@ -8,5 +8,5 @@ field_mapping: EventID: action_evtlog_event_id raw_log_fields: - - PipeName - - Image \ No newline at end of file + PipeName: regex + Image: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_powershell.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_powershell.yml index 6af38835..1d2e0ef4 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_powershell.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_powershell.yml @@ -10,10 +10,10 @@ field_mapping: raw_log_fields: - - CommandLine - - ScriptBlockText - - Payload - - HostApplication - - ContextInfo - - HostName - - EngineVersion \ No newline at end of file + CommandLine: regex + ScriptBlockText: regex + Payload: regex + HostApplication: regex + ContextInfo: regex + HostName: regex + EngineVersion: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_access.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_access.yml index 47a1033e..ab559df0 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_access.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_access.yml @@ -6,14 +6,15 @@ default_log_source: field_mapping: User: action_process_username + SourceUser: action_process_username raw_log_fields: - - SourceProcessGUID - - SourceProcessId - - SourceThreadId - - SourceImage - - TargetProcessGUID - - TargerProcessId - - TargetImage - - GrantedAccess - - CallTrace \ No newline at end of file + SourceProcessGUID: regex + SourceProcessId: regex + SourceThreadId: regex + SourceImage: regex + TargetProcessGUID: regex + TargerProcessId: regex + TargetImage: regex + GrantedAccess: regex + CallTrace: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_security.yml index a2abf004..bb77e430 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_security.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_security.yml @@ -9,140 +9,140 @@ field_mapping: Provider_Name: provider_name raw_log_fields: - - ParentImage - - AccessMask - - AccountName - - AllowedToDelegateTo - - AttributeLDAPDisplayName - - AuditPolicyChanges - - AuthenticationPackageName - - CallingProcessName - - Channel - - ComputerName - - EventType - - FailureReason - - FileName - - GrantedAccess - - Hashes - - HiveName - - IpAddress - - IpPort - - KeyLength - - LogonProcessName - - LogonType - - LinkName - - ProcessId - - PublishURLs - - ElevatedToken - - MemberName - - MemberSid - - NewProcessName - - ObjectClass - - ObjectName - - ObjectType - - ObjectValueName - - Path - - CommandLine - - OldUacValue - - CertIssuerName - - SubStatus - - DisplayName - - TaskContent - - ServiceSid - - CertThumbprint - - ClassName - - NotificationPackageName - - NewSd - - TestSigning - - TargetInfo - - ParentProcessId - - AccessList - - GroupMembership - - FilterName - - ChangeType - - LayerName - - ServiceAccount - - ClientProcessId - - AttributeValue - - SessionName - - TaskName - - ObjectDN - - TemplateContent - - NewTemplateContent - - SourcePort - - PasswordLastSet - - PrivilegeList - - DeviceDescription - - TargetServerName - - NewTargetUserName - - OperationType - - DestPort - - ServiceStartType - - OldTargetUserName - - UserPrincipalName - - Accesses - - DnsHostName - - DisableIntegrityChecks - - AuditSourceName - - Workstation - - DestAddress - - PreAuthType - - SecurityPackageName - - SubjectLogonId - - NewUacValue - - EnabledPrivilegeList - - RelativeTargetName - - CertSerialNumber - - SidHistory - - TargetLogonId - - KernelDebug - - CallerProcessName - - ProcessName - - Properties - - UserAccountControl - - RegistryValue - - SecurityID - - ServiceFileName - - SecurityDescriptor - - ServiceName - - ShareName - - NewValue - - Source - - Status - - SubjectDomainName - - SubjectUserName - - SubjectUserSid - - SourceAddr - - SourceAddress - - TargetName - - ServicePrincipalNames - - TargetDomainName - - TargetSid - - TargetUserName - - ObjectServer - - TargetUserSid - - TicketEncryptionType - - TicketOptions - - WorkstationName - - TransmittedServices - - AuthenticationAlgorithm - - LayerRTID - - BSSID - - BSSType - - CipherAlgorithm - - ConnectionId - - ConnectionMode - - InterfaceDescription - - InterfaceGuid - - OnexEnabled - - PHYType - - ProfileName - - SSID - - Domain - - ServiceType - - SourceName - - StartType - - UserID - - ParentProcessName - - ExceptionCode - - Service \ No newline at end of file + ParentImage: regex + AccessMask: regex + AccountName: regex + AllowedToDelegateTo: regex + AttributeLDAPDisplayName: regex + AuditPolicyChanges: regex + AuthenticationPackageName: regex + CallingProcessName: regex + Channel: regex + ComputerName: regex + EventType: regex + FailureReason: regex + FileName: regex + GrantedAccess: regex + Hashes: regex + HiveName: regex + IpAddress: regex + IpPort: regex + KeyLength: regex + LogonProcessName: regex + LogonType: regex + LinkName: regex + ProcessId: regex + PublishURLs: regex + ElevatedToken: regex + MemberName: regex + MemberSid: regex + NewProcessName: regex + ObjectClass: regex + ObjectName: regex + ObjectType: regex + ObjectValueName: regex + Path: regex + CommandLine: regex + OldUacValue: regex + CertIssuerName: regex + SubStatus: regex + DisplayName: regex + TaskContent: regex + ServiceSid: regex + CertThumbprint: regex + ClassName: regex + NotificationPackageName: regex + NewSd: regex + TestSigning: regex + TargetInfo: regex + ParentProcessId: regex + AccessList: regex + GroupMembership: regex + FilterName: regex + ChangeType: regex + LayerName: regex + ServiceAccount: regex + ClientProcessId: regex + AttributeValue: regex + SessionName: regex + TaskName: regex + ObjectDN: regex + TemplateContent: regex + NewTemplateContent: regex + SourcePort: regex + PasswordLastSet: regex + PrivilegeList: regex + DeviceDescription: regex + TargetServerName: regex + NewTargetUserName: regex + OperationType: regex + DestPort: regex + ServiceStartType: regex + OldTargetUserName: regex + UserPrincipalName: regex + Accesses: regex + DnsHostName: regex + DisableIntegrityChecks: regex + AuditSourceName: regex + Workstation: regex + DestAddress: regex + PreAuthType: regex + SecurityPackageName: regex + SubjectLogonId: regex + NewUacValue: regex + EnabledPrivilegeList: regex + RelativeTargetName: regex + CertSerialNumber: regex + SidHistory: regex + TargetLogonId: regex + KernelDebug: regex + CallerProcessName: regex + ProcessName: regex + Properties: regex + UserAccountControl: regex + RegistryValue: regex + SecurityID: regex + ServiceFileName: regex + SecurityDescriptor: regex + ServiceName: regex + ShareName: regex + NewValue: regex + Source: regex + Status: regex + SubjectDomainName: regex + SubjectUserName: regex + SubjectUserSid: regex + SourceAddr: regex + SourceAddress: regex + TargetName: regex + ServicePrincipalNames: regex + TargetDomainName: regex + TargetSid: regex + TargetUserName: regex + ObjectServer: regex + TargetUserSid: regex + TicketEncryptionType: regex + TicketOptions: regex + WorkstationName: regex + TransmittedServices: regex + AuthenticationAlgorithm: regex + LayerRTID: regex + BSSID: regex + BSSType: regex + CipherAlgorithm: regex + ConnectionId: regex + ConnectionMode: regex + InterfaceDescription: regex + InterfaceGuid: regex + OnexEnabled: regex + PHYType: regex + ProfileName: regex + SSID: regex + Domain: regex + ServiceType: regex + SourceName: regex + StartType: regex + UserID: regex + ParentProcessName: regex + ExceptionCode: regex + Service: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_sysmon.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_sysmon.yml index ebfac1ec..bc88c7c6 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_sysmon.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_sysmon.yml @@ -11,50 +11,50 @@ field_mapping: Description: action_evtlog_description raw_log_fields: - - CommandLine - - Image - - ParentImage - - CallTrace - - Company - - CurrentDirectory - - DestinationHostname - - DestinationIp - - DestinationIsIpv6 - - DestinationPort - - DestinationPortName - - Hashes - - Initiated - - IntegrityLevel - - ParentCommandLine - - Product - - Protocol - - RuleName - - SourceHostname - - SourceIp - - SourceIsIpv6 - - SourcePort - - SourcePortName - - TargetFilename - - User - - Signed - - Signature - - SignatureStatus - - TargetObject - - Details - - QueryName - - QueryResults - - QueryStatus - - IsExecutable - - PipeName - - ImageLoaded - - ImagePath - - Imphash - - SourceImage - - StartModule - - TargetImage - - Device - - ProcessID - - FileVersion - - StartAddress - - StartFunction - - EventType \ No newline at end of file + CommandLine: regex + Image: regex + ParentImage: regex + CallTrace: regex + Company: regex + CurrentDirectory: regex + DestinationHostname: regex + DestinationIp: regex + DestinationIsIpv6: regex + DestinationPort: regex + DestinationPortName: regex + Hashes: regex + Initiated: regex + IntegrityLevel: regex + ParentCommandLine: regex + Product: regex + Protocol: regex + RuleName: regex + SourceHostname: regex + SourceIp: regex + SourceIsIpv6: regex + SourcePort: regex + SourcePortName: regex + TargetFilename: regex + User: regex + Signed: regex + Signature: regex + SignatureStatus: regex + TargetObject: regex + Details: regex + QueryName: regex + QueryResults: regex + QueryStatus: regex + IsExecutable: regex + PipeName: regex + ImageLoaded: regex + ImagePath: regex + Imphash: regex + SourceImage: regex + StartModule: regex + TargetImage: regex + Device: regex + ProcessID: regex + FileVersion: regex + StartAddress: regex + StartFunction: regex + EventType: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_system.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_system.yml index d4bcb22a..889872e6 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_system.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_system.yml @@ -10,14 +10,14 @@ field_mapping: ImagePath: actor_process_image_path raw_log_fields: - - AccountName - - ServiceName - - ServiceType - - StartType - - Origin - - HiveName - - Caption - - param1 - - param2 - - Channel - - DeviceName \ No newline at end of file + AccountName: regex + ServiceName: regex + ServiceType: regex + StartType: regex + Origin: regex + HiveName: regex + Caption: regex + param1: regex + param2: regex + Channel: regex + DeviceName: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/sigma/aws_cloudtrail.yml b/uncoder-core/app/translator/mappings/platforms/sigma/aws_cloudtrail.yml index da509b89..7716ea12 100644 --- a/uncoder-core/app/translator/mappings/platforms/sigma/aws_cloudtrail.yml +++ b/uncoder-core/app/translator/mappings/platforms/sigma/aws_cloudtrail.yml @@ -13,7 +13,7 @@ default_log_source: field_mapping: eventSource: eventSource eventName: eventName - AdditionalEventData: additionalEventData.MFAUsed + AdditionalEventData.MFAUsed: additionalEventData.MFAUsed errorCode: errorCode errorMessage: errorMessage eventType: eventType diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index b0d1e0f7..0af8206a 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -136,9 +136,11 @@ class CortexXQLQueryRender(PlatformQueryRender): details: PlatformDetails = cortex_xql_query_details mappings: CortexXQLMappings = cortex_xql_mappings is_strict_mapping = True - raw_log_field_pattern = ( - '| alter {field} = regextract(to_json_string(action_evtlog_data_fields)->{field}{{}}, "\\"(.*)\\"")' - ) + raw_log_field_pattern_map = { + 'regex': '| alter {field} = regextract(to_json_string(action_evtlog_data_fields)->{field}{{}}, "\\"(.*)\\"")', + 'object': '| alter {field_name} = json_extract_scalar({field_object} , "$.{field_path}")', + 'list': '| alter {field_name} = arraystring(json_extract_array({field_object} , "$.{field_path}")," ")' + } platform_functions: CortexXQLFunctions = cortex_xql_functions or_token = "or" @@ -154,6 +156,18 @@ def __init__(self): super().__init__() self.platform_functions.manager.post_init_configure(self) + def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]: + raw_log_field_pattern = self.raw_log_field_pattern_map.get(field_type) + if raw_log_field_pattern is None: + return + if field_type == "regex": + field = field.replace(".", r"\.") + return raw_log_field_pattern.format(field=field) + if field_type in ("object", "list") and "." in field: + field_object, field_path = field.split(".", 1) + field_name = field.replace(".", "_") + return raw_log_field_pattern.format(field_name=field_name, field_object=field_object, field_path=field_path) + def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, functions_prefix: str = "") -> str: functions_prefix = f"{functions_prefix} | " if functions_prefix else "" return f"{functions_prefix}{log_source_signature}"