Merge branch 'opensearch-project:main' into main

src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java

@@ -174,13 +174,6 @@ private void doCreateMapping(
             List<String> missingPathsInIndex = validationResult.getLeft();
             List<String> presentPathsInIndex = validationResult.getRight();
 
-            // Filter out mappings of sourceIndex fields to which we're applying alias mappings
-            Map<String, Object> presentPathsMappings = MapperUtils.getFieldMappingsFlat(mappingMetadata, presentPathsInIndex);
-            // Filtered alias mappings -- contains only aliases for fields which are present in sourceIndex
-            Map<String, Object> filteredAliasMappings;
-            MappingsTraverser mappingsTraverser = new MappingsTraverser(aliasMappingsJSON, Set.of());
-            filteredAliasMappings = mappingsTraverser.traverseAndCopyAsFlat();
-
             if(missingPathsInIndex.size() > 0) {
                 // If user didn't allow partial apply, we should error out here
                 if (!partial) {
@@ -190,14 +183,18 @@ private void doCreateMapping(
                                             .collect(Collectors.joining(", ", "[", "]")))
                     );
                 }
-                // Filter out missing paths from alias mappings so that our PutMappings request succeeds
-                List<Pair<String, String>> pathsToSkip =
-                        missingPathsInIndex.stream()
-                                .map(e -> Pair.of(PATH, e))
-                                .collect(Collectors.toList());
-                mappingsTraverser = new MappingsTraverser(aliasMappingsJSON, pathsToSkip);
-                filteredAliasMappings = mappingsTraverser.traverseAndCopyAsFlat();
             }
+
+            // Filter out mappings of sourceIndex fields to which we're applying alias mappings
+            Map<String, Object> presentPathsMappings = MapperUtils.getFieldMappingsFlat(mappingMetadata, presentPathsInIndex);
+            // Filtered alias mappings -- contains only aliases which are applicable to index:
+            //      1. fields in path params exists in index
+            //      2. alias isn't named as one of existing fields in index
+            Map<String, Object> filteredAliasMappings = filterNonApplicableAliases(
+                    mappingMetadata,
+                    missingPathsInIndex,
+                    aliasMappingsJSON
+            );
             Map<String, Object> allMappings = new HashMap<>(presentPathsMappings);
             allMappings.putAll((Map<String, ?>) filteredAliasMappings.get(PROPERTIES));
 
@@ -227,6 +224,45 @@ public void onFailure(Exception e) {
         }
     }
 
+    private Map<String, Object> filterNonApplicableAliases(
+            MappingMetadata indexMappingMetadata,
+            List<String> missingPathsInIndex,
+            String aliasMappingsJSON
+    ) throws IOException {
+        // Parse aliasMappings JSON into Map
+        MappingsTraverser mappingsTraverser = new MappingsTraverser(aliasMappingsJSON, Set.of());
+        Map<String, Object> filteredAliasMappings = mappingsTraverser.traverseAndCopyAsFlat();
+
+        List<Pair<String, String>> propertiesToSkip = new ArrayList<>();
+        if(missingPathsInIndex.size() > 0) {
+            // Filter out missing paths from alias mappings so that our PutMappings request succeeds
+            propertiesToSkip.addAll(
+                    missingPathsInIndex.stream()
+                            .map(e -> Pair.of(PATH, e))
+                            .collect(Collectors.toList())
+            );
+        }
+        // Filter out all aliases which name already exists as field in index mappings
+        List<String> nonAliasIndexFields = MapperUtils.getAllNonAliasFieldsFromIndex(indexMappingMetadata);
+        List<String> aliasFields = MapperUtils.getAllAliases(aliasMappingsJSON);
+        Set<String> aliasesToInclude =
+                aliasFields.stream()
+                        .filter(e -> nonAliasIndexFields.contains(e) == false)
+                        .collect(Collectors.toSet());
+
+        boolean excludeSomeAliases = aliasesToInclude.size() < aliasFields.size();
+        // check if we need to filter out some properties/nodes in alias mapping
+        if (propertiesToSkip.size() > 0 || excludeSomeAliases) {
+            mappingsTraverser = new MappingsTraverser(aliasMappingsJSON, propertiesToSkip);
+            if (aliasesToInclude.size() > 0) {
+                filteredAliasMappings = mappingsTraverser.traverseAndCopyWithFilter(aliasesToInclude);
+            } else {
+                filteredAliasMappings = mappingsTraverser.traverseAndCopyAsFlat();
+            }
+        }
+        return filteredAliasMappings;
+    }
+
     public void updateMappingAction(String indexName, String field, String alias, ActionListener<AcknowledgedResponse> actionListener) {
         PutMappingRequest request = new PutMappingRequest(indexName).source(field, alias);
         indicesClient.putMapping(request, new ActionListener<>() {
@@ -384,7 +420,8 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
                             // Maintain list of found paths in index
                             applyableAliases.add(alias);
                             pathsOfApplyableAliases.add(path);
-                        } else {
+                        } else if (allFieldsFromIndex.contains(alias) == false)  {
+                            // we don't want to send back aliases which have same name as existing field in index
                             unmappedFieldAliases.add(alias);
                         }
                     }

src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java

@@ -27,6 +27,29 @@ public class MapperUtils {
     public static final String ALIAS = "alias";
     public static final String NESTED = "nested";
 
+    public static List<String> getAllAliases(String aliasMappingsJson) throws IOException {
+        MappingsTraverser mappingsTraverser = new MappingsTraverser(aliasMappingsJson, Set.of());
+        List<String> aliasFields = new ArrayList<>();
+        mappingsTraverser.addListener(new MappingsTraverser.MappingsTraverserListener() {
+            @Override
+            public void onLeafVisited(MappingsTraverser.Node node) {
+                // We'll ignore any irregularities in alias mappings here
+                if (node.getProperties().containsKey(PATH) == false ||
+                        node.getProperties().get(TYPE).equals(ALIAS) == false) {
+                    return;
+                }
+                aliasFields.add(node.currentPath);
+            }
+
+            @Override
+            public void onError(String error) {
+                throw new IllegalArgumentException(error);
+            }
+        });
+        mappingsTraverser.traverse();
+        return aliasFields;
+    }
+
     public static List<Pair<String, String>> getAllAliasPathPairs(String aliasMappingsJson) throws IOException {
         MappingsTraverser mappingsTraverser = new MappingsTraverser(aliasMappingsJson, Set.of());
         return getAllAliasPathPairs(mappingsTraverser);

src/main/java/org/opensearch/securityanalytics/model/Detector.java

@@ -226,6 +226,11 @@ public enum DetectorType {
         APACHE_ACCESS("apache_access"),
         CLOUDTRAIL("cloudtrail"),
         DNS("dns"),
+        GITHUB("github"),
+        M365("m365"),
+        GWORKSPACE("gworkspace"),
+        OKTA("okta"),
+        AZURE("azure"),
         S3("s3"),
         TEST_WINDOWS("test_windows");
 

src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java

@@ -182,6 +182,9 @@ public Map<String, Object> getQueryFields() {
 
     public void resetQueryFields() {
         queryFields.clear();
+        if (ruleQueryFields != null) {
+            ruleQueryFields.clear();
+        }
     }
 
     public abstract Object convertConditionAsInExpression(Either<ConditionAND, ConditionOR> condition);

src/main/resources/OSMapping/ad_ldap/fieldmappings.yml

@@ -1,3 +1,24 @@
 fieldmappings:
-    TargetUserName: winlog-event_data-TargetUserName
-    creationTime: timestamp
+    TargetUserName: azure-signinlogs-properties-user_id
+    creationTime: timestamp
+    Category: azure-activitylogs-category
+    OperationName: azure-platformlogs-operation_name
+    ModifiedProperties_NewValue: modified_properties-new_value
+    ResourceProviderValue: azure-resource-provider
+    conditionalAccessStatus: azure-signinlogs-properties-conditional_access_status
+    SearchFilter: SearchFilter
+    Operation: azure-platformlogs-operation_name
+    ResultType: azure-platformlogs-result_type
+    DeviceDetail_isCompliant: azure-signinlogs-properties-device_detail-is_compliant
+    ResourceDisplayName: resource_display_name
+    AuthenticationRequirement: azure-signinlogs-properties-authentication_requirement
+    TargetResources: target_resources
+    Workload: workload
+    DeviceDetail.deviceId: azure-signinlogs-properties-device_detail-device_id
+    OperationNameValue: azure-platformlogs-operation_name
+    ResourceId: azure-signinlogs-properties-resource_id
+    ResultDescription: azure-signinlogs-result_description
+    EventID: EventID
+    NetworkLocationDetails: azure-signinlogs-properties-network_location_details
+    CategoryValue: azure-activitylogs-category
+    ActivityDisplayName: azure-auditlogs-properties-activity_display_name

src/main/resources/OSMapping/ad_ldap/mappings.json

@@ -1,11 +1,79 @@
 {
   "properties": {
-    "winlog-event_data-TargetUserName": {
-      "path": "winlog.event_data.TargetUserName",
+    "azure-signinlogs-properties-user_id": {
+      "path": "azure.signinlogs.props.user_id",
+      "type": "alias"
+    },
+    "azure-activitylogs-category": {
+      "path": "azure.activitylogs.category",
+      "type": "alias"
+    },
+    "azure-platformlogs-operation_name": {
+      "path": "azure.platformlogs.operation_name",
+      "type": "alias"
+    },
+    "modified_properties-new_value": {
+      "path": "modified_properties.new_value",
+      "type": "alias"
+    },
+    "azure-resource-provider": {
+      "path": "azure.resource.provider",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-conditional_access_status": {
+      "path": "azure.signinlogs.props.conditional_access_status",
+      "type": "alias"
+    },
+    "SearchFilter": {
+      "path": "SearchFilter",
+      "type": "alias"
+    },
+    "azure-platformlogs-result_type": {
+      "path": "azure.platformlogs.result_type",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-device_detail-is_compliant": {
+      "path": "azure.signinlogs.props.device_detail.is_compliant",
+      "type": "alias"
+    },
+    "ResourceDisplayName": {
+      "path": "ResourceDisplayName",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-authentication_requirement": {
+      "path": "azure.signinlogs.props.authentication_requirement",
+      "type": "alias"
+    },
+    "TargetResources": {
+      "path": "TargetResources",
+      "type": "alias"
+    },
+    "Workload": {
+      "path": "Workload",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-device_detail-device_id": {
+      "path": "azure.signinlogs.props.device_detail.device_id",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-resource_id": {
+      "path": "azure.signinlogs.props.resource_id",
+      "type": "alias"
+    },
+    "EventID": {
+      "path": "EventID",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-network_location_details": {
+      "path": "azure.signinlogs.props.network_location_details",
+      "type": "alias"
+    },
+    "azure-auditlogs-properties-activity_display_name": {
+      "path": "azure.auditlogs.props.activity_display_name",
       "type": "alias"
     },
     "timestamp": {
-      "path": "creationTime",
+      "path": "@timestamp",
       "type": "alias"
     }
   }

src/main/resources/OSMapping/azure/fieldmappings.yml

@@ -0,0 +1,31 @@
+fieldmappings:
+  Resultdescription: azure-signinlogs-result_description
+  eventSource: eventSource
+  eventName: eventName
+  Status: azure-platformlogs-status
+  LoggedByService: azure-auditlogs-properties-logged_by_service
+  properties_message: properties_message
+  status: azure-platformlogs-status
+  TargetUserName: azure-signinlogs-properties-user_id
+  creationTime: timestamp
+  Category: azure-activitylogs-category
+  OperationName: azure-platformlogs-operation_name
+  ModifiedProperties_NewValue: modified_properties-new_value
+  ResourceProviderValue: azure-resource-provider
+  conditionalAccessStatus: azure-signinlogs-properties-conditional_access_status
+  SearchFilter: search_filter
+  Operation: azure-platformlogs-operation_name
+  ResultType: azure-platformlogs-result_type
+  DeviceDetail_isCompliant: azure-signinlogs-properties-device_detail-is_compliant
+  ResourceDisplayName: resource_display_name
+  AuthenticationRequirement: azure-signinlogs-properties-authentication_requirement
+  TargetResources: target_resources
+  Workload: Workload
+  DeviceDetail_deviceId: azure-signinlogs-properties-device_detail-device_id
+  OperationNameValue: azure-platformlogs-operation_name
+  ResourceId: azure-signinlogs-properties-resource_id
+  ResultDescription: azure-signinlogs-result-description
+  EventID: EventID
+  NetworkLocationDetails: azure-signinlogs-properties-network_location_details
+  CategoryValue: azure-activitylogs-category
+  ActivityDisplayName: azure-auditlogs-properties-activity_display_name

src/main/resources/OSMapping/azure/mappings.json

@@ -0,0 +1,104 @@
+{
+  "properties": {
+    "azure-signinlogs-properties-user_id": {
+      "path": "azure.signinlogs.props.user_id",
+      "type": "alias"
+    },
+    "azure-activitylogs-category": {
+      "path": "azure.activitylogs.category",
+      "type": "alias"
+    },
+    "azure-platformlogs-operation_name": {
+      "path": "azure.platformlogs.operation_name",
+      "type": "alias"
+    },
+    "modified_properties-new_value": {
+      "path": "modified_properties.new_value",
+      "type": "alias"
+    },
+    "azure-resource-provider": {
+      "path": "azure.resource.provider",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-conditional_access_status": {
+      "path": "azure.signinlogs.props.conditional_access_status",
+      "type": "alias"
+    },
+    "SearchFilter": {
+      "path": "SearchFilter",
+      "type": "alias"
+    },
+    "azure-platformlogs-result_type": {
+      "path": "azure.platformlogs.result_type",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-device_detail-is_compliant": {
+      "path": "azure.signinlogs.props.device_detail.is_compliant",
+      "type": "alias"
+    },
+    "ResourceDisplayName": {
+      "path": "ResourceDisplayName",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-authentication_requirement": {
+      "path": "azure.signinlogs.props.authentication_requirement",
+      "type": "alias"
+    },
+    "TargetResources": {
+      "path": "TargetResources",
+      "type": "alias"
+    },
+    "Workload": {
+      "path": "Workload",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-device_detail-device_id": {
+      "path": "azure.signinlogs.props.device_detail.device_id",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-resource_id": {
+      "path": "azure.signinlogs.props.resource_id",
+      "type": "alias"
+    },
+    "EventID": {
+      "path": "EventID",
+      "type": "alias"
+    },
+    "azure-signinlogs-properties-network_location_details": {
+      "path": "azure.signinlogs.props.network_location_details",
+      "type": "alias"
+    },
+    "azure-auditlogs-properties-activity_display_name": {
+      "path": "azure.auditlogs.props.activity_display_name",
+      "type": "alias"
+    },
+    "azure-signinlogs-result-description": {
+      "path": "azure.signinlogs.result-description",
+      "type": "alias"
+    },
+    "eventSource": {
+      "path": "eventSource",
+      "type": "alias"
+    },
+    "eventName": {
+      "path": "eventName",
+      "type": "alias"
+    },
+    "azure-platformlogs-status": {
+      "path": "azure.platformlogs.status",
+      "type": "alias"
+    },
+    "azure-auditlogs-properties-logged_by_service": {
+      "path": "azure.auditlogs.props.logged_by_service",
+      "type": "alias"
+    },
+    "properties_message": {
+      "path": "properties_message",
+      "type": "alias"
+    },
+    "timestamp": {
+      "path": "@timestamp",
+      "type": "alias"
+    }
+  }
+}