From e44e5d45f284280bd124c50955d6e237791d0d3b Mon Sep 17 00:00:00 2001 From: Riya <69919272+riysaxen-amzn@users.noreply.github.com> Date: Fri, 15 Mar 2024 18:26:08 -0700 Subject: [PATCH] Backport changes and Integ test fix (#935) * Fix duplicate ecs mappings which returns incorrect log index field in mapping view API (#786) (#788) * field mapping changes Signed-off-by: Joanne Wang * add integ test Signed-off-by: Joanne Wang * turn unmappedfieldaliases as set and add integ test Signed-off-by: Joanne Wang * add comments Signed-off-by: Joanne Wang * fix integ tests Signed-off-by: Joanne Wang * moved logic to method for better readability Signed-off-by: Joanne Wang --------- Signed-off-by: Joanne Wang * support object fields in aggregation based sigma rules (#789) Signed-off-by: Subhobrata Dey * Pass rule field names in doc level queries during monitor/creation. Remove blocking actionGet() calls (#873) * pass query field names in doc level queries during monitor creation/updation Signed-off-by: Surya Sashank Nistala * remove actionGet() and change get index mapping call to event driven flow Signed-off-by: Surya Sashank Nistala * fix chained findings monitor Signed-off-by: Surya Sashank Nistala * add finding mappings Signed-off-by: Surya Sashank Nistala * remove test messages from logs Signed-off-by: Surya Sashank Nistala * revert build.gradle change Signed-off-by: Surya Sashank Nistala --------- Signed-off-by: Surya Sashank Nistala * version bump Signed-off-by: Surya Sashank Nistala * AggregationBackendTests fix Signed-off-by: Riya Saxena --------- Signed-off-by: Joanne Wang Signed-off-by: Subhobrata Dey Signed-off-by: Surya Sashank Nistala Signed-off-by: Riya Saxena Co-authored-by: Joanne Wang Co-authored-by: Subhobrata Dey Co-authored-by: Surya Sashank Nistala --- build.gradle | 2 +- .../aggregation/AggregationLexer.java | 88 +-- src/main/grammars/Aggregation.g4 | 2 +- .../mapper/MapperService.java | 76 ++- .../securityanalytics/mapper/MapperUtils.java | 8 +- .../rules/backend/AggregationBuilders.java | 2 +- .../rules/backend/OSQueryBackend.java | 32 +- .../TransportIndexDetectorAction.java | 603 ++++++++++------- .../securityanalytics/TestHelpers.java | 611 ++++++++++++++++++ .../resthandler/DetectorMonitorRestApiIT.java | 60 +- .../resthandler/OCSFDetectorRestApiIT.java | 4 +- .../aggregation/AggregationBackendTests.java | 89 ++- 12 files changed, 1233 insertions(+), 344 deletions(-) diff --git a/build.gradle b/build.gradle index 1fe0c1b7a..357c3746d 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "2.9.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.9.1-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java index fcf7ae82a..115766bb1 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java @@ -129,50 +129,50 @@ public AggregationLexer(CharStream input) { "\u0001\u000f\u0001\u000f\u0000\u0000\u0010\u0001\u0001\u0003\u0002\u0005"+ "\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007\u000f\b\u0011\t\u0013\n"+ "\u0015\u000b\u0017\f\u0019\r\u001b\u000e\u001d\u000f\u001f\u0010\u0001"+ - "\u0000\u0004\u0001\u000009\u0004\u0000**AZ__az\u0004\u000009AZ__az\u0003"+ - "\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003\u0001"+ - "\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007\u0001"+ - "\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001\u0000"+ - "\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000"+ - "\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000"+ - "\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000"+ - "\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000"+ - "\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000"+ - "\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000\u0005"+ - "&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001\u0000"+ - "\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000\u0000"+ - "\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013@"+ - "\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001\u0000"+ - "\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000\u0000"+ - "\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!\"\u0005"+ - ">\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000$%\u0005"+ - "=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000\'\u0006"+ - "\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000*\b"+ - "\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000-\n"+ - "\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u000001\u0005"+ - "u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001\u0000"+ - "\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m\u0000"+ - "\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005i\u0000"+ - "\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005m\u0000"+ - "\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000\u0000"+ - "\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000\u0000"+ - "C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000\u0000"+ - "F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001\u0000"+ - "\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000KM\u0005"+ - "-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000MO\u0001"+ - "\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000\u0000"+ - "PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000\u0000"+ - "\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000\u0000"+ - "\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001\u0000"+ - "\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000YS\u0001"+ - "\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000\u0000"+ - "\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001\u0000"+ - "\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000_`\u0001"+ - "\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000\u0000"+ - "\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001\u0000"+ - "\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000fg\u0001"+ - "\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000\u0000"+ - "\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000"; + "\u0000\u0004\u0001\u000009\u0005\u0000**..AZ__az\u0005\u0000..09AZ__a"+ + "z\u0003\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003"+ + "\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007"+ + "\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001"+ + "\u0000\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000"+ + "\u0000\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000"+ + "\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000"+ + "\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000"+ + "\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000"+ + "\u0000\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000"+ + "\u0005&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001"+ + "\u0000\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000"+ + "\u0000\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013"+ + "@\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001"+ + "\u0000\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000"+ + "\u0000\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!"+ + "\"\u0005>\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000"+ + "$%\u0005=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000"+ + "\'\u0006\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000"+ + "*\b\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000"+ + "-\n\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u0000"+ + "01\u0005u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001"+ + "\u0000\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m"+ + "\u0000\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005"+ + "i\u0000\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005"+ + "m\u0000\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000"+ + "\u0000\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000"+ + "\u0000C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000"+ + "\u0000F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001"+ + "\u0000\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000"+ + "KM\u0005-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000"+ + "MO\u0001\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000"+ + "\u0000PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000"+ + "\u0000\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000"+ + "\u0000\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001"+ + "\u0000\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000"+ + "YS\u0001\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000"+ + "\u0000\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001"+ + "\u0000\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000"+ + "_`\u0001\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000"+ + "\u0000\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001"+ + "\u0000\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000"+ + "fg\u0001\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000"+ + "\u0000\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/src/main/grammars/Aggregation.g4 b/src/main/grammars/Aggregation.g4 index 00303b68a..be395c5ae 100644 --- a/src/main/grammars/Aggregation.g4 +++ b/src/main/grammars/Aggregation.g4 @@ -21,7 +21,7 @@ RPAREN : ')' ; DECIMAL : '-'?[0-9]+('.'[0-9]+)? ; -IDENTIFIER : [a-zA-Z*_][a-zA-Z_0-9]* ; +IDENTIFIER : [a-zA-Z*_.][a-zA-Z_0-9.]* ; WS : [ \r\t\u000C\n]+ -> skip ; comparison_expr : comparison_operand comp_operator comparison_operand # ComparisonExpressionWithOperator diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index 0a53e3098..77b5dca80 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -5,20 +5,9 @@ package org.opensearch.securityanalytics.mapper; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.OpenSearchStatusException; import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.get.GetIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexResponse; @@ -44,6 +33,16 @@ import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import static org.opensearch.securityanalytics.mapper.MapperUtils.PATH; import static org.opensearch.securityanalytics.mapper.MapperUtils.PROPERTIES; @@ -78,9 +77,11 @@ public void createMappingAction(String indexName, String logType, String aliasMa // since you can't update documents in non-write indices String index = indexName; boolean shouldUpsertIndexTemplate = IndexUtils.isConcreteIndex(indexName, this.clusterService.state()) == false; - if (IndexUtils.isDataStream(indexName, this.clusterService.state())) { + if (IndexUtils.isDataStream(indexName, this.clusterService.state()) || IndexUtils.isAlias(indexName, this.clusterService.state())) { + log.debug("{} is an alias or datastream. Fetching write index for create mapping action.", indexName); String writeIndex = IndexUtils.getWriteIndex(indexName, this.clusterService.state()); if (writeIndex != null) { + log.debug("Write index for {} is {}", indexName, writeIndex); index = writeIndex; } } @@ -92,6 +93,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { applyAliasMappings(getMappingsResponse.getMappings(), logType, aliasMappings, partial, new ActionListener<>() { @Override public void onResponse(Collection createMappingResponse) { + log.debug("Completed create mappings for {}", indexName); // We will return ack==false if one of the requests returned that // else return ack==true Optional notAckd = createMappingResponse.stream() @@ -110,6 +112,7 @@ public void onResponse(Collection createMappingResponse) { @Override public void onFailure(Exception e) { + log.debug("Failed to create mappings for {}", indexName ); actionListener.onFailure(e); } }); @@ -478,13 +481,16 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { String rawPath = requiredField.getRawField(); String ocsfPath = requiredField.getOcsf(); if (allFieldsFromIndex.contains(rawPath)) { - if (alias != null) { - // Maintain list of found paths in index - applyableAliases.add(alias); - } else { - applyableAliases.add(rawPath); + // if the alias was already added into applyable aliases, then skip to avoid duplicates + if (!applyableAliases.contains(alias) && !applyableAliases.contains(rawPath)) { + if (alias != null) { + // Maintain list of found paths in index + applyableAliases.add(alias); + } else { + applyableAliases.add(rawPath); + } + pathsOfApplyableAliases.add(rawPath); } - pathsOfApplyableAliases.add(rawPath); } else if (allFieldsFromIndex.contains(ocsfPath)) { applyableAliases.add(alias); pathsOfApplyableAliases.add(ocsfPath); @@ -498,13 +504,23 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { } } + // turn unmappedFieldAliases into a set to remove duplicates + Set setOfUnmappedFieldAliases = new HashSet<>(unmappedFieldAliases); + + // filter out aliases that were included in applyableAliases already + List filteredUnmappedFieldAliases = setOfUnmappedFieldAliases.stream() + .filter(e -> false == applyableAliases.contains(e)) + .collect(Collectors.toList()); + Map> aliasMappingFields = new HashMap<>(); XContentBuilder aliasMappingsObj = XContentFactory.jsonBuilder().startObject(); for (LogType.Mapping mapping: requiredFields) { if (allFieldsFromIndex.contains(mapping.getOcsf())) { aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf())); } else if (mapping.getEcs() != null) { - aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField())); + shouldUpdateEcsMappingAndMaybeUpdates(mapping, aliasMappingFields, pathsOfApplyableAliases); + } else if (mapping.getEcs() == null) { + aliasMappingFields.put(mapping.getRawField(), Map.of("type", "alias", "path", mapping.getRawField())); } } aliasMappingsObj.field("properties", aliasMappingFields); @@ -519,7 +535,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { .collect(Collectors.toList()); actionListener.onResponse( - new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, unmappedFieldAliases) + new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, filteredUnmappedFieldAliases) ); } catch (Exception e) { actionListener.onFailure(e); @@ -533,6 +549,26 @@ public void onFailure(Exception e) { }); } + /** + * Only updates the alias mapping fields if the ecs key has not been mapped yet + * or if pathOfApplyableAliases contains the raw field + * + * @param mapping + * @param aliasMappingFields + * @param pathsOfApplyableAliases + */ + private static void shouldUpdateEcsMappingAndMaybeUpdates(LogType.Mapping mapping, Map> aliasMappingFields, List pathsOfApplyableAliases) { + // check if aliasMappingFields already contains a key + if (aliasMappingFields.containsKey(mapping.getEcs())) { + // if the pathOfApplyableAliases contains the raw field, then override the existing map + if (pathsOfApplyableAliases.contains(mapping.getRawField())) { + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField())); + } + } else { + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField())); + } + } + /** * Given index name, resolves it to single concrete index, depending on what initial indexName is. * In case of Datastream or Alias, WriteIndex would be returned. In case of index pattern, newest index by creation date would be returned. diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java index 72dd36d11..8c8bf353f 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java @@ -5,6 +5,10 @@ package org.opensearch.securityanalytics.mapper; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -12,9 +16,6 @@ import java.util.Locale; import java.util.Map; import java.util.Set; -import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.cluster.metadata.MappingMetadata; -import org.opensearch.securityanalytics.util.SecurityAnalyticsException; public class MapperUtils { @@ -246,7 +247,6 @@ public void onError(String error) { } }); mappingsTraverser.traverse(); - return presentPathsMappings; } } diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java index 3927186fb..c0f6bbb7a 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java @@ -46,7 +46,7 @@ public static AggregationBuilder getAggregationBuilderByFunction(String aggregat aggregationBuilder = new TermsAggregationBuilder(name).field(name); break; case "count": - aggregationBuilder = new ValueCountAggregationBuilder(name).field(name); + aggregationBuilder = new ValueCountAggregationBuilder(name.replace(".", "_")).field(name); break; default: throw new NotImplementedException(String.format(Locale.getDefault(), "Aggregation %s not supported by the backend", aggregationFunction)); diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index bf8b883ff..44c06abe6 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -331,9 +331,12 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre @Override public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError { + String field = getFinalValueField(); + ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); SigmaString value = (SigmaString) condition.getValue(); boolean containsWildcard = value.containsWildcard(); - return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue())); + return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), + this.convertValueStr((SigmaString) condition.getValue())); } @Override @@ -361,14 +364,15 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) { BucketSelectorExtAggregationBuilder condition; String bucketTriggerSelectorId = UUIDs.base64UUID(); - if (aggregation.getAggFunction().equals("count")) { + if (aggregation.getAggFunction().equals("count") && aggregation.getAggField().equals("*")) { String fieldName; - if (aggregation.getAggField().equals("*") && aggregation.getGroupByField() == null) { + if (aggregation.getGroupByField() == null) { fieldName = "_index"; fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", "_index"); } else { - fieldName = aggregation.getGroupByField(); - fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", aggregation.getGroupByField()); + String mappedGroupByField = getMappedField(aggregation.getGroupByField()); + fieldName = mappedGroupByField; + fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", mappedGroupByField); } aggBuilder.field(fieldName); fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, "_cnt", "_cnt", "result_agg", "_cnt", aggregation.getCompOperator(), aggregation.getThreshold()); @@ -376,17 +380,23 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) { Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, "_cnt", aggregation.getCompOperator(), aggregation.getThreshold())); condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap("_cnt", "_cnt"), script, "result_agg", null); } else { - fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", aggregation.getGroupByField(), aggregation.getAggField(), aggregation.getAggFunction(), aggregation.getAggField()); - fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, aggregation.getAggField(), aggregation.getAggField(), "result_agg", aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold()); + /** + * removing dots to eliminate dots in aggregation names + */ + String mappedAggField = getFinalField(aggregation.getAggField()); + String mappedAggFieldUpdated = mappedAggField.replace(".", "_"); + String mappedGroupByField = getMappedField(aggregation.getGroupByField()); + fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", mappedGroupByField, mappedAggFieldUpdated, aggregation.getAggFunction().equals("count")? "value_count": aggregation.getAggFunction(), mappedAggField); + fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, mappedAggFieldUpdated, mappedAggField, "result_agg", mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold()); // Add subaggregation - AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), aggregation.getAggField()); + AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), mappedAggField); if (subAgg != null) { - aggBuilder.field(aggregation.getGroupByField()).subAggregation(subAgg); + aggBuilder.field(mappedGroupByField).subAggregation(subAgg); } - Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold())); - condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(aggregation.getAggField(), aggregation.getAggField()), script, "result_agg", null); + Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold())); + condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(mappedAggFieldUpdated, mappedAggFieldUpdated), script, "result_agg", null); } AggregationQueries aggregationQueries = new AggregationQueries(); diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 6b74e16af..663ead35d 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -8,7 +8,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.search.join.ScoreMode; -import org.opensearch.common.SetOnce; import org.opensearch.OpenSearchStatusException; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionRunnable; @@ -32,7 +31,6 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.CheckedConsumer; import org.opensearch.common.SetOnce; import org.opensearch.common.inject.Inject; import org.opensearch.common.io.stream.NamedWriteableRegistry; @@ -41,7 +39,6 @@ import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.commons.alerting.AlertingPluginInterface; import org.opensearch.commons.alerting.action.DeleteMonitorRequest; @@ -60,8 +57,9 @@ import org.opensearch.commons.authuser.User; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.RangeQueryBuilder; @@ -90,6 +88,7 @@ import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.Value; +import org.opensearch.securityanalytics.rules.aggregation.AggregationItem; import org.opensearch.securityanalytics.rules.backend.OSQueryBackend; import org.opensearch.securityanalytics.rules.backend.OSQueryBackend.AggregationQueries; import org.opensearch.securityanalytics.rules.backend.QueryBackend; @@ -109,9 +108,11 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -205,15 +206,18 @@ private void checkIndicesAndExecute( ) { String [] detectorIndices = request.getDetector().getInputs().stream().flatMap(detectorInput -> detectorInput.getIndices().stream()).toArray(String[]::new); SearchRequest searchRequest = new SearchRequest(detectorIndices).source(SearchSourceBuilder.searchSource().size(1).query(QueryBuilders.matchAllQuery()));; + searchRequest.setCancelAfterTimeInterval(TimeValue.timeValueSeconds(30)); client.search(searchRequest, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { + log.debug("check indices and execute completed. Took {} millis", searchResponse.getTook().millis()); AsyncIndexDetectorsAction asyncAction = new AsyncIndexDetectorsAction(user, task, request, listener); asyncAction.start(); } @Override public void onFailure(Exception e) { + log.debug("check indices and execute failed", e); if (e instanceof OpenSearchStatusException) { listener.onFailure(SecurityAnalyticsException.wrap( new OpenSearchStatusException(String.format(Locale.getDefault(), "User doesn't have read permissions for one or more configured index %s", detectorIndices), RestStatus.FORBIDDEN) @@ -230,26 +234,69 @@ public void onFailure(Exception e) { }); } - private void createMonitorFromQueries(String index, List> rulesById, Detector detector, ActionListener> listener, WriteRequest.RefreshPolicy refreshPolicy) throws SigmaError, IOException { - List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( - Collectors.toList()); - List> bucketLevelRules = rulesById.stream().filter(it -> it.getRight().isAggregationRule()).collect( - Collectors.toList()); + private void createMonitorFromQueries(List> rulesById, Detector detector, ActionListener> listener, WriteRequest.RefreshPolicy refreshPolicy, + List queryFieldNames) { + try { + List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( + Collectors.toList()); + List> bucketLevelRules = rulesById.stream().filter(it -> it.getRight().isAggregationRule()).collect( + Collectors.toList()); - List monitorRequests = new ArrayList<>(); + List monitorRequests = new ArrayList<>(); - if (!docLevelRules.isEmpty()) { - monitorRequests.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); - } + if (!docLevelRules.isEmpty()) { + monitorRequests.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST, queryFieldNames)); + } - if (!bucketLevelRules.isEmpty()) { - StepListener> bucketLevelMonitorRequests = new StepListener<>(); - buildBucketLevelMonitorRequests(bucketLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST, bucketLevelMonitorRequests); - bucketLevelMonitorRequests.whenComplete(indexMonitorRequests -> { - monitorRequests.addAll(indexMonitorRequests); - // Do nothing if detector doesn't have any monitor + if (!bucketLevelRules.isEmpty()) { + StepListener> bucketLevelMonitorRequests = new StepListener<>(); + buildBucketLevelMonitorRequests(bucketLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST, bucketLevelMonitorRequests); + bucketLevelMonitorRequests.whenComplete(indexMonitorRequests -> { + monitorRequests.addAll(indexMonitorRequests); + // Do nothing if detector doesn't have any monitor + if (monitorRequests.isEmpty()) { + listener.onResponse(Collections.emptyList()); + return; + } + + List monitorResponses = new ArrayList<>(); + StepListener addFirstMonitorStep = new StepListener(); + + // Indexing monitors in two steps in order to prevent all shards failed error from alerting + // https://github.com/opensearch-project/alerting/issues/646 + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(0), namedWriteableRegistry, addFirstMonitorStep); + addFirstMonitorStep.whenComplete(addedFirstMonitorResponse -> { + monitorResponses.add(addedFirstMonitorResponse); + int numberOfUnprocessedResponses = monitorRequests.size() - 1; + if (numberOfUnprocessedResponses == 0) { + listener.onResponse(monitorResponses); + } else { + GroupedActionListener monitorResponseListener = new GroupedActionListener( + new ActionListener>() { + @Override + public void onResponse(Collection indexMonitorResponse) { + monitorResponses.addAll(indexMonitorResponse.stream().collect(Collectors.toList())); + listener.onResponse(monitorResponses); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, numberOfUnprocessedResponses); + + for (int i = 1; i < monitorRequests.size(); i++) { + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(i), namedWriteableRegistry, monitorResponseListener); + } + } + }, + listener::onFailure + ); + }, listener::onFailure); + } else { + // Failure if detector doesn't have any monitor if (monitorRequests.isEmpty()) { - listener.onResponse(Collections.emptyList()); + listener.onFailure(new OpenSearchStatusException("Detector cannot be created as no compatible rules were provided", RestStatus.BAD_REQUEST)); return; } @@ -286,55 +333,22 @@ public void onFailure(Exception e) { }, listener::onFailure ); - }, listener::onFailure); - } else { - // Failure if detector doesn't have any monitor - if (monitorRequests.isEmpty()) { - listener.onFailure(new OpenSearchStatusException("Detector cannot be created as no compatible rules were provided", RestStatus.BAD_REQUEST)); - return; } - - List monitorResponses = new ArrayList<>(); - StepListener addFirstMonitorStep = new StepListener(); - - // Indexing monitors in two steps in order to prevent all shards failed error from alerting - // https://github.com/opensearch-project/alerting/issues/646 - AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(0), namedWriteableRegistry, addFirstMonitorStep); - addFirstMonitorStep.whenComplete(addedFirstMonitorResponse -> { - monitorResponses.add(addedFirstMonitorResponse); - int numberOfUnprocessedResponses = monitorRequests.size() - 1; - if (numberOfUnprocessedResponses == 0) { - listener.onResponse(monitorResponses); - } else { - GroupedActionListener monitorResponseListener = new GroupedActionListener( - new ActionListener>() { - @Override - public void onResponse(Collection indexMonitorResponse) { - monitorResponses.addAll(indexMonitorResponse.stream().collect(Collectors.toList())); - listener.onResponse(monitorResponses); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }, numberOfUnprocessedResponses); - - for (int i = 1; i < monitorRequests.size(); i++) { - AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(i), namedWriteableRegistry, monitorResponseListener); - } - } - }, - listener::onFailure - ); + } catch (Exception e) { + listener.onFailure(e); } } - private void updateMonitorFromQueries(String index, List> rulesById, Detector detector, ActionListener> listener, WriteRequest.RefreshPolicy refreshPolicy) throws SigmaError, IOException { + private void updateMonitorFromQueries(List> rulesById, + Detector detector, + ActionListener> listener, + WriteRequest.RefreshPolicy refreshPolicy, + List queryFieldNames) { List monitorsToBeUpdated = new ArrayList<>(); List> bucketLevelRules = rulesById.stream().filter(it -> it.getRight().isAggregationRule()).collect( Collectors.toList()); + List monitorsToBeAdded = new ArrayList<>(); // Process bucket level monitors if (!bucketLevelRules.isEmpty()) { @@ -352,48 +366,78 @@ public void onResponse(Map> ruleFieldMappings) { // Pair of RuleId - MonitorId for existing monitors of the detector Map monitorPerRule = detector.getRuleIdMonitorIdMap(); + GroupedActionListener groupedActionListener = new GroupedActionListener<>( + new ActionListener<>() { + @Override + public void onResponse(Collection indexMonitorRequests) { + onIndexMonitorRequestCreation( + monitorsToBeUpdated, + monitorsToBeAdded, + rulesById, + detector, + refreshPolicy, + queryFieldNames, + listener + ); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, bucketLevelRules.size() + ); for (Pair query : bucketLevelRules) { Rule rule = query.getRight(); if (rule.getAggregationQueries() != null) { // Detect if the monitor should be added or updated if (monitorPerRule.containsKey(rule.getId())) { String monitorId = monitorPerRule.get(rule.getId()); - monitorsToBeUpdated.add(createBucketLevelMonitorRequest(query.getRight(), + createBucketLevelMonitorRequest(query.getRight(), detector, refreshPolicy, monitorId, Method.PUT, - queryBackendMap.get(rule.getCategory()))); + queryBackendMap.get(rule.getCategory()), + new ActionListener<>() { + @Override + public void onResponse(IndexMonitorRequest indexMonitorRequest) { + monitorsToBeUpdated.add(indexMonitorRequest); + groupedActionListener.onResponse(indexMonitorRequest); + } + + @Override + public void onFailure(Exception e) { + log.error("Failed to create bucket level monitor request", e); + listener.onFailure(e); + } + }); } else { - monitorsToBeAdded.add(createBucketLevelMonitorRequest(query.getRight(), + createBucketLevelMonitorRequest(query.getRight(), detector, refreshPolicy, Monitor.NO_ID, Method.POST, - queryBackendMap.get(rule.getCategory()))); - } - } - } + queryBackendMap.get(rule.getCategory()), + new ActionListener<>() { + @Override + public void onResponse(IndexMonitorRequest indexMonitorRequest) { + monitorsToBeAdded.add(indexMonitorRequest); + groupedActionListener.onResponse(indexMonitorRequest); - List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( - Collectors.toList()); + } - // Process doc level monitors - if (!docLevelRules.isEmpty()) { - if (detector.getDocLevelMonitorId() == null) { - monitorsToBeAdded.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); - } else { - monitorsToBeUpdated.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, detector.getDocLevelMonitorId(), Method.PUT)); + @Override + public void onFailure(Exception e) { + log.error("Failed to create bucket level monitor request", e); + listener.onFailure(e); + } + }); + } } } - List monitorIdsToBeDeleted = detector.getRuleIdMonitorIdMap().values().stream().collect(Collectors.toList()); - monitorIdsToBeDeleted.removeAll(monitorsToBeUpdated.stream().map(IndexMonitorRequest::getMonitorId).collect( - Collectors.toList())); - - updateAlertingMonitors(monitorsToBeAdded, monitorsToBeUpdated, monitorIdsToBeDeleted, refreshPolicy, listener); - } catch (IOException | SigmaError ex) { + } catch (Exception ex) { listener.onFailure(ex); } } @@ -404,26 +448,45 @@ public void onFailure(Exception e) { } }); } else { - List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( - Collectors.toList()); + onIndexMonitorRequestCreation( + monitorsToBeUpdated, + monitorsToBeAdded, + rulesById, + detector, + refreshPolicy, + queryFieldNames, + listener + ); + } + } - // Process doc level monitors - if (!docLevelRules.isEmpty()) { - if (detector.getDocLevelMonitorId() == null) { - monitorsToBeAdded.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); - } else { - monitorsToBeUpdated.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, detector.getDocLevelMonitorId(), Method.PUT)); - } + private void onIndexMonitorRequestCreation(List monitorsToBeUpdated, + List monitorsToBeAdded, + List> rulesById, + Detector detector, + RefreshPolicy refreshPolicy, + List queryFieldNames, + ActionListener> listener) { + List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( + Collectors.toList()); + + // Process doc level monitors + if (!docLevelRules.isEmpty()) { + if (detector.getDocLevelMonitorId() == null) { + monitorsToBeAdded.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST, queryFieldNames)); + } else { + monitorsToBeUpdated.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, detector.getDocLevelMonitorId(), Method.PUT, queryFieldNames)); } + } - List monitorIdsToBeDeleted = detector.getRuleIdMonitorIdMap().values().stream().collect(Collectors.toList()); - monitorIdsToBeDeleted.removeAll(monitorsToBeUpdated.stream().map(IndexMonitorRequest::getMonitorId).collect( - Collectors.toList())); + List monitorIdsToBeDeleted = detector.getRuleIdMonitorIdMap().values().stream().collect(Collectors.toList()); + monitorIdsToBeDeleted.removeAll(monitorsToBeUpdated.stream().map(IndexMonitorRequest::getMonitorId).collect( + Collectors.toList())); - updateAlertingMonitors(monitorsToBeAdded, monitorsToBeUpdated, monitorIdsToBeDeleted, refreshPolicy, listener); - } + updateAlertingMonitors(rulesById, detector, monitorsToBeAdded, monitorsToBeUpdated, monitorIdsToBeDeleted, refreshPolicy, listener); } + /** * Update list of monitors for the given detector * Executed in a steps: @@ -438,6 +501,8 @@ public void onFailure(Exception e) { * @param listener Listener that accepts the list of updated monitors if the action was successful */ private void updateAlertingMonitors( + List> rulesById, + Detector detector, List monitorsToBeAdded, List monitorsToBeUpdated, List monitorsToBeDeleted, @@ -459,25 +524,25 @@ private void updateAlertingMonitors( executeMonitorActionRequest(monitorsToBeUpdated, updateMonitorsStep); // 2. Update existing alerting monitors (based on the common rules) updateMonitorsStep.whenComplete(updateMonitorResponse -> { - if (updateMonitorResponse!=null && !updateMonitorResponse.isEmpty()) { - updatedMonitors.addAll(updateMonitorResponse); - } + if (updateMonitorResponse != null && !updateMonitorResponse.isEmpty()) { + updatedMonitors.addAll(updateMonitorResponse); + } - StepListener> deleteMonitorStep = new StepListener<>(); - deleteAlertingMonitors(monitorsToBeDeleted, refreshPolicy, deleteMonitorStep); - // 3. Delete alerting monitors (rules that are not provided by the user) - deleteMonitorStep.whenComplete(deleteMonitorResponses -> - // Return list of all updated + newly added monitors - listener.onResponse(updatedMonitors), - // Handle delete monitors (step 3) - listener::onFailure); - }, // Handle update monitor failed (step 2) + StepListener> deleteMonitorStep = new StepListener<>(); + deleteAlertingMonitors(monitorsToBeDeleted, refreshPolicy, deleteMonitorStep); + // 3. Delete alerting monitors (rules that are not provided by the user) + deleteMonitorStep.whenComplete(deleteMonitorResponses -> + // Return list of all updated + newly added monitors + listener.onResponse(updatedMonitors), + // Handle delete monitors (step 3) + listener::onFailure); + }, // Handle update monitor failed (step 2) listener::onFailure); // Handle add failed (step 1) }, listener::onFailure); } - private IndexMonitorRequest createDocLevelMonitorRequest(List> queries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod) { + private IndexMonitorRequest createDocLevelMonitorRequest(List> queries, Detector detector, RefreshPolicy refreshPolicy, String monitorId, Method restMethod, List queryFieldNames) { List docLevelMonitorInputs = new ArrayList<>(); List docLevelQueries = new ArrayList<>(); @@ -528,39 +593,71 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } - private void buildBucketLevelMonitorRequests(List> queries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod, ActionListener> listener) throws IOException, SigmaError { - + private void buildBucketLevelMonitorRequests(List> queries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod, ActionListener> listener) throws Exception { + log.debug("bucket level monitor request starting"); + log.debug("get rule field mappings request being made"); logTypeService.getRuleFieldMappings(new ActionListener<>() { @Override public void onResponse(Map> ruleFieldMappings) { - try { - List ruleCategories = queries.stream().map(Pair::getRight).map(Rule::getCategory).distinct().collect( - Collectors.toList()); - Map queryBackendMap = new HashMap<>(); - for(String category: ruleCategories) { - Map fieldMappings = ruleFieldMappings.get(category); + log.debug("got rule field mapping success"); + List ruleCategories = queries.stream().map(Pair::getRight).map(Rule::getCategory).distinct().collect( + Collectors.toList()); + Map queryBackendMap = new HashMap<>(); + for(String category: ruleCategories) { + Map fieldMappings = ruleFieldMappings.get(category); + try { queryBackendMap.put(category, new OSQueryBackend(fieldMappings, true, true)); + } catch (IOException e) { + logger.error("Failed to create OSQueryBackend from field mappings", e); + listener.onFailure(e); } + } + + List monitorRequests = new ArrayList<>(); + GroupedActionListener bucketLevelMonitorRequestsListener = new GroupedActionListener<>( + new ActionListener<>() { + @Override + public void onResponse(Collection indexMonitorRequests) { + listener.onResponse(monitorRequests); + } - List monitorRequests = new ArrayList<>(); + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, queries.size() + ); + for (Pair query: queries) { + Rule rule = query.getRight(); + + // Creating bucket level monitor per each aggregation rule + if (rule.getAggregationQueries() != null) { + createBucketLevelMonitorRequest( + query.getRight(), + detector, + refreshPolicy, + monitorId, + restMethod, + queryBackendMap.get(rule.getCategory()), + new ActionListener<>() { + @Override + public void onResponse(IndexMonitorRequest indexMonitorRequest) { + monitorRequests.add(indexMonitorRequest); + bucketLevelMonitorRequestsListener.onResponse(indexMonitorRequest); + } - for (Pair query: queries) { - Rule rule = query.getRight(); - // Creating bucket level monitor per each aggregation rule - if (rule.getAggregationQueries() != null){ - monitorRequests.add(createBucketLevelMonitorRequest( - query.getRight(), - detector, - refreshPolicy, - Monitor.NO_ID, - Method.POST, - queryBackendMap.get(rule.getCategory()))); - } + @Override + public void onFailure(Exception e) { + logger.error("Failed to build bucket level monitor requests", e); + bucketLevelMonitorRequestsListener.onFailure(e); + } + }); + + } else { + log.debug("Aggregation query is null in rule {}", rule.getId()); + bucketLevelMonitorRequestsListener.onResponse(null); } - listener.onResponse(monitorRequests); - } catch (IOException | SigmaError ex) { - listener.onFailure(ex); } } @@ -571,93 +668,110 @@ public void onFailure(Exception e) { }); } - private IndexMonitorRequest createBucketLevelMonitorRequest( + private void createBucketLevelMonitorRequest( Rule rule, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod, - QueryBackend queryBackend - ) throws SigmaError { - + QueryBackend queryBackend, + ActionListener listener + ) { + log.debug(":create bucket level monitor response starting"); List indices = detector.getInputs().get(0).getIndices(); - - AggregationQueries aggregationQueries = queryBackend.convertAggregation(rule.getAggregationItemsFromRule().get(0)); - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() - .seqNoAndPrimaryTerm(true) - .version(true) - // Build query string filter - .query(QueryBuilders.queryStringQuery(rule.getQueries().get(0).getValue())) - .aggregation(aggregationQueries.getAggBuilder()); - // input index can also be an index pattern or alias so we have to resolve it to concrete index - String concreteIndex = IndexUtils.getNewIndexByCreationDate( - clusterService.state(), - indexNameExpressionResolver, - indices.get(0) // taking first one is fine because we expect that all indices in list share same mappings - ); try { - GetIndexMappingsResponse getIndexMappingsResponse = client.execute( + AggregationItem aggItem = rule.getAggregationItemsFromRule().get(0); + AggregationQueries aggregationQueries = queryBackend.convertAggregation(aggItem); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .seqNoAndPrimaryTerm(true) + .version(true) + // Build query string filter + .query(QueryBuilders.queryStringQuery(rule.getQueries().get(0).getValue())) + .aggregation(aggregationQueries.getAggBuilder()); + // input index can also be an index pattern or alias so we have to resolve it to concrete index + String concreteIndex = IndexUtils.getNewIndexByCreationDate( + clusterService.state(), + indexNameExpressionResolver, + indices.get(0) // taking first one is fine because we expect that all indices in list share same mappings + ); + client.execute( GetIndexMappingsAction.INSTANCE, - new GetIndexMappingsRequest(concreteIndex)) - .actionGet(); - MappingMetadata mappingMetadata = getIndexMappingsResponse.mappings().get(concreteIndex); - List> pairs = MapperUtils.getAllAliasPathPairs(mappingMetadata); - boolean timeStampAliasPresent = pairs. - stream() - .anyMatch(p -> - TIMESTAMP_FIELD_ALIAS.equals(p.getLeft()) || TIMESTAMP_FIELD_ALIAS.equals(p.getRight())); - if(timeStampAliasPresent) { - BoolQueryBuilder boolQueryBuilder = searchSourceBuilder.query() == null - ? new BoolQueryBuilder() - : QueryBuilders.boolQuery().must(searchSourceBuilder.query()); - RangeQueryBuilder timeRangeFilter = QueryBuilders.rangeQuery(TIMESTAMP_FIELD_ALIAS) - .gt("{{period_end}}||-1h") - .lte("{{period_end}}") - .format("epoch_millis"); - boolQueryBuilder.must(timeRangeFilter); - searchSourceBuilder.query(boolQueryBuilder); - } - } catch (Exception e) { - log.error( - String.format(Locale.getDefault(), - "Unable to verify presence of timestamp alias for index [%s] in detector [%s]. Not setting time range filter for bucket level monitor.", - concreteIndex, detector.getName()), e); - } - - List bucketLevelMonitorInputs = new ArrayList<>(); - bucketLevelMonitorInputs.add(new SearchInput(indices, searchSourceBuilder)); - - List triggers = new ArrayList<>(); - BucketLevelTrigger bucketLevelTrigger = new BucketLevelTrigger(rule.getId(), rule.getTitle(), rule.getLevel(), aggregationQueries.getCondition(), - Collections.emptyList()); - triggers.add(bucketLevelTrigger); - - /** TODO - Think how to use detector trigger - List detectorTriggers = detector.getTriggers(); - for (DetectorTrigger detectorTrigger: detectorTriggers) { - String id = detectorTrigger.getId(); - String name = detectorTrigger.getName(); - String severity = detectorTrigger.getSeverity(); - List actions = detectorTrigger.getActions(); - Script condition = detectorTrigger.convertToCondition(); - - BucketLevelTrigger bucketLevelTrigger1 = new BucketLevelTrigger(id, name, severity, condition, actions); - triggers.add(bucketLevelTrigger1); - } **/ - - Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), detector.getEnabled(), detector.getSchedule(), detector.getLastUpdateTime(), detector.getEnabledTime(), - MonitorType.BUCKET_LEVEL_MONITOR, detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), - new DataSources(detector.getRuleIndex(), - detector.getFindingsIndex(), - detector.getFindingsIndexPattern(), - detector.getAlertsIndex(), - detector.getAlertsHistoryIndex(), - detector.getAlertsHistoryIndexPattern(), - DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + new GetIndexMappingsRequest(concreteIndex), + new ActionListener<>() { + @Override + public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { + MappingMetadata mappingMetadata = getIndexMappingsResponse.mappings().get(concreteIndex); + List> pairs = null; + try { + pairs = MapperUtils.getAllAliasPathPairs(mappingMetadata); + } catch (IOException e) { + logger.debug("Failed to get alias path pairs from mapping metadata", e); + onFailure(e); + } + boolean timeStampAliasPresent = pairs. + stream() + .anyMatch(p -> + TIMESTAMP_FIELD_ALIAS.equals(p.getLeft()) || TIMESTAMP_FIELD_ALIAS.equals(p.getRight())); + if (timeStampAliasPresent) { + BoolQueryBuilder boolQueryBuilder = searchSourceBuilder.query() == null + ? new BoolQueryBuilder() + : QueryBuilders.boolQuery().must(searchSourceBuilder.query()); + RangeQueryBuilder timeRangeFilter = QueryBuilders.rangeQuery(TIMESTAMP_FIELD_ALIAS) + .gt("{{period_end}}||-1h") + .lte("{{period_end}}") + .format("epoch_millis"); + boolQueryBuilder.must(timeRangeFilter); + searchSourceBuilder.query(boolQueryBuilder); + } + List bucketLevelMonitorInputs = new ArrayList<>(); + bucketLevelMonitorInputs.add(new SearchInput(indices, searchSourceBuilder)); + + List triggers = new ArrayList<>(); + BucketLevelTrigger bucketLevelTrigger = new BucketLevelTrigger(rule.getId(), rule.getTitle(), rule.getLevel(), aggregationQueries.getCondition(), + Collections.emptyList()); + triggers.add(bucketLevelTrigger); + + /** TODO - Think how to use detector trigger + List detectorTriggers = detector.getTriggers(); + for (DetectorTrigger detectorTrigger: detectorTriggers) { + String id = detectorTrigger.getId(); + String name = detectorTrigger.getName(); + String severity = detectorTrigger.getSeverity(); + List actions = detectorTrigger.getActions(); + Script condition = detectorTrigger.convertToCondition(); + + BucketLevelTrigger bucketLevelTrigger1 = new BucketLevelTrigger(id, name, severity, condition, actions); + triggers.add(bucketLevelTrigger1); + } **/ + + Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), false, detector.getSchedule(), detector.getLastUpdateTime(), null, + MonitorType.BUCKET_LEVEL_MONITOR, detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), + new DataSources(detector.getRuleIndex(), + detector.getFindingsIndex(), + detector.getFindingsIndexPattern(), + detector.getAlertsIndex(), + detector.getAlertsHistoryIndex(), + detector.getAlertsHistoryIndexPattern(), + DetectorMonitorConfig.getRuleIndexMappingsByType(), + true), PLUGIN_OWNER_FIELD); + + listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null)); + } - return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); + @Override + public void onFailure(Exception e) { + log.error( + String.format(Locale.getDefault(), + "Unable to verify presence of timestamp alias for index [%s] in detector [%s]. Not setting time range filter for bucket level monitor.", + concreteIndex, detector.getName()), e); + listener.onFailure(e); + } + }); + } catch (SigmaError e) { + log.error("Failed to create bucket level monitor request", e); + listener.onFailure(e); + } } /** @@ -1162,13 +1276,9 @@ public void onResponse(SearchResponse response) { } else if (detectorInput.getCustomRules().size() > 0) { onFailures(new OpenSearchStatusException("Custom Rule Index not found", RestStatus.NOT_FOUND)); } else { - if (request.getMethod() == RestRequest.Method.POST) { - createMonitorFromQueries(logIndex, queries, detector, listener, request.getRefreshPolicy()); - } else if (request.getMethod() == RestRequest.Method.PUT) { - updateMonitorFromQueries(logIndex, queries, detector, listener, request.getRefreshPolicy()); - } + resolveRuleFieldNamesAndUpsertMonitorFromQueries(queries, detector, logIndex, listener); } - } catch (IOException | SigmaError e) { + } catch (Exception e) { onFailures(e); } } @@ -1214,13 +1324,8 @@ public void onResponse(SearchResponse response) { queries.add(Pair.of(id, rule)); } - - if (request.getMethod() == RestRequest.Method.POST) { - createMonitorFromQueries(logIndex, queries, detector, listener, request.getRefreshPolicy()); - } else if (request.getMethod() == RestRequest.Method.PUT) { - updateMonitorFromQueries(logIndex, queries, detector, listener, request.getRefreshPolicy()); - } - } catch (IOException | SigmaError ex) { + resolveRuleFieldNamesAndUpsertMonitorFromQueries(queries, detector, logIndex, listener); + } catch (Exception ex) { onFailures(ex); } } @@ -1232,6 +1337,56 @@ public void onFailure(Exception e) { }); } + private void resolveRuleFieldNamesAndUpsertMonitorFromQueries(List> queries, Detector detector, String logIndex, ActionListener> listener) { + logger.error("PERF_DEBUG_SAP: Fetching alias path pairs to construct rule_field_names"); + long start = System.currentTimeMillis(); + Set ruleFieldNames = new HashSet<>(); + for (Pair query : queries) { + List queryFieldNames = query.getValue().getQueryFieldNames().stream().map(Value::getValue).collect(Collectors.toList()); + ruleFieldNames.addAll(queryFieldNames); + } + client.execute(GetIndexMappingsAction.INSTANCE, new GetIndexMappingsRequest(logIndex), new ActionListener<>() { + @Override + public void onResponse(GetIndexMappingsResponse getMappingsViewResponse) { + try { + List> aliasPathPairs; + + aliasPathPairs = MapperUtils.getAllAliasPathPairs(getMappingsViewResponse.getMappings().get(logIndex)); + for (Pair aliasPathPair : aliasPathPairs) { + if (ruleFieldNames.contains(aliasPathPair.getLeft())) { + ruleFieldNames.remove(aliasPathPair.getLeft()); + ruleFieldNames.add(aliasPathPair.getRight()); + } + } + long took = System.currentTimeMillis() - start; + log.debug("completed collecting rule_field_names in {} millis", took); + + } catch (Exception e) { + logger.error("Failure in parsing rule field names/aliases while " + + detector.getId() == null ? "creating" : "updating" + + " detector. Not optimizing detector queries with relevant fields", e); + ruleFieldNames.clear(); + } + upsertMonitorQueries(queries, detector, listener, ruleFieldNames, logIndex); + + } + + private void upsertMonitorQueries(List> queries, Detector detector, ActionListener> listener, Set ruleFieldNames, String logIndex) { + if (request.getMethod() == Method.POST) { + createMonitorFromQueries(queries, detector, listener, request.getRefreshPolicy(), new ArrayList<>(ruleFieldNames)); + } else if (request.getMethod() == Method.PUT) { + updateMonitorFromQueries(queries, detector, listener, request.getRefreshPolicy(), new ArrayList<>(ruleFieldNames)); + } + } + + @Override + public void onFailure(Exception e) { + log.error("Failed to fetch mappings view response for log index " + logIndex, e); + listener.onFailure(e); + } + }); + } + public void indexDetector() throws IOException { IndexRequest indexRequest; if (request.getMethod() == RestRequest.Method.POST) { diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index d6a866baf..deef78a0f 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -239,6 +239,29 @@ public static String randomNullRule() { "level: high"; } + public static String randomCloudtrailRuleForCorrelations(String value) { + return "id: 5f92fff9-82e2-48ab-8fc1-8b133556a551\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: AWS User Created\n" + + "description: AWS User Created\n" + + "tags:\n" + + " - attack.test1\n" + + "falsepositives:\n" + + " - Legit User Account Administration\n" + + "level: high\n" + + "date: 2022/01/01\n" + + "status: experimental\n" + + "references:\n" + + " - 'https://github.com/RhinoSecurityLabs/AWS-IAM-Privilege-Escalation'\n" + + "author: toffeebr33k\n" + + "detection:\n" + + " condition: selection_source\n" + + " selection_source:\n" + + " EventName:\n" + + " - " + value; + } + public static String randomRuleForMappingView(String field) { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -726,6 +749,24 @@ public static String productIndexAvgAggRule(){ " condition: sel | avg(fieldA) by fieldC > 110"; } + public static String productIndexCountAggRule(){ + return " title: Test\n" + + " id: 39f918f3-981b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " timeframe: 5m\n" + + " sel:\n" + + " name: laptop\n" + + " condition: sel | count(*) by name > 2"; + } + public static String randomAggregationRule(String aggFunction, String signAndValue) { String rule = "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -786,6 +827,109 @@ public static String randomAggregationRule(String aggFunction, String signAndVa return String.format(Locale.ROOT, rule, opCode, aggFunction, signAndValue); } + public static String randomCloudtrailAggrRule() { + return "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " EventName:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by AccountName >= 2"; + } + + public static String randomCloudtrailAggrRuleWithDotFields() { + return "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(api.operation) by cloud.region > 1\n" + + " selection1:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078"; + } + + public static String randomCloudtrailAggrRuleWithEcsFields() { + return "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(eventName) by awsRegion > 1\n" + + " selection1:\n" + + " eventSource:\n" + + " - lambda.amazonaws.com\n" + + " eventName:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " eventSource:\n" + + " - lambda.amazonaws.com\n" + + " eventName: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078"; + } + + public static String cloudtrailOcsfMappings() { + return "\"properties\": {\n" + + " \"time\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"cloud.region\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"api\": {\n" + + " \"properties\": {\n" + + " \"operation\": {\"type\": \"keyword\"},\n" + + " \"service\": {\n" + + " \"properties\": {\n" + + " \"name\": {\"type\": \"text\"}\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } + public static String windowsIndexMapping() { return "\"properties\": {\n" + " \"@timestamp\": {\"type\":\"date\"},\n" + @@ -1629,6 +1773,13 @@ public static String randomDoc() { "}"; } + public static String randomCloudtrailAggrDoc(String eventType, String accountId) { + return "{\n" + + " \"AccountName\": \"" + accountId + "\",\n" + + " \"EventType\": \"" + eventType + "\"\n" + + "}"; + } + public static String randomVpcFlowDoc() { return "{\n" + " \"version\": 1,\n" + @@ -1651,6 +1802,160 @@ public static String randomAdLdapDoc() { "}"; } + public static String randomCloudtrailOcsfDoc() { + return "{\n" + + " \"activity_id\": 8,\n" + + " \"activity_name\": \"Detach Policy\",\n" + + " \"actor\": {\n" + + " \"idp\": {\n" + + " \"name\": null\n" + + " },\n" + + " \"invoked_by\": null,\n" + + " \"session\": {\n" + + " \"created_time\": 1702510696000,\n" + + " \"issuer\": \"arn\",\n" + + " \"mfa\": false\n" + + " },\n" + + " \"user\": {\n" + + " \"account_uid\": \"\",\n" + + " \"credential_uid\": \"\",\n" + + " \"name\": null,\n" + + " \"type\": \"AssumedRole\",\n" + + " \"uid\": \"\",\n" + + " \"uuid\": \"\"\n" + + " }\n" + + " },\n" + + " \"api\": {\n" + + " \"operation\": \"CreateFunction\",\n" + + " \"request\": {\n" + + " \"uid\": \"0966237c-6279-43f4-a9d7-1eb416fca17d\"\n" + + " },\n" + + " \"response\": {\n" + + " \"error\": null,\n" + + " \"message\": null\n" + + " },\n" + + " \"service\": {\n" + + " \"name\": \"lambda.amazonaws.com\"\n" + + " },\n" + + " \"version\": null\n" + + " },\n" + + " \"category_name\": \"Audit Activity\",\n" + + " \"category_uid\": 3,\n" + + " \"class_name\": \"account_change\",\n" + + " \"class_uid\": 3001,\n" + + " \"cloud\": {\n" + + " \"provider\": \"AWS\",\n" + + " \"region\": \"us-east-1\"\n" + + " },\n" + + " \"dst_endpoint\": null,\n" + + " \"http_request\": {\n" + + " \"user_agent\": \"Boto3/1.26.90 Python/3.7.17 Linux/test.amzn2.x86_64 exec-env/AWS_Lambda_python3.7 Botocore/1.29.90\"\n" + + " },\n" + + " \"metadata\": {\n" + + " \"product\": {\n" + + " \"feature\": {\n" + + " \"name\": \"Management\"\n" + + " },\n" + + " \"name\": \"cloudtrail\",\n" + + " \"vendor_name\": \"AWS\",\n" + + " \"version\": \"1.08\"\n" + + " },\n" + + " \"profiles\": [\n" + + " \"cloud\"\n" + + " ],\n" + + " \"uid\": \"\",\n" + + " \"version\": \"1.0.0-rc.2\"\n" + + " },\n" + + " \"mfa\": null,\n" + + " \"resources\": null,\n" + + " \"severity\": \"Informational\",\n" + + " \"severity_id\": 1,\n" + + " \"src_endpoint\": {\n" + + " \"domain\": null,\n" + + " \"ip\": \"\",\n" + + " \"uid\": null\n" + + " },\n" + + " \"status\": \"Success\",\n" + + " \"status_id\": 1,\n" + + " \"time\": 1702952105000,\n" + + " \"type_name\": \"Account Change: Detach Policy\",\n" + + " \"type_uid\": 300108,\n" + + " \"unmapped\": {\n" + + " \"eventType\": \"AwsApiCall\",\n" + + " \"managementEvent\": \"true\",\n" + + " \"readOnly\": \"false\",\n" + + " \"recipientAccountId\": \"\",\n" + + " \"requestParameters.instanceProfileName\": \"\",\n" + + " \"tlsDetails.cipherSuite\": \"\",\n" + + " \"tlsDetails.clientProvidedHostHeader\": \"iam.amazonaws.com\",\n" + + " \"tlsDetails.tlsVersion\": \"TLSv1.2\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.accountId\": \"\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.principalId\": \"\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.type\": \"Role\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.userName\": \"\"\n" + + " },\n" + + " \"user\": {\n" + + " \"name\": \"\",\n" + + " \"uid\": null,\n" + + " \"uuid\": null\n" + + " }\n" + + "}"; + } + + public static String randomCloudtrailDoc(String user, String event) { + return "{\n" + + " \"eventVersion\": \"1.08\",\n" + + " \"userIdentity\": {\n" + + " \"type\": \"IAMUser\",\n" + + " \"principalId\": \"AIDA6ON6E4XEGITEXAMPLE\",\n" + + " \"arn\": \"arn:aws:iam::888888888888:user/Mary\",\n" + + " \"accountId\": \"888888888888\",\n" + + " \"accessKeyId\": \"AKIAIOSFODNN7EXAMPLE\",\n" + + " \"userName\": \"Mary\",\n" + + " \"sessionContext\": {\n" + + " \"sessionIssuer\": {},\n" + + " \"webIdFederationData\": {},\n" + + " \"attributes\": {\n" + + " \"creationDate\": \"2023-07-19T21:11:57Z\",\n" + + " \"mfaAuthenticated\": \"false\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventTime\": \"2023-07-19T21:25:09Z\",\n" + + " \"eventSource\": \"iam.amazonaws.com\",\n" + + " \"EventName\": \"" + event + "\",\n" + + " \"awsRegion\": \"us-east-1\",\n" + + " \"sourceIPAddress\": \"192.0.2.0\",\n" + + " \"AccountName\": \"" + user + "\",\n" + + " \"userAgent\": \"aws-cli/2.13.5 Python/3.11.4 Linux/4.14.255-314-253.539.amzn2.x86_64 exec-env/CloudShell exe/x86_64.amzn.2 prompt/off command/iam.create-user\",\n" + + " \"requestParameters\": {\n" + + " \"userName\": \"" + user + "\"\n" + + " },\n" + + " \"responseElements\": {\n" + + " \"user\": {\n" + + " \"path\": \"/\",\n" + + " \"arn\": \"arn:aws:iam::888888888888:user/Richard\",\n" + + " \"userId\": \"AIDA6ON6E4XEP7EXAMPLE\",\n" + + " \"createDate\": \"Jul 19, 2023 9:25:09 PM\",\n" + + " \"userName\": \"Richard\"\n" + + " }\n" + + " },\n" + + " \"requestID\": \"2d528c76-329e-410b-9516-EXAMPLE565dc\",\n" + + " \"eventID\": \"ba0801a1-87ec-4d26-be87-EXAMPLE75bbb\",\n" + + " \"readOnly\": false,\n" + + " \"eventType\": \"AwsApiCall\",\n" + + " \"managementEvent\": true,\n" + + " \"recipientAccountId\": \"888888888888\",\n" + + " \"eventCategory\": \"Management\",\n" + + " \"tlsDetails\": {\n" + + " \"tlsVersion\": \"TLSv1.2\",\n" + + " \"cipherSuite\": \"ECDHE-RSA-AES128-GCM-SHA256\",\n" + + " \"clientProvidedHostHeader\": \"iam.amazonaws.com\"\n" + + " },\n" + + " \"sessionCredentialFromConsole\": \"true\"\n" + + "}"; + } + public static String randomAppLogDoc() { return "{\n" + " \"endpoint\": \"/customer_records.txt\",\n" + @@ -1681,6 +1986,312 @@ public static String adLdapLogMappings() { " }"; } + public static String cloudtrailMappings() { + return "\"properties\": {\n" + + " \"Records\": {\n" + + " \"properties\": {\n" + + " \"awsRegion\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventCategory\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventID\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventSource\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventTime\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"eventType\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventVersion\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"managementEvent\": {\n" + + " \"type\": \"boolean\"\n" + + " },\n" + + " \"readOnly\": {\n" + + " \"type\": \"boolean\"\n" + + " },\n" + + " \"recipientAccountId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"requestID\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"requestParameters\": {\n" + + " \"properties\": {\n" + + " \"userName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"responseElements\": {\n" + + " \"properties\": {\n" + + " \"user\": {\n" + + " \"properties\": {\n" + + " \"arn\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"createDate\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"path\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sessionCredentialFromConsole\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sourceIPAddress\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"tlsDetails\": {\n" + + " \"properties\": {\n" + + " \"cipherSuite\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"clientProvidedHostHeader\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"tlsVersion\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userAgent\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userIdentity\": {\n" + + " \"properties\": {\n" + + " \"accessKeyId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"accountId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"arn\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"principalId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sessionContext\": {\n" + + " \"properties\": {\n" + + " \"attributes\": {\n" + + " \"properties\": {\n" + + " \"creationDate\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"mfaAuthenticated\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sessionIssuer\": {\n" + + " \"type\": \"object\"\n" + + " },\n" + + " \"webIdFederationData\": {\n" + + " \"type\": \"object\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }}"; + } + public static String s3AccessLogMappings() { return " \"properties\": {" + " \"aws.cloudtrail.eventSource\": {" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 47e517f3c..d36b9fcd8 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -4,35 +4,6 @@ */ package org.opensearch.securityanalytics.resthandler; -import static java.util.Collections.emptyList; -import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; -import static org.opensearch.securityanalytics.TestHelpers.randomDetector; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; -import static org.opensearch.securityanalytics.TestHelpers.randomDoc; -import static org.opensearch.securityanalytics.TestHelpers.randomDocWithNullField; -import static org.opensearch.securityanalytics.TestHelpers.randomIndex; -import static org.opensearch.securityanalytics.TestHelpers.randomNullRule; -import static org.opensearch.securityanalytics.TestHelpers.randomRule; -import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; -import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndText; -import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithDateKeywords; -import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithKeywords; -import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithStringKeywords; -import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndDate; -import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndDate; -import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndText; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; import org.apache.http.HttpStatus; import org.junit.Assert; import org.opensearch.action.search.SearchResponse; @@ -47,9 +18,38 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.model.Rule; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; +import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; +import static org.opensearch.securityanalytics.TestHelpers.randomDetector; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; +import static org.opensearch.securityanalytics.TestHelpers.randomDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndDate; +import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndText; +import static org.opensearch.securityanalytics.TestHelpers.randomDocWithNullField; +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomNullRule; +import static org.opensearch.securityanalytics.TestHelpers.randomRule; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithDateKeywords; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithKeywords; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithStringKeywords; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndDate; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndText; + public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { /** * 1. Creates detector with 5 doc prepackaged level rules and one doc level monitor based on the given rules diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java index 62e4287c3..d523a0b0a 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java @@ -436,7 +436,7 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException { assertEquals(20, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(25, unmappedFieldAliases.size()); + assertEquals(24, unmappedFieldAliases.size()); } @SuppressWarnings("unchecked") @@ -502,7 +502,7 @@ public void testRawCloudtrailGetMappingsViewApi() throws IOException { assertEquals(17, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(26, unmappedFieldAliases.size()); + assertEquals(25, unmappedFieldAliases.size()); } @SuppressWarnings("unchecked") diff --git a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java index 43db549c8..36901e76d 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java @@ -80,7 +80,7 @@ public void testCountAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_cnt\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 1.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -113,7 +113,7 @@ public void testSumAggregationWithGroupBy() throws IOException, SigmaError { // inputs.query.aggregations -> Query - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery); // triggers.bucket_level_trigger.condition -> Condition Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -145,7 +145,7 @@ public void testMinAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -176,7 +176,7 @@ public void testMaxAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -207,7 +207,84 @@ public void testAvgAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } -} \ No newline at end of file + + + public void testCloudtrailAggregationRule() throws IOException, SigmaError { + OSQueryBackend queryBackend = new OSQueryBackend(Map.of(), true, true); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " event:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by accountid > 2", true)); + + String query = queries.get(0).toString(); + Assert.assertEquals("(event: \"CREATED\") OR (event: \"DELETED\")", query); + + OSQueryBackend.AggregationQueries aggQueries = (OSQueryBackend.AggregationQueries) queries.get(1); + String aggQuery = aggQueries.getAggQuery(); + String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); + + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"accountid\"}}}", aggQuery); + Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_cnt\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 2.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); + } + + public void testCloudtrailAggregationRuleWithDotFields() throws IOException, SigmaError { + OSQueryBackend queryBackend = new OSQueryBackend(Map.of(), true, true); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(api.operation) by cloud.region > 1\n" + + " selection1:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078", true)); + + String query = queries.get(0).toString(); + Assert.assertEquals("((api.service.name: \"lambda.amazonaws.com\") AND (api.operation: \"CreateFunction\")) OR ((api.service.name: \"lambda.amazonaws.com\") AND (api.operation: \"Invoke\"))", query); + + OSQueryBackend.AggregationQueries aggQueries = (OSQueryBackend.AggregationQueries) queries.get(1); + String aggQuery = aggQueries.getAggQuery(); + String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); + + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"cloud.region\"},\"aggs\":{\"api_operation\":{\"value_count\":{\"field\":\"api.operation\"}}}}}", aggQuery); + Assert.assertEquals("{\"buckets_path\":{\"api_operation\":\"api.operation\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.api_operation > 1.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); + } +}