From eac45d7e30baeb7e0ee49857802e028a330f8e23 Mon Sep 17 00:00:00 2001 From: Raj Chakravarthi Date: Thu, 1 Dec 2022 14:54:27 -0500 Subject: [PATCH 1/5] search returns detector type in CAPS fix and integration tests Signed-off-by: Raj Chakravarthi --- .../securityanalytics/model/Detector.java | 2 +- .../resthandler/DetectorRestApiIT.java | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/securityanalytics/model/Detector.java b/src/main/java/org/opensearch/securityanalytics/model/Detector.java index 5cc391e22..a05f04b81 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/Detector.java +++ b/src/main/java/org/opensearch/securityanalytics/model/Detector.java @@ -248,7 +248,7 @@ private XContentBuilder createXContentBuilder(XContentBuilder builder, ToXConten } builder.field(TYPE_FIELD, type) .field(NAME_FIELD, name) - .field(DETECTOR_TYPE_FIELD, detectorType); + .field(DETECTOR_TYPE_FIELD, detectorType.getDetectorType()); if (!secure) { if (user == null) { diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 90fe3b0d2..0be554c32 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -85,6 +85,9 @@ public void testCreatingADetector() throws IOException { Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType(), detectorTypeInResponse); + String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + @@ -182,6 +185,9 @@ public void testGettingADetector() throws IOException { Map responseBody = asMap(getResponse); Assert.assertEquals(createdId, responseBody.get("_id")); Assert.assertNotNull(responseBody.get("detector")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType(), detectorTypeInResponse); } @SuppressWarnings("unchecked") @@ -218,6 +224,11 @@ public void testSearchingDetectors() throws IOException { Map searchResponseHits = (Map) searchResponseBody.get("hits"); Map searchResponseTotal = (Map) searchResponseHits.get("total"); Assert.assertEquals(1, searchResponseTotal.get("value")); + + List> hits = ((List>) ((Map) searchResponseBody.get("hits")).get("hits")); + Map hit = hits.get(0); + String detectorTypeInResponse = (String) ((Map) hit.get("_source")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", detectorTypeInResponse, randomDetectorType()); } @SuppressWarnings("unchecked") @@ -274,6 +285,9 @@ public void testCreatingADetectorWithCustomRules() throws IOException { List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); + String detectorType = (String) ((Map) hit.getSourceAsMap().get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", detectorType, randomDetectorType()); + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); indexDoc(index, "1", randomDoc()); @@ -430,6 +444,9 @@ public void testUpdateADetector() throws IOException { Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(updatedDetector)); Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + String detectorTypeInResponse = (String) ((Map) (asMap(updateResponse).get("detector"))).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType(), detectorTypeInResponse); + request = "{\n" + " \"query\" : {\n" + " \"match_all\":{\n" + From ba63cc6b7b2933935b37c0e24cba8cc4be1959c0 Mon Sep 17 00:00:00 2001 From: Raj Chakravarthi Date: Thu, 1 Dec 2022 16:46:43 -0500 Subject: [PATCH 2/5] nestedquery change at transport layer for detector type Signed-off-by: Raj Chakravarthi --- .../securityanalytics/transport/TransportGetAlertsAction.java | 2 +- .../securityanalytics/transport/TransportGetFindingsAction.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java index 4598e89c5..bdbf7c345 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java @@ -73,7 +73,7 @@ protected void doExecute(Task task, GetAlertsRequest request, ActionListener Date: Thu, 1 Dec 2022 22:40:43 -0500 Subject: [PATCH 3/5] removed unwanted commented code Signed-off-by: Raj Chakravarthi --- .../securityanalytics/transport/TransportGetAlertsAction.java | 2 +- .../securityanalytics/transport/TransportGetFindingsAction.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java index bdbf7c345..9e3e18abc 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java @@ -73,7 +73,7 @@ protected void doExecute(Task task, GetAlertsRequest request, ActionListener Date: Tue, 27 Dec 2022 22:49:04 -0500 Subject: [PATCH 4/5] make detector type in request case insensitive Signed-off-by: Raj Chakravarthi --- .../action/IndexRuleRequest.java | 2 +- .../monitors/DetectorMonitorConfig.java | 32 +++++++++---------- .../mapper/MapperTopicStore.java | 5 +-- .../model/DetectorTrigger.java | 5 ++- .../transport/TransportIndexRuleAction.java | 2 +- .../SecurityAnalyticsRestTestCase.java | 2 +- .../securityanalytics/TestHelpers.java | 2 +- .../resthandler/DetectorRestApiIT.java | 10 +++--- .../resthandler/RuleRestApiIT.java | 6 ++-- 9 files changed, 35 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java b/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java index 2f0e53037..0702b7ac2 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java @@ -64,7 +64,7 @@ public IndexRuleRequest( super(); this.ruleId = ruleId; this.refreshPolicy = refreshPolicy; - this.logType = logType; + this.logType = logType.toLowerCase(Locale.ROOT); this.method = method; this.rule = rule; this.forced = forced; diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index f77ade3b5..02258c2aa 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -58,32 +58,32 @@ public class DetectorMonitorConfig { } public static String getRuleIndex(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType) ? - detectorTypeToIndicesMapping.get(detectorType).getRuleIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? + detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getRuleIndex() : OPENSEARCH_DEFAULT_RULE_INDEX; } public static String getAlertsIndex(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType) ? - detectorTypeToIndicesMapping.get(detectorType).getAlertsIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? + detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAlertsIndex() : OPENSEARCH_DEFAULT_ALERT_INDEX; } public static String getAlertsHistoryIndex(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType) ? - detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? + detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAlertsHistoryIndex() : OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX; } public static String getAlertsHistoryIndexPattern(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType) ? - detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndexPattern() : + return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? + detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAlertsHistoryIndexPattern() : OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN; } public static String getAllAlertsIndicesPattern(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType) ? - detectorTypeToIndicesMapping.get(detectorType).getAllAlertsIndicesPattern() : + return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? + detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAllAlertsIndicesPattern() : OPENSEARCH_DEFAULT_ALL_ALERT_INDICES_PATTERN; } @@ -95,14 +95,14 @@ public static List getAllAlertsIndicesPatternForAllTypes() { } public static String getFindingsIndex(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType) ? - detectorTypeToIndicesMapping.get(detectorType).getFindingsIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? + detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getFindingsIndex() : OPENSEARCH_DEFAULT_FINDINGS_INDEX; } public static String getAllFindingsIndicesPattern(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType) ? - detectorTypeToIndicesMapping.get(detectorType).getAllFindingsIndicesPattern() : + return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? + detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAllFindingsIndicesPattern() : OPENSEARCH_DEFAULT_ALL_FINDINGS_INDICES_PATTERN; } @@ -114,8 +114,8 @@ public static List getAllFindingsIndicesPatternForAllTypes() { } public static String getFindingsIndexPattern(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType) ? - detectorTypeToIndicesMapping.get(detectorType).getFindingsIndexPattern() : + return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? + detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getFindingsIndexPattern() : OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN; } diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperTopicStore.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperTopicStore.java index 1b46df14d..d2f399917 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperTopicStore.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperTopicStore.java @@ -9,6 +9,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -54,11 +55,11 @@ private MapperTopicStore() { } public static String aliasMappings(String mapperTopic) throws IOException { - if (INSTANCE.mapperMap.containsKey(mapperTopic)) { + if (INSTANCE.mapperMap.containsKey(mapperTopic.toLowerCase(Locale.ROOT))) { return new String(Objects.requireNonNull( INSTANCE.getClass().getClassLoader().getResourceAsStream(INSTANCE. - mapperMap.get(mapperTopic))).readAllBytes(), + mapperMap.get(mapperTopic.toLowerCase(Locale.ROOT)))).readAllBytes(), StandardCharsets.UTF_8); } throw new IllegalArgumentException("Mapper not found: [" + mapperTopic + "]"); diff --git a/src/main/java/org/opensearch/securityanalytics/model/DetectorTrigger.java b/src/main/java/org/opensearch/securityanalytics/model/DetectorTrigger.java index f1309d570..33e381558 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/DetectorTrigger.java +++ b/src/main/java/org/opensearch/securityanalytics/model/DetectorTrigger.java @@ -27,6 +27,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; public class DetectorTrigger implements Writeable, ToXContentObject { @@ -66,7 +67,9 @@ public DetectorTrigger(String id, String name, String severity, List rul this.id = id == null? UUIDs.base64UUID(): id; this.name = name; this.severity = severity; - this.ruleTypes = ruleTypes; + this.ruleTypes = ruleTypes.stream() + .map( e -> e.toLowerCase(Locale.ROOT)) + .collect(Collectors.toList()); this.ruleIds = ruleIds; this.ruleSeverityLevels = ruleSeverityLevels; this.tags = tags; diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java index 5eb178fe4..d9dff94aa 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java @@ -171,7 +171,7 @@ public void onFailure(Exception e) { void prepareRuleIndexing() { String rule = request.getRule(); - String category = request.getLogType(); + String category = request.getLogType().toLowerCase(Locale.ROOT); try { SigmaRule parsedRule = SigmaRule.fromYaml(rule, true); diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 086457225..f62d7ade7 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -239,7 +239,7 @@ protected List getRandomPrePackagedRules() throws IOException { " \"query\": {\n" + " \"bool\": {\n" + " \"must\": [\n" + - " { \"match\": {\"rule.category\": \"" + TestHelpers.randomDetectorType() + "\"}}\n" + + " { \"match\": {\"rule.category\": \"" + TestHelpers.randomDetectorType().toLowerCase(Locale.ROOT) + "\"}}\n" + " ]\n" + " }\n" + " }\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 3a5529278..434e34487 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -307,7 +307,7 @@ public static User randomUserEmpty() { } public static String randomDetectorType() { - return "test_windows"; + return "TEST_WINDOWS".toUpperCase(Locale.ROOT); } public static DetectorInput randomDetectorInput() { diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 0be554c32..e34b271e3 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -86,7 +86,7 @@ public void testCreatingADetector() throws IOException { Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", randomDetectorType(), detectorTypeInResponse); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); String request = "{\n" + " \"query\" : {\n" + @@ -187,7 +187,7 @@ public void testGettingADetector() throws IOException { Assert.assertNotNull(responseBody.get("detector")); String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", randomDetectorType(), detectorTypeInResponse); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); } @SuppressWarnings("unchecked") @@ -228,7 +228,7 @@ public void testSearchingDetectors() throws IOException { List> hits = ((List>) ((Map) searchResponseBody.get("hits")).get("hits")); Map hit = hits.get(0); String detectorTypeInResponse = (String) ((Map) hit.get("_source")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", detectorTypeInResponse, randomDetectorType()); + Assert.assertEquals("Detector type incorrect", detectorTypeInResponse, randomDetectorType().toLowerCase(Locale.ROOT)); } @SuppressWarnings("unchecked") @@ -286,7 +286,7 @@ public void testCreatingADetectorWithCustomRules() throws IOException { SearchHit hit = hits.get(0); String detectorType = (String) ((Map) hit.getSourceAsMap().get("detector")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", detectorType, randomDetectorType()); + Assert.assertEquals("Detector type incorrect", detectorType, randomDetectorType().toLowerCase(Locale.ROOT)); String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); @@ -445,7 +445,7 @@ public void testUpdateADetector() throws IOException { Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); String detectorTypeInResponse = (String) ((Map) (asMap(updateResponse).get("detector"))).get("detector_type"); - Assert.assertEquals("Detector type incorrect", randomDetectorType(), detectorTypeInResponse); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); request = "{\n" + " \"query\" : {\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index d8a214d84..83e3fe745 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -74,7 +74,7 @@ public void testCreatingARule() throws IOException { " \"query\": {\n" + " \"bool\": {\n" + " \"must\": [\n" + - " { \"match\": {\"rule.category\": \"" + randomDetectorType() + "\"}}\n" + + " { \"match\": {\"rule.category\": \"" + randomDetectorType().toLowerCase(Locale.ROOT) + "\"}}\n" + " ]\n" + " }\n" + " }\n" + @@ -180,7 +180,7 @@ public void testSearchingPrepackagedRules() throws IOException { " \"query\": {\n" + " \"bool\": {\n" + " \"must\": [\n" + - " { \"match\": {\"rule.category\": \"" + randomDetectorType() + "\"}}\n" + + " { \"match\": {\"rule.category\": \"" + randomDetectorType().toLowerCase(Locale.ROOT) + "\"}}\n" + " ]\n" + " }\n" + " }\n" + @@ -288,7 +288,7 @@ public void testSearchingCustomRules() throws IOException { " \"query\": {\n" + " \"bool\": {\n" + " \"must\": [\n" + - " { \"match\": {\"rule.category\": \"" + randomDetectorType() + "\"}}\n" + + " { \"match\": {\"rule.category\": \"" + randomDetectorType().toLowerCase(Locale.ROOT) + "\"}}\n" + " ]\n" + " }\n" + " }\n" + From dea69bc32685fe5327713d96bc2fa3c9a0bc16a4 Mon Sep 17 00:00:00 2001 From: Raj Chakravarthi Date: Tue, 27 Dec 2022 23:02:25 -0500 Subject: [PATCH 5/5] removed toUpperCase in randomDetectorType Signed-off-by: Raj Chakravarthi --- src/test/java/org/opensearch/securityanalytics/TestHelpers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 434e34487..1ea6c984d 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -307,7 +307,7 @@ public static User randomUserEmpty() { } public static String randomDetectorType() { - return "TEST_WINDOWS".toUpperCase(Locale.ROOT); + return "TEST_WINDOWS"; } public static DetectorInput randomDetectorInput() {