diff --git a/src/main/resources/mappings/finding_mapping.json b/src/main/resources/mappings/finding_mapping.json index c9386b2ef..421dc202c 100644 --- a/src/main/resources/mappings/finding_mapping.json +++ b/src/main/resources/mappings/finding_mapping.json @@ -1,7 +1,7 @@ { "dynamic": "strict", "_meta" : { - "schema_version": 1 + "schema_version": 2 }, "properties": { "schema_version": { @@ -51,6 +51,15 @@ }, "timestamp": { "type": "long" + }, + "correlated_doc_ids": { + "type" : "text", + "analyzer": "whitespace", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } } } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 4947c3543..bfae03c62 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -79,6 +79,14 @@ import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule; import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; import static org.opensearch.securityanalytics.util.RuleTopicIndices.ruleTopicIndexSettings; public class SecurityAnalyticsRestTestCase extends OpenSearchRestTestCase { @@ -1628,4 +1636,18 @@ protected void createSampleDatastream(String datastreamName, String mappings, bo createDatastreamAPI(datastreamName); } + + + protected void restoreAlertsFindingsIMSettings() throws IOException { + updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "720m"); + updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "100000"); + updateClusterSetting(ALERT_HISTORY_INDEX_MAX_AGE.getKey(), "60d"); + updateClusterSetting(ALERT_HISTORY_RETENTION_PERIOD.getKey(), "60d"); + + updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "720m"); + updateClusterSetting(FINDING_HISTORY_MAX_DOCS.getKey(), "100000"); + updateClusterSetting(FINDING_HISTORY_INDEX_MAX_AGE.getKey(), "60d"); + updateClusterSetting(FINDING_HISTORY_RETENTION_PERIOD.getKey(), "60d"); + + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index b373b8211..c50abfa82 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -654,6 +654,7 @@ public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedExc } assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + restoreAlertsFindingsIMSettings(); } public void testAlertHistoryRollover_maxAge_low_retention() throws IOException, InterruptedException { @@ -732,6 +733,8 @@ public void testAlertHistoryRollover_maxAge_low_retention() throws IOException, } assertTrue("Did not find 3 alert indices", alertIndices.size() == 1); + + restoreAlertsFindingsIMSettings(); } public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedException { @@ -814,6 +817,7 @@ public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedEx } assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + restoreAlertsFindingsIMSettings(); } public void testGetAlertsFromAllIndices() throws IOException, InterruptedException { @@ -909,5 +913,7 @@ public void testGetAlertsFromAllIndices() throws IOException, InterruptedExcepti getAlertsBody = asMap(getAlertsResponse); // 1 from alertIndex and 1 from history index Assert.assertEquals(2, getAlertsBody.get("total_alerts")); + + restoreAlertsFindingsIMSettings(); } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index 7b34b7d94..daac0bb87 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -304,9 +304,21 @@ public void testGetFindings_rolloverByMaxAge_success() throws IOException, Inter String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + // Execute monitor first time to create findings index/alias indexDoc(index, "1", randomDoc()); - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + + // Wait for findings index to rollover first, to make sure that our rollover applied correct settings/mappings + List findingIndices = getFindingIndices(detector.getDetectorType()); + while(findingIndices.size() < 2) { + findingIndices = getFindingIndices(detector.getDetectorType()); + Thread.sleep(1000); + } + assertTrue("Did not find more then 2 finding indices", findingIndices.size() >= 2); + + // Execute monitor second time to insert finding in new rollover'd index + indexDoc(index, "2", randomDoc()); + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); @@ -316,14 +328,9 @@ public void testGetFindings_rolloverByMaxAge_success() throws IOException, Inter params.put("detector_id", detectorId); Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); Map getFindingsBody = entityAsMap(getFindingsResponse); - Assert.assertEquals(1, getFindingsBody.get("total_findings")); + Assert.assertEquals(2, getFindingsBody.get("total_findings")); - List findingIndices = getFindingIndices(detector.getDetectorType()); - while(findingIndices.size() < 2) { - findingIndices = getFindingIndices(detector.getDetectorType()); - Thread.sleep(1000); - } - assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); + restoreAlertsFindingsIMSettings(); } public void testGetFindings_rolloverByMaxDoc_success() throws IOException, InterruptedException { @@ -387,6 +394,8 @@ public void testGetFindings_rolloverByMaxDoc_success() throws IOException, Inter Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); + + restoreAlertsFindingsIMSettings(); } public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IOException, InterruptedException { @@ -472,5 +481,7 @@ public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IO getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); + + restoreAlertsFindingsIMSettings(); } }