Findings index mappings fix (#409) (#415)

Signed-off-by: Petar Dzepina <petar.dzepina@gmail.com>
opensearch-project · May 2, 2023 · 4202306 · 4202306
1 parent cbb56f6
commit 4202306
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 9 deletions.
diff --git a/src/main/resources/mappings/finding_mapping.json b/src/main/resources/mappings/finding_mapping.json
@@ -1,7 +1,7 @@
 {
   "dynamic": "strict",
   "_meta" : {
-    "schema_version": 1
+    "schema_version": 2
   },
   "properties": {
     "schema_version": {
@@ -51,6 +51,15 @@
     },
     "timestamp": {
       "type": "long"
+    },
+    "correlated_doc_ids": {
+      "type" : "text",
+      "analyzer": "whitespace",
+      "fields" : {
+        "keyword" : {
+          "type" : "keyword"
+        }
+      }
     }
   }
 }
diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java
@@ -79,6 +79,14 @@
 import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule;
 import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule;
 import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD;
+import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD;
 import static org.opensearch.securityanalytics.util.RuleTopicIndices.ruleTopicIndexSettings;
 
 public class SecurityAnalyticsRestTestCase extends OpenSearchRestTestCase {
@@ -1628,4 +1636,18 @@ protected void createSampleDatastream(String datastreamName, String mappings, bo
 
         createDatastreamAPI(datastreamName);
     }
+
+
+    protected void restoreAlertsFindingsIMSettings() throws IOException {
+        updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "720m");
+        updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "100000");
+        updateClusterSetting(ALERT_HISTORY_INDEX_MAX_AGE.getKey(), "60d");
+        updateClusterSetting(ALERT_HISTORY_RETENTION_PERIOD.getKey(), "60d");
+
+        updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "720m");
+        updateClusterSetting(FINDING_HISTORY_MAX_DOCS.getKey(), "100000");
+        updateClusterSetting(FINDING_HISTORY_INDEX_MAX_AGE.getKey(), "60d");
+        updateClusterSetting(FINDING_HISTORY_RETENTION_PERIOD.getKey(), "60d");
+
+    }
 }
diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java
@@ -654,6 +654,7 @@ public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedExc
         }
         assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3);
 
+        restoreAlertsFindingsIMSettings();
     }
 
     public void testAlertHistoryRollover_maxAge_low_retention() throws IOException, InterruptedException {
@@ -732,6 +733,8 @@ public void testAlertHistoryRollover_maxAge_low_retention() throws IOException,
         }
 
         assertTrue("Did not find 3 alert indices", alertIndices.size() == 1);
+
+        restoreAlertsFindingsIMSettings();
     }
 
     public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedException {
@@ -814,6 +817,7 @@ public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedEx
         }
         assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3);
 
+        restoreAlertsFindingsIMSettings();
     }
 
     public void testGetAlertsFromAllIndices() throws IOException, InterruptedException {
@@ -909,5 +913,7 @@ public void testGetAlertsFromAllIndices() throws IOException, InterruptedExcepti
         getAlertsBody = asMap(getAlertsResponse);
         // 1 from alertIndex and 1 from history index
         Assert.assertEquals(2, getAlertsBody.get("total_alerts"));
+
+        restoreAlertsFindingsIMSettings();
     }
 }
diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java
@@ -304,9 +304,21 @@ public void testGetFindings_rolloverByMaxAge_success() throws IOException, Inter
 
         String monitorId = ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
 
+        // Execute monitor first time to create findings index/alias
         indexDoc(index, "1", randomDoc());
-
         Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
+
+        // Wait for findings index to rollover first, to make sure that our rollover applied correct settings/mappings
+        List<String> findingIndices = getFindingIndices(detector.getDetectorType());
+        while(findingIndices.size() < 2) {
+            findingIndices = getFindingIndices(detector.getDetectorType());
+            Thread.sleep(1000);
+        }
+        assertTrue("Did not find more then 2 finding indices", findingIndices.size() >= 2);
+
+        // Execute monitor second time to insert finding in new rollover'd index
+        indexDoc(index, "2", randomDoc());
+        executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
         Map<String, Object> executeResults = entityAsMap(executeResponse);
 
         int noOfSigmaRuleMatches = ((List<Map<String, Object>>) ((Map<String, Object>) executeResults.get("input_results")).get("results")).get(0).size();
@@ -316,14 +328,9 @@ public void testGetFindings_rolloverByMaxAge_success() throws IOException, Inter
         params.put("detector_id", detectorId);
         Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null);
         Map<String, Object> getFindingsBody = entityAsMap(getFindingsResponse);
-        Assert.assertEquals(1, getFindingsBody.get("total_findings"));
+        Assert.assertEquals(2, getFindingsBody.get("total_findings"));
 
-        List<String> findingIndices = getFindingIndices(detector.getDetectorType());
-        while(findingIndices.size() < 2) {
-            findingIndices = getFindingIndices(detector.getDetectorType());
-            Thread.sleep(1000);
-        }
-        assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2);
+        restoreAlertsFindingsIMSettings();
     }
 
     public void testGetFindings_rolloverByMaxDoc_success() throws IOException, InterruptedException {
@@ -387,6 +394,8 @@ public void testGetFindings_rolloverByMaxDoc_success() throws IOException, Inter
             Thread.sleep(1000);
         }
         assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2);
+
+        restoreAlertsFindingsIMSettings();
     }
 
     public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IOException, InterruptedException {
@@ -472,5 +481,7 @@ public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IO
         getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null);
         getFindingsBody = entityAsMap(getFindingsResponse);
         Assert.assertEquals(1, getFindingsBody.get("total_findings"));
+
+        restoreAlertsFindingsIMSettings();
     }
 }