Rollover for alerts/findings history indices

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -5,6 +5,7 @@
 package org.opensearch.securityanalytics;
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.function.Supplier;
 import org.opensearch.action.ActionRequest;
@@ -13,6 +14,7 @@
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.component.LifecycleComponent;
 import org.opensearch.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.IndexScopedSettings;
@@ -39,6 +41,7 @@
 import org.opensearch.securityanalytics.action.IndexDetectorAction;
 import org.opensearch.securityanalytics.action.SearchDetectorAction;
 import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction;
+import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService;
 import org.opensearch.securityanalytics.mapper.MapperService;
 import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction;
 import org.opensearch.securityanalytics.resthandler.RestGetFindingsAction;
@@ -99,6 +102,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin {
 
     private RuleIndices ruleIndices;
 
+    private DetectorIndexManagementService detectorIndexManagementService;
+
     @Override
     public Collection<Object> createComponents(Client client,
                                                ClusterService clusterService,
@@ -118,6 +123,11 @@ public Collection<Object> createComponents(Client client,
         return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService);
     }
 
+    @Override
+    public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
+        return Collections.singletonList(DetectorIndexManagementService.class);
+    }
+
     @Override
     public List<RestHandler> getRestHandlers(Settings settings,
                                              RestController restController,
@@ -156,7 +166,20 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
     @Override
     public List<Setting<?>> getSettings() {
         return List.of(
-                SecurityAnalyticsSettings.INDEX_TIMEOUT
+                SecurityAnalyticsSettings.INDEX_TIMEOUT,
+                SecurityAnalyticsSettings.ALERT_HISTORY_ENABLED,
+                SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD,
+                SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE,
+                SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS,
+                SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD,
+                SecurityAnalyticsSettings.REQUEST_TIMEOUT,
+                SecurityAnalyticsSettings.MAX_ACTION_THROTTLE_VALUE,
+                SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES,
+                SecurityAnalyticsSettings.FINDING_HISTORY_ENABLED,
+                SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS,
+                SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE,
+                SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD,
+                SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD
         );
     }
 

src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java

@@ -243,7 +243,7 @@ public void getAlerts(List<String> alertIds,
                 "ALL",
                 "ALL",
                 null,
-                DetectorMonitorConfig.getAlertsIndex(detector.getDetectorType()),
+                DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()),
                 null,
                 alertIds);
         AlertingPluginInterface.INSTANCE.getAlerts(

src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java

@@ -4,6 +4,8 @@
  */
 package org.opensearch.securityanalytics.config.monitors;
 
+import java.util.ArrayList;
+import java.util.List;
 import org.opensearch.securityanalytics.model.Detector;
 
 import java.util.Arrays;
@@ -13,71 +15,80 @@
 
 
 public class DetectorMonitorConfig {
+
     public static final String OPENSEARCH_DEFAULT_RULE_INDEX = ".opensearch-sap-detectors-queries-default";
     public static final String OPENSEARCH_DEFAULT_ALERT_INDEX = ".opensearch-sap-alerts-default";
     public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX = ".opensearch-sap-alerts-history-default";
     public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN = "<.opensearch-sap-alerts-history-default-{now/d}-1>";
     public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX = ".opensearch-sap-findings-default";
     public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN = "<.opensearch-sap-findings-default-{now/d}-1>";
 
-    private static Map<String, MonitorConfig> ruleIndexByDetectorTypeMap;
+    private static Map<String, MonitorConfig> detectorTypeToIndicesMapping;
 
     static {
-        ruleIndexByDetectorTypeMap = new HashMap<>();
+        detectorTypeToIndicesMapping = new HashMap<>();
         Arrays.stream(Detector.DetectorType.values()).forEach(
                 detectorType -> {
                     String ruleIndex = String.format(
-                            Locale.getDefault(), ".opensearch-sap-detectors-queries-%s", detectorType.getDetectorType());
+                            Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", detectorType.getDetectorType());
                     String alertsIndex = String.format(
-                            Locale.getDefault(), ".opensearch-sap-alerts-%s", detectorType.getDetectorType());
+                            Locale.getDefault(), ".opensearch-sap-%s-alerts", detectorType.getDetectorType());
                     String alertsHistoryIndex = String.format(
-                            Locale.getDefault(), ".opensearch-sap-alerts-history-%s", detectorType.getDetectorType());
+                            Locale.getDefault(), ".opensearch-sap-%s-alerts-history", detectorType.getDetectorType());
                     String alertsHistoryIndexPattern = String.format(
-                            Locale.getDefault(), "<.opensearch-sap-alerts-history-%s-{now/d}-1>", detectorType.getDetectorType());
+                            Locale.getDefault(), "<.opensearch-sap-%s-alerts-history-{now/d}-1>", detectorType.getDetectorType());
+                    String allAlertsIndicesPattern = String.format(
+                            Locale.getDefault(), ".opensearch-sap-%s-alerts*", detectorType.getDetectorType());
                     String findingsIndex = String.format(
-                            Locale.getDefault(), ".opensearch-sap-findings-%s", detectorType.getDetectorType());
+                            Locale.getDefault(), ".opensearch-sap-%s-findings", detectorType.getDetectorType());
                     String findingsIndexPattern = String.format(
-                            Locale.getDefault(), "<.opensearch-sap-findings-%s-{now/d}-1>", detectorType.getDetectorType());
+                            Locale.getDefault(), "<.opensearch-sap-%s-findings-{now/d}-1>", detectorType.getDetectorType());
 
-                    MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, findingsIndex, findingsIndexPattern, ruleIndex);
-                    ruleIndexByDetectorTypeMap.put(detectorType.getDetectorType(), monitor);
+                    MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, allAlertsIndicesPattern, findingsIndex, findingsIndexPattern, ruleIndex);
+                    detectorTypeToIndicesMapping.put(detectorType.getDetectorType(), monitor);
                 });
     }
 
     public static String getRuleIndex(String detectorType) {
-        return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
-                ruleIndexByDetectorTypeMap.get(detectorType).getRuleIndex() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
+                detectorTypeToIndicesMapping.get(detectorType).getRuleIndex() :
                 OPENSEARCH_DEFAULT_RULE_INDEX;
     }
 
     public static String getAlertsIndex(String detectorType) {
-        return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
-                ruleIndexByDetectorTypeMap.get(detectorType).getAlertsIndex() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
+                detectorTypeToIndicesMapping.get(detectorType).getAlertsIndex() :
                 OPENSEARCH_DEFAULT_ALERT_INDEX;
     }
 
     public static String getAlertsHistoryIndex(String detectorType) {
-        return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
-                ruleIndexByDetectorTypeMap.get(detectorType).getAlertsHistoryIndex() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
+                detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndex() :
                 OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX;
     }
 
     public static String getAlertsHistoryIndexPattern(String detectorType) {
-        return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
-                ruleIndexByDetectorTypeMap.get(detectorType).getAlertsHistoryIndexPattern() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
+                detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndexPattern() :
                 OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN;
     }
 
+    public static String getAllAlertsIndicesPattern(String detectorType) {
+        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
+                detectorTypeToIndicesMapping.get(detectorType).getAllAlertsIndicesPattern() :
+                "*";
+    }
+
     public static String getFindingsIndex(String detectorType) {
-        return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
-                ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndex() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
+                detectorTypeToIndicesMapping.get(detectorType).getFindingsIndex() :
                 OPENSEARCH_DEFAULT_FINDINGS_INDEX;
     }
 
     public static String getFindingsIndexPattern(String detectorType) {
-        return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
-                ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndexPattern() :
-                OPENSEARCH_DEFAULT_FINDINGS_INDEX;
+        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
+                detectorTypeToIndicesMapping.get(detectorType).getFindingsIndexPattern() :
+                OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN;
     }
 
     public static Map<String, Map<String, String>> getRuleIndexMappingsByType(String detectorType) {
@@ -88,10 +99,11 @@ public static Map<String, Map<String, String>> getRuleIndexMappingsByType(String
         return fieldMappingProperties;
     }
 
-    private static class MonitorConfig {
+    public static class MonitorConfig {
         private final String alertsIndex;
         private final String alertsHistoryIndex;
         private final String alertsHistoryIndexPattern;
+        private final String allAlertsIndicesPattern;
         private final String findingIndex;
         private final String findingsIndexPattern;
         private final String ruleIndex;
@@ -100,13 +112,15 @@ private MonitorConfig(
                 String alertsIndex,
                 String alertsHistoryIndex,
                 String alertsHistoryIndexPattern,
+                String allAlertsIndicesPattern,
                 String findingsIndex,
                 String findingsIndexPattern,
                 String ruleIndex
         ) {
             this.alertsIndex = alertsIndex;
             this.alertsHistoryIndex = alertsHistoryIndex;
             this.alertsHistoryIndexPattern = alertsHistoryIndexPattern;
+            this.allAlertsIndicesPattern = allAlertsIndicesPattern;
             this.findingIndex = findingsIndex;
             this.findingsIndexPattern = findingsIndexPattern;
             this.ruleIndex = ruleIndex;
@@ -124,6 +138,10 @@ public String getAlertsHistoryIndexPattern() {
             return alertsHistoryIndexPattern;
         }
 
+        public String getAllAlertsIndicesPattern() {
+            return allAlertsIndicesPattern;
+        }
+
         public String getFindingsIndex() {
             return findingIndex;
         }