Skip to content

Commit

Permalink
Rollover for alerts/findings history indices (#82)
Browse files Browse the repository at this point in the history 
Signed-off-by: Petar Dzepina <petar.dzepina@gmail.com>
  • Loading branch information
@petardz
petardz committed Nov 3, 2022
1 parent bfb2b23 commit 95f8d9a
Show file tree
Hide file tree
Showing 11 changed files with 1,277 additions and 30 deletions.
25 changes: 24 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package org.opensearch.securityanalytics;

import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.function.Supplier;
import org.opensearch.action.ActionRequest;
Expand All @@ -13,6 +14,7 @@
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.node.DiscoveryNodes;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.component.LifecycleComponent;
import org.opensearch.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.IndexScopedSettings;
Expand All @@ -39,6 +41,7 @@
import org.opensearch.securityanalytics.action.IndexDetectorAction;
import org.opensearch.securityanalytics.action.SearchDetectorAction;
import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction;
import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService;
import org.opensearch.securityanalytics.action.ValidateRulesAction;
import org.opensearch.securityanalytics.mapper.MapperService;
import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction;
Expand Down Expand Up @@ -102,6 +105,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin {

private RuleIndices ruleIndices;

private DetectorIndexManagementService detectorIndexManagementService;

@Override
public Collection<Object> createComponents(Client client,
ClusterService clusterService,
Expand All @@ -121,6 +126,11 @@ public Collection<Object> createComponents(Client client,
return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService);
}

@Override
public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
return Collections.singletonList(DetectorIndexManagementService.class);
}

@Override
public List<RestHandler> getRestHandlers(Settings settings,
RestController restController,
Expand Down Expand Up @@ -160,7 +170,20 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
@Override
public List<Setting<?>> getSettings() {
return List.of(
SecurityAnalyticsSettings.INDEX_TIMEOUT
SecurityAnalyticsSettings.INDEX_TIMEOUT,
SecurityAnalyticsSettings.ALERT_HISTORY_ENABLED,
SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD,
SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE,
SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS,
SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD,
SecurityAnalyticsSettings.REQUEST_TIMEOUT,
SecurityAnalyticsSettings.MAX_ACTION_THROTTLE_VALUE,
SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES,
SecurityAnalyticsSettings.FINDING_HISTORY_ENABLED,
SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS,
SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE,
SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD,
SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD
);
}

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ public void getAlerts(List<String> alertIds,
"ALL",
"ALL",
null,
DetectorMonitorConfig.getAlertsIndex(detector.getDetectorType()),
DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()),
null,
alertIds);
AlertingPluginInterface.INSTANCE.getAlerts(
Expand Down
66 changes: 42 additions & 24 deletions src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
*/
package org.opensearch.securityanalytics.config.monitors;

import java.util.ArrayList;
import java.util.List;
import org.opensearch.securityanalytics.model.Detector;

import java.util.Arrays;
Expand All @@ -13,71 +15,80 @@


public class DetectorMonitorConfig {

public static final String OPENSEARCH_DEFAULT_RULE_INDEX = ".opensearch-sap-detectors-queries-default";
public static final String OPENSEARCH_DEFAULT_ALERT_INDEX = ".opensearch-sap-alerts-default";
public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX = ".opensearch-sap-alerts-history-default";
public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN = "<.opensearch-sap-alerts-history-default-{now/d}-1>";
public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX = ".opensearch-sap-findings-default";
public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN = "<.opensearch-sap-findings-default-{now/d}-1>";

private static Map<String, MonitorConfig> ruleIndexByDetectorTypeMap;
private static Map<String, MonitorConfig> detectorTypeToIndicesMapping;

static {
ruleIndexByDetectorTypeMap = new HashMap<>();
detectorTypeToIndicesMapping = new HashMap<>();
Arrays.stream(Detector.DetectorType.values()).forEach(
detectorType -> {
String ruleIndex = String.format(
Locale.getDefault(), ".opensearch-sap-detectors-queries-%s", detectorType.getDetectorType());
Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", detectorType.getDetectorType());
String alertsIndex = String.format(
Locale.getDefault(), ".opensearch-sap-alerts-%s", detectorType.getDetectorType());
Locale.getDefault(), ".opensearch-sap-%s-alerts", detectorType.getDetectorType());
String alertsHistoryIndex = String.format(
Locale.getDefault(), ".opensearch-sap-alerts-history-%s", detectorType.getDetectorType());
Locale.getDefault(), ".opensearch-sap-%s-alerts-history", detectorType.getDetectorType());
String alertsHistoryIndexPattern = String.format(
Locale.getDefault(), "<.opensearch-sap-alerts-history-%s-{now/d}-1>", detectorType.getDetectorType());
Locale.getDefault(), "<.opensearch-sap-%s-alerts-history-{now/d}-1>", detectorType.getDetectorType());
String allAlertsIndicesPattern = String.format(
Locale.getDefault(), ".opensearch-sap-%s-alerts*", detectorType.getDetectorType());
String findingsIndex = String.format(
Locale.getDefault(), ".opensearch-sap-findings-%s", detectorType.getDetectorType());
Locale.getDefault(), ".opensearch-sap-%s-findings", detectorType.getDetectorType());
String findingsIndexPattern = String.format(
Locale.getDefault(), "<.opensearch-sap-findings-%s-{now/d}-1>", detectorType.getDetectorType());
Locale.getDefault(), "<.opensearch-sap-%s-findings-{now/d}-1>", detectorType.getDetectorType());

MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, findingsIndex, findingsIndexPattern, ruleIndex);
ruleIndexByDetectorTypeMap.put(detectorType.getDetectorType(), monitor);
MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, allAlertsIndicesPattern, findingsIndex, findingsIndexPattern, ruleIndex);
detectorTypeToIndicesMapping.put(detectorType.getDetectorType(), monitor);
});
}

public static String getRuleIndex(String detectorType) {
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
ruleIndexByDetectorTypeMap.get(detectorType).getRuleIndex() :
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
detectorTypeToIndicesMapping.get(detectorType).getRuleIndex() :
OPENSEARCH_DEFAULT_RULE_INDEX;
}

public static String getAlertsIndex(String detectorType) {
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
ruleIndexByDetectorTypeMap.get(detectorType).getAlertsIndex() :
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
detectorTypeToIndicesMapping.get(detectorType).getAlertsIndex() :
OPENSEARCH_DEFAULT_ALERT_INDEX;
}

public static String getAlertsHistoryIndex(String detectorType) {
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
ruleIndexByDetectorTypeMap.get(detectorType).getAlertsHistoryIndex() :
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndex() :
OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX;
}

public static String getAlertsHistoryIndexPattern(String detectorType) {
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
ruleIndexByDetectorTypeMap.get(detectorType).getAlertsHistoryIndexPattern() :
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndexPattern() :
OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN;
}

public static String getAllAlertsIndicesPattern(String detectorType) {
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
detectorTypeToIndicesMapping.get(detectorType).getAllAlertsIndicesPattern() :
"*";
}

public static String getFindingsIndex(String detectorType) {
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndex() :
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
detectorTypeToIndicesMapping.get(detectorType).getFindingsIndex() :
OPENSEARCH_DEFAULT_FINDINGS_INDEX;
}

public static String getFindingsIndexPattern(String detectorType) {
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndexPattern() :
OPENSEARCH_DEFAULT_FINDINGS_INDEX;
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
detectorTypeToIndicesMapping.get(detectorType).getFindingsIndexPattern() :
OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN;
}

public static Map<String, Map<String, String>> getRuleIndexMappingsByType(String detectorType) {
Expand All @@ -88,10 +99,11 @@ public static Map<String, Map<String, String>> getRuleIndexMappingsByType(String
return fieldMappingProperties;
}

private static class MonitorConfig {
public static class MonitorConfig {
private final String alertsIndex;
private final String alertsHistoryIndex;
private final String alertsHistoryIndexPattern;
private final String allAlertsIndicesPattern;
private final String findingIndex;
private final String findingsIndexPattern;
private final String ruleIndex;
Expand All @@ -100,13 +112,15 @@ private MonitorConfig(
String alertsIndex,
String alertsHistoryIndex,
String alertsHistoryIndexPattern,
String allAlertsIndicesPattern,
String findingsIndex,
String findingsIndexPattern,
String ruleIndex
) {
this.alertsIndex = alertsIndex;
this.alertsHistoryIndex = alertsHistoryIndex;
this.alertsHistoryIndexPattern = alertsHistoryIndexPattern;
this.allAlertsIndicesPattern = allAlertsIndicesPattern;
this.findingIndex = findingsIndex;
this.findingsIndexPattern = findingsIndexPattern;
this.ruleIndex = ruleIndex;
Expand All @@ -124,6 +138,10 @@ public String getAlertsHistoryIndexPattern() {
return alertsHistoryIndexPattern;
}

public String getAllAlertsIndicesPattern() {
return allAlertsIndicesPattern;
}

public String getFindingsIndex() {
return findingIndex;
}
Expand Down
Loading

0 comments on commit 95f8d9a

Please sign in to comment.