diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 73654d73c..19ba64c79 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -5,6 +5,7 @@ package org.opensearch.securityanalytics; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.function.Supplier; import org.opensearch.action.ActionRequest; @@ -13,6 +14,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.component.LifecycleComponent; import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; @@ -39,6 +41,7 @@ import org.opensearch.securityanalytics.action.IndexDetectorAction; import org.opensearch.securityanalytics.action.SearchDetectorAction; import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction; +import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService; import org.opensearch.securityanalytics.action.ValidateRulesAction; import org.opensearch.securityanalytics.mapper.MapperService; import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction; @@ -102,6 +105,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin { private RuleIndices ruleIndices; + private DetectorIndexManagementService detectorIndexManagementService; + @Override public Collection createComponents(Client client, ClusterService clusterService, @@ -121,6 +126,11 @@ public Collection createComponents(Client client, return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService); } + @Override + public Collection> getGuiceServiceClasses() { + return Collections.singletonList(DetectorIndexManagementService.class); + } + @Override public List getRestHandlers(Settings settings, RestController restController, @@ -160,7 +170,20 @@ public List getNamedXContent() { @Override public List> getSettings() { return List.of( - SecurityAnalyticsSettings.INDEX_TIMEOUT + SecurityAnalyticsSettings.INDEX_TIMEOUT, + SecurityAnalyticsSettings.ALERT_HISTORY_ENABLED, + SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD, + SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE, + SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS, + SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD, + SecurityAnalyticsSettings.REQUEST_TIMEOUT, + SecurityAnalyticsSettings.MAX_ACTION_THROTTLE_VALUE, + SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, + SecurityAnalyticsSettings.FINDING_HISTORY_ENABLED, + SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS, + SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE, + SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD, + SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD ); } diff --git a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java index 2508786f0..a2840adbc 100644 --- a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java +++ b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java @@ -243,7 +243,7 @@ public void getAlerts(List alertIds, "ALL", "ALL", null, - DetectorMonitorConfig.getAlertsIndex(detector.getDetectorType()), + DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()), null, alertIds); AlertingPluginInterface.INSTANCE.getAlerts( diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index a24750ae2..c3129317c 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -4,6 +4,8 @@ */ package org.opensearch.securityanalytics.config.monitors; +import java.util.ArrayList; +import java.util.List; import org.opensearch.securityanalytics.model.Detector; import java.util.Arrays; @@ -13,6 +15,7 @@ public class DetectorMonitorConfig { + public static final String OPENSEARCH_DEFAULT_RULE_INDEX = ".opensearch-sap-detectors-queries-default"; public static final String OPENSEARCH_DEFAULT_ALERT_INDEX = ".opensearch-sap-alerts-default"; public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX = ".opensearch-sap-alerts-history-default"; @@ -20,64 +23,72 @@ public class DetectorMonitorConfig { public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX = ".opensearch-sap-findings-default"; public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN = "<.opensearch-sap-findings-default-{now/d}-1>"; - private static Map ruleIndexByDetectorTypeMap; + private static Map detectorTypeToIndicesMapping; static { - ruleIndexByDetectorTypeMap = new HashMap<>(); + detectorTypeToIndicesMapping = new HashMap<>(); Arrays.stream(Detector.DetectorType.values()).forEach( detectorType -> { String ruleIndex = String.format( - Locale.getDefault(), ".opensearch-sap-detectors-queries-%s", detectorType.getDetectorType()); + Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", detectorType.getDetectorType()); String alertsIndex = String.format( - Locale.getDefault(), ".opensearch-sap-alerts-%s", detectorType.getDetectorType()); + Locale.getDefault(), ".opensearch-sap-%s-alerts", detectorType.getDetectorType()); String alertsHistoryIndex = String.format( - Locale.getDefault(), ".opensearch-sap-alerts-history-%s", detectorType.getDetectorType()); + Locale.getDefault(), ".opensearch-sap-%s-alerts-history", detectorType.getDetectorType()); String alertsHistoryIndexPattern = String.format( - Locale.getDefault(), "<.opensearch-sap-alerts-history-%s-{now/d}-1>", detectorType.getDetectorType()); + Locale.getDefault(), "<.opensearch-sap-%s-alerts-history-{now/d}-1>", detectorType.getDetectorType()); + String allAlertsIndicesPattern = String.format( + Locale.getDefault(), ".opensearch-sap-%s-alerts*", detectorType.getDetectorType()); String findingsIndex = String.format( - Locale.getDefault(), ".opensearch-sap-findings-%s", detectorType.getDetectorType()); + Locale.getDefault(), ".opensearch-sap-%s-findings", detectorType.getDetectorType()); String findingsIndexPattern = String.format( - Locale.getDefault(), "<.opensearch-sap-findings-%s-{now/d}-1>", detectorType.getDetectorType()); + Locale.getDefault(), "<.opensearch-sap-%s-findings-{now/d}-1>", detectorType.getDetectorType()); - MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, findingsIndex, findingsIndexPattern, ruleIndex); - ruleIndexByDetectorTypeMap.put(detectorType.getDetectorType(), monitor); + MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, allAlertsIndicesPattern, findingsIndex, findingsIndexPattern, ruleIndex); + detectorTypeToIndicesMapping.put(detectorType.getDetectorType(), monitor); }); } public static String getRuleIndex(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getRuleIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getRuleIndex() : OPENSEARCH_DEFAULT_RULE_INDEX; } public static String getAlertsIndex(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getAlertsIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getAlertsIndex() : OPENSEARCH_DEFAULT_ALERT_INDEX; } public static String getAlertsHistoryIndex(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getAlertsHistoryIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndex() : OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX; } public static String getAlertsHistoryIndexPattern(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getAlertsHistoryIndexPattern() : + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndexPattern() : OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN; } + public static String getAllAlertsIndicesPattern(String detectorType) { + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getAllAlertsIndicesPattern() : + "*"; + } + public static String getFindingsIndex(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getFindingsIndex() : OPENSEARCH_DEFAULT_FINDINGS_INDEX; } public static String getFindingsIndexPattern(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndexPattern() : - OPENSEARCH_DEFAULT_FINDINGS_INDEX; + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getFindingsIndexPattern() : + OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN; } public static Map> getRuleIndexMappingsByType(String detectorType) { @@ -88,10 +99,11 @@ public static Map> getRuleIndexMappingsByType(String return fieldMappingProperties; } - private static class MonitorConfig { + public static class MonitorConfig { private final String alertsIndex; private final String alertsHistoryIndex; private final String alertsHistoryIndexPattern; + private final String allAlertsIndicesPattern; private final String findingIndex; private final String findingsIndexPattern; private final String ruleIndex; @@ -100,6 +112,7 @@ private MonitorConfig( String alertsIndex, String alertsHistoryIndex, String alertsHistoryIndexPattern, + String allAlertsIndicesPattern, String findingsIndex, String findingsIndexPattern, String ruleIndex @@ -107,6 +120,7 @@ private MonitorConfig( this.alertsIndex = alertsIndex; this.alertsHistoryIndex = alertsHistoryIndex; this.alertsHistoryIndexPattern = alertsHistoryIndexPattern; + this.allAlertsIndicesPattern = allAlertsIndicesPattern; this.findingIndex = findingsIndex; this.findingsIndexPattern = findingsIndexPattern; this.ruleIndex = ruleIndex; @@ -124,6 +138,10 @@ public String getAlertsHistoryIndexPattern() { return alertsHistoryIndexPattern; } + public String getAllAlertsIndicesPattern() { + return allAlertsIndicesPattern; + } + public String getFindingsIndex() { return findingIndex; } diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java new file mode 100644 index 000000000..ad6bbd240 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -0,0 +1,560 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.indexmanagment; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.ActionListener; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.admin.indices.rollover.RolloverRequest; +import org.opensearch.action.admin.indices.rollover.RolloverResponse; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterStateListener; +import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.threadpool.Scheduler; +import org.opensearch.threadpool.ThreadPool; + + +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ENABLED; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ENABLED; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; + +public class DetectorIndexManagementService extends AbstractLifecycleComponent implements ClusterStateListener { + + private Logger logger = LogManager.getLogger(DetectorIndexManagementService.class); + + private static final String ALERT_HISTORY_ALL = ".opensearch-sap-alerts-history-*"; + private static final String FINDING_HISTORY_ALL = ".opensearch-sap-findings-*"; + + private final Client client; + private final ThreadPool threadPool; + private final ClusterService clusterService; + private Settings settings; + + private volatile Boolean alertHistoryEnabled; + private volatile Boolean findingHistoryEnabled; + + private volatile Long alertHistoryMaxDocs; + private volatile Long findingHistoryMaxDocs; + + private volatile TimeValue alertHistoryMaxAge; + private volatile TimeValue findingHistoryMaxAge; + + private volatile TimeValue alertHistoryRolloverPeriod; + private volatile TimeValue findingHistoryRolloverPeriod; + + private volatile TimeValue alertHistoryRetentionPeriod; + private volatile TimeValue findingHistoryRetentionPeriod; + + private volatile boolean isClusterManager = false; + + private Scheduler.Cancellable scheduledAlertsRollover = null; + private Scheduler.Cancellable scheduledFindingsRollover = null; + + List alertHistoryIndices = new ArrayList<>(); + List findingHistoryIndices = new ArrayList<>(); + + @Inject + public DetectorIndexManagementService(Settings settings, Client client, ThreadPool threadPool, ClusterService clusterService) { + this.settings = settings; + this.client = client; + this.threadPool = threadPool; + this.clusterService = clusterService; + + clusterService.addListener(this); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(ALERT_HISTORY_ENABLED, this::setAlertHistoryEnabled); + clusterService.getClusterSettings().addSettingsUpdateConsumer(ALERT_HISTORY_MAX_DOCS, maxDocs -> { + setAlertHistoryMaxDocs(maxDocs); + for (HistoryIndexInfo h : alertHistoryIndices) { + h.maxDocs = maxDocs; + } + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(ALERT_HISTORY_INDEX_MAX_AGE, maxAge -> { + setAlertHistoryMaxAge(maxAge); + for (HistoryIndexInfo h : alertHistoryIndices) { + h.maxAge = maxAge; + } + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(ALERT_HISTORY_ROLLOVER_PERIOD, timeValue -> { + DetectorIndexManagementService.this.alertHistoryRolloverPeriod = timeValue; + rescheduleAlertRollover(); + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(ALERT_HISTORY_RETENTION_PERIOD, this::setAlertHistoryRetentionPeriod); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_ENABLED, this::setFindingHistoryEnabled); + clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_MAX_DOCS, maxDocs -> { + setFindingHistoryMaxDocs(maxDocs); + for (HistoryIndexInfo h : findingHistoryIndices) { + h.maxDocs = maxDocs; + } + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_INDEX_MAX_AGE, maxAge -> { + setFindingHistoryMaxAge(maxAge); + for (HistoryIndexInfo h : findingHistoryIndices) { + h.maxAge = maxAge; + } + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_ROLLOVER_PERIOD, timeValue -> { + DetectorIndexManagementService.this.findingHistoryRolloverPeriod = timeValue; + rescheduleFindingRollover(); + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_RETENTION_PERIOD, this::setFindingHistoryRetentionPeriod); + + initFromClusterSettings(); + + initAllIndexLists(); + } + + private void initAllIndexLists() { + Arrays.stream(Detector.DetectorType.values()).forEach( + detectorType -> { + + String alertsHistoryIndex = DetectorMonitorConfig.getAlertsHistoryIndex(detectorType.getDetectorType()); + String alertsHistoryIndexPattern = DetectorMonitorConfig.getAlertsHistoryIndexPattern(detectorType.getDetectorType()); + + alertHistoryIndices.add(new HistoryIndexInfo( + alertsHistoryIndex, + alertsHistoryIndexPattern, + alertMapping(), + alertHistoryMaxDocs, + alertHistoryMaxAge, + false + )); + + String findingsIndex = DetectorMonitorConfig.getFindingsIndex(detectorType.getDetectorType()); + String findingsIndexPattern = DetectorMonitorConfig.getFindingsIndexPattern(detectorType.getDetectorType()); + + findingHistoryIndices.add(new HistoryIndexInfo( + findingsIndex, + findingsIndexPattern, + findingMapping(), + findingHistoryMaxDocs, + findingHistoryMaxAge, + false + )); + }); + } + + private void initFromClusterSettings() { + alertHistoryEnabled = ALERT_HISTORY_ENABLED.get(settings); + findingHistoryEnabled = FINDING_HISTORY_ENABLED.get(settings); + alertHistoryMaxDocs = ALERT_HISTORY_MAX_DOCS.get(settings); + findingHistoryMaxDocs = FINDING_HISTORY_MAX_DOCS.get(settings); + alertHistoryMaxAge = ALERT_HISTORY_INDEX_MAX_AGE.get(settings); + findingHistoryMaxAge = FINDING_HISTORY_INDEX_MAX_AGE.get(settings); + alertHistoryRolloverPeriod = ALERT_HISTORY_ROLLOVER_PERIOD.get(settings); + findingHistoryRolloverPeriod = FINDING_HISTORY_ROLLOVER_PERIOD.get(settings); + alertHistoryRetentionPeriod = ALERT_HISTORY_RETENTION_PERIOD.get(settings); + findingHistoryRetentionPeriod = FINDING_HISTORY_RETENTION_PERIOD.get(settings); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + // Instead of using a LocalNodeClusterManagerListener to track master changes, this service will + // track them here to avoid conditions where master listener events run after other + // listeners that depend on what happened in the master listener + if (this.isClusterManager != event.localNodeClusterManager()) { + this.isClusterManager = event.localNodeClusterManager(); + if (this.isClusterManager) { + onMaster(); + } else { + offMaster(); + } + } + for (HistoryIndexInfo h : alertHistoryIndices) { + h.isInitialized = event.state().metadata().hasAlias(h.indexAlias); + } + for (HistoryIndexInfo h : findingHistoryIndices) { + h.isInitialized = event.state().metadata().hasAlias(h.indexAlias); + } + } + + private void onMaster() { + try { + // try to rollover immediately as we might be restarting the cluster + rolloverAlertHistoryIndices(); + rolloverFindingHistoryIndices(); + // schedule the next rollover for approx MAX_AGE later + scheduledAlertsRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteAlertHistoryIndices(), alertHistoryRolloverPeriod, executorName()); + scheduledFindingsRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteFindingHistoryIndices(), findingHistoryRolloverPeriod, executorName()); + } catch (Exception e) { + // This should be run on cluster startup + logger.error( + "Error creating alert/finding indices. " + + "Alerts/Findings can't be recorded until master node is restarted.", + e + ); + } + } + + private void offMaster() { + if (scheduledAlertsRollover != null) { + scheduledAlertsRollover.cancel(); + } + if (scheduledFindingsRollover != null) { + scheduledFindingsRollover.cancel(); + } + } + + private String executorName() { + return ThreadPool.Names.MANAGEMENT; + } + + private void deleteOldIndices(String tag, String indices) { + logger.error("info deleteOldIndices"); + ClusterStateRequest clusterStateRequest = new ClusterStateRequest() + .clear() + .indices(indices) + .metadata(true) + .local(true) + .indicesOptions(IndicesOptions.strictExpand()); + client.admin().cluster().state( + clusterStateRequest, + new ActionListener<>() { + @Override + public void onResponse(ClusterStateResponse clusterStateResponse) { + if (!clusterStateResponse.getState().metadata().getIndices().isEmpty()) { + List indicesToDelete = getIndicesToDelete(clusterStateResponse); + logger.info("Deleting old " + tag + " indices viz $indicesToDelete"); + deleteAllOldHistoryIndices(indicesToDelete); + } else { + logger.info("No Old " + tag + " Indices to delete"); + } + } + + @Override + public void onFailure(Exception e) { + logger.error("Error fetching cluster state"); + } + } + ); + } + + private List getIndicesToDelete(ClusterStateResponse clusterStateResponse) { + List indicesToDelete = new ArrayList<>(); + for (ObjectCursor in : clusterStateResponse.getState().metadata().indices().values()) { + IndexMetadata indexMetaData = in.value; + indicesToDelete.add( + getHistoryIndexToDelete(indexMetaData, alertHistoryRetentionPeriod.millis(), alertHistoryIndices, alertHistoryEnabled) + ); + indicesToDelete.add( + getHistoryIndexToDelete(indexMetaData, findingHistoryRetentionPeriod.millis(), findingHistoryIndices, findingHistoryEnabled) + ); + } + return indicesToDelete; + } + + private String getHistoryIndexToDelete( + IndexMetadata indexMetadata, + Long retentionPeriodMillis, + List historyIndices, + Boolean historyEnabled + ) { + long creationTime = indexMetadata.getCreationDate(); + if ((Instant.now().toEpochMilli() - creationTime) > retentionPeriodMillis) { + String alias = null; + for (ObjectCursor aliasMetadata : indexMetadata.getAliases().values()) { + Optional historyIndexInfoOptional = historyIndices + .stream() + .filter(e -> e.indexAlias.equals(aliasMetadata.value.alias())) + .findFirst(); + if (historyIndexInfoOptional.isPresent()) { + alias = historyIndexInfoOptional.get().indexAlias; + break; + } + } + if (alias != null) { + if (historyEnabled) { + // If the index has the write alias and history is enabled, don't delete the index + return null; + } + } + return indexMetadata.getIndex().getName(); + } + return null; + } + + private void deleteAllOldHistoryIndices(List indicesToDelete) { + if (indicesToDelete.size() > 0) { + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])); + client.admin().indices().delete( + deleteIndexRequest, + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse deleteIndicesResponse) { + if (!deleteIndicesResponse.isAcknowledged()) { + logger.error( + "Could not delete one or more Alerting/Finding history indices: $indicesToDelete. Retrying one by one." + ); + deleteOldHistoryIndex(indicesToDelete); + } + } + + @Override + public void onFailure(Exception e) { + logger.error("Delete for Alerting/Finding History Indices $indicesToDelete Failed. Retrying one By one."); + deleteOldHistoryIndex(indicesToDelete); + } + } + ); + } + } + + private void deleteOldHistoryIndex(List indicesToDelete) { + for (String index : indicesToDelete) { + final DeleteIndexRequest singleDeleteRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])); + + client.admin().indices().delete( + singleDeleteRequest, + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + if (!acknowledgedResponse.isAcknowledged()) { + logger.error("Could not delete one or more Alerting/Finding history indices: " + index); + } + } + + @Override + public void onFailure(Exception e) { + logger.debug("Exception ${e.message} while deleting the index " + index); + } + } + ); + } + } + + private void rolloverAndDeleteAlertHistoryIndices() { + if (alertHistoryEnabled) rolloverAlertHistoryIndices(); + deleteOldIndices("History", ALERT_HISTORY_ALL); + } + + private void rolloverAndDeleteFindingHistoryIndices() { + if (findingHistoryEnabled) rolloverFindingHistoryIndices(); + deleteOldIndices("Finding", FINDING_HISTORY_ALL); + } + + private void rolloverIndex( + Boolean initialized, + String index, + String pattern, + String map, + Long docsCondition, + TimeValue ageCondition + ) { + if (!initialized) { + return; + } + + // We have to pass null for newIndexName in order to get Elastic to increment the index count. + RolloverRequest request = new RolloverRequest(index, null); + request.getCreateIndexRequest().index(pattern) + .mapping(map) + .settings(Settings.builder().put("index.hidden", true).build()); + request.addMaxIndexDocsCondition(docsCondition); + request.addMaxIndexAgeCondition(ageCondition); + client.admin().indices().rolloverIndex( + request, + new ActionListener<>() { + @Override + public void onResponse(RolloverResponse rolloverResponse) { + if (!rolloverResponse.isRolledOver()) { + logger.info(index + "not rolled over. Conditions were: ${response.conditionStatus}"); + } + } + + @Override + public void onFailure(Exception e) { + logger.error(index + " not roll over failed."); + } + } + ); + } + + private void rolloverAlertHistoryIndices() { + for(HistoryIndexInfo h : alertHistoryIndices) { + rolloverIndex( + h.isInitialized, h.indexAlias, + h.indexPattern, h.indexMappings, + h.maxDocs, h.maxAge + ); + } + } + private void rolloverFindingHistoryIndices() { + for (HistoryIndexInfo h : findingHistoryIndices) { + rolloverIndex( + h.isInitialized, h.indexAlias, + h.indexPattern, h.indexMappings, + h.maxDocs, h.maxAge + ); + } + } + + private void rescheduleAlertRollover() { + if (clusterService.state().getNodes().isLocalNodeElectedMaster()) { + if (scheduledAlertsRollover != null) { + scheduledAlertsRollover.cancel(); + } + scheduledAlertsRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteAlertHistoryIndices(), alertHistoryRolloverPeriod, executorName()); + } + } + + private void rescheduleFindingRollover() { + if (clusterService.state().getNodes().isLocalNodeElectedMaster()) { + if (scheduledFindingsRollover != null) { + scheduledFindingsRollover.cancel(); + } + scheduledFindingsRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteFindingHistoryIndices(), findingHistoryRolloverPeriod, executorName()); + } + } + + private String alertMapping() { + String alertMapping = null; + try ( + InputStream is = DetectorIndexManagementService.class.getClassLoader().getResourceAsStream("mappings/alert_mapping.json") + ) { + alertMapping = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + logger.error(e.getMessage()); + } + return alertMapping; + } + + private String findingMapping() { + String findingMapping = null; + try ( + InputStream is = DetectorIndexManagementService.class.getClassLoader().getResourceAsStream("mappings/finding_mapping.json") + ) { + findingMapping = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + logger.error(e.getMessage()); + } + return findingMapping; + } + + // Setters + + public void setAlertHistoryEnabled(Boolean alertHistoryEnabled) { + this.alertHistoryEnabled = alertHistoryEnabled; + } + + public void setFindingHistoryEnabled(Boolean findingHistoryEnabled) { + this.findingHistoryEnabled = findingHistoryEnabled; + } + + public void setAlertHistoryMaxDocs(Long alertHistoryMaxDocs) { + this.alertHistoryMaxDocs = alertHistoryMaxDocs; + } + + public void setFindingHistoryMaxDocs(Long findingHistoryMaxDocs) { + this.findingHistoryMaxDocs = findingHistoryMaxDocs; + } + + public void setAlertHistoryMaxAge(TimeValue alertHistoryMaxAge) { + this.alertHistoryMaxAge = alertHistoryMaxAge; + } + + public void setFindingHistoryMaxAge(TimeValue findingHistoryMaxAge) { + this.findingHistoryMaxAge = findingHistoryMaxAge; + } + + public void setAlertHistoryRolloverPeriod(TimeValue alertHistoryRolloverPeriod) { + this.alertHistoryRolloverPeriod = alertHistoryRolloverPeriod; + } + + public void setFindingHistoryRolloverPeriod(TimeValue findingHistoryRolloverPeriod) { + this.findingHistoryRolloverPeriod = findingHistoryRolloverPeriod; + } + + public void setAlertHistoryRetentionPeriod(TimeValue alertHistoryRetentionPeriod) { + this.alertHistoryRetentionPeriod = alertHistoryRetentionPeriod; + } + + public void setFindingHistoryRetentionPeriod(TimeValue findingHistoryRetentionPeriod) { + this.findingHistoryRetentionPeriod = findingHistoryRetentionPeriod; + } + + public void setClusterManager(boolean clusterManager) { + isClusterManager = clusterManager; + } + + @Override + protected void doStart() { + + } + + @Override + protected void doStop() { + if (scheduledAlertsRollover != null) { + scheduledAlertsRollover.cancel(); + } + if (scheduledFindingsRollover != null) { + scheduledFindingsRollover.cancel(); + } + } + + @Override + protected void doClose() { + if (scheduledAlertsRollover != null) { + scheduledAlertsRollover.cancel(); + } + if (scheduledFindingsRollover != null) { + scheduledFindingsRollover.cancel(); + } + } + + private static class HistoryIndexInfo { + + String indexAlias; + String indexPattern; + String indexMappings; + Long maxDocs; + TimeValue maxAge; + boolean isInitialized; + + public HistoryIndexInfo(String indexAlias, String indexPattern, String indexMappings, Long maxDocs, TimeValue maxAge, boolean isInitialized) { + this.indexAlias = indexAlias; + this.indexPattern = indexPattern; + this.indexMappings = indexMappings; + this.maxDocs = maxDocs; + this.maxAge = maxAge; + this.isInitialized = isInitialized; + } + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index 693f6af05..92b990eb6 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -4,6 +4,7 @@ */ package org.opensearch.securityanalytics.settings; +import java.util.concurrent.TimeUnit; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; @@ -12,4 +13,86 @@ public class SecurityAnalyticsSettings { public static Setting INDEX_TIMEOUT = Setting.positiveTimeSetting("plugins.security_analytics.index_timeout", TimeValue.timeValueSeconds(60), Setting.Property.NodeScope, Setting.Property.Dynamic); + + public static final Long DEFAULT_MAX_ACTIONABLE_ALERT_COUNT = 50L; + + public static final Setting ALERT_HISTORY_ENABLED = Setting.boolSetting( + "plugins.security_analytics.alert_history_enabled", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FINDING_HISTORY_ENABLED = Setting.boolSetting( + "plugins.security_analytics.alert_finding_enabled", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting ALERT_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.alert_history_rollover_period", + TimeValue.timeValueHours(12), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FINDING_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.alert_finding_rollover_period", + TimeValue.timeValueHours(12), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting ALERT_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( + "plugins.security_analytics.alert_history_max_age", + new TimeValue(30, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FINDING_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( + "plugins.security_analytics.finding_history_max_age", + new TimeValue(30, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting ALERT_HISTORY_MAX_DOCS = Setting.longSetting( + "plugins.security_analytics.alert_history_max_docs", + 1000L, + 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FINDING_HISTORY_MAX_DOCS = Setting.longSetting( + "plugins.security_analytics.alert_finding_max_docs", + 1000L, + 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated + ); + + public static final Setting ALERT_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.alert_history_retention_period", + new TimeValue(60, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FINDING_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.finding_history_retention_period", + new TimeValue(60, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting REQUEST_TIMEOUT = Setting.positiveTimeSetting( + "plugins.security_analytics.request_timeout", + TimeValue.timeValueSeconds(10), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting MAX_ACTION_THROTTLE_VALUE = Setting.positiveTimeSetting( + "plugins.security_analytics.action_throttle_max_value", + TimeValue.timeValueHours(24), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FILTER_BY_BACKEND_ROLES = Setting.boolSetting( + "plugins.security_analytics.filter_by_backend_roles", + false, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); } \ No newline at end of file diff --git a/src/main/resources/mappings/alert_mapping.json b/src/main/resources/mappings/alert_mapping.json new file mode 100644 index 000000000..fcb1d1c94 --- /dev/null +++ b/src/main/resources/mappings/alert_mapping.json @@ -0,0 +1,157 @@ +{ + "dynamic": "strict", + "_routing": { + "required": true + }, + "_meta" : { + "schema_version": 4 + }, + "properties": { + "schema_version": { + "type": "integer" + }, + "monitor_id": { + "type": "keyword" + }, + "monitor_version": { + "type": "long" + }, + "id": { + "type": "keyword" + }, + "version": { + "type": "long" + }, + "severity": { + "type": "keyword" + }, + "monitor_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "monitor_user": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "backend_roles": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "roles": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "custom_attribute_names": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + } + } + }, + "trigger_id": { + "type": "keyword" + }, + "trigger_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "finding_ids": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "related_doc_ids": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "state": { + "type": "keyword" + }, + "start_time": { + "type": "date" + }, + "last_notification_time": { + "type": "date" + }, + "acknowledged_time": { + "type": "date" + }, + "end_time": { + "type": "date" + }, + "error_message": { + "type": "text" + }, + "alert_history": { + "type": "nested", + "properties": { + "timestamp": { + "type": "date" + }, + "message": { + "type": "text" + } + } + }, + "action_execution_results": { + "type": "nested", + "properties": { + "action_id": { + "type": "keyword" + }, + "last_execution_time": { + "type": "date" + }, + "throttled_count": { + "type": "integer" + } + } + }, + "agg_alert_content": { + "dynamic": true, + "properties": { + "parent_bucket_path": { + "type": "text" + }, + "bucket_key": { + "type": "text" + } + } + } + } +} \ No newline at end of file diff --git a/src/main/resources/mappings/finding_mapping.json b/src/main/resources/mappings/finding_mapping.json new file mode 100644 index 000000000..c9386b2ef --- /dev/null +++ b/src/main/resources/mappings/finding_mapping.json @@ -0,0 +1,56 @@ +{ + "dynamic": "strict", + "_meta" : { + "schema_version": 1 + }, + "properties": { + "schema_version": { + "type": "integer" + }, + "related_doc_ids": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "monitor_id": { + "type": "keyword" + }, + "monitor_name": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "index": { + "type": "keyword" + }, + "queries" : { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "query": { + "type": "text" + }, + "tags": { + "type": "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + } + } + }, + "timestamp": { + "type": "long" + } + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index abb655156..501ae5d92 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -4,6 +4,8 @@ */ package org.opensearch.securityanalytics; +import java.util.ArrayList; +import java.util.function.BiConsumer; import java.io.File; import java.nio.file.Path; import org.apache.http.Header; @@ -40,6 +42,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.action.AlertDto; import org.opensearch.securityanalytics.action.CreateIndexMappingsRequest; import org.opensearch.securityanalytics.action.UpdateIndexMappingsRequest; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; @@ -888,6 +891,68 @@ private String alertingScheduledJobMappings() { " }"; } + public List getAlertIndices(String detectorType) throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getAllAlertsIndicesPattern(detectorType) + "?format=json")); + XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); + List responseList = xcp.list(); + List indices = new ArrayList<>(); + for (Object o : responseList) { + if (o instanceof Map) { + ((Map) o).forEach((BiConsumer) + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); + } + } + return indices; + } + + public List getFindingIndices(String detectorType) throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getFindingsIndex(detectorType) + "?format=json")); + XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); + List responseList = xcp.list(); + List indices = new ArrayList<>(); + for (Object o : responseList) { + if (o instanceof Map) { + ((Map) o).forEach((BiConsumer) + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); + } + } + return indices; + } + + public void updateClusterSetting(String setting, String value) throws IOException { + String settingJson = "{\n" + + " \"persistent\" : {" + + " \"%s\": \"%s\"" + + " }" + + "}"; + settingJson = String.format(settingJson, setting, value); + makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), new StringEntity(settingJson, ContentType.APPLICATION_JSON), new BasicHeader("Content-Type", "application/json")); + } + + public void acknowledgeAlert(String alertId, String detectorId) throws IOException { + String body = String.format(Locale.getDefault(), "{\"alerts\":[\"%s\"]}", alertId); + Request post = new Request("POST", String.format( + Locale.getDefault(), + "%s/%s/_acknowledge/alerts", + SecurityAnalyticsPlugin.DETECTOR_BASE_URI, + detectorId)); + post.setJsonEntity(body); + Response ackAlertsResponse = client().performRequest(post); + assertNotNull(ackAlertsResponse); + Map ackAlertsResponseMap = entityAsMap(ackAlertsResponse); + assertTrue(((ArrayList) ackAlertsResponseMap.get("missing")).isEmpty()); + assertTrue(((ArrayList) ackAlertsResponseMap.get("failed")).isEmpty()); + assertEquals(((ArrayList) ackAlertsResponseMap.get("acknowledged")).size(), 1); + } + protected void createNetflowLogIndex(String indexName) throws IOException { String indexMapping = " \"properties\": {" + diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index 035e39434..75906c073 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -13,7 +13,6 @@ import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; - import org.apache.http.HttpStatus; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; @@ -40,6 +39,9 @@ import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.randomRule; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD; public class AlertsIT extends SecurityAnalyticsRestTestCase { @@ -162,7 +164,7 @@ public void testGetAlerts_success() throws IOException { @SuppressWarnings("unchecked") - public void testAckAlerts_WithInvalidDetectorAlertsCombination() throws IOException, InterruptedException { + public void testAckAlerts_WithInvalidDetectorAlertsCombination() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -448,4 +450,158 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx Assert.assertEquals(1, getAlertsBody.get("total_alerts")); } + + public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedException { + updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "1000"); + updateClusterSetting(ALERT_HISTORY_INDEX_MAX_AGE.getKey(), "1s"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("windows"), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + client().performRequest(new Request("POST", "_refresh")); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + hits = new ArrayList<>(); + + while (hits.size() == 0) { + hits = executeSearch(DetectorMonitorConfig.getAlertsIndex("windows"), request); + } + + List alertIndices = getAlertIndices(detector.getDetectorType()); + while(alertIndices.size() < 3) { + alertIndices = getAlertIndices(detector.getDetectorType()); + Thread.sleep(1000); + } + assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + + } + + public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedException { + updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "1"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("windows"), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + client().performRequest(new Request("POST", "_refresh")); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + hits = new ArrayList<>(); + + while (hits.size() == 0) { + hits = executeSearch(DetectorMonitorConfig.getAlertsIndex("windows"), request); + } + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + // TODO enable asserts here when able + Assert.assertEquals(1, getAlertsBody.get("total_alerts")); + String alertId = (String) ((ArrayList>) getAlertsBody.get("alerts")).get(0).get("id"); + String _detectorId = (String) ((ArrayList>) getAlertsBody.get("alerts")).get(0).get("detector_id"); + + // Ack alert to move it to history index + acknowledgeAlert(alertId, detectorId); + + List alertIndices = getAlertIndices(detector.getDetectorType()); + while(alertIndices.size() < 3) { + alertIndices = getAlertIndices(detector.getDetectorType()); + Thread.sleep(1000); + } + assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + + } + } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index ee274e0ce..6937e0931 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -10,7 +10,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import java.util.stream.Collectors; import org.apache.http.HttpStatus; import org.junit.Assert; @@ -30,6 +29,9 @@ import static org.opensearch.securityanalytics.TestHelpers.randomDoc; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; public class FindingIT extends SecurityAnalyticsRestTestCase { @@ -243,4 +245,130 @@ public void testGetFindings_byDetectorType_success() throws IOException { getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); } + + public void testGetFindings_rolloverByMaxAge_success() throws IOException, InterruptedException { + + updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(FINDING_HISTORY_INDEX_MAX_AGE.getKey(), "1s"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("windows"), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + + List findingIndices = getAlertIndices(detector.getDetectorType()); + while(findingIndices.size() < 2) { + findingIndices = getAlertIndices(detector.getDetectorType()); + Thread.sleep(1000); + } + assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); + } + + public void testGetFindings_rolloverByMaxDoc_success() throws IOException, InterruptedException { + + updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(FINDING_HISTORY_MAX_DOCS.getKey(), "1"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("windows"), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + + List findingIndices = getAlertIndices(detector.getDetectorType()); + while(findingIndices.size() < 2) { + findingIndices = getAlertIndices(detector.getDetectorType()); + Thread.sleep(1000); + } + assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); + } } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index f97e21a11..372a75a8a 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -328,7 +328,8 @@ public void testDeletingADetector() throws IOException { Assert.assertFalse(alertingMonitorExists(monitorId)); - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-detectors-queries-%s", "windows"))); + // todo: change to assertFalse when alerting bug is fixed. https://github.com/opensearch-project/alerting/issues/581 + Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", "windows"))); hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size());