From b49c41dccc29360cdd6d52b59edba2b73992f8a1 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Thu, 27 Oct 2022 00:06:54 +0200 Subject: [PATCH 01/18] initial commit Signed-off-by: Petar Dzepina --- .../SecurityAnalyticsPlugin.java | 20 +- .../monitors/DetectorMonitorConfig.java | 2 +- .../DetectorIndexManagementService.java | 546 ++++++++++++++++++ .../settings/SecurityAnalyticsSettings.java | 91 +++ .../resources/mappings/alert_mapping.json | 157 +++++ .../resources/mappings/finding_mapping.json | 56 ++ 6 files changed, 870 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java create mode 100644 src/main/resources/mappings/alert_mapping.json create mode 100644 src/main/resources/mappings/finding_mapping.json diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index e0a974740..5d6fe6116 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -39,6 +39,7 @@ import org.opensearch.securityanalytics.action.IndexDetectorAction; import org.opensearch.securityanalytics.action.SearchDetectorAction; import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction; +import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService; import org.opensearch.securityanalytics.mapper.MapperService; import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction; import org.opensearch.securityanalytics.resthandler.RestGetFindingsAction; @@ -99,6 +100,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin { private RuleIndices ruleIndices; + private DetectorIndexManagementService detectorIndexManagementService; + @Override public Collection createComponents(Client client, ClusterService clusterService, @@ -115,6 +118,7 @@ public Collection createComponents(Client client, ruleTopicIndices = new RuleTopicIndices(client, clusterService); mapperService = new MapperService(client.admin().indices()); ruleIndices = new RuleIndices(client, clusterService, threadPool); + detectorIndexManagementService = new DetectorIndexManagementService(environment.settings(), client, threadPool, clusterService); return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService); } @@ -156,7 +160,21 @@ public List getNamedXContent() { @Override public List> getSettings() { return List.of( - SecurityAnalyticsSettings.INDEX_TIMEOUT + SecurityAnalyticsSettings.INDEX_TIMEOUT, + SecurityAnalyticsSettings.ALERT_HISTORY_ENABLED, + SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD, + SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE, + SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS, + SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD, + SecurityAnalyticsSettings.REQUEST_TIMEOUT, + SecurityAnalyticsSettings.MAX_ACTION_THROTTLE_VALUE, + SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, + SecurityAnalyticsSettings.MAX_ACTIONABLE_ALERT_COUNT, + SecurityAnalyticsSettings.FINDING_HISTORY_ENABLED, + SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS, + SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE, + SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD, + SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD ); } diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index a24750ae2..4a48587fd 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -77,7 +77,7 @@ public static String getFindingsIndex(String detectorType) { public static String getFindingsIndexPattern(String detectorType) { return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndexPattern() : - OPENSEARCH_DEFAULT_FINDINGS_INDEX; + OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN; } public static Map> getRuleIndexMappingsByType(String detectorType) { diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java new file mode 100644 index 000000000..9a0c2156c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -0,0 +1,546 @@ +package org.opensearch.securityanalytics.indexmanagment; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.ActionListener; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.admin.indices.rollover.RolloverRequest; +import org.opensearch.action.admin.indices.rollover.RolloverResponse; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterStateListener; +import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.threadpool.Scheduler; +import org.opensearch.threadpool.ThreadPool; + + +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ENABLED; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ENABLED; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; + +public class DetectorIndexManagementService implements ClusterStateListener { + + private Logger logger = LogManager.getLogger(DetectorIndexManagementService.class); + + private static final String ALERT_HISTORY_ALL = ".opensearch-sap-alerts-history-*"; + private static final String FINDING_HISTORY_ALL = ".opensearch-sap-findings-*"; + + private final Client client; + private final ThreadPool threadPool; + private final ClusterService clusterService; + private Settings settings; + + private volatile Boolean alertHistoryEnabled; + private volatile Boolean findingHistoryEnabled; + + private volatile Long alertHistoryMaxDocs; + private volatile Long findingHistoryMaxDocs; + + private volatile TimeValue alertHistoryMaxAge; + private volatile TimeValue findingHistoryMaxAge; + + private volatile TimeValue alertHistoryRolloverPeriod; + private volatile TimeValue findingHistoryRolloverPeriod; + + private volatile TimeValue alertHistoryRetentionPeriod; + private volatile TimeValue findingHistoryRetentionPeriod; + + private volatile boolean isClusterManager = false; + + ///// probably not needed + TimeValue lastRolloverTime = null; + + private Boolean alertHistoryIndexInitialized = false; + + private Boolean findingHistoryIndexInitialized = false; + + private Boolean alertIndexInitialized = false; + + private Scheduler.Cancellable scheduledRollover = null; + + List alertHistoryIndices = new ArrayList<>(); + List findingHistoryIndices = new ArrayList<>(); + ////// + + public DetectorIndexManagementService(Settings settings, Client client, ThreadPool threadPool, ClusterService clusterService) { + this.settings = settings; + this.client = client; + this.threadPool = threadPool; + this.clusterService = clusterService; + + clusterService.addListener(this); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(ALERT_HISTORY_ENABLED, this::setAlertHistoryEnabled); + clusterService.getClusterSettings().addSettingsUpdateConsumer(ALERT_HISTORY_MAX_DOCS, maxDocs -> { + setAlertHistoryMaxDocs(maxDocs); + for (HistoryIndexInfo h : alertHistoryIndices) { + h.maxDocs = maxDocs; + } + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(ALERT_HISTORY_INDEX_MAX_AGE, maxAge -> { + setAlertHistoryMaxAge(maxAge); + for (HistoryIndexInfo h : alertHistoryIndices) { + h.maxAge = maxAge; + } + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(ALERT_HISTORY_ROLLOVER_PERIOD, timeValue -> { + DetectorIndexManagementService.this.alertHistoryRolloverPeriod = timeValue; + rescheduleAlertRollover(); + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(ALERT_HISTORY_RETENTION_PERIOD, this::setAlertHistoryRetentionPeriod); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_ENABLED, this::setFindingHistoryEnabled); + clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_MAX_DOCS, maxDocs -> { + setFindingHistoryMaxDocs(maxDocs); + for (HistoryIndexInfo h : findingHistoryIndices) { + h.maxDocs = maxDocs; + } + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_INDEX_MAX_AGE, new Consumer() { + @Override + public void accept(TimeValue maxAge) { + setFindingHistoryMaxAge(maxAge); + for (HistoryIndexInfo h : findingHistoryIndices) { + h.maxAge = maxAge; + } + } + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_ROLLOVER_PERIOD, timeValue -> { + DetectorIndexManagementService.this.findingHistoryRolloverPeriod = timeValue; + rescheduleFindingRollover(); + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_RETENTION_PERIOD, this::setFindingHistoryRetentionPeriod); + + initAllIndexLists(); + + initFromSettings(); + } + + private void initAllIndexLists() { + Arrays.stream(Detector.DetectorType.values()).forEach( + detectorType -> { + String alertsHistoryIndex = String.format( + Locale.getDefault(), ".opensearch-sap-alerts-history-%s", detectorType.getDetectorType()); + String alertsHistoryIndexPattern = String.format( + Locale.getDefault(), "<.opensearch-sap-alerts-history-%s-{now/d}-1>", detectorType.getDetectorType()); + + alertHistoryIndices.add(new HistoryIndexInfo( + alertsHistoryIndex, + alertsHistoryIndexPattern, + alertMapping(), + alertHistoryMaxDocs, + alertHistoryMaxAge, + false + )); + + String findingsIndex = String.format( + Locale.getDefault(), ".opensearch-sap-findings-%s", detectorType.getDetectorType()); + String findingsIndexPattern = String.format( + Locale.getDefault(), "<.opensearch-sap-findings-%s-{now/d}-1>", detectorType.getDetectorType()); + + findingHistoryIndices.add(new HistoryIndexInfo( + findingsIndex, + findingsIndexPattern, + findingMapping(), + findingHistoryMaxDocs, + findingHistoryMaxAge, + false + )); + }); + } + + private void initFromSettings() { + alertHistoryEnabled = ALERT_HISTORY_ENABLED.get(settings); + findingHistoryEnabled = FINDING_HISTORY_ENABLED.get(settings); + alertHistoryMaxDocs = ALERT_HISTORY_MAX_DOCS.get(settings); + findingHistoryMaxDocs = FINDING_HISTORY_MAX_DOCS.get(settings); + alertHistoryMaxAge = ALERT_HISTORY_INDEX_MAX_AGE.get(settings); + findingHistoryMaxAge = FINDING_HISTORY_INDEX_MAX_AGE.get(settings); + alertHistoryRolloverPeriod = ALERT_HISTORY_ROLLOVER_PERIOD.get(settings); + findingHistoryRolloverPeriod = FINDING_HISTORY_ROLLOVER_PERIOD.get(settings); + alertHistoryRetentionPeriod = ALERT_HISTORY_RETENTION_PERIOD.get(settings); + findingHistoryRetentionPeriod = FINDING_HISTORY_RETENTION_PERIOD.get(settings); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + // Instead of using a LocalNodeClusterManagerListener to track master changes, this service will + // track them here to avoid conditions where master listener events run after other + // listeners that depend on what happened in the master listener + if (this.isClusterManager != event.localNodeClusterManager()) { + this.isClusterManager = event.localNodeClusterManager(); + if (this.isClusterManager) { + onMaster(); + } else { + offMaster(); + } + } + for (HistoryIndexInfo h : alertHistoryIndices) { + h.isInitialized = event.state().metadata().hasAlias(h.indexAlias); + } + for (HistoryIndexInfo h : findingHistoryIndices) { + h.isInitialized = event.state().metadata().hasAlias(h.indexAlias); + } + } + + + private void onMaster() { + try { + // try to rollover immediately as we might be restarting the cluster + rolloverAlertHistoryIndex(); + rolloverFindingHistoryIndex(); + // schedule the next rollover for approx MAX_AGE later + scheduledRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteAlertHistoryIndices(), alertHistoryRolloverPeriod, executorName()); + scheduledRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteFindingHistoryIndices(), findingHistoryRolloverPeriod, executorName()); + } catch (Exception e) { + // This should be run on cluster startup + logger.error( + "Error creating alert/finding indices. " + + "Alerts/Findings can't be recorded until master node is restarted.", + e + ); + } + } + + private void offMaster() { + if (scheduledRollover != null) { + scheduledRollover.cancel(); + } + } + + private String executorName() { + return ThreadPool.Names.MANAGEMENT; + } + + private void deleteOldIndices(String tag, String indices) { + logger.error("info deleteOldIndices"); + ClusterStateRequest clusterStateRequest = new ClusterStateRequest() + .clear() + .indices(indices) + .metadata(true) + .local(true) + .indicesOptions(IndicesOptions.strictExpand()); + client.admin().cluster().state( + clusterStateRequest, + new ActionListener<>() { + @Override + public void onResponse(ClusterStateResponse clusterStateResponse) { + if (!clusterStateResponse.getState().metadata().getIndices().isEmpty()) { + List indicesToDelete = getIndicesToDelete(clusterStateResponse); + logger.info("Deleting old " + tag + " indices viz $indicesToDelete"); + deleteAllOldHistoryIndices(indicesToDelete); + } else { + logger.info("No Old " + tag + " Indices to delete"); + } + } + + @Override + public void onFailure(Exception e) { + logger.error("Error fetching cluster state"); + } + } + ); + } + + private List getIndicesToDelete(ClusterStateResponse clusterStateResponse) { + List indicesToDelete = new ArrayList<>(); + for (ObjectCursor in : clusterStateResponse.getState().metadata().indices().values()) { + IndexMetadata indexMetaData = in.value; + indicesToDelete.add( + getHistoryIndexToDelete(indexMetaData, alertHistoryRetentionPeriod.millis(), alertHistoryIndices, alertHistoryEnabled) + ); + indicesToDelete.add( + getHistoryIndexToDelete(indexMetaData, findingHistoryRetentionPeriod.millis(), findingHistoryIndices, findingHistoryEnabled) + ); + } + return indicesToDelete; + } + + private String getHistoryIndexToDelete( + IndexMetadata indexMetadata, + Long retentionPeriodMillis, + List historyIndices, + Boolean historyEnabled + ) { + long creationTime = indexMetadata.getCreationDate(); + if ((Instant.now().toEpochMilli() - creationTime) > retentionPeriodMillis) { + String alias = null; + for (ObjectCursor aliasMetadata : indexMetadata.getAliases().values()) { + Optional historyIndexInfoOptional = historyIndices + .stream() + .filter(e -> e.indexAlias.equals(aliasMetadata.value.alias())) + .findFirst(); + if (historyIndexInfoOptional.isPresent()) { + alias = historyIndexInfoOptional.get().indexAlias; + break; + } + } + if (alias != null) { + if (historyEnabled) { + // If the index has the write alias and history is enabled, don't delete the index + return null; + } + } + return indexMetadata.getIndex().getName(); + } + return null; + } + + private void deleteAllOldHistoryIndices(List indicesToDelete) { + if (indicesToDelete.size() > 0) { + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])); + client.admin().indices().delete( + deleteIndexRequest, + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse deleteIndicesResponse) { + if (!deleteIndicesResponse.isAcknowledged()) { + logger.error( + "Could not delete one or more Alerting/Finding history indices: $indicesToDelete. Retrying one by one." + ); + deleteOldHistoryIndex(indicesToDelete); + } + } + + @Override + public void onFailure(Exception e) { + logger.error("Delete for Alerting/Finding History Indices $indicesToDelete Failed. Retrying one By one."); + deleteOldHistoryIndex(indicesToDelete); + } + } + ); + } + } + + private void deleteOldHistoryIndex(List indicesToDelete) { + for (String index : indicesToDelete) { + final DeleteIndexRequest singleDeleteRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])); + + client.admin().indices().delete( + singleDeleteRequest, + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + if (!acknowledgedResponse.isAcknowledged()) { + logger.error("Could not delete one or more Alerting/Finding history indices: " + index); + } + } + + @Override + public void onFailure(Exception e) { + logger.debug("Exception ${e.message} while deleting the index " + index); + } + } + ); + } + } + + private void rolloverAndDeleteAlertHistoryIndices() { + if (alertHistoryEnabled) rolloverAlertHistoryIndex(); + deleteOldIndices("History", ALERT_HISTORY_ALL); + } + + private void rolloverAndDeleteFindingHistoryIndices() { + if (findingHistoryEnabled) rolloverFindingHistoryIndex(); + deleteOldIndices("Finding", FINDING_HISTORY_ALL); + } + + private void rolloverIndex( + Boolean initialized, + String index, + String pattern, + String map, + Long docsCondition, + TimeValue ageCondition + ) { + if (!initialized) { + return; + } + + // We have to pass null for newIndexName in order to get Elastic to increment the index count. + RolloverRequest request = new RolloverRequest(index, null); + request.getCreateIndexRequest().index(pattern) + .mapping(map) + .settings(Settings.builder().put("index.hidden", true).build()); + request.addMaxIndexDocsCondition(docsCondition); + request.addMaxIndexAgeCondition(ageCondition); + client.admin().indices().rolloverIndex( + request, + new ActionListener<>() { + @Override + public void onResponse(RolloverResponse rolloverResponse) { + if (!rolloverResponse.isRolledOver()) { + logger.info(index + "not rolled over. Conditions were: ${response.conditionStatus}"); + } else { + lastRolloverTime = TimeValue.timeValueMillis(threadPool.absoluteTimeInMillis()); + } + } + + @Override + public void onFailure(Exception e) { + logger.error(index + "not roll over failed."); + } + } + ); + } + + private void rolloverAlertHistoryIndex() { + for(HistoryIndexInfo h : alertHistoryIndices) { + rolloverIndex( + h.isInitialized, h.indexAlias, + h.indexPattern, h.indexMappings, + h.maxDocs, h.maxAge + ); + } + } + private void rolloverFindingHistoryIndex() { + for (HistoryIndexInfo h : findingHistoryIndices) { + rolloverIndex( + h.isInitialized, h.indexAlias, + h.indexPattern, h.indexMappings, + h.maxDocs, h.maxAge + ); + } + } + + private void rescheduleAlertRollover() { + if (clusterService.state().getNodes().isLocalNodeElectedMaster()) { + if (scheduledRollover != null) { + scheduledRollover.cancel(); + } + scheduledRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteAlertHistoryIndices(), alertHistoryRolloverPeriod, executorName()); + } + } + + private void rescheduleFindingRollover() { + if (clusterService.state().getNodes().isLocalNodeElectedMaster()) { + if (scheduledRollover != null) { + scheduledRollover.cancel(); + } + scheduledRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteFindingHistoryIndices(), findingHistoryRolloverPeriod, executorName()); + } + } + + + + private String alertMapping() { + String alertMapping = null; + try ( + InputStream is = DetectorIndexManagementService.class.getClassLoader().getResourceAsStream("mappings/alert_mapping.json") + ) { + alertMapping = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + logger.error(e); + } + return alertMapping; + } + + private String findingMapping() { + String findingMapping = null; + try ( + InputStream is = DetectorIndexManagementService.class.getClassLoader().getResourceAsStream("mappings/finding_mapping.json") + ) { + findingMapping = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + logger.error(e); + } + return findingMapping; + } + + // Setters + + public void setAlertHistoryEnabled(Boolean alertHistoryEnabled) { + this.alertHistoryEnabled = alertHistoryEnabled; + } + + public void setFindingHistoryEnabled(Boolean findingHistoryEnabled) { + this.findingHistoryEnabled = findingHistoryEnabled; + } + + public void setAlertHistoryMaxDocs(Long alertHistoryMaxDocs) { + this.alertHistoryMaxDocs = alertHistoryMaxDocs; + } + + public void setFindingHistoryMaxDocs(Long findingHistoryMaxDocs) { + this.findingHistoryMaxDocs = findingHistoryMaxDocs; + } + + public void setAlertHistoryMaxAge(TimeValue alertHistoryMaxAge) { + this.alertHistoryMaxAge = alertHistoryMaxAge; + } + + public void setFindingHistoryMaxAge(TimeValue findingHistoryMaxAge) { + this.findingHistoryMaxAge = findingHistoryMaxAge; + } + + public void setAlertHistoryRolloverPeriod(TimeValue alertHistoryRolloverPeriod) { + this.alertHistoryRolloverPeriod = alertHistoryRolloverPeriod; + } + + public void setFindingHistoryRolloverPeriod(TimeValue findingHistoryRolloverPeriod) { + this.findingHistoryRolloverPeriod = findingHistoryRolloverPeriod; + } + + public void setAlertHistoryRetentionPeriod(TimeValue alertHistoryRetentionPeriod) { + this.alertHistoryRetentionPeriod = alertHistoryRetentionPeriod; + } + + public void setFindingHistoryRetentionPeriod(TimeValue findingHistoryRetentionPeriod) { + this.findingHistoryRetentionPeriod = findingHistoryRetentionPeriod; + } + + public void setClusterManager(boolean clusterManager) { + isClusterManager = clusterManager; + } + + private static class HistoryIndexInfo { + + String indexAlias; + String indexPattern; + String indexMappings; + Long maxDocs; + TimeValue maxAge; + boolean isInitialized; + + public HistoryIndexInfo(String indexAlias, String indexPattern, String indexMappings, Long maxDocs, TimeValue maxAge, boolean isInitialized) { + this.indexAlias = indexAlias; + this.indexPattern = indexPattern; + this.indexMappings = indexMappings; + this.maxDocs = maxDocs; + this.maxAge = maxAge; + this.isInitialized = isInitialized; + } + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index 693f6af05..2ee8f8d30 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -4,6 +4,7 @@ */ package org.opensearch.securityanalytics.settings; +import java.util.concurrent.TimeUnit; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; @@ -12,4 +13,94 @@ public class SecurityAnalyticsSettings { public static Setting INDEX_TIMEOUT = Setting.positiveTimeSetting("plugins.security_analytics.index_timeout", TimeValue.timeValueSeconds(60), Setting.Property.NodeScope, Setting.Property.Dynamic); + + public static final Long DEFAULT_MAX_ACTIONABLE_ALERT_COUNT = 50L; + + public static final Setting ALERT_HISTORY_ENABLED = Setting.boolSetting( + "plugins.security_analytics.alert_history_enabled", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FINDING_HISTORY_ENABLED = Setting.boolSetting( + "plugins.security_analytics.alert_finding_enabled", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting ALERT_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.alert_history_rollover_period", + TimeValue.timeValueHours(12), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FINDING_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.alert_finding_rollover_period", + TimeValue.timeValueHours(12), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting ALERT_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( + "plugins.security_analytics.alert_history_max_age", + new TimeValue(30, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FINDING_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( + "plugins.security_analytics.finding_history_max_age", + new TimeValue(30, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting ALERT_HISTORY_MAX_DOCS = Setting.longSetting( + "plugins.security_analytics.alert_history_max_docs", + 1000L, + 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FINDING_HISTORY_MAX_DOCS = Setting.longSetting( + "plugins.security_analytics.alert_finding_max_docs", + 1000L, + 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated + ); + + public static final Setting ALERT_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.alert_history_retention_period", + new TimeValue(60, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FINDING_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.finding_history_retention_period", + new TimeValue(60, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting REQUEST_TIMEOUT = Setting.positiveTimeSetting( + "plugins.security_analytics.request_timeout", + TimeValue.timeValueSeconds(10), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting MAX_ACTION_THROTTLE_VALUE = Setting.positiveTimeSetting( + "plugins.security_analytics.action_throttle_max_value", + TimeValue.timeValueHours(24), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting FILTER_BY_BACKEND_ROLES = Setting.boolSetting( + "plugins.security_analytics.filter_by_backend_roles", + false, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting MAX_ACTIONABLE_ALERT_COUNT = Setting.longSetting( + "plugins.security_analytics.max_actionable_alert_count", + DEFAULT_MAX_ACTIONABLE_ALERT_COUNT, + -1L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + } \ No newline at end of file diff --git a/src/main/resources/mappings/alert_mapping.json b/src/main/resources/mappings/alert_mapping.json new file mode 100644 index 000000000..fcb1d1c94 --- /dev/null +++ b/src/main/resources/mappings/alert_mapping.json @@ -0,0 +1,157 @@ +{ + "dynamic": "strict", + "_routing": { + "required": true + }, + "_meta" : { + "schema_version": 4 + }, + "properties": { + "schema_version": { + "type": "integer" + }, + "monitor_id": { + "type": "keyword" + }, + "monitor_version": { + "type": "long" + }, + "id": { + "type": "keyword" + }, + "version": { + "type": "long" + }, + "severity": { + "type": "keyword" + }, + "monitor_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "monitor_user": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "backend_roles": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "roles": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "custom_attribute_names": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + } + } + }, + "trigger_id": { + "type": "keyword" + }, + "trigger_name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "finding_ids": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "related_doc_ids": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "state": { + "type": "keyword" + }, + "start_time": { + "type": "date" + }, + "last_notification_time": { + "type": "date" + }, + "acknowledged_time": { + "type": "date" + }, + "end_time": { + "type": "date" + }, + "error_message": { + "type": "text" + }, + "alert_history": { + "type": "nested", + "properties": { + "timestamp": { + "type": "date" + }, + "message": { + "type": "text" + } + } + }, + "action_execution_results": { + "type": "nested", + "properties": { + "action_id": { + "type": "keyword" + }, + "last_execution_time": { + "type": "date" + }, + "throttled_count": { + "type": "integer" + } + } + }, + "agg_alert_content": { + "dynamic": true, + "properties": { + "parent_bucket_path": { + "type": "text" + }, + "bucket_key": { + "type": "text" + } + } + } + } +} \ No newline at end of file diff --git a/src/main/resources/mappings/finding_mapping.json b/src/main/resources/mappings/finding_mapping.json new file mode 100644 index 000000000..c9386b2ef --- /dev/null +++ b/src/main/resources/mappings/finding_mapping.json @@ -0,0 +1,56 @@ +{ + "dynamic": "strict", + "_meta" : { + "schema_version": 1 + }, + "properties": { + "schema_version": { + "type": "integer" + }, + "related_doc_ids": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "monitor_id": { + "type": "keyword" + }, + "monitor_name": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "index": { + "type": "keyword" + }, + "queries" : { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "query": { + "type": "text" + }, + "tags": { + "type": "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + } + } + }, + "timestamp": { + "type": "long" + } + } +} \ No newline at end of file From e07a40f4cbe167323464dab56f89aad1b8f7ecb2 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Thu, 27 Oct 2022 00:12:55 +0200 Subject: [PATCH 02/18] rename Signed-off-by: Petar Dzepina --- .../DetectorIndexManagementService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index 9a0c2156c..b324a3f19 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -215,8 +215,8 @@ public void clusterChanged(ClusterChangedEvent event) { private void onMaster() { try { // try to rollover immediately as we might be restarting the cluster - rolloverAlertHistoryIndex(); - rolloverFindingHistoryIndex(); + rolloverAlertHistoryIndices(); + rolloverFindingHistoryIndices(); // schedule the next rollover for approx MAX_AGE later scheduledRollover = threadPool .scheduleWithFixedDelay(() -> rolloverAndDeleteAlertHistoryIndices(), alertHistoryRolloverPeriod, executorName()); @@ -366,12 +366,12 @@ public void onFailure(Exception e) { } private void rolloverAndDeleteAlertHistoryIndices() { - if (alertHistoryEnabled) rolloverAlertHistoryIndex(); + if (alertHistoryEnabled) rolloverAlertHistoryIndices(); deleteOldIndices("History", ALERT_HISTORY_ALL); } private void rolloverAndDeleteFindingHistoryIndices() { - if (findingHistoryEnabled) rolloverFindingHistoryIndex(); + if (findingHistoryEnabled) rolloverFindingHistoryIndices(); deleteOldIndices("Finding", FINDING_HISTORY_ALL); } @@ -414,7 +414,7 @@ public void onFailure(Exception e) { ); } - private void rolloverAlertHistoryIndex() { + private void rolloverAlertHistoryIndices() { for(HistoryIndexInfo h : alertHistoryIndices) { rolloverIndex( h.isInitialized, h.indexAlias, @@ -423,7 +423,7 @@ private void rolloverAlertHistoryIndex() { ); } } - private void rolloverFindingHistoryIndex() { + private void rolloverFindingHistoryIndices() { for (HistoryIndexInfo h : findingHistoryIndices) { rolloverIndex( h.isInitialized, h.indexAlias, From c3e027bf30047f06a1fa7f3f6f4fd90aec68c8b8 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Thu, 27 Oct 2022 23:32:45 +0200 Subject: [PATCH 03/18] more tests; Added allAlertsIndicesPattern when calling GetAlerts API Signed-off-by: Petar Dzepina --- .../alerts/AlertsService.java | 2 +- .../monitors/DetectorMonitorConfig.java | 21 ++- .../DetectorIndexManagementService.java | 4 +- .../SecurityAnalyticsRestTestCase.java | 68 ++++++++ .../securityanalytics/alerts/AlertsIT.java | 157 +++++++++++++++++- .../securityanalytics/findings/FindingIT.java | 130 ++++++++++++++- 6 files changed, 375 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java index 2508786f0..a2840adbc 100644 --- a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java +++ b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java @@ -243,7 +243,7 @@ public void getAlerts(List alertIds, "ALL", "ALL", null, - DetectorMonitorConfig.getAlertsIndex(detector.getDetectorType()), + DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()), null, alertIds); AlertingPluginInterface.INSTANCE.getAlerts( diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index 4a48587fd..101ff7708 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -13,6 +13,10 @@ public class DetectorMonitorConfig { + + public static final String OPENSEARCH_DEFAULT_ALL_ALERT_INDEX_PATTERN = ".opensearch-sap-alerts*"; + public static final String OPENSEARCH_DEFAULT_ALL_FINDING_INDEX_PATTERN = ".opensearch-sap-findings*"; + public static final String OPENSEARCH_DEFAULT_RULE_INDEX = ".opensearch-sap-detectors-queries-default"; public static final String OPENSEARCH_DEFAULT_ALERT_INDEX = ".opensearch-sap-alerts-default"; public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX = ".opensearch-sap-alerts-history-default"; @@ -34,12 +38,14 @@ public class DetectorMonitorConfig { Locale.getDefault(), ".opensearch-sap-alerts-history-%s", detectorType.getDetectorType()); String alertsHistoryIndexPattern = String.format( Locale.getDefault(), "<.opensearch-sap-alerts-history-%s-{now/d}-1>", detectorType.getDetectorType()); + String allAlertsIndicesPattern = String.format( + Locale.getDefault(), "<.opensearch-sap-alerts-history-%s-*", detectorType.getDetectorType()); String findingsIndex = String.format( Locale.getDefault(), ".opensearch-sap-findings-%s", detectorType.getDetectorType()); String findingsIndexPattern = String.format( Locale.getDefault(), "<.opensearch-sap-findings-%s-{now/d}-1>", detectorType.getDetectorType()); - MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, findingsIndex, findingsIndexPattern, ruleIndex); + MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, allAlertsIndicesPattern, findingsIndex, findingsIndexPattern, ruleIndex); ruleIndexByDetectorTypeMap.put(detectorType.getDetectorType(), monitor); }); } @@ -68,6 +74,12 @@ public static String getAlertsHistoryIndexPattern(String detectorType) { OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN; } + public static String getAllAlertsIndicesPattern(String detectorType) { + return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? + ruleIndexByDetectorTypeMap.get(detectorType).getAllAlertsIndicesPattern() : + OPENSEARCH_DEFAULT_ALL_ALERT_INDEX_PATTERN; + } + public static String getFindingsIndex(String detectorType) { return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndex() : @@ -92,6 +104,7 @@ private static class MonitorConfig { private final String alertsIndex; private final String alertsHistoryIndex; private final String alertsHistoryIndexPattern; + private final String allAlertsIndicesPattern; private final String findingIndex; private final String findingsIndexPattern; private final String ruleIndex; @@ -100,6 +113,7 @@ private MonitorConfig( String alertsIndex, String alertsHistoryIndex, String alertsHistoryIndexPattern, + String allAlertsIndicesPattern, String findingsIndex, String findingsIndexPattern, String ruleIndex @@ -107,6 +121,7 @@ private MonitorConfig( this.alertsIndex = alertsIndex; this.alertsHistoryIndex = alertsHistoryIndex; this.alertsHistoryIndexPattern = alertsHistoryIndexPattern; + this.allAlertsIndicesPattern = allAlertsIndicesPattern; this.findingIndex = findingsIndex; this.findingsIndexPattern = findingsIndexPattern; this.ruleIndex = ruleIndex; @@ -124,6 +139,10 @@ public String getAlertsHistoryIndexPattern() { return alertsHistoryIndexPattern; } + public String getAllAlertsIndicesPattern() { + return allAlertsIndicesPattern; + } + public String getFindingsIndex() { return findingIndex; } diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index b324a3f19..ba343c461 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -139,9 +139,9 @@ public void accept(TimeValue maxAge) { }); clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_RETENTION_PERIOD, this::setFindingHistoryRetentionPeriod); - initAllIndexLists(); - initFromSettings(); + + initAllIndexLists(); } private void initAllIndexLists() { diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index a3511b862..ca156e334 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -4,6 +4,8 @@ */ package org.opensearch.securityanalytics; +import java.util.ArrayList; +import java.util.function.BiConsumer; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; @@ -30,12 +32,14 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.commons.alerting.model.ScheduledJob; import org.opensearch.commons.alerting.util.IndexUtilsKt; import org.opensearch.index.mapper.MapperService; import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.action.AlertDto; import org.opensearch.securityanalytics.action.CreateIndexMappingsRequest; import org.opensearch.securityanalytics.action.UpdateIndexMappingsRequest; import org.opensearch.securityanalytics.model.Detector; @@ -52,6 +56,8 @@ import java.util.stream.Collectors; import static org.opensearch.action.admin.indices.create.CreateIndexRequest.MAPPINGS; +import static org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig.OPENSEARCH_DEFAULT_ALL_ALERT_INDEX_PATTERN; +import static org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig.OPENSEARCH_DEFAULT_ALL_FINDING_INDEX_PATTERN; public class SecurityAnalyticsRestTestCase extends OpenSearchRestTestCase { @@ -847,4 +853,66 @@ private String alertingScheduledJobMappings() { " }\n" + " }"; } + + public List getAlertIndices() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + OPENSEARCH_DEFAULT_ALL_ALERT_INDEX_PATTERN + "?format=json")); + XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); + List responseList = xcp.list(); + List indices = new ArrayList<>(); + for (Object o : responseList) { + if (o instanceof Map) { + ((Map) o).forEach((BiConsumer) + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); + } + } + return indices; + } + + public List getFindingIndices() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + OPENSEARCH_DEFAULT_ALL_FINDING_INDEX_PATTERN + "?format=json")); + XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); + List responseList = xcp.list(); + List indices = new ArrayList<>(); + for (Object o : responseList) { + if (o instanceof Map) { + ((Map) o).forEach((BiConsumer) + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); + } + } + return indices; + } + + public void updateClusterSetting(String setting, String value) throws IOException { + String settingJson = "{\n" + + " \"persistent\" : {" + + " \"%s\": \"%s\"" + + " }" + + "}"; + settingJson = String.format(settingJson, setting, value); + makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), new StringEntity(settingJson, ContentType.APPLICATION_JSON), new BasicHeader("Content-Type", "application/json")); + } + + public void acknowledgeAlert(String alertId, String detectorId) throws IOException { + String body = String.format(Locale.getDefault(), "{\"alerts\":[\"%s\"]}", alertId); + Request post = new Request("POST", String.format( + Locale.getDefault(), + "%s/%s/_acknowledge/alerts", + SecurityAnalyticsPlugin.DETECTOR_BASE_URI, + detectorId)); + post.setJsonEntity(body); + Response ackAlertsResponse = client().performRequest(post); + assertNotNull(ackAlertsResponse); + Map ackAlertsResponseMap = entityAsMap(ackAlertsResponse); + assertTrue(((ArrayList) ackAlertsResponseMap.get("missing")).isEmpty()); + assertTrue(((ArrayList) ackAlertsResponseMap.get("failed")).isEmpty()); + assertEquals(((ArrayList) ackAlertsResponseMap.get("acknowledged")).size(), 1); + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index cc48fa03f..7a63942ea 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -12,9 +12,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - import org.apache.http.HttpStatus; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; @@ -41,6 +39,8 @@ import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.randomRule; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD; public class AlertsIT extends SecurityAnalyticsRestTestCase { @@ -447,4 +447,157 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx Assert.assertEquals(1, getAlertsBody.get("total_alerts")); } + + public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedException { + updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "1s"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("windows"), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + client().performRequest(new Request("POST", "_refresh")); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + hits = new ArrayList<>(); + + while (hits.size() == 0) { + hits = executeSearch(DetectorMonitorConfig.getAlertsIndex("windows"), request); + } + + List alertIndices = getAlertIndices(); + while(alertIndices.size() < 3) { + alertIndices = getAlertIndices(); + Thread.sleep(1000); + } + assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + + } + + public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedException { + updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "1"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("windows"), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + client().performRequest(new Request("POST", "_refresh")); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + hits = new ArrayList<>(); + + while (hits.size() == 0) { + hits = executeSearch(DetectorMonitorConfig.getAlertsIndex("windows"), request); + } + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + // TODO enable asserts here when able + Assert.assertEquals(1, getAlertsBody.get("total_alerts")); + String alertId = (String) ((ArrayList>) getAlertsBody.get("alerts")).get(0).get("id"); + String _detectorId = (String) ((ArrayList>) getAlertsBody.get("alerts")).get(0).get("detector_id"); + + // Ack alert to move it to history index + acknowledgeAlert(alertId, detectorId); + + List alertIndices = getAlertIndices(); + while(alertIndices.size() < 3) { + alertIndices = getAlertIndices(); + Thread.sleep(1000); + } + assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + + } + } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index c019363dd..f6b194649 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -10,7 +10,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import java.util.stream.Collectors; import org.apache.http.HttpStatus; import org.junit.Assert; @@ -30,6 +29,9 @@ import static org.opensearch.securityanalytics.TestHelpers.randomDoc; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; public class FindingIT extends SecurityAnalyticsRestTestCase { @@ -243,4 +245,130 @@ public void testGetFindings_byDetectorType_success() throws IOException { getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); } + + public void testGetFindings_rolloverByMaxAge_success() throws IOException, InterruptedException { + + updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(FINDING_HISTORY_INDEX_MAX_AGE.getKey(), "1s"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("windows"), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + + List findingIndices = getAlertIndices(); + while(findingIndices.size() < 2) { + findingIndices = getAlertIndices(); + Thread.sleep(1000); + } + assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); + } + + public void testGetFindings_rolloverByMaxDoc_success() throws IOException, InterruptedException { + + updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(FINDING_HISTORY_MAX_DOCS.getKey(), "1"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("windows"), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + + List findingIndices = getAlertIndices(); + while(findingIndices.size() < 2) { + findingIndices = getAlertIndices(); + Thread.sleep(1000); + } + assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); + } } From b39f5883098e9614b6602fe2da047e00a957f9a9 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Fri, 28 Oct 2022 00:07:44 +0200 Subject: [PATCH 04/18] forbiddenAPI task fix Signed-off-by: Petar Dzepina --- .../indexmanagment/DetectorIndexManagementService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index ba343c461..c31e134ea 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -462,7 +462,7 @@ private String alertMapping() { ) { alertMapping = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); } catch (IOException e) { - logger.error(e); + logger.error(e.getMessage()); } return alertMapping; } @@ -474,7 +474,7 @@ private String findingMapping() { ) { findingMapping = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); } catch (IOException e) { - logger.error(e); + logger.error(e.getMessage()); } return findingMapping; } From 4ecea2a1b802d14ee465ce6aeae128b5cf6eb70a Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Fri, 28 Oct 2022 00:18:50 +0200 Subject: [PATCH 05/18] refactor Signed-off-by: Petar Dzepina --- .../SecurityAnalyticsPlugin.java | 3 +- .../DetectorIndexManagementService.java | 38 +++++++------------ .../settings/SecurityAnalyticsSettings.java | 8 ---- 3 files changed, 15 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 5d6fe6116..17b437bb5 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -118,7 +118,7 @@ public Collection createComponents(Client client, ruleTopicIndices = new RuleTopicIndices(client, clusterService); mapperService = new MapperService(client.admin().indices()); ruleIndices = new RuleIndices(client, clusterService, threadPool); - detectorIndexManagementService = new DetectorIndexManagementService(environment.settings(), client, threadPool, clusterService); + DetectorIndexManagementService.Init(environment.settings(), client, threadPool, clusterService); return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService); } @@ -169,7 +169,6 @@ public List> getSettings() { SecurityAnalyticsSettings.REQUEST_TIMEOUT, SecurityAnalyticsSettings.MAX_ACTION_THROTTLE_VALUE, SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, - SecurityAnalyticsSettings.MAX_ACTIONABLE_ALERT_COUNT, SecurityAnalyticsSettings.FINDING_HISTORY_ENABLED, SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS, SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE, diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index c31e134ea..bbbc24a3f 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -11,7 +11,6 @@ import java.util.Locale; import java.util.Objects; import java.util.Optional; -import java.util.function.Consumer; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; @@ -53,6 +52,8 @@ public class DetectorIndexManagementService implements ClusterStateListener { private static final String ALERT_HISTORY_ALL = ".opensearch-sap-alerts-history-*"; private static final String FINDING_HISTORY_ALL = ".opensearch-sap-findings-*"; + public static DetectorIndexManagementService INSTANCE; + private final Client client; private final ThreadPool threadPool; private final ClusterService clusterService; @@ -75,22 +76,16 @@ public class DetectorIndexManagementService implements ClusterStateListener { private volatile boolean isClusterManager = false; - ///// probably not needed - TimeValue lastRolloverTime = null; - - private Boolean alertHistoryIndexInitialized = false; - - private Boolean findingHistoryIndexInitialized = false; - - private Boolean alertIndexInitialized = false; - private Scheduler.Cancellable scheduledRollover = null; List alertHistoryIndices = new ArrayList<>(); List findingHistoryIndices = new ArrayList<>(); - ////// - public DetectorIndexManagementService(Settings settings, Client client, ThreadPool threadPool, ClusterService clusterService) { + public static void Init(Settings settings, Client client, ThreadPool threadPool, ClusterService clusterService) { + INSTANCE = new DetectorIndexManagementService(settings, client, threadPool, clusterService); + } + + private DetectorIndexManagementService(Settings settings, Client client, ThreadPool threadPool, ClusterService clusterService) { this.settings = settings; this.client = client; this.threadPool = threadPool; @@ -124,13 +119,10 @@ public DetectorIndexManagementService(Settings settings, Client client, ThreadPo h.maxDocs = maxDocs; } }); - clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_INDEX_MAX_AGE, new Consumer() { - @Override - public void accept(TimeValue maxAge) { - setFindingHistoryMaxAge(maxAge); - for (HistoryIndexInfo h : findingHistoryIndices) { - h.maxAge = maxAge; - } + clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_INDEX_MAX_AGE, maxAge -> { + setFindingHistoryMaxAge(maxAge); + for (HistoryIndexInfo h : findingHistoryIndices) { + h.maxAge = maxAge; } }); clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_ROLLOVER_PERIOD, timeValue -> { @@ -139,7 +131,7 @@ public void accept(TimeValue maxAge) { }); clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_RETENTION_PERIOD, this::setFindingHistoryRetentionPeriod); - initFromSettings(); + initFromClusterSettings(); initAllIndexLists(); } @@ -177,7 +169,7 @@ private void initAllIndexLists() { }); } - private void initFromSettings() { + private void initFromClusterSettings() { alertHistoryEnabled = ALERT_HISTORY_ENABLED.get(settings); findingHistoryEnabled = FINDING_HISTORY_ENABLED.get(settings); alertHistoryMaxDocs = ALERT_HISTORY_MAX_DOCS.get(settings); @@ -401,14 +393,12 @@ private void rolloverIndex( public void onResponse(RolloverResponse rolloverResponse) { if (!rolloverResponse.isRolledOver()) { logger.info(index + "not rolled over. Conditions were: ${response.conditionStatus}"); - } else { - lastRolloverTime = TimeValue.timeValueMillis(threadPool.absoluteTimeInMillis()); } } @Override public void onFailure(Exception e) { - logger.error(index + "not roll over failed."); + logger.error(index + " not roll over failed."); } } ); diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index 2ee8f8d30..92b990eb6 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -95,12 +95,4 @@ public class SecurityAnalyticsSettings { false, Setting.Property.NodeScope, Setting.Property.Dynamic ); - - public static final Setting MAX_ACTIONABLE_ALERT_COUNT = Setting.longSetting( - "plugins.security_analytics.max_actionable_alert_count", - DEFAULT_MAX_ACTIONABLE_ALERT_COUNT, - -1L, - Setting.Property.NodeScope, Setting.Property.Dynamic - ); - } \ No newline at end of file From 28e5121bc29cf37e8e2082d3f59313fada35372f Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Fri, 28 Oct 2022 03:31:57 +0200 Subject: [PATCH 06/18] fixes Signed-off-by: Petar Dzepina --- .../monitors/DetectorMonitorConfig.java | 57 +++++++++---------- .../DetectorIndexManagementService.java | 14 ++--- .../SecurityAnalyticsRestTestCase.java | 11 ++-- .../securityanalytics/alerts/AlertsIT.java | 12 ++-- .../securityanalytics/findings/FindingIT.java | 8 +-- 5 files changed, 49 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index 101ff7708..c3129317c 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -4,6 +4,8 @@ */ package org.opensearch.securityanalytics.config.monitors; +import java.util.ArrayList; +import java.util.List; import org.opensearch.securityanalytics.model.Detector; import java.util.Arrays; @@ -14,9 +16,6 @@ public class DetectorMonitorConfig { - public static final String OPENSEARCH_DEFAULT_ALL_ALERT_INDEX_PATTERN = ".opensearch-sap-alerts*"; - public static final String OPENSEARCH_DEFAULT_ALL_FINDING_INDEX_PATTERN = ".opensearch-sap-findings*"; - public static final String OPENSEARCH_DEFAULT_RULE_INDEX = ".opensearch-sap-detectors-queries-default"; public static final String OPENSEARCH_DEFAULT_ALERT_INDEX = ".opensearch-sap-alerts-default"; public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX = ".opensearch-sap-alerts-history-default"; @@ -24,71 +23,71 @@ public class DetectorMonitorConfig { public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX = ".opensearch-sap-findings-default"; public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN = "<.opensearch-sap-findings-default-{now/d}-1>"; - private static Map ruleIndexByDetectorTypeMap; + private static Map detectorTypeToIndicesMapping; static { - ruleIndexByDetectorTypeMap = new HashMap<>(); + detectorTypeToIndicesMapping = new HashMap<>(); Arrays.stream(Detector.DetectorType.values()).forEach( detectorType -> { String ruleIndex = String.format( - Locale.getDefault(), ".opensearch-sap-detectors-queries-%s", detectorType.getDetectorType()); + Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", detectorType.getDetectorType()); String alertsIndex = String.format( - Locale.getDefault(), ".opensearch-sap-alerts-%s", detectorType.getDetectorType()); + Locale.getDefault(), ".opensearch-sap-%s-alerts", detectorType.getDetectorType()); String alertsHistoryIndex = String.format( - Locale.getDefault(), ".opensearch-sap-alerts-history-%s", detectorType.getDetectorType()); + Locale.getDefault(), ".opensearch-sap-%s-alerts-history", detectorType.getDetectorType()); String alertsHistoryIndexPattern = String.format( - Locale.getDefault(), "<.opensearch-sap-alerts-history-%s-{now/d}-1>", detectorType.getDetectorType()); + Locale.getDefault(), "<.opensearch-sap-%s-alerts-history-{now/d}-1>", detectorType.getDetectorType()); String allAlertsIndicesPattern = String.format( - Locale.getDefault(), "<.opensearch-sap-alerts-history-%s-*", detectorType.getDetectorType()); + Locale.getDefault(), ".opensearch-sap-%s-alerts*", detectorType.getDetectorType()); String findingsIndex = String.format( - Locale.getDefault(), ".opensearch-sap-findings-%s", detectorType.getDetectorType()); + Locale.getDefault(), ".opensearch-sap-%s-findings", detectorType.getDetectorType()); String findingsIndexPattern = String.format( - Locale.getDefault(), "<.opensearch-sap-findings-%s-{now/d}-1>", detectorType.getDetectorType()); + Locale.getDefault(), "<.opensearch-sap-%s-findings-{now/d}-1>", detectorType.getDetectorType()); MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, allAlertsIndicesPattern, findingsIndex, findingsIndexPattern, ruleIndex); - ruleIndexByDetectorTypeMap.put(detectorType.getDetectorType(), monitor); + detectorTypeToIndicesMapping.put(detectorType.getDetectorType(), monitor); }); } public static String getRuleIndex(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getRuleIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getRuleIndex() : OPENSEARCH_DEFAULT_RULE_INDEX; } public static String getAlertsIndex(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getAlertsIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getAlertsIndex() : OPENSEARCH_DEFAULT_ALERT_INDEX; } public static String getAlertsHistoryIndex(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getAlertsHistoryIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndex() : OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX; } public static String getAlertsHistoryIndexPattern(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getAlertsHistoryIndexPattern() : + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndexPattern() : OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN; } public static String getAllAlertsIndicesPattern(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getAllAlertsIndicesPattern() : - OPENSEARCH_DEFAULT_ALL_ALERT_INDEX_PATTERN; + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getAllAlertsIndicesPattern() : + "*"; } public static String getFindingsIndex(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndex() : + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getFindingsIndex() : OPENSEARCH_DEFAULT_FINDINGS_INDEX; } public static String getFindingsIndexPattern(String detectorType) { - return ruleIndexByDetectorTypeMap.containsKey(detectorType) ? - ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndexPattern() : + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getFindingsIndexPattern() : OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN; } @@ -100,7 +99,7 @@ public static Map> getRuleIndexMappingsByType(String return fieldMappingProperties; } - private static class MonitorConfig { + public static class MonitorConfig { private final String alertsIndex; private final String alertsHistoryIndex; private final String alertsHistoryIndexPattern; diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index bbbc24a3f..99b255aaf 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -29,6 +29,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; @@ -139,10 +140,9 @@ private DetectorIndexManagementService(Settings settings, Client client, ThreadP private void initAllIndexLists() { Arrays.stream(Detector.DetectorType.values()).forEach( detectorType -> { - String alertsHistoryIndex = String.format( - Locale.getDefault(), ".opensearch-sap-alerts-history-%s", detectorType.getDetectorType()); - String alertsHistoryIndexPattern = String.format( - Locale.getDefault(), "<.opensearch-sap-alerts-history-%s-{now/d}-1>", detectorType.getDetectorType()); + + String alertsHistoryIndex = DetectorMonitorConfig.getAlertsHistoryIndex(detectorType.getDetectorType()); + String alertsHistoryIndexPattern = DetectorMonitorConfig.getAlertsHistoryIndexPattern(detectorType.getDetectorType()); alertHistoryIndices.add(new HistoryIndexInfo( alertsHistoryIndex, @@ -153,10 +153,8 @@ private void initAllIndexLists() { false )); - String findingsIndex = String.format( - Locale.getDefault(), ".opensearch-sap-findings-%s", detectorType.getDetectorType()); - String findingsIndexPattern = String.format( - Locale.getDefault(), "<.opensearch-sap-findings-%s-{now/d}-1>", detectorType.getDetectorType()); + String findingsIndex = DetectorMonitorConfig.getFindingsIndex(detectorType.getDetectorType()); + String findingsIndexPattern = DetectorMonitorConfig.getFindingsIndexPattern(detectorType.getDetectorType()); findingHistoryIndices.add(new HistoryIndexInfo( findingsIndex, diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index ca156e334..23c4faee5 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -42,6 +42,7 @@ import org.opensearch.securityanalytics.action.AlertDto; import org.opensearch.securityanalytics.action.CreateIndexMappingsRequest; import org.opensearch.securityanalytics.action.UpdateIndexMappingsRequest; +import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -56,8 +57,6 @@ import java.util.stream.Collectors; import static org.opensearch.action.admin.indices.create.CreateIndexRequest.MAPPINGS; -import static org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig.OPENSEARCH_DEFAULT_ALL_ALERT_INDEX_PATTERN; -import static org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig.OPENSEARCH_DEFAULT_ALL_FINDING_INDEX_PATTERN; public class SecurityAnalyticsRestTestCase extends OpenSearchRestTestCase { @@ -854,8 +853,8 @@ private String alertingScheduledJobMappings() { " }"; } - public List getAlertIndices() throws IOException { - Response response = client().performRequest(new Request("GET", "/_cat/indices/" + OPENSEARCH_DEFAULT_ALL_ALERT_INDEX_PATTERN + "?format=json")); + public List getAlertIndices(String detectorType) throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getAllAlertsIndicesPattern(detectorType) + "?format=json")); XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); List responseList = xcp.list(); List indices = new ArrayList<>(); @@ -872,8 +871,8 @@ public List getAlertIndices() throws IOException { return indices; } - public List getFindingIndices() throws IOException { - Response response = client().performRequest(new Request("GET", "/_cat/indices/" + OPENSEARCH_DEFAULT_ALL_FINDING_INDEX_PATTERN + "?format=json")); + public List getFindingIndices(String detectorType) throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getFindingsIndex(detectorType) + "?format=json")); XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); List responseList = xcp.list(); List indices = new ArrayList<>(); diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index 7a63942ea..d7e435f0d 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -161,7 +161,7 @@ public void testGetAlerts_success() throws IOException { @SuppressWarnings("unchecked") - public void testAckAlerts_WithInvalidDetectorAlertsCombination() throws IOException, InterruptedException { + public void testAckAlerts_WithInvalidDetectorAlertsCombination() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -450,7 +450,7 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedException { updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); - updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "1s"); + updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "1"); String index = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -509,9 +509,9 @@ public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedExc hits = executeSearch(DetectorMonitorConfig.getAlertsIndex("windows"), request); } - List alertIndices = getAlertIndices(); + List alertIndices = getAlertIndices(detector.getDetectorType()); while(alertIndices.size() < 3) { - alertIndices = getAlertIndices(); + alertIndices = getAlertIndices(detector.getDetectorType()); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); @@ -591,9 +591,9 @@ public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedEx // Ack alert to move it to history index acknowledgeAlert(alertId, detectorId); - List alertIndices = getAlertIndices(); + List alertIndices = getAlertIndices(detector.getDetectorType()); while(alertIndices.size() < 3) { - alertIndices = getAlertIndices(); + alertIndices = getAlertIndices(detector.getDetectorType()); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index f6b194649..8fa8efab4 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -301,9 +301,9 @@ public void testGetFindings_rolloverByMaxAge_success() throws IOException, Inter Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); - List findingIndices = getAlertIndices(); + List findingIndices = getAlertIndices(detector.getDetectorType()); while(findingIndices.size() < 2) { - findingIndices = getAlertIndices(); + findingIndices = getAlertIndices(detector.getDetectorType()); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); @@ -364,9 +364,9 @@ public void testGetFindings_rolloverByMaxDoc_success() throws IOException, Inter Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); - List findingIndices = getAlertIndices(); + List findingIndices = getAlertIndices(detector.getDetectorType()); while(findingIndices.size() < 2) { - findingIndices = getAlertIndices(); + findingIndices = getAlertIndices(detector.getDetectorType()); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); From b4ca7b9b0cc199374ce876bece68986a4ff46bbe Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Fri, 28 Oct 2022 03:38:35 +0200 Subject: [PATCH 07/18] IT fix Signed-off-by: Petar Dzepina --- .../org/opensearch/securityanalytics/alerts/AlertsIT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index d7e435f0d..8c94db090 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -39,6 +39,7 @@ import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.randomRule; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD; @@ -450,7 +451,8 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedException { updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); - updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "1"); + updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "1000"); + updateClusterSetting(ALERT_HISTORY_INDEX_MAX_AGE.getKey(), "1s"); String index = createTestIndex(randomIndex(), windowsIndexMapping()); From ba905dd100d207b478c3609c95882a0d400281cd Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Fri, 28 Oct 2022 03:59:05 +0200 Subject: [PATCH 08/18] ITs fix Signed-off-by: Petar Dzepina --- .../org/opensearch/securityanalytics/findings/FindingIT.java | 2 +- .../securityanalytics/resthandler/DetectorRestApiIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index 8fa8efab4..ad7b06dec 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -293,7 +293,7 @@ public void testGetFindings_rolloverByMaxAge_success() throws IOException, Inter Map executeResults = entityAsMap(executeResponse); int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(5, noOfSigmaRuleMatches); + Assert.assertEquals(1, noOfSigmaRuleMatches); // Call GetFindings API Map params = new HashMap<>(); params.put("detector_id", detectorId); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 9b58ae084..38350568e 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -329,7 +329,7 @@ public void testDeletingADetector() throws IOException { Assert.assertFalse(alertingMonitorExists(monitorId)); // todo: change to assertFalse when alerting bug is fixed. https://github.com/opensearch-project/alerting/issues/581 - Assert.assertTrue(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-detectors-queries-%s", "windows"))); + Assert.assertTrue(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", "windows"))); hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); From 1edc6d183b32e0a748c023671de7406cc3208a46 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Fri, 28 Oct 2022 18:56:08 +0200 Subject: [PATCH 09/18] empty commit Signed-off-by: Petar Dzepina From 97716ec5da7b6fcdb5c3c4aebdcd8bc84dbcc809 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Fri, 28 Oct 2022 19:07:56 +0200 Subject: [PATCH 10/18] fix IT Signed-off-by: Petar Dzepina --- .../org/opensearch/securityanalytics/findings/FindingIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index ad7b06dec..8fa8efab4 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -293,7 +293,7 @@ public void testGetFindings_rolloverByMaxAge_success() throws IOException, Inter Map executeResults = entityAsMap(executeResponse); int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - Assert.assertEquals(1, noOfSigmaRuleMatches); + Assert.assertEquals(5, noOfSigmaRuleMatches); // Call GetFindings API Map params = new HashMap<>(); params.put("detector_id", detectorId); From b99dda2d24bcb65c0098f1f20a3753149dae8db7 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Fri, 28 Oct 2022 19:50:16 +0200 Subject: [PATCH 11/18] licence fix Signed-off-by: Petar Dzepina --- .../indexmanagment/DetectorIndexManagementService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index 99b255aaf..4fd413bb8 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -1,3 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ package org.opensearch.securityanalytics.indexmanagment; import com.carrotsearch.hppc.cursors.ObjectCursor; @@ -8,7 +12,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Locale; import java.util.Objects; import java.util.Optional; import org.apache.logging.log4j.LogManager; From 6793bd64159bb512cd0d0f28f61daab9445a7b40 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Fri, 28 Oct 2022 20:31:10 +0200 Subject: [PATCH 12/18] renamed indices for consistent naming Signed-off-by: Petar Dzepina --- .../java/org/opensearch/securityanalytics/model/Detector.java | 2 +- .../java/org/opensearch/securityanalytics/model/Rule.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/model/Detector.java b/src/main/java/org/opensearch/securityanalytics/model/Detector.java index ecf655f78..4dffe6bd6 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/Detector.java +++ b/src/main/java/org/opensearch/securityanalytics/model/Detector.java @@ -57,7 +57,7 @@ public class Detector implements Writeable, ToXContentObject { private static final String FINDINGS_INDEX = "findings_index"; private static final String FINDINGS_INDEX_PATTERN = "findings_index_pattern"; - public static final String DETECTORS_INDEX = ".opensearch-detectors-config"; + public static final String DETECTORS_INDEX = ".opensearch-sap-detectors-config"; public static final NamedXContentRegistry.Entry XCONTENT_REGISTRY = new NamedXContentRegistry.Entry( Detector.class, diff --git a/src/main/java/org/opensearch/securityanalytics/model/Rule.java b/src/main/java/org/opensearch/securityanalytics/model/Rule.java index d5c90d02e..2e5343aca 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/Rule.java +++ b/src/main/java/org/opensearch/securityanalytics/model/Rule.java @@ -52,8 +52,8 @@ public class Rule implements Writeable, ToXContentObject { private static final String QUERIES = "queries"; public static final String RULE = "rule"; - public static final String PRE_PACKAGED_RULES_INDEX = ".opensearch-pre-packaged-rules-config"; - public static final String CUSTOM_RULES_INDEX = ".opensearch-custom-rules-config"; + public static final String PRE_PACKAGED_RULES_INDEX = ".opensearch-sap-pre-packaged-rules-config"; + public static final String CUSTOM_RULES_INDEX = ".opensearch-sap-custom-rules-config"; public static final NamedXContentRegistry.Entry XCONTENT_REGISTRY = new NamedXContentRegistry.Entry( Rule.class, From 3eaf0d02ae2ef4ecef0f1478f7f87413da402d2b Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Sat, 29 Oct 2022 01:00:02 +0200 Subject: [PATCH 13/18] empty commit Signed-off-by: Petar Dzepina From 38dbd380a4481b64a4a1e36b86137631194efe1f Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Sat, 29 Oct 2022 01:27:25 +0200 Subject: [PATCH 14/18] DetectorIndexManagmentService is LifecycleComponent now Signed-off-by: Petar Dzepina --- .../SecurityAnalyticsPlugin.java | 8 +++++- .../DetectorIndexManagementService.java | 28 +++++++++++++------ .../securityanalytics/findings/FindingIT.java | 6 ++-- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 17b437bb5..d3df02155 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -5,6 +5,7 @@ package org.opensearch.securityanalytics; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.function.Supplier; import org.opensearch.action.ActionRequest; @@ -13,6 +14,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.component.LifecycleComponent; import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; @@ -118,10 +120,14 @@ public Collection createComponents(Client client, ruleTopicIndices = new RuleTopicIndices(client, clusterService); mapperService = new MapperService(client.admin().indices()); ruleIndices = new RuleIndices(client, clusterService, threadPool); - DetectorIndexManagementService.Init(environment.settings(), client, threadPool, clusterService); return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService); } + @Override + public Collection> getGuiceServiceClasses() { + return Collections.singletonList(DetectorIndexManagementService.class); + } + @Override public List getRestHandlers(Settings settings, RestController restController, diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index 4fd413bb8..1e08e20ea 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -30,6 +30,8 @@ import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; @@ -49,15 +51,13 @@ import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; -public class DetectorIndexManagementService implements ClusterStateListener { +public class DetectorIndexManagementService extends AbstractLifecycleComponent implements ClusterStateListener { private Logger logger = LogManager.getLogger(DetectorIndexManagementService.class); private static final String ALERT_HISTORY_ALL = ".opensearch-sap-alerts-history-*"; private static final String FINDING_HISTORY_ALL = ".opensearch-sap-findings-*"; - public static DetectorIndexManagementService INSTANCE; - private final Client client; private final ThreadPool threadPool; private final ClusterService clusterService; @@ -85,11 +85,8 @@ public class DetectorIndexManagementService implements ClusterStateListener { List alertHistoryIndices = new ArrayList<>(); List findingHistoryIndices = new ArrayList<>(); - public static void Init(Settings settings, Client client, ThreadPool threadPool, ClusterService clusterService) { - INSTANCE = new DetectorIndexManagementService(settings, client, threadPool, clusterService); - } - - private DetectorIndexManagementService(Settings settings, Client client, ThreadPool threadPool, ClusterService clusterService) { + @Inject + public DetectorIndexManagementService(Settings settings, Client client, ThreadPool threadPool, ClusterService clusterService) { this.settings = settings; this.client = client; this.threadPool = threadPool; @@ -516,6 +513,21 @@ public void setClusterManager(boolean clusterManager) { isClusterManager = clusterManager; } + @Override + protected void doStart() { + + } + + @Override + protected void doStop() { + scheduledRollover.cancel(); + } + + @Override + protected void doClose() { + scheduledRollover.cancel(); + } + private static class HistoryIndexInfo { String indexAlias; diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index 8fa8efab4..6937e0931 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -134,7 +134,7 @@ public void testGetFindings_byDetectorType_oneDetector_success() throws IOExcept Assert.assertEquals(5, noOfSigmaRuleMatches); // Call GetFindings API Map params = new HashMap<>(); - params.put("detectorType", detector.getDetectorType().toUpperCase()); + params.put("detectorType", detector.getDetectorType()); Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); @@ -234,13 +234,13 @@ public void testGetFindings_byDetectorType_success() throws IOException { // Call GetFindings API for first detector Map params = new HashMap<>(); - params.put("detectorType", detector1.getDetectorType().toUpperCase()); + params.put("detectorType", detector1.getDetectorType()); Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); // Call GetFindings API for second detector params.clear(); - params.put("detectorType", detector2.getDetectorType().toUpperCase()); + params.put("detectorType", detector2.getDetectorType()); getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); From 7e5139cb588232024278710fd164365691f0d560 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Sun, 30 Oct 2022 15:09:20 +0100 Subject: [PATCH 15/18] empty commit Signed-off-by: Petar Dzepina From b7f19be4dc3d5a05fc3594de889958289520faa9 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Tue, 1 Nov 2022 01:33:14 +0100 Subject: [PATCH 16/18] minor fixes Signed-off-by: Petar Dzepina --- .../DetectorIndexManagementService.java | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index 1e08e20ea..ad6bbd240 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -80,7 +80,8 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private volatile boolean isClusterManager = false; - private Scheduler.Cancellable scheduledRollover = null; + private Scheduler.Cancellable scheduledAlertsRollover = null; + private Scheduler.Cancellable scheduledFindingsRollover = null; List alertHistoryIndices = new ArrayList<>(); List findingHistoryIndices = new ArrayList<>(); @@ -201,16 +202,15 @@ public void clusterChanged(ClusterChangedEvent event) { } } - private void onMaster() { try { // try to rollover immediately as we might be restarting the cluster rolloverAlertHistoryIndices(); rolloverFindingHistoryIndices(); // schedule the next rollover for approx MAX_AGE later - scheduledRollover = threadPool + scheduledAlertsRollover = threadPool .scheduleWithFixedDelay(() -> rolloverAndDeleteAlertHistoryIndices(), alertHistoryRolloverPeriod, executorName()); - scheduledRollover = threadPool + scheduledFindingsRollover = threadPool .scheduleWithFixedDelay(() -> rolloverAndDeleteFindingHistoryIndices(), findingHistoryRolloverPeriod, executorName()); } catch (Exception e) { // This should be run on cluster startup @@ -223,8 +223,11 @@ private void onMaster() { } private void offMaster() { - if (scheduledRollover != null) { - scheduledRollover.cancel(); + if (scheduledAlertsRollover != null) { + scheduledAlertsRollover.cancel(); + } + if (scheduledFindingsRollover != null) { + scheduledFindingsRollover.cancel(); } } @@ -423,26 +426,24 @@ private void rolloverFindingHistoryIndices() { private void rescheduleAlertRollover() { if (clusterService.state().getNodes().isLocalNodeElectedMaster()) { - if (scheduledRollover != null) { - scheduledRollover.cancel(); + if (scheduledAlertsRollover != null) { + scheduledAlertsRollover.cancel(); } - scheduledRollover = threadPool + scheduledAlertsRollover = threadPool .scheduleWithFixedDelay(() -> rolloverAndDeleteAlertHistoryIndices(), alertHistoryRolloverPeriod, executorName()); } } private void rescheduleFindingRollover() { if (clusterService.state().getNodes().isLocalNodeElectedMaster()) { - if (scheduledRollover != null) { - scheduledRollover.cancel(); + if (scheduledFindingsRollover != null) { + scheduledFindingsRollover.cancel(); } - scheduledRollover = threadPool + scheduledFindingsRollover = threadPool .scheduleWithFixedDelay(() -> rolloverAndDeleteFindingHistoryIndices(), findingHistoryRolloverPeriod, executorName()); } } - - private String alertMapping() { String alertMapping = null; try ( @@ -520,12 +521,22 @@ protected void doStart() { @Override protected void doStop() { - scheduledRollover.cancel(); + if (scheduledAlertsRollover != null) { + scheduledAlertsRollover.cancel(); + } + if (scheduledFindingsRollover != null) { + scheduledFindingsRollover.cancel(); + } } @Override protected void doClose() { - scheduledRollover.cancel(); + if (scheduledAlertsRollover != null) { + scheduledAlertsRollover.cancel(); + } + if (scheduledFindingsRollover != null) { + scheduledFindingsRollover.cancel(); + } } private static class HistoryIndexInfo { From b05c7d8d7974429186dd8452297b6b3a53e1df0b Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Tue, 1 Nov 2022 17:57:43 +0100 Subject: [PATCH 17/18] empty commit Signed-off-by: Petar Dzepina From 642d5b343f7c590373c8ba47680e9a357f10632f Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Tue, 1 Nov 2022 19:15:51 +0100 Subject: [PATCH 18/18] empty commit Signed-off-by: Petar Dzepina