From 81957f19690bb054010450275e0e3202431d15c2 Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Mon, 3 Jun 2024 14:35:03 -0700 Subject: [PATCH] index threat intel monitor api Signed-off-by: Surya Sashank Nistala --- .../SecurityAnalyticsPlugin.java | 135 ++++++++++++---- .../resthandler/RestIndexDetectorAction.java | 2 +- .../IndexThreatIntelMonitorAction.java | 17 +++ .../monitor/IocScanMonitorFanOutAction.java | 19 +++ .../IndexThreatIntelMonitorRequest.java | 59 +++++++ .../IndexThreatIntelMonitorResponse.java | 89 +++++++++++ .../iocscan/dto/PerIocTypeScanInput.java | 122 +++++++++++++++ .../model/SATIFSourceConfigDto.java | 2 +- .../RestIndexIocScanMonitorAction.java | 79 ++++++++++ .../IndexTIFSourceConfigResponse.java | 4 +- .../IndexIocScanMonitorResponseInterface.java | 8 + .../IndexTIFSourceConfigRequestInterface.java | 4 + .../monitor/ThreatIntelMonitorActions.java | 6 + .../monitor/ThreatIntelMonitorDto.java | 144 ++++++++++++++++++ .../ThreatIntelMonitorDtoInterface.java | 4 + ...ransportIndexThreatIntelMonitorAction.java | 134 ++++++++++++++++ .../TransportIndexDetectorAction.java | 8 +- .../securityanalytics/util/DetectorUtils.java | 2 +- .../SecurityAnalyticsRestTestCase.java | 9 ++ .../alerts/AlertingServiceTests.java | 4 +- .../ThreatIntelMonitorRestApiIT.java | 51 +++++++ 21 files changed, 863 insertions(+), 39 deletions(-) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IndexThreatIntelMonitorAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IocScanMonitorFanOutAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/IndexThreatIntelMonitorRequest.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/IndexThreatIntelMonitorResponse.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInput.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestIndexIocScanMonitorAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexIocScanMonitorResponseInterface.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexTIFSourceConfigRequestInterface.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorActions.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDtoInterface.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java create mode 100644 src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index e1d730801..39ece99cd 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -4,20 +4,11 @@ */ package org.opensearch.securityanalytics; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.function.Supplier; -import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.core.action.ActionListener; import org.opensearch.action.ActionRequest; -import org.opensearch.core.action.ActionResponse; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -29,6 +20,8 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; import org.opensearch.commons.alerting.action.AlertingActions; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; @@ -54,7 +47,30 @@ import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; import org.opensearch.script.ScriptService; -import org.opensearch.securityanalytics.action.*; +import org.opensearch.securityanalytics.action.AckAlertsAction; +import org.opensearch.securityanalytics.action.CorrelatedFindingAction; +import org.opensearch.securityanalytics.action.CreateIndexMappingsAction; +import org.opensearch.securityanalytics.action.DeleteCorrelationRuleAction; +import org.opensearch.securityanalytics.action.DeleteCustomLogTypeAction; +import org.opensearch.securityanalytics.action.DeleteDetectorAction; +import org.opensearch.securityanalytics.action.DeleteRuleAction; +import org.opensearch.securityanalytics.action.GetAlertsAction; +import org.opensearch.securityanalytics.action.GetAllRuleCategoriesAction; +import org.opensearch.securityanalytics.action.GetDetectorAction; +import org.opensearch.securityanalytics.action.GetFindingsAction; +import org.opensearch.securityanalytics.action.GetIndexMappingsAction; +import org.opensearch.securityanalytics.action.GetMappingsViewAction; +import org.opensearch.securityanalytics.action.IndexCorrelationRuleAction; +import org.opensearch.securityanalytics.action.IndexCustomLogTypeAction; +import org.opensearch.securityanalytics.action.IndexDetectorAction; +import org.opensearch.securityanalytics.action.IndexRuleAction; +import org.opensearch.securityanalytics.action.ListCorrelationsAction; +import org.opensearch.securityanalytics.action.SearchCorrelationRuleAction; +import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction; +import org.opensearch.securityanalytics.action.SearchDetectorAction; +import org.opensearch.securityanalytics.action.SearchRuleAction; +import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction; +import org.opensearch.securityanalytics.action.ValidateRulesAction; import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService; import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper; import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder; @@ -64,32 +80,82 @@ import org.opensearch.securityanalytics.mapper.IndexTemplateManager; import org.opensearch.securityanalytics.mapper.MapperService; import org.opensearch.securityanalytics.model.CustomLogType; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.IocDao; +import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; -import org.opensearch.securityanalytics.resthandler.*; +import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction; +import org.opensearch.securityanalytics.resthandler.RestCreateIndexMappingsAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteCorrelationRuleAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteCustomLogTypeAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteRuleAction; +import org.opensearch.securityanalytics.resthandler.RestGetAlertsAction; +import org.opensearch.securityanalytics.resthandler.RestGetAllRuleCategoriesAction; +import org.opensearch.securityanalytics.resthandler.RestGetDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestGetFindingsAction; +import org.opensearch.securityanalytics.resthandler.RestGetIndexMappingsAction; +import org.opensearch.securityanalytics.resthandler.RestGetMappingsViewAction; +import org.opensearch.securityanalytics.resthandler.RestIndexCorrelationRuleAction; +import org.opensearch.securityanalytics.resthandler.RestIndexCustomLogTypeAction; +import org.opensearch.securityanalytics.resthandler.RestIndexDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestIndexRuleAction; +import org.opensearch.securityanalytics.resthandler.RestListCorrelationAction; +import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationAction; +import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationRuleAction; +import org.opensearch.securityanalytics.resthandler.RestSearchCustomLogTypeAction; +import org.opensearch.securityanalytics.resthandler.RestSearchDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestSearchRuleAction; +import org.opensearch.securityanalytics.resthandler.RestUpdateIndexMappingsAction; +import org.opensearch.securityanalytics.resthandler.RestValidateRulesAction; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; -import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.threatIntel.action.monitor.IndexThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestIndexIocScanMonitorAction; import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; -import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; -import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; -import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; -import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; -import org.opensearch.securityanalytics.transport.*; -import org.opensearch.securityanalytics.model.Rule; -import org.opensearch.securityanalytics.model.Detector; -import org.opensearch.securityanalytics.model.DetectorInput; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.transport.monitor.TransportIndexThreatIntelMonitorAction; +import org.opensearch.securityanalytics.transport.TransportAcknowledgeAlertsAction; +import org.opensearch.securityanalytics.transport.TransportCorrelateFindingAction; +import org.opensearch.securityanalytics.transport.TransportCreateIndexMappingsAction; +import org.opensearch.securityanalytics.transport.TransportDeleteCorrelationRuleAction; +import org.opensearch.securityanalytics.transport.TransportDeleteCustomLogTypeAction; +import org.opensearch.securityanalytics.transport.TransportDeleteDetectorAction; +import org.opensearch.securityanalytics.transport.TransportDeleteRuleAction; +import org.opensearch.securityanalytics.transport.TransportGetAlertsAction; +import org.opensearch.securityanalytics.transport.TransportGetAllRuleCategoriesAction; +import org.opensearch.securityanalytics.transport.TransportGetDetectorAction; +import org.opensearch.securityanalytics.transport.TransportGetFindingsAction; +import org.opensearch.securityanalytics.transport.TransportGetIndexMappingsAction; +import org.opensearch.securityanalytics.transport.TransportGetMappingsViewAction; +import org.opensearch.securityanalytics.transport.TransportIndexCorrelationRuleAction; +import org.opensearch.securityanalytics.transport.TransportIndexCustomLogTypeAction; +import org.opensearch.securityanalytics.transport.TransportIndexDetectorAction; +import org.opensearch.securityanalytics.transport.TransportIndexRuleAction; +import org.opensearch.securityanalytics.transport.TransportListCorrelationAction; +import org.opensearch.securityanalytics.transport.TransportSearchCorrelationAction; +import org.opensearch.securityanalytics.transport.TransportSearchCorrelationRuleAction; +import org.opensearch.securityanalytics.transport.TransportSearchCustomLogTypeAction; +import org.opensearch.securityanalytics.transport.TransportSearchDetectorAction; +import org.opensearch.securityanalytics.transport.TransportSearchRuleAction; +import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction; +import org.opensearch.securityanalytics.transport.TransportValidateRulesAction; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.securityanalytics.util.CorrelationRuleIndices; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; @@ -99,6 +165,14 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.FEED_SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; @@ -118,6 +192,7 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; public static final String THREAT_INTEL_BASE_URI = PLUGINS_BASE_URI + "/threat_intel"; public static final String THREAT_INTEL_SOURCE_URI = PLUGINS_BASE_URI + "/threat_intel/source"; + public static final String THREAT_INTEL_MONITOR_URI = PLUGINS_BASE_URI + "/threat_intel/monitor"; public static final String IOC_BASE_URI = PLUGINS_BASE_URI + "/ioc"; public static final String IOC_FETCH_BASE_URI = IOC_BASE_URI + "/fetch"; @@ -154,7 +229,7 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private LogTypeService logTypeService; - private SATIFSourceConfigService SaTifSourceConfigService; + private SATIFSourceConfigService saTifSourceConfigService; @Override public Collection getSystemIndexDescriptors(Settings settings){ @@ -192,8 +267,8 @@ public Collection createComponents(Client client, TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); - SaTifSourceConfigService = new SATIFSourceConfigService(client, clusterService, threadPool, xContentRegistry, threatIntelLockService); - SATIFSourceConfigManagementService SaTifSourceConfigManagementService = new SATIFSourceConfigManagementService(SaTifSourceConfigService, threatIntelLockService); + saTifSourceConfigService = new SATIFSourceConfigService(client, clusterService, threadPool, xContentRegistry, threatIntelLockService); + SATIFSourceConfigManagementService saTifSourceConfigManagementService = new SATIFSourceConfigManagementService(saTifSourceConfigService, threatIntelLockService); TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); @@ -201,7 +276,7 @@ public Collection createComponents(Client client, return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, - tifJobUpdateService, tifJobParameterService, threatIntelLockService, SaTifSourceConfigService, SaTifSourceConfigManagementService); + tifJobUpdateService, tifJobParameterService, threatIntelLockService, saTifSourceConfigService, saTifSourceConfigManagementService); } @Override @@ -243,7 +318,8 @@ public List getRestHandlers(Settings settings, new RestSearchCustomLogTypeAction(), new RestDeleteCustomLogTypeAction(), new RestIndexTIFSourceConfigAction(), - new RestGetTIFSourceConfigAction() + new RestGetTIFSourceConfigAction(), + new RestIndexIocScanMonitorAction() ); } @@ -379,6 +455,7 @@ public List> getSettings() { new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class), new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class), new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class), + new ActionHandler<>(IndexThreatIntelMonitorAction.INSTANCE, TransportIndexThreatIntelMonitorAction.class), new ActionHandler<>(SAIndexTIFSourceConfigAction.INSTANCE, TransportIndexTIFSourceConfigAction.class), new ActionHandler<>(SAGetTIFSourceConfigAction.INSTANCE, TransportGetTIFSourceConfigAction.class) ); diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexDetectorAction.java index 6fac7a078..6de3787cd 100644 --- a/src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexDetectorAction.java @@ -32,7 +32,7 @@ import java.util.List; import java.util.Locale; -public class RestIndexDetectorAction extends BaseRestHandler { +public class RestIndexDetectorAction extends BaseRestHandler { private static final Logger log = LogManager.getLogger(RestIndexDetectorAction.class); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IndexThreatIntelMonitorAction.java new file mode 100644 index 000000000..e85ef09bf --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IndexThreatIntelMonitorAction.java @@ -0,0 +1,17 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor; + +import org.opensearch.action.ActionType; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.IndexThreatIntelMonitorResponse; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorActions.INDEX_THREAT_INTEL_MONITOR_ACTION_NAME; + + +public class IndexThreatIntelMonitorAction extends ActionType { + + public static final IndexThreatIntelMonitorAction INSTANCE = new IndexThreatIntelMonitorAction(); + public static final String NAME = INDEX_THREAT_INTEL_MONITOR_ACTION_NAME; + + private IndexThreatIntelMonitorAction() { + super(NAME, IndexThreatIntelMonitorResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IocScanMonitorFanOutAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IocScanMonitorFanOutAction.java new file mode 100644 index 000000000..eb3665992 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/IocScanMonitorFanOutAction.java @@ -0,0 +1,19 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor; + +import org.opensearch.action.ActionType; +import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutResponse; +import org.opensearch.core.common.io.stream.Writeable; + +/** + * Ioc Scan Monitor fan out action that distributes the monitor runner logic to mutliple data node. + */ +public class IocScanMonitorFanOutAction extends ActionType { + /** + * @param name The name of the action, must be unique across actions. + * @param docLevelMonitorFanOutResponseReader A reader for the response type + */ + public IocScanMonitorFanOutAction(String name, Writeable.Reader docLevelMonitorFanOutResponseReader) { + super(name, docLevelMonitorFanOutResponseReader); + } + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/IndexThreatIntelMonitorRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/IndexThreatIntelMonitorRequest.java new file mode 100644 index 000000000..64d4a433b --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/request/IndexThreatIntelMonitorRequest.java @@ -0,0 +1,59 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.request; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.IndexTIFSourceConfigRequestInterface; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; + +import java.io.IOException; + +public class IndexThreatIntelMonitorRequest extends ActionRequest implements IndexTIFSourceConfigRequestInterface { + + public static final String THREAT_INTEL_MONITOR_ID = "threat_intel_monitor_id"; + + private final String id; + private final RestRequest.Method method; + private final ThreatIntelMonitorDto threatIntelMonitor; + + public IndexThreatIntelMonitorRequest(String id, RestRequest.Method method, ThreatIntelMonitorDto threatIntelMonitor) { + super(); + this.id = id; + this.method = method; + this.threatIntelMonitor = threatIntelMonitor; + } + + public IndexThreatIntelMonitorRequest(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readEnum(RestRequest.Method.class), // method + ThreatIntelMonitorDto.readFrom(sin) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeEnum(method); + threatIntelMonitor.writeTo(out); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public String getId() { + return id; + } + + public RestRequest.Method getMethod() { + return method; + } + + public ThreatIntelMonitorDto getThreatIntelMonitor() { + return threatIntelMonitor; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/IndexThreatIntelMonitorResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/IndexThreatIntelMonitorResponse.java new file mode 100644 index 000000000..05c0bdb06 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/monitor/response/IndexThreatIntelMonitorResponse.java @@ -0,0 +1,89 @@ +package org.opensearch.securityanalytics.threatIntel.action.monitor.response; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.IndexIocScanMonitorResponseInterface; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; + +import java.io.IOException; + +/** + * Response obect resturned for request that indexes ioc scan monitor + */ +public class IndexThreatIntelMonitorResponse extends ActionResponse implements ToXContentObject, IndexIocScanMonitorResponseInterface { + private static final String ID = "id"; + private static final String NAME = "version"; + private static final String SEQ_NO = "seq_no"; + private static final String PRIMARY_TERM = "primary_term"; + private static final String MONITOR = "monitor"; + + private final String id; + private final long version; + private final long seqNo; + private final long primaryTerm; + private final ThreatIntelMonitorDto iocScanMonitor; + + public IndexThreatIntelMonitorResponse(String id, long version, long seqNo, long primaryTerm, ThreatIntelMonitorDto monitor) { + this.id = id; + this.version = version; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; + this.iocScanMonitor = monitor; + } + + public IndexThreatIntelMonitorResponse(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readLong(), // version + sin.readLong(), // seqNo + sin.readLong(), // primaryTerm + ThreatIntelMonitorDto.readFrom(sin) // monitor + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeLong(seqNo); + out.writeLong(primaryTerm); + iocScanMonitor.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + return builder.startObject() + .field(ID, id) + .field(NAME, version) + .field(SEQ_NO, seqNo) + .field(PRIMARY_TERM, primaryTerm) + .field(MONITOR, iocScanMonitor) + .endObject(); + } + + @Override + public String getId() { + return id; + } + + public Long getVersion() { + return version; + } + + public long getSeqNo() { + return seqNo; + } + + public long getPrimaryTerm() { + return primaryTerm; + } + + @Override + public ThreatIntelMonitorDto getIocScanMonitor() { + return iocScanMonitor; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInput.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInput.java new file mode 100644 index 000000000..a1a8f4906 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInput.java @@ -0,0 +1,122 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dto; + +import org.opensearch.commons.alerting.model.Input; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * DTO that contains information about an Ioc type, the indices storing iocs of that ioc type and + * list of fields in each index that contain values of the given ioc type like Ip addresss contain fields. + * List of indices is optional. If indices is empty we scan the feed config and get the list of indices + */ +public class PerIocTypeScanInput implements Writeable, ToXContentObject, Input { + + private static final String IOC_TYPE = "ioc_type"; + private static final String INDEX_TO_FIELDS_MAP = "index_to_fields_map"; + private static final String INDICES = "indices"; + private final String iocType; + private final Map> indexToFieldsMap; + private final List indices; + + public PerIocTypeScanInput(String iocType, Map> indexToFieldsMap, List indices) { + this.iocType = iocType; + this.indexToFieldsMap = indexToFieldsMap; + this.indices = indices; + } + + public PerIocTypeScanInput(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readMapOfLists(StreamInput::readString, StreamInput::readString), + sin.readStringList() + ); + } + + public String getIocType() { + return iocType; + } + + public Map> getIndexToFieldsMap() { + return indexToFieldsMap; + } + + public List getIndices() { + return indices; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(iocType); + out.writeMapOfLists(indexToFieldsMap, StreamOutput::writeString, StreamOutput::writeString); + out.writeStringCollection(indices); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(IOC_TYPE, iocType) + .field(INDEX_TO_FIELDS_MAP, indexToFieldsMap) + .field(INDICES, indices) + .endObject(); + } + + public static PerIocTypeScanInput parse(XContentParser xcp) throws IOException { + String iocType = null; + Map> indexToFieldsMap = new HashMap<>(); + List indices = new ArrayList<>(); + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case IOC_TYPE: + iocType = xcp.text(); + break; + case INDEX_TO_FIELDS_MAP: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + indexToFieldsMap = null; + } else { + indexToFieldsMap = xcp.map(HashMap::new, p -> { + List fields = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + fields.add(xcp.text()); + } + return fields; + }); + } + break; + case INDICES: + List strings = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + strings.add(xcp.text()); + } + indices = strings; + break; + default: + xcp.skipChildren(); + } + } + return new PerIocTypeScanInput(iocType, indexToFieldsMap, indices); + } + + @Override + public String name() { + return ThreatIntelMonitorDto.PER_IOC_TYPE_SCAN_INPUT_FIELD; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java index 89dc80d17..9d89e67ac 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java @@ -93,7 +93,7 @@ public SATIFSourceConfigDto(SATIFSourceConfig SaTifSourceConfig) { this.enabledTime = SaTifSourceConfig.getEnabledTime(); this.lastUpdateTime = SaTifSourceConfig.getLastUpdateTime(); this.schedule = SaTifSourceConfig.getSchedule(); - this.state = SaTifSourceConfig.getState();; + this.state = SaTifSourceConfig.getState(); this.refreshType = SaTifSourceConfig.getRefreshType(); this.lastRefreshedTime = SaTifSourceConfig.getLastRefreshedTime(); this.lastRefreshedUser = SaTifSourceConfig.getLastRefreshedUser(); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestIndexIocScanMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestIndexIocScanMonitorAction.java new file mode 100644 index 000000000..47ef4e214 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestIndexIocScanMonitorAction.java @@ -0,0 +1,79 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler.monitor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.monitor.IndexThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.IndexThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.IndexThreatIntelMonitorResponse; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +public class RestIndexIocScanMonitorAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestIndexIocScanMonitorAction.class); + + @Override + public String getName() { + return "index_ioc_scan_monitor_action"; + } + + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI), + new Route(RestRequest.Method.PUT, String.format(Locale.getDefault(), "%s/{%s}", + SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, IndexThreatIntelMonitorRequest.THREAT_INTEL_MONITOR_ID)) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI)); + + String id = request.param(IndexThreatIntelMonitorRequest.THREAT_INTEL_MONITOR_ID, null); + + XContentParser xcp = request.contentParser(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + + ThreatIntelMonitorDto iocScanMonitor = ThreatIntelMonitorDto.parse(xcp, id, null); + + IndexThreatIntelMonitorRequest indexThreatIntelMonitorRequest = new IndexThreatIntelMonitorRequest(id, request.method(), iocScanMonitor); + return channel -> client.execute(IndexThreatIntelMonitorAction.INSTANCE, indexThreatIntelMonitorRequest, getListener(channel, request.method())); + } + + private RestResponseListener getListener(RestChannel channel, RestRequest.Method restMethod) { + return new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(IndexThreatIntelMonitorResponse response) throws Exception { + RestStatus returnStatus = RestStatus.CREATED; + if (restMethod == RestRequest.Method.PUT) { + returnStatus = RestStatus.OK; + } + + BytesRestResponse restResponse = new BytesRestResponse(returnStatus, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + + if (restMethod == RestRequest.Method.POST) { + String location = String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, response.getId()); + restResponse.addHeader("Location", location); + } + + return restResponse; + } + }; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java index 5a9e4daa6..fc04ddc34 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java @@ -8,4 +8,6 @@ public interface IndexTIFSourceConfigResponse { String getTIFConfigId(); Long getVersion(); TIFSourceConfigDto getTIFConfigDto(); -} \ No newline at end of file +} + + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexIocScanMonitorResponseInterface.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexIocScanMonitorResponseInterface.java new file mode 100644 index 000000000..bf5be489c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexIocScanMonitorResponseInterface.java @@ -0,0 +1,8 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +public interface IndexIocScanMonitorResponseInterface { + String getId(); + + ThreatIntelMonitorDto getIocScanMonitor(); +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexTIFSourceConfigRequestInterface.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexTIFSourceConfigRequestInterface.java new file mode 100644 index 000000000..60f233899 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/IndexTIFSourceConfigRequestInterface.java @@ -0,0 +1,4 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +public interface IndexTIFSourceConfigRequestInterface { +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorActions.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorActions.java new file mode 100644 index 000000000..b173f8959 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorActions.java @@ -0,0 +1,6 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +public class ThreatIntelMonitorActions { + public static final String INDEX_THREAT_INTEL_MONITOR_ACTION_NAME = "cluster:admin/security_analytics/threatIntel/monitor/write"; + public static final String GET_THREAT_INTEL_MONITOR_ACTION_NAME = "cluster:admin/security_analytics/threatIntel/monitor/get"; +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java new file mode 100644 index 000000000..a6c83883a --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java @@ -0,0 +1,144 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +import org.apache.commons.lang3.StringUtils; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.Schedule; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeScanInput; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +public class ThreatIntelMonitorDto implements Writeable, ToXContentObject, ThreatIntelMonitorDtoInterface { + + private static final String ID = "id"; + public static final String PER_IOC_TYPE_SCAN_INPUT_FIELD = "per_ioc_type_scan_input"; + private final String id; + private final String name; + private final List perIocTypeScanInputList; + private final Schedule schedule; + private final boolean enabled; + private final User user; + + public ThreatIntelMonitorDto(String id, String name, List perIocTypeScanInputList, Schedule schedule, boolean enabled, User user) { + this.id = StringUtils.isBlank(id) ? UUID.randomUUID().toString() : id; + this.name = name; + this.perIocTypeScanInputList = perIocTypeScanInputList; + this.schedule = schedule; + this.enabled = enabled; + this.user = user; + } + + public ThreatIntelMonitorDto(StreamInput sin) throws IOException { + this( + sin.readOptionalString(), + sin.readString(), + sin.readList(PerIocTypeScanInput::new), + Schedule.readFrom(sin), + sin.readBoolean(), + sin.readBoolean() ? new User(sin) : null + ); + } + + public static ThreatIntelMonitorDto readFrom(StreamInput sin) throws IOException { + return new ThreatIntelMonitorDto(sin); + } + + public static ThreatIntelMonitorDto parse(XContentParser xcp, String id, Long version) throws IOException { + String name = null; + List inputs = new ArrayList<>(); + Schedule schedule = null; + Boolean enabled = null; + User user = null; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case ID: + id = xcp.text(); + break; + case Monitor.NAME_FIELD: + name = xcp.text(); + break; + case PER_IOC_TYPE_SCAN_INPUT_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + PerIocTypeScanInput input = PerIocTypeScanInput.parse(xcp); + inputs.add(input); + } + break; + case Monitor.SCHEDULE_FIELD: + schedule = Schedule.parse(xcp); + break; + case Monitor.ENABLED_FIELD: + enabled = xcp.booleanValue(); + break; + case Monitor.USER_FIELD: + user = xcp.currentToken() == XContentParser.Token.VALUE_NULL ? null : User.parse(xcp); + break; + default: + xcp.skipChildren(); + break; + } + } + + return new ThreatIntelMonitorDto(id, name, inputs, schedule, enabled != null ? enabled : false, user); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(id); + out.writeString(name); + out.writeList(perIocTypeScanInputList); + schedule.writeTo(out); + out.writeBoolean(enabled); + user.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(ID, id) + .field(Monitor.NAME_FIELD, name) + .field(PER_IOC_TYPE_SCAN_INPUT_FIELD, perIocTypeScanInputList) + .field(Monitor.SCHEDULE_FIELD, schedule) + .field(Monitor.ENABLED_FIELD, enabled) + .field(Monitor.USER_FIELD, user) + .endObject(); + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public List getPerIocTypeScanInputList() { + return perIocTypeScanInputList; + } + + public Schedule getSchedule() { + return schedule; + } + + public boolean isEnabled() { + return enabled; + } + + public User getUser() { + return user; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDtoInterface.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDtoInterface.java new file mode 100644 index 000000000..f0cd154cd --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDtoInterface.java @@ -0,0 +1,4 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; + +public interface ThreatIntelMonitorDtoInterface { +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java new file mode 100644 index 000000000..0c576760f --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java @@ -0,0 +1,134 @@ +package org.opensearch.securityanalytics.threatIntel.transport.monitor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.commons.alerting.AlertingPluginInterface; +import org.opensearch.commons.alerting.action.IndexMonitorRequest; +import org.opensearch.commons.alerting.model.DataSources; +import org.opensearch.commons.alerting.model.DocLevelMonitorInput; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.monitor.IndexThreatIntelMonitorAction; +import org.opensearch.securityanalytics.threatIntel.action.monitor.request.IndexThreatIntelMonitorRequest; +import org.opensearch.securityanalytics.threatIntel.action.monitor.response.IndexThreatIntelMonitorResponse; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeScanInput; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.time.Instant; +import java.util.Collections; +import java.util.List; + +import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; + +public class TransportIndexThreatIntelMonitorAction extends HandledTransportAction implements SecureTransportAction { + private static final Logger log = LogManager.getLogger(TransportIndexThreatIntelMonitorAction.class); + + private final ThreadPool threadPool; + private final Settings settings; + private final NamedWriteableRegistry namedWriteableRegistry; + private final Client client; + private volatile Boolean filterByEnabled; + private final TimeValue indexTimeout; + + @Inject + public TransportIndexThreatIntelMonitorAction( + final TransportService transportService, + final ActionFilters actionFilters, + final ThreadPool threadPool, + final Settings settings, + final Client client, + final NamedWriteableRegistry namedWriteableRegistry + ) { + super(IndexThreatIntelMonitorAction.NAME, transportService, actionFilters, IndexThreatIntelMonitorRequest::new); + this.threadPool = threadPool; + this.settings = settings; + this.namedWriteableRegistry = namedWriteableRegistry; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings); + this.client = client; + } + + @Override + protected void doExecute(Task task, IndexThreatIntelMonitorRequest request, ActionListener listener) { + // validate user + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); + return; + } + //create + String id = request.getMethod() == RestRequest.Method.POST ? Monitor.NO_ID : request.getId(); + IndexMonitorRequest indexMonitorRequest = new IndexMonitorRequest( + id, + SequenceNumbers.UNASSIGNED_SEQ_NO, + SequenceNumbers.UNASSIGNED_PRIMARY_TERM, + WriteRequest.RefreshPolicy.IMMEDIATE, + request.getMethod(), + getMonitor(request), + null + ); + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, indexMonitorRequest, namedWriteableRegistry, ActionListener.wrap( + r -> { + listener.onResponse(new IndexThreatIntelMonitorResponse(r.getId(), r.getVersion(), r.getSeqNo(), r.getPrimaryTerm(), + new ThreatIntelMonitorDto( + r.getId(), + r.getMonitor().getName(), + request.getThreatIntelMonitor().getPerIocTypeScanInputList(), + r.getMonitor().getSchedule(), + r.getMonitor().getEnabled(), + user) + )); + }, e -> { + log.error("failed to creat custom monitor", e); + listener.onFailure(e); + } + )); + } + + private static Monitor getMonitor(IndexThreatIntelMonitorRequest request) { + //TODO replace with threat intel monitor + return new Monitor( + request.getMethod() == RestRequest.Method.POST ? Monitor.NO_ID : request.getId(), + Monitor.NO_VERSION, + request.getThreatIntelMonitor().getName(), + request.getThreatIntelMonitor().isEnabled(), + request.getThreatIntelMonitor().getSchedule(), + Instant.now(), + Instant.now(), +// "CUSTOM_" + + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), + request.getThreatIntelMonitor().getUser(), + 1, + List.of(new DocLevelMonitorInput("", List.of("*"), Collections.emptyList())), + Collections.emptyList(), + Collections.emptyMap(), + new DataSources(), + PLUGIN_OWNER_FIELD + ); + } + + private PerIocTypeScanInput getPerIocTypeScanInput(Monitor monitor) { + return null; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index f9fb9b198..ca3b39bf2 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -785,7 +785,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List } Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), false, detector.getSchedule(), detector.getLastUpdateTime(), null, - Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), new DataSources(detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), @@ -886,7 +886,7 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( } Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null, - Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), new DataSources(detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), @@ -1060,7 +1060,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { } **/ Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), false, detector.getSchedule(), detector.getLastUpdateTime(), null, - MonitorType.BUCKET_LEVEL_MONITOR, detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), + MonitorType.BUCKET_LEVEL_MONITOR.getValue(), detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), new DataSources(detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), @@ -1782,7 +1782,7 @@ private Map mapMonitorIds(List monitorResp Collectors.toMap( // In the case of bucket level monitors rule id is trigger id it -> { - if (MonitorType.BUCKET_LEVEL_MONITOR == it.getMonitor().getMonitorType()) { + if (MonitorType.BUCKET_LEVEL_MONITOR.getValue() == it.getMonitor().getMonitorType()) { return it.getMonitor().getTriggers().get(0).getId(); } else { if (it.getMonitor().getName().contains("_chained_findings")) { diff --git a/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java b/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java index 14c241f83..2a3bb041a 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java @@ -109,7 +109,7 @@ public static List getBucketLevelMonitorIds( ) { return monitorResponses.stream().filter( // In the case of bucket level monitors rule id is trigger id - it -> Monitor.MonitorType.BUCKET_LEVEL_MONITOR == it.getMonitor().getMonitorType() + it -> Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(it.getMonitor().getMonitorType()) ).map(IndexMonitorResponse::getId).collect(Collectors.toList()); } public static List getAggRuleIdsConfiguredToTrigger(Detector detector, List> rulesById) { diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 91289a91e..c392bca4e 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -68,6 +68,7 @@ import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -666,6 +667,9 @@ protected HttpEntity toHttpEntity(CorrelationRule rule) throws IOException { protected HttpEntity toHttpEntity(SATIFSourceConfigDto SaTifSourceConfigDto) throws IOException { return new StringEntity(toJsonString(SaTifSourceConfigDto), ContentType.APPLICATION_JSON); } + protected HttpEntity toHttpEntity(ThreatIntelMonitorDto threatIntelMonitorDto) throws IOException { + return new StringEntity(toJsonString(threatIntelMonitorDto), ContentType.APPLICATION_JSON); + } protected RestStatus restStatus(Response response) { return RestStatus.fromCode(response.getStatusLine().getStatusCode()); @@ -715,6 +719,11 @@ private String toJsonString(SATIFSourceConfigDto SaTifSourceConfigDto) throws IO return IndexUtilsKt.string(shuffleXContent(SaTifSourceConfigDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); } + private String toJsonString(ThreatIntelMonitorDto threatIntelMonitorDto) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(threatIntelMonitorDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + private String alertingScheduledJobMappings() { return " \"_meta\" : {\n" + " \"schema_version\": 5\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index 1c928d2fc..06e464d34 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -88,7 +88,7 @@ public void testGetAlerts_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Monitor.MonitorType.DOC_LEVEL_MONITOR, + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), null, 1, List.of(), @@ -122,7 +122,7 @@ public void testGetAlerts_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Monitor.MonitorType.DOC_LEVEL_MONITOR, + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), null, 1, List.of(), diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java new file mode 100644 index 000000000..76dfdea67 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -0,0 +1,51 @@ +package org.opensearch.securityanalytics.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeScanInput; +import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class ThreatIntelMonitorRestApiIT extends SecurityAnalyticsRestTestCase { + private static final Logger log = LogManager.getLogger(ThreatIntelMonitorRestApiIT.class); + + public void testCreateThreatIntelMonitor() throws IOException { + String monitorName = "test_monitor_name"; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + final String createdId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, createdId); + + Response alertingMonitorResponse = getAlertingMonitor(client(), createdId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + } + + private ThreatIntelMonitorDto randomIocScanMonitorDto() { + return new ThreatIntelMonitorDto( + Monitor.NO_ID, + randomAlphaOfLength(10), + List.of(new PerIocTypeScanInput("IP", Map.of("abc", List.of("abc")), Collections.emptyList())), + new org.opensearch.commons.alerting.model.IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), + true, + null + ); + } +} +