Skip to content

Commit

Permalink
index threat intel monitor api
Browse files Browse the repository at this point in the history
Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>
  • Loading branch information
eirsep committed Jun 6, 2024
1 parent 4fc0d84 commit a62f243
Show file tree
Hide file tree
Showing 22 changed files with 858 additions and 36 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,11 @@
*/
package org.opensearch.securityanalytics;

import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Supplier;
import java.util.Optional;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.core.action.ActionListener;
import org.opensearch.action.ActionRequest;
import org.opensearch.core.action.ActionResponse;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.cluster.node.DiscoveryNodes;
Expand All @@ -29,6 +20,8 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.common.settings.SettingsFilter;
import org.opensearch.commons.alerting.action.AlertingActions;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.action.ActionResponse;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentParser;
Expand All @@ -54,7 +47,30 @@
import org.opensearch.rest.RestController;
import org.opensearch.rest.RestHandler;
import org.opensearch.script.ScriptService;
import org.opensearch.securityanalytics.action.*;
import org.opensearch.securityanalytics.action.AckAlertsAction;
import org.opensearch.securityanalytics.action.CorrelatedFindingAction;
import org.opensearch.securityanalytics.action.CreateIndexMappingsAction;
import org.opensearch.securityanalytics.action.DeleteCorrelationRuleAction;
import org.opensearch.securityanalytics.action.DeleteCustomLogTypeAction;
import org.opensearch.securityanalytics.action.DeleteDetectorAction;
import org.opensearch.securityanalytics.action.DeleteRuleAction;
import org.opensearch.securityanalytics.action.GetAlertsAction;
import org.opensearch.securityanalytics.action.GetAllRuleCategoriesAction;
import org.opensearch.securityanalytics.action.GetDetectorAction;
import org.opensearch.securityanalytics.action.GetFindingsAction;
import org.opensearch.securityanalytics.action.GetIndexMappingsAction;
import org.opensearch.securityanalytics.action.GetMappingsViewAction;
import org.opensearch.securityanalytics.action.IndexCorrelationRuleAction;
import org.opensearch.securityanalytics.action.IndexCustomLogTypeAction;
import org.opensearch.securityanalytics.action.IndexDetectorAction;
import org.opensearch.securityanalytics.action.IndexRuleAction;
import org.opensearch.securityanalytics.action.ListCorrelationsAction;
import org.opensearch.securityanalytics.action.SearchCorrelationRuleAction;
import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction;
import org.opensearch.securityanalytics.action.SearchDetectorAction;
import org.opensearch.securityanalytics.action.SearchRuleAction;
import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction;
import org.opensearch.securityanalytics.action.ValidateRulesAction;
import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService;
import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper;
import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder;
Expand All @@ -64,32 +80,82 @@
import org.opensearch.securityanalytics.mapper.IndexTemplateManager;
import org.opensearch.securityanalytics.mapper.MapperService;
import org.opensearch.securityanalytics.model.CustomLogType;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
import org.opensearch.securityanalytics.model.IocDao;
import org.opensearch.securityanalytics.model.Rule;
import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
import org.opensearch.securityanalytics.resthandler.*;
import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction;
import org.opensearch.securityanalytics.resthandler.RestCreateIndexMappingsAction;
import org.opensearch.securityanalytics.resthandler.RestDeleteCorrelationRuleAction;
import org.opensearch.securityanalytics.resthandler.RestDeleteCustomLogTypeAction;
import org.opensearch.securityanalytics.resthandler.RestDeleteDetectorAction;
import org.opensearch.securityanalytics.resthandler.RestDeleteRuleAction;
import org.opensearch.securityanalytics.resthandler.RestGetAlertsAction;
import org.opensearch.securityanalytics.resthandler.RestGetAllRuleCategoriesAction;
import org.opensearch.securityanalytics.resthandler.RestGetDetectorAction;
import org.opensearch.securityanalytics.resthandler.RestGetFindingsAction;
import org.opensearch.securityanalytics.resthandler.RestGetIndexMappingsAction;
import org.opensearch.securityanalytics.resthandler.RestGetMappingsViewAction;
import org.opensearch.securityanalytics.resthandler.RestIndexCorrelationRuleAction;
import org.opensearch.securityanalytics.resthandler.RestIndexCustomLogTypeAction;
import org.opensearch.securityanalytics.resthandler.RestIndexDetectorAction;
import org.opensearch.securityanalytics.resthandler.RestIndexRuleAction;
import org.opensearch.securityanalytics.resthandler.RestListCorrelationAction;
import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationAction;
import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationRuleAction;
import org.opensearch.securityanalytics.resthandler.RestSearchCustomLogTypeAction;
import org.opensearch.securityanalytics.resthandler.RestSearchDetectorAction;
import org.opensearch.securityanalytics.resthandler.RestSearchRuleAction;
import org.opensearch.securityanalytics.resthandler.RestUpdateIndexMappingsAction;
import org.opensearch.securityanalytics.resthandler.RestValidateRulesAction;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.action.monitor.IndexThreatIntelMonitorAction;
import org.opensearch.securityanalytics.threatIntel.common.TIFLockService;
import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao;
import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader;
import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestIndexIocScanMonitorAction;
import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService;
import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService;
import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService;
import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService;
import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.common.TIFLockService;
import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader;
import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService;
import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner;
import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService;
import org.opensearch.securityanalytics.transport.*;
import org.opensearch.securityanalytics.model.Rule;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.threatIntel.transport.monitor.TransportIndexThreatIntelMonitorAction;
import org.opensearch.securityanalytics.transport.TransportAcknowledgeAlertsAction;
import org.opensearch.securityanalytics.transport.TransportCorrelateFindingAction;
import org.opensearch.securityanalytics.transport.TransportCreateIndexMappingsAction;
import org.opensearch.securityanalytics.transport.TransportDeleteCorrelationRuleAction;
import org.opensearch.securityanalytics.transport.TransportDeleteCustomLogTypeAction;
import org.opensearch.securityanalytics.transport.TransportDeleteDetectorAction;
import org.opensearch.securityanalytics.transport.TransportDeleteRuleAction;
import org.opensearch.securityanalytics.transport.TransportGetAlertsAction;
import org.opensearch.securityanalytics.transport.TransportGetAllRuleCategoriesAction;
import org.opensearch.securityanalytics.transport.TransportGetDetectorAction;
import org.opensearch.securityanalytics.transport.TransportGetFindingsAction;
import org.opensearch.securityanalytics.transport.TransportGetIndexMappingsAction;
import org.opensearch.securityanalytics.transport.TransportGetMappingsViewAction;
import org.opensearch.securityanalytics.transport.TransportIndexCorrelationRuleAction;
import org.opensearch.securityanalytics.transport.TransportIndexCustomLogTypeAction;
import org.opensearch.securityanalytics.transport.TransportIndexDetectorAction;
import org.opensearch.securityanalytics.transport.TransportIndexRuleAction;
import org.opensearch.securityanalytics.transport.TransportListCorrelationAction;
import org.opensearch.securityanalytics.transport.TransportSearchCorrelationAction;
import org.opensearch.securityanalytics.transport.TransportSearchCorrelationRuleAction;
import org.opensearch.securityanalytics.transport.TransportSearchCustomLogTypeAction;
import org.opensearch.securityanalytics.transport.TransportSearchDetectorAction;
import org.opensearch.securityanalytics.transport.TransportSearchRuleAction;
import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction;
import org.opensearch.securityanalytics.transport.TransportValidateRulesAction;
import org.opensearch.securityanalytics.util.CorrelationIndices;
import org.opensearch.securityanalytics.util.CorrelationRuleIndices;
import org.opensearch.securityanalytics.util.CustomLogTypeIndices;
Expand All @@ -99,6 +165,14 @@
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.watcher.ResourceWatcherService;

import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;

import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.FEED_SOURCE_CONFIG_FIELD;
import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX;

Expand All @@ -118,6 +192,7 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map
public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules";
public static final String THREAT_INTEL_BASE_URI = PLUGINS_BASE_URI + "/threat_intel";
public static final String THREAT_INTEL_SOURCE_URI = PLUGINS_BASE_URI + "/threat_intel/source";
public static final String THREAT_INTEL_MONITOR_URI = PLUGINS_BASE_URI + "/threat_intel/monitor";
public static final String IOC_BASE_URI = PLUGINS_BASE_URI + "/ioc";
public static final String IOC_FETCH_BASE_URI = IOC_BASE_URI + "/fetch";

Expand Down Expand Up @@ -243,7 +318,8 @@ public List<RestHandler> getRestHandlers(Settings settings,
new RestSearchCustomLogTypeAction(),
new RestDeleteCustomLogTypeAction(),
new RestIndexTIFSourceConfigAction(),
new RestGetTIFSourceConfigAction()
new RestGetTIFSourceConfigAction(),
new RestIndexIocScanMonitorAction()
);
}

Expand Down Expand Up @@ -379,6 +455,7 @@ public List<Setting<?>> getSettings() {
new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class),
new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class),
new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class),
new ActionHandler<>(IndexThreatIntelMonitorAction.INSTANCE, TransportIndexThreatIntelMonitorAction.class),
new ActionHandler<>(SAIndexTIFSourceConfigAction.INSTANCE, TransportIndexTIFSourceConfigAction.class),
new ActionHandler<>(SAGetTIFSourceConfigAction.INSTANCE, TransportGetTIFSourceConfigAction.class)
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import java.util.List;
import java.util.Locale;

public class RestIndexDetectorAction extends BaseRestHandler {
public class RestIndexDetectorAction extends BaseRestHandler {

private static final Logger log = LogManager.getLogger(RestIndexDetectorAction.class);

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package org.opensearch.securityanalytics.threatIntel.action.monitor;

import org.opensearch.action.ActionType;
import org.opensearch.securityanalytics.threatIntel.action.monitor.response.IndexThreatIntelMonitorResponse;

import static org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorActions.INDEX_THREAT_INTEL_MONITOR_ACTION_NAME;


public class IndexThreatIntelMonitorAction extends ActionType<IndexThreatIntelMonitorResponse> {

public static final IndexThreatIntelMonitorAction INSTANCE = new IndexThreatIntelMonitorAction();
public static final String NAME = INDEX_THREAT_INTEL_MONITOR_ACTION_NAME;

private IndexThreatIntelMonitorAction() {
super(NAME, IndexThreatIntelMonitorResponse::new);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package org.opensearch.securityanalytics.threatIntel.action.monitor;

import org.opensearch.action.ActionType;
import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutResponse;
import org.opensearch.core.common.io.stream.Writeable;

/**
* Ioc Scan Monitor fan out action that distributes the monitor runner logic to mutliple data node.
*/
public class IocScanMonitorFanOutAction extends ActionType<DocLevelMonitorFanOutResponse> {
/**
* @param name The name of the action, must be unique across actions.
* @param docLevelMonitorFanOutResponseReader A reader for the response type
*/
public IocScanMonitorFanOutAction(String name, Writeable.Reader<DocLevelMonitorFanOutResponse> docLevelMonitorFanOutResponseReader) {
super(name, docLevelMonitorFanOutResponseReader);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package org.opensearch.securityanalytics.threatIntel.action.monitor.request;

import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionRequestValidationException;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.rest.RestRequest;
import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.IndexTIFSourceConfigRequestInterface;
import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto;

import java.io.IOException;

public class IndexThreatIntelMonitorRequest extends ActionRequest implements IndexTIFSourceConfigRequestInterface {

public static final String THREAT_INTEL_MONITOR_ID = "threat_intel_monitor_id";

private final String id;
private final RestRequest.Method method;
private final ThreatIntelMonitorDto threatIntelMonitor;

public IndexThreatIntelMonitorRequest(String id, RestRequest.Method method, ThreatIntelMonitorDto threatIntelMonitor) {
super();
this.id = id;
this.method = method;
this.threatIntelMonitor = threatIntelMonitor;
}

public IndexThreatIntelMonitorRequest(StreamInput sin) throws IOException {
this(
sin.readString(),
sin.readEnum(RestRequest.Method.class), // method
ThreatIntelMonitorDto.readFrom(sin)
);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
out.writeEnum(method);
threatIntelMonitor.writeTo(out);
}

@Override
public ActionRequestValidationException validate() {
return null;
}

public String getId() {
return id;
}

public RestRequest.Method getMethod() {
return method;
}

public ThreatIntelMonitorDto getThreatIntelMonitor() {
return threatIntelMonitor;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package org.opensearch.securityanalytics.threatIntel.action.monitor.response;

import org.opensearch.core.action.ActionResponse;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.IndexIocScanMonitorResponseInterface;
import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto;

import java.io.IOException;

/**
* Response obect resturned for request that indexes ioc scan monitor
*/
public class IndexThreatIntelMonitorResponse extends ActionResponse implements ToXContentObject, IndexIocScanMonitorResponseInterface {
private static final String ID = "id";
private static final String NAME = "version";
private static final String SEQ_NO = "seq_no";
private static final String PRIMARY_TERM = "primary_term";
private static final String MONITOR = "monitor";

private final String id;
private final long version;
private final long seqNo;
private final long primaryTerm;
private final ThreatIntelMonitorDto iocScanMonitor;

public IndexThreatIntelMonitorResponse(String id, long version, long seqNo, long primaryTerm, ThreatIntelMonitorDto monitor) {
this.id = id;
this.version = version;
this.seqNo = seqNo;
this.primaryTerm = primaryTerm;
this.iocScanMonitor = monitor;
}

public IndexThreatIntelMonitorResponse(StreamInput sin) throws IOException {
this(
sin.readString(),
sin.readLong(), // version
sin.readLong(), // seqNo
sin.readLong(), // primaryTerm
ThreatIntelMonitorDto.readFrom(sin) // monitor
);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
out.writeLong(version);
out.writeLong(seqNo);
out.writeLong(primaryTerm);
iocScanMonitor.writeTo(out);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
return builder.startObject()
.field(ID, id)
.field(NAME, version)
.field(SEQ_NO, seqNo)
.field(PRIMARY_TERM, primaryTerm)
.field(MONITOR, iocScanMonitor)
.endObject();
}

@Override
public String getId() {
return id;
}

public Long getVersion() {
return version;
}

public long getSeqNo() {
return seqNo;
}

public long getPrimaryTerm() {
return primaryTerm;
}

@Override
public ThreatIntelMonitorDto getIocScanMonitor() {
return iocScanMonitor;
}
}
Loading

0 comments on commit a62f243

Please sign in to comment.