diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index e7fe43106..a49c82852 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -7,6 +7,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Supplier; import java.util.Optional; @@ -30,6 +31,8 @@ import org.opensearch.commons.alerting.action.AlertingActions; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; @@ -61,23 +64,58 @@ import org.opensearch.securityanalytics.mapper.IndexTemplateManager; import org.opensearch.securityanalytics.mapper.MapperService; import org.opensearch.securityanalytics.model.CustomLogType; +import org.opensearch.securityanalytics.model.IocDao; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; -import org.opensearch.securityanalytics.resthandler.*; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; +import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction; +import org.opensearch.securityanalytics.resthandler.RestCreateIndexMappingsAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteCorrelationRuleAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteCustomLogTypeAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestDeleteRuleAction; +import org.opensearch.securityanalytics.resthandler.RestGetAlertsAction; +import org.opensearch.securityanalytics.resthandler.RestGetAllRuleCategoriesAction; +import org.opensearch.securityanalytics.resthandler.RestGetDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestGetFindingsAction; +import org.opensearch.securityanalytics.resthandler.RestGetIndexMappingsAction; +import org.opensearch.securityanalytics.resthandler.RestGetMappingsViewAction; +import org.opensearch.securityanalytics.resthandler.RestIndexCorrelationRuleAction; +import org.opensearch.securityanalytics.resthandler.RestIndexCustomLogTypeAction; +import org.opensearch.securityanalytics.resthandler.RestIndexDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestIndexRuleAction; +import org.opensearch.securityanalytics.resthandler.RestListCorrelationAction; +import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationAction; +import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationRuleAction; +import org.opensearch.securityanalytics.resthandler.RestSearchCustomLogTypeAction; +import org.opensearch.securityanalytics.resthandler.RestSearchDetectorAction; +import org.opensearch.securityanalytics.resthandler.RestSearchRuleAction; +import org.opensearch.securityanalytics.resthandler.RestUpdateIndexMappingsAction; +import org.opensearch.securityanalytics.resthandler.RestValidateRulesAction; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; -import org.opensearch.securityanalytics.threatIntel.action.TransportPutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; +import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.securityanalytics.transport.*; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.securityanalytics.util.CorrelationRuleIndices; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; @@ -87,7 +125,8 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.FEED_SOURCE_CONFIG_FIELD; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension { @@ -103,10 +142,21 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate"; public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations"; public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; + public static final String THREAT_INTEL_BASE_URI = PLUGINS_BASE_URI + "/threat_intel"; + public static final String THREAT_INTEL_SOURCE_URI = PLUGINS_BASE_URI + "/threat_intel/source"; + public static final String IOC_BASE_URI = PLUGINS_BASE_URI + "/ioc"; + public static final String IOC_FETCH_BASE_URI = IOC_BASE_URI + "/fetch"; public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; + public static final String JOB_TYPE = "opensearch_sap_job"; + public static final Map TIF_JOB_INDEX_SETTING = Map.of(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1, IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all", IndexMetadata.SETTING_INDEX_HIDDEN, true); + public static final String IOC_INDEX_NAME_BASE = ".opensearch-sap-ioc"; + public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*"; + public static final String IOC_DOMAIN_INDEX_NAME = IOC_INDEX_NAME_BASE + IocDao.IocType.DOMAIN.name().toLowerCase(Locale.ROOT); + public static final String IOC_HASH_INDEX_NAME = IOC_INDEX_NAME_BASE + IocDao.IocType.HASH.name().toLowerCase(Locale.ROOT); + public static final String IOC_IP_INDEX_NAME = IOC_INDEX_NAME_BASE + IocDao.IocType.IP.name().toLowerCase(Locale.ROOT); private CorrelationRuleIndices correlationRuleIndices; @@ -129,6 +179,9 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private BuiltinLogTypeLoader builtinLogTypeLoader; private LogTypeService logTypeService; + + private SATIFSourceConfigDao SaTifSourceConfigDao; + @Override public Collection getSystemIndexDescriptors(Settings settings){ return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); @@ -165,13 +218,16 @@ public Collection createComponents(Client client, TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); + SaTifSourceConfigDao = new SATIFSourceConfigDao(client, clusterService, threadPool, xContentRegistry, threatIntelLockService); + SATIFSourceConfigService SaTifSourceConfigService = new SATIFSourceConfigService(SaTifSourceConfigDao, threatIntelLockService); + TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, - tifJobUpdateService, tifJobParameterService, threatIntelLockService); + tifJobUpdateService, tifJobParameterService, threatIntelLockService, SaTifSourceConfigDao, SaTifSourceConfigService); } @Override @@ -211,13 +267,16 @@ public List getRestHandlers(Settings settings, new RestSearchCorrelationRuleAction(), new RestIndexCustomLogTypeAction(), new RestSearchCustomLogTypeAction(), - new RestDeleteCustomLogTypeAction() + new RestDeleteCustomLogTypeAction(), + new RestIndexTIFSourceConfigAction(), + new RestGetTIFSourceConfigAction(), + new RestGetIocFindingsAction() ); } @Override public String getJobType() { - return "opensearch_sap_job"; + return JOB_TYPE; } @Override @@ -232,7 +291,21 @@ public ScheduledJobRunner getJobRunner() { @Override public ScheduledJobParser getJobParser() { - return (parser, id, jobDocVersion) -> TIFJobParameter.PARSER.parse(parser, null); + return (xcp, id, jobDocVersion) -> { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case FEED_SOURCE_CONFIG_FIELD: + return SATIFSourceConfig.parse(xcp, id, null); + default: + log.error("Job parser failed for [{}] in security analytics job registration", fieldName); + xcp.skipChildren(); + } + } + return null; + }; } @Override @@ -332,7 +405,10 @@ public List> getSettings() { new ActionHandler<>(IndexCustomLogTypeAction.INSTANCE, TransportIndexCustomLogTypeAction.class), new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class), new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class), - new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class) + new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class), + new ActionHandler<>(SAIndexTIFSourceConfigAction.INSTANCE, TransportIndexTIFSourceConfigAction.class), + new ActionHandler<>(SAGetTIFSourceConfigAction.INSTANCE, TransportGetTIFSourceConfigAction.class), + new ActionHandler<>(GetIocFindingsAction.INSTANCE, TransportGetIocFindingsAction.class) ); } diff --git a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java index a61fe9d35..03b12a2fd 100644 --- a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java +++ b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java @@ -133,16 +133,7 @@ public void getAlertsByMonitorIds( ) { org.opensearch.commons.alerting.action.GetAlertsRequest req = - new org.opensearch.commons.alerting.action.GetAlertsRequest( - table, - severityLevel, - alertState, - null, - alertIndex, - monitorIds, - null, - null - ); + null; AlertingPluginInterface.INSTANCE.getAlerts((NodeClient) client, req, new ActionListener<>() { @Override @@ -247,15 +238,7 @@ public void getAlerts(List alertIds, Detector detector, Table table, ActionListener actionListener) { - GetAlertsRequest request = new GetAlertsRequest( - table, - "ALL", - "ALL", - null, - DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()), - null, - null, - alertIds); + GetAlertsRequest request = null; AlertingPluginInterface.INSTANCE.getAlerts( (NodeClient) client, request, actionListener); diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java index 0d28bce4d..20cc34ffd 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java @@ -10,6 +10,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -69,8 +70,11 @@ public void ensureLogTypesLoaded() { private List loadBuiltinLogTypes() throws URISyntaxException, IOException { List logTypes = new ArrayList<>(); - final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); + String pathurl = Paths.get(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH).toURI()).toString(); + final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); + logger.error("SASHANK Path url is {}", pathurl); + logger.error("SASHANK currently used url is {}", url); Path dirPath = null; if (url.contains("!")) { final String[] paths = url.split("!"); diff --git a/src/main/java/org/opensearch/securityanalytics/model/IocDao.java b/src/main/java/org/opensearch/securityanalytics/model/IocDao.java new file mode 100644 index 000000000..6719af006 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/IocDao.java @@ -0,0 +1,334 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_DOMAIN_INDEX_NAME; +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_HASH_INDEX_NAME; +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_IP_INDEX_NAME; + +public class IocDao implements Writeable, ToXContentObject { + private static final Logger logger = LogManager.getLogger(IocDao.class); + + public static final String NO_ID = ""; + + static final String ID_FIELD = "id"; + static final String NAME_FIELD = "name"; + static final String TYPE_FIELD = "type"; + static final String VALUE_FIELD = "value"; + static final String SEVERITY_FIELD = "severity"; + static final String SPEC_VERSION_FIELD = "spec_version"; + static final String CREATED_FIELD = "created"; + static final String MODIFIED_FIELD = "modified"; + static final String DESCRIPTION_FIELD = "description"; + static final String LABELS_FIELD = "labels"; + static final String FEED_ID_FIELD = "feed_id"; + + private String id; + private String name; + private IocType type; + private String value; + private String severity; + private String specVersion; + private Instant created; + private Instant modified; + private String description; + private List labels; + private String feedId; + + public IocDao( + String id, + String name, + IocType type, + String value, + String severity, + String specVersion, + Instant created, + Instant modified, + String description, + List labels, + String feedId + ) { + this.id = id == null ? NO_ID : id; + this.name = name; + this.type = type; + this.value = value; + this.severity = severity; + this.specVersion = specVersion; + this.created = created; + this.modified = modified; + this.description = description; + this.labels = labels == null ? Collections.emptyList() : labels; + this.feedId = feedId; + validate(); + } + + public IocDao(StreamInput sin) throws IOException { + this( + sin.readString(), // id + sin.readString(), // name + sin.readEnum(IocType.class), // type + sin.readString(), // value + sin.readString(), // severity + sin.readString(), // specVersion + sin.readInstant(), // created + sin.readInstant(), // modified + sin.readString(), // description + sin.readStringList(), // labels + sin.readString() // feedId + ); + } + + public IocDao(IocDto iocDto) { + this( + iocDto.getId(), + iocDto.getName(), + iocDto.getType(), + iocDto.getValue(), + iocDto.getSeverity(), + iocDto.getSpecVersion(), + iocDto.getCreated(), + iocDto.getModified(), + iocDto.getDescription(), + iocDto.getLabels(), + iocDto.getFeedId() + ); + } + + public static IocDao readFrom(StreamInput sin) throws IOException { + return new IocDao(sin); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeString(name); + out.writeEnum(type); + out.writeString(value); + out.writeString(severity); + out.writeString(specVersion); + out.writeInstant(created); + out.writeInstant(modified); + out.writeString(description); + out.writeStringCollection(labels); + out.writeString(feedId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(ID_FIELD, id) + .field(NAME_FIELD, name) + .field(TYPE_FIELD, type) + .field(VALUE_FIELD, value) + .field(SEVERITY_FIELD, severity) + .field(SPEC_VERSION_FIELD, specVersion) + .timeField(CREATED_FIELD, created) + .timeField(MODIFIED_FIELD, modified) + .field(DESCRIPTION_FIELD, description) + .field(LABELS_FIELD, labels) + .field(FEED_ID_FIELD, feedId) + .endObject(); + } + + public static IocDao parse(XContentParser xcp, String id) throws IOException { + if (id == null) { + id = NO_ID; + } + + String name = null; + IocType type = null; + String value = null; + String severity = null; + String specVersion = null; + Instant created = null; + Instant modified = null; + String description = null; + List labels = Collections.emptyList(); + String feedId = null; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case NAME_FIELD: + name = xcp.text(); + break; + case TYPE_FIELD: + type = IocType.valueOf(xcp.text().toUpperCase(Locale.ROOT)); + break; + case VALUE_FIELD: + value = xcp.text(); + break; + case SEVERITY_FIELD: + severity = xcp.text(); + break; + case SPEC_VERSION_FIELD: + specVersion = xcp.text(); + break; + case CREATED_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + created = null; + } else if (xcp.currentToken().isValue()) { + created = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + created = null; + } + break; + case MODIFIED_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + modified = null; + } else if (xcp.currentToken().isValue()) { + modified = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + modified = null; + } + break; + case DESCRIPTION_FIELD: + description = xcp.text(); + break; + case LABELS_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + String entry = xcp.textOrNull(); + if (entry != null) { + labels.add(entry); + } + } + break; + case FEED_ID_FIELD: + feedId = xcp.text(); + break; + default: + xcp.skipChildren(); + } + } + + return new IocDao( + id, + name, + type, + value, + severity, + specVersion, + created, + modified, + description, + labels, + feedId + ); + } + + /** + * Validates required fields. + * @throws IllegalArgumentException + */ + public void validate() throws IllegalArgumentException { + if (type == null) { + throw new IllegalArgumentException(String.format("[%s] is required.", TYPE_FIELD)); + } else if (!Arrays.asList(IocType.values()).contains(type)) { + logger.debug("Unsupported IocType: {}", type); + throw new IllegalArgumentException(String.format("[%s] is not supported.", TYPE_FIELD)); + } + + if (value == null || value.isEmpty()) { + throw new IllegalArgumentException(String.format("[%s] is required.", VALUE_FIELD)); + } + + if (feedId == null || feedId.isEmpty()) { + throw new IllegalArgumentException(String.format("[%s] is required.", FEED_ID_FIELD)); + } + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public IocType getType() { + return type; + } + + public String getValue() { + return value; + } + + public String getSeverity() { + return severity; + } + + public String getSpecVersion() { + return specVersion; + } + + public Instant getCreated() { + return created; + } + + public Instant getModified() { + return modified; + } + + public String getDescription() { + return description; + } + + public List getLabels() { + return labels; + } + + public String getFeedId() { + return feedId; + } + + public enum IocType { + DOMAIN("domain") { + @Override + public String getSystemIndexName() { + return IOC_DOMAIN_INDEX_NAME; + } + }, + HASH("hash") { // TODO placeholder + @Override + public String getSystemIndexName() { + return IOC_HASH_INDEX_NAME; + } + }, + IP("ip") { + @Override + public String getSystemIndexName() { + return IOC_IP_INDEX_NAME; + } + }; + + IocType(String type) {} + + public abstract String getSystemIndexName(); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/model/IocDto.java b/src/main/java/org/opensearch/securityanalytics/model/IocDto.java new file mode 100644 index 000000000..ca9163cf8 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/IocDto.java @@ -0,0 +1,140 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +public class IocDto implements Writeable, ToXContentObject { + private static final Logger logger = LogManager.getLogger(IocDto.class); + + private String id; + private String name; + private IocDao.IocType type; + private String value; + private String severity; + private String specVersion; + private Instant created; + private Instant modified; + private String description; + private List labels; + private String feedId; + + public IocDto(IocDao iocDao) { + this.id = iocDao.getId(); + this.name = iocDao.getName(); + this.type = iocDao.getType(); + this.value = iocDao.getValue(); + this.severity = iocDao.getSeverity(); + this.specVersion = iocDao.getSpecVersion(); + this.created = iocDao.getCreated(); + this.modified = iocDao.getModified(); + this.description = iocDao.getDescription(); + this.labels = iocDao.getLabels(); + this.feedId = iocDao.getFeedId(); + } + + public IocDto(StreamInput sin) throws IOException { + this(new IocDao(sin)); + } + + public static IocDto readFrom(StreamInput sin) throws IOException { + return new IocDto(sin); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeString(name); + out.writeEnum(type); + out.writeString(value); + out.writeString(severity); + out.writeString(specVersion); + out.writeInstant(created); + out.writeInstant(modified); + out.writeString(description); + out.writeStringCollection(labels); + out.writeString(feedId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(IocDao.ID_FIELD, id) + .field(IocDao.NAME_FIELD, name) + .field(IocDao.TYPE_FIELD, type) + .field(IocDao.VALUE_FIELD, value) + .field(IocDao.SEVERITY_FIELD, severity) + .field(IocDao.SPEC_VERSION_FIELD, specVersion) + .timeField(IocDao.CREATED_FIELD, created) + .timeField(IocDao.MODIFIED_FIELD, modified) + .field(IocDao.DESCRIPTION_FIELD, description) + .field(IocDao.LABELS_FIELD, labels) + .field(IocDao.FEED_ID_FIELD, feedId) + .endObject(); + } + + public static IocDto parse(XContentParser xcp, String id) throws IOException { + return new IocDto(IocDao.parse(xcp, id)); + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public IocDao.IocType getType() { + return type; + } + + public String getValue() { + return value; + } + + public String getSeverity() { + return severity; + } + + public String getSpecVersion() { + return specVersion; + } + + public Instant getCreated() { + return created; + } + + public Instant getModified() { + return modified; + } + + public String getDescription() { + return description; + } + + public List getLabels() { + return labels; + } + + public String getFeedId() { + return feedId; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java new file mode 100644 index 000000000..43ecd53ad --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocFinding.java @@ -0,0 +1,234 @@ +package org.opensearch.securityanalytics.model.threatintel; + +import org.apache.commons.lang3.StringUtils; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job + * It's the inverse of an IoC finding which maps a document to list of IoC's + */ +public class IocFinding implements Writeable, ToXContent { + //TODO implement IoC_Match interface from security-analytics-commons + public static final String ID_FIELD = "id"; + public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids"; + public static final String FEED_IDS_FIELD = "feed_ids"; + public static final String IOC_SCAN_JOB_ID_FIELD = "ioc_scan_job_id"; + public static final String IOC_SCAN_JOB_NAME_FIELD = "ioc_scan_job_name"; + public static final String IOC_VALUE_FIELD = "ioc_value"; + public static final String IOC_TYPE_FIELD = "ioc_type"; + public static final String TIMESTAMP_FIELD = "timestamp"; + public static final String EXECUTION_ID_FIELD = "execution_id"; + + private final String id; + private final List relatedDocIds; + private final List feedIds; + private final String iocScanJobId; + private final String iocScanJobName; + private final String iocValue; + private final String iocType; + private final Instant timestamp; + private final String executionId; + + public IocFinding(String id, List relatedDocIds, List feedIds, String iocScanJobId, + String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) { + validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds); + this.id = id; + this.relatedDocIds = relatedDocIds; + this.feedIds = feedIds; + this.iocScanJobId = iocScanJobId; + this.iocScanJobName = iocScanJobName; + this.iocValue = iocValue; + this.iocType = iocType; + this.timestamp = timestamp; + this.executionId = executionId; + } + + public IocFinding(StreamInput in) throws IOException { + id = in.readString(); + relatedDocIds = in.readStringList(); + feedIds = in.readStringList(); + iocScanJobId = in.readString(); + iocScanJobName = in.readString(); + iocValue = in.readString(); + iocType = in.readString(); + timestamp = in.readInstant(); + executionId = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeStringCollection(relatedDocIds); + out.writeStringCollection(feedIds); + out.writeString(iocScanJobId); + out.writeString(iocScanJobName); + out.writeString(iocValue); + out.writeString(iocType); + out.writeInstant(timestamp); + out.writeOptionalString(executionId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(ID_FIELD, id) + .field(RELATED_DOC_IDS_FIELD, relatedDocIds) + .field(FEED_IDS_FIELD, feedIds) + .field(IOC_SCAN_JOB_ID_FIELD, iocScanJobId) + .field(IOC_SCAN_JOB_NAME_FIELD, iocScanJobName) + .field(IOC_VALUE_FIELD, iocValue) + .field(IOC_TYPE_FIELD, iocType) + .field(TIMESTAMP_FIELD, timestamp.toEpochMilli()) + .field(EXECUTION_ID_FIELD, executionId) + .endObject(); + return builder; + } + + public String getId() { + return id; + } + + public List getRelatedDocIds() { + return relatedDocIds; + } + + public List getFeedIds() { + return feedIds; + } + + public String getIocScanJobId() { + return iocScanJobId; + } + + public String getIocScanJobName() { + return iocScanJobName; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } + + public Instant getTimestamp() { + return timestamp; + } + + public String getExecutionId() { + return executionId; + } + + public static IocFinding parse(XContentParser xcp) throws IOException { + String id = null; + List relatedDocIds = new ArrayList<>(); + List feedIds = new ArrayList<>(); + String iocScanJobId = null; + String iocScanName = null; + String iocValue = null; + String iocType = null; + Instant timestamp = null; + String executionId = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case ID_FIELD: + id = xcp.text(); + break; + case RELATED_DOC_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + relatedDocIds.add(xcp.text()); + } + break; + case FEED_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + feedIds.add(xcp.text()); + } + break; + case IOC_SCAN_JOB_ID_FIELD: + iocScanJobId = xcp.textOrNull(); + break; + case IOC_SCAN_JOB_NAME_FIELD: + iocScanName = xcp.textOrNull(); + break; + case IOC_VALUE_FIELD: + iocValue = xcp.textOrNull(); + break; + case IOC_TYPE_FIELD: + iocType = xcp.textOrNull(); + break; + case TIMESTAMP_FIELD: + try { + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + timestamp = null; + } else if (xcp.currentToken().isValue()) { + timestamp = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + timestamp = null; + } + break; + } catch (Exception e) { + throw new IllegalArgumentException("failed to parse timestamp in IoC Match object"); + } + case EXECUTION_ID_FIELD: + executionId = xcp.textOrNull(); + break; + } + } + + return new IocFinding(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId); + } + + public static IocFinding readFrom(StreamInput in) throws IOException { + return new IocFinding(in); + } + + + private static void validateIoCMatch(String id, String iocScanJobId, String iocScanName, String iocValue, Instant timestamp, String executionId, List relatedDocIds) { + if (StringUtils.isBlank(id)) { + throw new IllegalArgumentException("id cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocValue)) { + throw new IllegalArgumentException("ioc_value cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocValue)) { + throw new IllegalArgumentException("ioc_value cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocScanJobId)) { + throw new IllegalArgumentException("ioc_scan_job_id cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocScanName)) { + throw new IllegalArgumentException("ioc_scan_job_name cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(executionId)) { + throw new IllegalArgumentException("execution_id cannot be empty in IoC_Match Object"); + } + if (timestamp == null) { + throw new IllegalArgumentException("timestamp cannot be null in IoC_Match Object"); + } + if(relatedDocIds == null || relatedDocIds.isEmpty()) { + throw new IllegalArgumentException("related_doc_ids cannot be null or empty in IoC_Match Object"); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java new file mode 100644 index 000000000..f92f3350e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +public class GetIocFindingsAction extends ActionType { + + public static final GetIocFindingsAction INSTANCE = new GetIocFindingsAction(); + public static final String NAME = "cluster:admin/opensearch/securityanalytics/ioc/findings/get"; + + public GetIocFindingsAction() { + super(NAME, GetIocFindingsResponse::new); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java new file mode 100644 index 000000000..a33b69401 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java @@ -0,0 +1,81 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ValidateActions; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +public class GetIocFindingsRequest extends ActionRequest { + + private List findingIds; + + private Instant startTime; + + private Instant endTime; + + private Table table; + + public GetIocFindingsRequest(StreamInput sin) throws IOException { + this( + sin.readOptionalStringList(), + sin.readOptionalInstant(), + sin.readOptionalInstant(), + Table.readFrom(sin) + ); + } + + public GetIocFindingsRequest(List findingIds, + Instant startTime, + Instant endTime, + Table table) { + this.findingIds = findingIds; + this.startTime = startTime; + this.endTime = endTime; + this.table = table; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (startTime != null && endTime != null && startTime.isAfter(endTime)) { + validationException = ValidateActions.addValidationError(String.format(Locale.getDefault(), + "startTime should be less than endTime"), validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalStringCollection(findingIds); + out.writeOptionalInstant(startTime); + out.writeOptionalInstant(endTime); + table.writeTo(out); + } + + public List getFindingIds() { + return findingIds; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } + + public Table getTable() { + return table; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java new file mode 100644 index 000000000..50ae08dd4 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class GetIocFindingsResponse extends ActionResponse implements ToXContentObject { + + private static final String TOTAL_IOC_FINDINGS_FIELD = "total_findings"; + + private static final String IOC_FINDINGS_FIELD = "ioc_findings"; + + private Integer totalFindings; + + private List iocFindings; + + public GetIocFindingsResponse(Integer totalFindings, List iocFindings) { + super(); + this.totalFindings = totalFindings; + this.iocFindings = iocFindings; + } + + public GetIocFindingsResponse(StreamInput sin) throws IOException { + this( + sin.readInt(), + Collections.unmodifiableList(sin.readList(IocFinding::new)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(totalFindings); + out.writeCollection(iocFindings); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(TOTAL_IOC_FINDINGS_FIELD, totalFindings) + .field(IOC_FINDINGS_FIELD, iocFindings); + return builder.endObject(); + } + + public Integer getTotalFindings() { + return totalFindings; + } + + public List getIocFindings() { + return iocFindings; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigAction.java new file mode 100644 index 000000000..f2a0099e7 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigAction.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.GET_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Get TIF Source Config Action + */ +public class SAGetTIFSourceConfigAction extends ActionType { + + public static final SAGetTIFSourceConfigAction INSTANCE = new SAGetTIFSourceConfigAction(); + public static final String NAME = GET_TIF_SOURCE_CONFIG_ACTION_NAME; + private SAGetTIFSourceConfigAction() { + super(NAME, SAGetTIFSourceConfigResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigRequest.java new file mode 100644 index 000000000..6f64809bd --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigRequest.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Locale; + +import static org.opensearch.action.ValidateActions.addValidationError; + +/** + * Get threat intel feed source config request + */ +public class SAGetTIFSourceConfigRequest extends ActionRequest { + private final String id; + private final Long version; + public static final String TIF_SOURCE_CONFIG_ID = "tif_source_config_id"; + + public SAGetTIFSourceConfigRequest(String id, Long version) { + super(); + this.id = id; + this.version = version; + } + + public SAGetTIFSourceConfigRequest(StreamInput sin) throws IOException { + this(sin.readString(), // id + sin.readLong()); // version + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + } + + public String getId() { + return id; + } + + public Long getVersion() { + return version; + } + + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (id == null || id.isEmpty()) { + validationException = addValidationError(String.format(Locale.getDefault(), "%s is missing", TIF_SOURCE_CONFIG_ID), validationException); + } + return validationException; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java new file mode 100644 index 000000000..e239b87af --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java @@ -0,0 +1,101 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; + +public class SAGetTIFSourceConfigResponse extends ActionResponse implements ToXContentObject { + private final String id; + + private final Long version; + + private final RestStatus status; + + private final SATIFSourceConfigDto SaTifSourceConfigDto; + + + public SAGetTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto SaTifSourceConfigDto) { + super(); + this.id = id; + this.version = version; + this.status = status; + this.SaTifSourceConfigDto = SaTifSourceConfigDto; + } + + public SAGetTIFSourceConfigResponse(StreamInput sin) throws IOException { + this( + sin.readString(), // id + sin.readLong(), // version + sin.readEnum(RestStatus.class), // status + sin.readBoolean()? SATIFSourceConfigDto.readFrom(sin) : null // SA tif config dto + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeEnum(status); + if (SaTifSourceConfigDto != null) { + out.writeBoolean((true)); + SaTifSourceConfigDto.writeTo(out); + } else { + out.writeBoolean(false); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(_ID, id) + .field(_VERSION, version); + builder.startObject("tif_config") + .field(SATIFSourceConfigDto.FEED_NAME_FIELD, SaTifSourceConfigDto.getName()) + .field(SATIFSourceConfigDto.FEED_FORMAT_FIELD, SaTifSourceConfigDto.getFeedFormat()) + .field(SATIFSourceConfigDto.FEED_TYPE_FIELD, SaTifSourceConfigDto.getFeedType()) + .field(SATIFSourceConfigDto.STATE_FIELD, SaTifSourceConfigDto.getState()) + .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, SaTifSourceConfigDto.getEnabledTime()) + .field(SATIFSourceConfigDto.ENABLED_FIELD, SaTifSourceConfigDto.isEnabled()) + .field(SATIFSourceConfigDto.CREATED_AT_FIELD, SaTifSourceConfigDto.getCreatedAt()) + .field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, SaTifSourceConfigDto.getLastUpdateTime()) + .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, SaTifSourceConfigDto.getLastRefreshedTime()) + .field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, SaTifSourceConfigDto.getRefreshType()) + .field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, SaTifSourceConfigDto.getLastRefreshedUser()) + .field(SATIFSourceConfigDto.SCHEDULE_FIELD, SaTifSourceConfigDto.getSchedule()) + // source + .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, SaTifSourceConfigDto.getCreatedByUser()) + .field(SATIFSourceConfigDto.IOC_MAP_STORE_FIELD, SaTifSourceConfigDto.getIocMapStore()) + .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, SaTifSourceConfigDto.getIocTypes()) + .endObject(); + return builder.endObject(); + } + + public String getId() { + return id; + } + + public Long getVersion() { + return version; + } + + public RestStatus getStatus() { + return status; + } + + public SATIFSourceConfigDto getSaTifSourceConfigDto() { + return SaTifSourceConfigDto; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java new file mode 100644 index 000000000..1b4acd80e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Threat intel tif job creation action + */ +public class SAIndexTIFSourceConfigAction extends ActionType { + + public static final SAIndexTIFSourceConfigAction INSTANCE = new SAIndexTIFSourceConfigAction(); + public static final String NAME = INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; + private SAIndexTIFSourceConfigAction() { + super(NAME, SAIndexTIFSourceConfigResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java new file mode 100644 index 000000000..a44a412ae --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigRequest; + +import java.io.IOException; +import java.util.List; + +/** + * Threat intel feed config creation request + */ +public class SAIndexTIFSourceConfigRequest extends ActionRequest implements IndexTIFSourceConfigRequest { + private static final ParameterValidator VALIDATOR = new ParameterValidator(); + private String tifSourceConfigId; + private final WriteRequest.RefreshPolicy refreshPolicy; + private final RestRequest.Method method; + private SATIFSourceConfigDto SaTifSourceConfigDto; + + public SAIndexTIFSourceConfigRequest(String tifSourceConfigId, + WriteRequest.RefreshPolicy refreshPolicy, + RestRequest.Method method, + SATIFSourceConfigDto SaTifSourceConfigDto) { + super(); + this.tifSourceConfigId = tifSourceConfigId; + this.refreshPolicy = refreshPolicy; + this.method = method; + this.SaTifSourceConfigDto = SaTifSourceConfigDto; + } + + public SAIndexTIFSourceConfigRequest(StreamInput sin) throws IOException { + this( + sin.readString(), // tif config id + WriteRequest.RefreshPolicy.readFrom(sin), // refresh policy + sin.readEnum(RestRequest.Method.class), // method + SATIFSourceConfigDto.readFrom(sin) // SA tif config dto + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(tifSourceConfigId); + refreshPolicy.writeTo(out); + out.writeEnum(method); + SaTifSourceConfigDto.writeTo(out); + } + + @Override + public String getTIFConfigId() { + return tifSourceConfigId; + } + + public void setTIFConfigId(String tifConfigId) { + this.tifSourceConfigId = tifConfigId; + } + + @Override + public SATIFSourceConfigDto getTIFConfigDto() { + return SaTifSourceConfigDto; + } + + public void setTIFConfigDto(SATIFSourceConfigDto SaTifSourceConfigDto) { + this.SaTifSourceConfigDto = SaTifSourceConfigDto; + } + + public WriteRequest.RefreshPolicy getRefreshPolicy() { + return refreshPolicy; + } + + public RestRequest.Method getMethod() { + return method; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException errors = new ActionRequestValidationException(); + List errorMsgs = VALIDATOR.validateTIFJobName(SaTifSourceConfigDto.getName()); + if (errorMsgs.isEmpty() == false) { + errorMsgs.forEach(errors::addValidationError); + } + return errors.validationErrors().isEmpty() ? null : errors; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java new file mode 100644 index 000000000..b4ea1b9c0 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfigDto; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; + +public class SAIndexTIFSourceConfigResponse extends ActionResponse implements ToXContentObject, IndexTIFSourceConfigResponse { + private final String id; + private final Long version; + private final RestStatus status; + private final SATIFSourceConfigDto SaTifSourceConfigDto; + + public SAIndexTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto SaTifSourceConfigDto) { + super(); + this.id = id; + this.version = version; + this.status = status; + this.SaTifSourceConfigDto = SaTifSourceConfigDto; + } + + public SAIndexTIFSourceConfigResponse(StreamInput sin) throws IOException { + this( + sin.readString(), // tif config id + sin.readLong(), // version + sin.readEnum(RestStatus.class), // status + SATIFSourceConfigDto.readFrom(sin) // SA tif config dto + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeEnum(status); + SaTifSourceConfigDto.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(_ID, id) + .field(_VERSION, version); + + builder.startObject("tif_config") + .field(SATIFSourceConfigDto.FEED_FORMAT_FIELD, SaTifSourceConfigDto.getFeedFormat()) + .field(SATIFSourceConfigDto.FEED_NAME_FIELD, SaTifSourceConfigDto.getName()) + .field(SATIFSourceConfigDto.FEED_TYPE_FIELD, SaTifSourceConfigDto.getFeedType()) + .field(SATIFSourceConfigDto.STATE_FIELD, SaTifSourceConfigDto.getState()) + .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, SaTifSourceConfigDto.getEnabledTime()) + .field(SATIFSourceConfigDto.ENABLED_FIELD, SaTifSourceConfigDto.isEnabled()) + .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, SaTifSourceConfigDto.getLastRefreshedTime()) + .field(SATIFSourceConfigDto.SCHEDULE_FIELD, SaTifSourceConfigDto.getSchedule()) + // source + .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, SaTifSourceConfigDto.getCreatedByUser()) + .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, SaTifSourceConfigDto.getIocTypes()) + .endObject(); + + return builder.endObject(); + } + @Override + public String getTIFConfigId() { + return id; + } + @Override + public Long getVersion() { + return version; + } + @Override + public TIFSourceConfigDto getTIFConfigDto() { + return SaTifSourceConfigDto; + } + public RestStatus getStatus() { + return status; + } + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/FeedType.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/FeedType.java new file mode 100644 index 000000000..606f9f1ec --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/FeedType.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.common; + +/** + * Types of feeds threat intel can support + * Feed types include: licensed, open-sourced, custom, and internal + */ +public enum FeedType { + + LICENSED, + + OPEN_SOURCED, + + CUSTOM, + + INTERNAL +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/RefreshType.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/RefreshType.java new file mode 100644 index 000000000..0ac915781 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/RefreshType.java @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.common; + +/** + * Refresh Types: Full + * TODO: Add other refresh types such as the delta + */ +public enum RefreshType { + + FULL +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java index 22ffee3e9..db72ac757 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java @@ -33,5 +33,15 @@ public enum TIFJobState { /** * tif job is being deleted */ - DELETING + DELETING, + + /** + * tif associated iocs are being refreshed + */ + REFRESHING, + + /** + * tif refresh job failed + */ + REFRESH_FAILED } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java new file mode 100644 index 000000000..3e3cfa311 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java @@ -0,0 +1,216 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.StepListener; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.threadpool.ThreadPool; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +/** + * CRUD for threat intel feeds source config object + */ +public class SATIFSourceConfigDao { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigDao.class); + private final Client client; + private final ClusterService clusterService; + private final ClusterSettings clusterSettings; + private final ThreadPool threadPool; + private final NamedXContentRegistry xContentRegistry; + private final TIFLockService lockService; + + + public SATIFSourceConfigDao(final Client client, + final ClusterService clusterService, + ThreadPool threadPool, + NamedXContentRegistry xContentRegistry, + final TIFLockService lockService + ) { + this.client = client; + this.clusterService = clusterService; + this.clusterSettings = clusterService.getClusterSettings(); + this.threadPool = threadPool; + this.xContentRegistry = xContentRegistry; + this.lockService = lockService; + } + + public void indexTIFSourceConfig(SATIFSourceConfig SaTifSourceConfig, + TimeValue indexTimeout, + final LockModel lock, + final ActionListener actionListener + ) { + StepListener createIndexStepListener = new StepListener<>(); + createIndexStepListener.whenComplete(v -> { + try { + IndexRequest indexRequest = new IndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(SaTifSourceConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .timeout(indexTimeout); + log.debug("Indexing tif source config"); + client.index(indexRequest, ActionListener.wrap(response -> { + log.debug("Threat intel source config with id [{}] indexed success.", response.getId()); + SATIFSourceConfig responseSaTifSourceConfig = createSATIFSourceConfig(SaTifSourceConfig, response); + actionListener.onResponse(responseSaTifSourceConfig); + }, actionListener::onFailure)); + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + }, exception -> { + lockService.releaseLock(lock); + log.error("Failed to release lock", exception); + actionListener.onFailure(exception); + }); + createJobIndexIfNotExists(createIndexStepListener); + } + + private static SATIFSourceConfig createSATIFSourceConfig(SATIFSourceConfig SaTifSourceConfig, IndexResponse response) { + return new SATIFSourceConfig( + response.getId(), + SaTifSourceConfig.getVersion(), + SaTifSourceConfig.getName(), + SaTifSourceConfig.getFeedFormat(), + SaTifSourceConfig.getFeedType(), + SaTifSourceConfig.getCreatedByUser(), + SaTifSourceConfig.getCreatedAt(), + SaTifSourceConfig.getEnabledTime(), + SaTifSourceConfig.getLastUpdateTime(), + SaTifSourceConfig.getSchedule(), + SaTifSourceConfig.getState(), + SaTifSourceConfig.getRefreshType(), + SaTifSourceConfig.getLastRefreshedTime(), + SaTifSourceConfig.getLastRefreshedUser(), + SaTifSourceConfig.isEnabled(), + SaTifSourceConfig.getIocMapStore(), + SaTifSourceConfig.getIocTypes() + ); + } + + // Get the job config index mapping + private String getIndexMapping() { + try { + try (InputStream is = SATIFSourceConfigDao.class.getResourceAsStream("/mappings/threat_intel_job_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Failed to get the threat intel index mapping", e); + throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + + // Create TIF source config index + /** + * Index name: .opensearch-sap--job + * Mapping: /mappings/threat_intel_job_mapping.json + * + * @param stepListener setup listener + */ + public void createJobIndexIfNotExists(final StepListener stepListener) { + // check if job index exists + if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == true) { + stepListener.onResponse(null); + return; + } + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); + StashedThreadContext.run(client, () -> client.admin().indices().create(createIndexRequest, new ActionListener<>() { + @Override + public void onResponse(final CreateIndexResponse createIndexResponse) { + log.debug("[{}] index created", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + } + + @Override + public void onFailure(final Exception e) { + if (e instanceof ResourceAlreadyExistsException) { + log.info("Index [{}] already exists", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + return; + } + log.error("Failed to create [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME, e); + stepListener.onFailure(e); + } + })); + } + + + // Get TIF source config + public void getTIFSourceConfig( + String tifSourceConfigId, + Long version, + ActionListener actionListener + ) { + GetRequest getRequest = new GetRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME, tifSourceConfigId).version(version); + client.get(getRequest, new ActionListener<>() { + @Override + public void onResponse(GetResponse response) { + try { + if (!response.isExists()) { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config not found.", RestStatus.NOT_FOUND))); + return; + } + SATIFSourceConfig SaTifSourceConfig = null; + if (!response.isSourceEmpty()) { + XContentParser xcp = XContentHelper.createParser( + xContentRegistry, LoggingDeprecationHandler.INSTANCE, + response.getSourceAsBytesRef(), XContentType.JSON + ); + SaTifSourceConfig = SATIFSourceConfig.docParse(xcp, response.getId(), response.getVersion()); + assert SaTifSourceConfig != null; + } + log.debug("Threat intel source config with id [{}] fetched.", response.getId()); + actionListener.onResponse(SaTifSourceConfig); + } catch (IOException ex) { + log.error("Failed to fetch threat intel source config document", ex); + actionListener.onFailure(ex); + } + } + @Override + public void onFailure(Exception e) { + log.error("Failed to fetch threat intel source config document " + tifSourceConfigId, e); + actionListener.onFailure(e); + } + }); + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java index 6b84e9fe9..2b5856999 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java @@ -10,7 +10,7 @@ import org.opensearch.common.settings.SettingsException; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import org.opensearch.securityanalytics.util.FileUtils; import java.io.IOException; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java new file mode 100644 index 000000000..94f2ef540 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java @@ -0,0 +1,204 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.get.MultiGetRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Data layer to perform CRUD operations for threat intel ioc match : store in system index. + */ +public class IocFindingService { + //TODO manage index rollover + public static final String INDEX_NAME = ".opensearch-sap-iocmatch"; + private static final Logger log = LogManager.getLogger(IocFindingService.class); + private final Client client; + private final ClusterService clusterService; + + private final NamedXContentRegistry xContentRegistry; + + public IocFindingService(final Client client, final ClusterService clusterService, final NamedXContentRegistry xContentRegistry) { + this.client = client; + this.clusterService = clusterService; + this.xContentRegistry = xContentRegistry; + } + + public void indexIocMatches(List iocFindings, + final ActionListener actionListener) { + try { + Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + createIndexIfNotExists(ActionListener.wrap( + r -> { + List bulkRequestList = new ArrayList<>(); + BulkRequest bulkRequest = new BulkRequest(INDEX_NAME); + for (int i = 0; i < iocFindings.size(); i++) { + IocFinding iocFinding = iocFindings.get(i); + try { + IndexRequest indexRequest = new IndexRequest(INDEX_NAME) + .source(iocFinding.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .opType(DocWriteRequest.OpType.CREATE); + bulkRequest.add(indexRequest); + if ( + bulkRequest.requests().size() == batchSize + && i != iocFindings.size() - 1 // final bulk request will be added outside for loop with refresh policy none + ) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } catch (IOException e) { + log.error(String.format("Failed to create index request for ioc match %s moving on to next"), e); + } + } + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestList.add(bulkRequest); + GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { + int idx = 0; + for (BulkResponse response : bulkResponses) { + BulkRequest request = bulkRequestList.get(idx); + if (response.hasFailures()) { + log.error("Failed to bulk index {} Ioc Matches. Failure: {}", request.batchSize(), response.buildFailureMessage()); + } + } + actionListener.onResponse(null); + }, actionListener::onFailure), bulkRequestList.size()); + for (BulkRequest req : bulkRequestList) { + try { + client.bulk(req, groupedListener); //todo why stash context here? + } catch (Exception e) { + log.error("Failed to save ioc matches.", e); + } + } + }, e -> { + log.error("Failed to create System Index"); + actionListener.onFailure(e); + })); + + + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + } + + private String getIndexMapping() { + try { + try (InputStream is = IocFindingService.class.getResourceAsStream("/mappings/ioc_match_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Failed to get the threat intel ioc match index mapping", e); + throw new SecurityAnalyticsException("Failed to get the threat intel ioc match index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + + /** + * Index name: .opensearch-sap-iocmatch + * Mapping: /mappings/ioc_match_mapping.json + * + * @param listener setup listener + */ + public void createIndexIfNotExists(final ActionListener listener) { + // check if job index exists + try { + if (clusterService.state().metadata().hasIndex(INDEX_NAME) == true) { + listener.onResponse(null); + return; + } + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); + client.admin().indices().create(createIndexRequest, ActionListener.wrap( + r -> { + log.debug("Ioc match index created"); + listener.onResponse(null); + }, e -> { + if (e instanceof ResourceAlreadyExistsException) { + log.debug("index {} already exist", INDEX_NAME); + listener.onResponse(null); + return; + } + log.error("Failed to create security analytics threat intel job index", e); + listener.onFailure(e); + } + )); + } catch (Exception e) { + log.error("Failure in creating ioc_match index", e); + listener.onFailure(e); + } + } + + public void searchIocMatches(SearchSourceBuilder searchSourceBuilder, final ActionListener actionListener) { + SearchRequest searchRequest = new SearchRequest() + .source(searchSourceBuilder) + .indices(INDEX_NAME); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse searchResponse) { + try { + long totalIocFindingsCount = searchResponse.getHits().getTotalHits().value; + List iocFindings = new ArrayList<>(); + + for (SearchHit hit: searchResponse.getHits()) { + XContentParser xcp = XContentType.JSON.xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString()); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + IocFinding iocFinding = IocFinding.parse(xcp); + iocFindings.add(iocFinding); + } + actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocFindings)); + } catch (Exception ex) { + this.onFailure(ex); + } + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + actionListener.onResponse(new GetIocFindingsResponse(0, List.of())); + return; + } + actionListener.onFailure(e); + } + }); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java new file mode 100644 index 000000000..3e77f2d28 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java @@ -0,0 +1,23 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dto; + +import org.opensearch.securityanalytics.threatIntel.iocscan.model.IocScanMonitor; + +import java.util.List; + +public class IocScanContext { + IocScanMonitor monitor; + boolean dryRun; + List data; + + public IocScanMonitor getMonitor() { + return monitor; + } + + public boolean isDryRun() { + return dryRun; + } + + public List getData() { + return data; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeFieldMappings.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeFieldMappings.java new file mode 100644 index 000000000..e101d2c5e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeFieldMappings.java @@ -0,0 +1,26 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dto; + +import java.util.List; +import java.util.Map; + +/** + * DTO that contains information about an Ioc type and the list of fields in each index that map to the given + */ +public class PerIocTypeFieldMappings { + + private final String iocType; + private final Map> indexToFieldsMap; + + public PerIocTypeFieldMappings(String iocType, Map> indexToFieldsMap) { + this.iocType = iocType; + this.indexToFieldsMap = indexToFieldsMap; + } + + public String getIocType() { + return iocType; + } + + public Map> getIndexToFieldsMap() { + return indexToFieldsMap; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/Ioc.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/Ioc.java new file mode 100644 index 000000000..5e83b7157 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/Ioc.java @@ -0,0 +1,26 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.model; + +public class Ioc { + + private final String feedId; + private final String iocValue; + private final String iocType; + + public Ioc(String feedId, String iocValue, String iocType) { + this.feedId = feedId; + this.iocValue = iocValue; + this.iocType = iocType; + } + + public String getFeedId() { + return feedId; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/IocScanMonitor.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/IocScanMonitor.java new file mode 100644 index 000000000..17b1c3e48 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/IocScanMonitor.java @@ -0,0 +1,30 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.model; + +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeFieldMappings; + +import java.util.List; +import java.util.Map; + + +public class IocScanMonitor { + String id; + String name; + List iocTypeToIndexFieldMappings; + Map> perIoCTypeThreatIntelIndices; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public List getIocTypeToIndexFieldMappings() { + return iocTypeToIndexFieldMappings; + } + + public Map> getPerIoCTypeThreatIntelIndices() { + return perIoCTypeThreatIntelIndices; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java new file mode 100644 index 000000000..294e03e65 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java @@ -0,0 +1,235 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.commons.alerting.model.Finding; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeFieldMappings; +import org.opensearch.securityanalytics.threatIntel.iocscan.model.Ioc; +import org.opensearch.securityanalytics.threatIntel.iocscan.model.IocScanMonitor; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.BiConsumer; + + +public abstract class IoCScanService implements IoCScanServiceInterface { + private static final Logger log = LogManager.getLogger(IoCScanService.class); + + @Override + public void scanIoCs(IocScanContext iocScanContext, + BiConsumer scanCallback + ) { + List data = iocScanContext.getData(); + IocScanMonitor iocScanMonitor = iocScanContext.getMonitor(); + + long start = System.currentTimeMillis(); + // log.debug("beginning to scan IoC's") + IocLookupDtos iocLookupDtos = extractIocPerTypeSet(data, iocScanMonitor.getIocTypeToIndexFieldMappings()); + BiConsumer, Exception> iocScanResultConsumer = (List maliciousIocs, Exception e) -> { + if (e == null) { + createIoCMatches(maliciousIocs, iocLookupDtos.iocValueToDocIdMap, iocScanContext, + new BiConsumer, Exception>() { + @Override + public void accept(List iocs, Exception e) { + createFindings(maliciousIocs, iocLookupDtos.docIdToIocsMap, iocScanMonitor); + } + } + ); + + } else { + // onIocMatchFailure(e, iocScanMonitor); + + } + }; + matchAgainstThreatIntelAndReturnMaliciousIocs(iocLookupDtos.getIocsPerIocTypeMap(), iocScanMonitor, iocScanResultConsumer); + } + + abstract void matchAgainstThreatIntelAndReturnMaliciousIocs( + Map> iocPerTypeSet, + IocScanMonitor iocScanMonitor, + BiConsumer, Exception> callback); + + /** + * For each doc, we extract the list of + */ + private IocLookupDtos extractIocPerTypeSet(List data, List iocTypeToIndexFieldMappings) { + Map> iocsPerIocTypeMap = new HashMap<>(); + Map> iocValueToDocIdMap = new HashMap<>(); + Map> docIdToIocsMap = new HashMap<>(); + for (Data datum : data) { + for (PerIocTypeFieldMappings iocTypeToIndexFieldMapping : iocTypeToIndexFieldMappings) { + String iocType = iocTypeToIndexFieldMapping.getIocType(); + String index = getIndexName(datum); + List fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); + for (String field : fields) { + List vals = getValuesAsStringList(datum, field); + String id = getId(datum); + String indexName = getIndexName(datum); + String docId = id + ":" + indexName; + Set iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>()); + iocs.addAll(vals); + docIdToIocsMap.put(docId, iocs); + for (String ioc : vals) { + Set docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>()); + docIds.add(docId); + iocValueToDocIdMap.put(ioc, docIds); + } + if (false == vals.isEmpty()) { + iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + iocs.addAll(vals); + iocsPerIocTypeMap.put(iocType, iocs); + } + } + } + } + return new IocLookupDtos(iocsPerIocTypeMap, iocValueToDocIdMap, docIdToIocsMap); + } + + public abstract List getValuesAsStringList(Data datum, String field); + + public abstract String getIndexName(Data datum); + + public abstract String getId(Data datum); + + public void createIoCMatches(List iocs, Map> iocValueToDocIdMap, IocScanContext iocScanContext, BiConsumer, Exception> callback) { + try { + Instant timestamp = Instant.now(); + IocScanMonitor iocScanMonitor = iocScanContext.getMonitor(); + // Map to collect unique IocValue with their respective FeedIds + Map> iocValueToFeedIds = new HashMap<>(); + + for (Ioc ioc : iocs) { + String iocValue = ioc.getIocValue(); + iocValueToFeedIds + .computeIfAbsent(iocValue, k -> new HashSet<>()) + .add(ioc.getFeedId()); + } + + List iocFindings = new ArrayList<>(); + + for (Map.Entry> entry : iocValueToFeedIds.entrySet()) { + String iocValue = entry.getKey(); + Set feedIds = entry.getValue(); + + List relatedDocIds = new ArrayList<>(iocValueToDocIdMap.getOrDefault(iocValue, new HashSet<>())); + List feedIdsList = new ArrayList<>(feedIds); + try { + IocFinding iocFinding = new IocFinding( + UUID.randomUUID().toString(), // Generating a unique ID + relatedDocIds, + feedIdsList, + iocScanMonitor.getId(), + iocScanMonitor.getName(), + iocValue, + iocs.stream().filter(i -> i.getIocValue().equals(iocValue)).findFirst().orElseThrow().getIocType(), + timestamp, + UUID.randomUUID().toString() // TODO execution ID + ); + iocFindings.add(iocFinding); + } catch (Exception e) { + log.error(String.format("skipping creating ioc match for %s due to unexpected failure.", entry.getKey()), e); + } + } + saveIocs(iocs, callback); + } catch (Exception e) { + log.error(() -> new ParameterizedMessage("Failed to create ioc matches due to unexpected error {}", iocScanContext.getMonitor().getId()), e); + callback.accept(null, e); + } + } + + abstract void saveIocs(List iocs, BiConsumer, Exception> callback); + + public List createFindings(List iocs, Map> docIdToIocsMap, IocScanMonitor iocScanMonitor) { + List findings = new ArrayList<>(); + + for (Map.Entry> entry : docIdToIocsMap.entrySet()) { + String docId = entry.getKey(); + Set iocValues = entry.getValue(); + + List iocStrings = new ArrayList<>(iocValues); + + Finding finding = new Finding( + UUID.randomUUID().toString(), // Generating a unique ID + Collections.singletonList(docId), // Singleton list for relatedDocIds + Collections.emptyList(), // Empty list for correlatedDocIds + iocScanMonitor.getId(), + iocScanMonitor.getName(), + "", // Index value, you may need to provide an actual value + Collections.emptyList(), // Empty list for docLevelQueries +// iocStrings, TODO add field in findings + Instant.now(), // Current timestamp + null // Setting executionId as null, you may adjust accordingly + ); + + findings.add(finding); + } + + return findings; + } + + + private static class IocMatchDto { + private final String iocValue; + private final String iocType; + private final List iocs; + private final List docIdsContainingIoc; + + public IocMatchDto(String iocValue, String iocType, List iocs, List docIdsContainingIoc) { + this.iocValue = iocValue; + this.iocType = iocType; + this.iocs = iocs; + this.docIdsContainingIoc = docIdsContainingIoc; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } + + public List getIocs() { + return iocs; + } + + public List getDocIdsContainingIoc() { + return docIdsContainingIoc; + } + } + + private static class IocLookupDtos { + private final Map> iocsPerIocTypeMap; + private final Map> iocValueToDocIdMap; + private final Map> docIdToIocsMap; + + public IocLookupDtos(Map> iocsPerIocTypeMap, Map> iocValueToDocIdMap, Map> docIdToIocsMap) { + this.iocsPerIocTypeMap = iocsPerIocTypeMap; + this.iocValueToDocIdMap = iocValueToDocIdMap; + this.docIdToIocsMap = docIdToIocsMap; + } + + public Map> getIocsPerIocTypeMap() { + return iocsPerIocTypeMap; + } + + public Map> getIocValueToDocIdMap() { + return iocValueToDocIdMap; + } + + public Map> getDocIdToIocsMap() { + return docIdToIocsMap; + } + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java new file mode 100644 index 000000000..1826824d3 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java @@ -0,0 +1,13 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; + +import java.util.function.BiConsumer; + +public interface IoCScanServiceInterface { + + void scanIoCs( + IocScanContext iocScanContext, + BiConsumer scanCallback + ); +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java index 1d8d8643f..65d7e46e5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java @@ -11,21 +11,21 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.action.ActionListener; import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import java.time.Instant; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.threadpool.ThreadPool; /** diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java new file mode 100644 index 000000000..dc9420381 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java @@ -0,0 +1,524 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfig; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Implementation of TIF Config to store the feed configuration metadata and to schedule it onto the job scheduler + */ +public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJobParameter { + + private static final Logger log = LogManager.getLogger(SATIFSourceConfig.class); + + /** + * Prefix of indices having threatIntel data + */ + public static final String THREAT_INTEL_DATA_INDEX_NAME_PREFIX = ".opensearch-sap-threat-intel"; + public static final String FEED_SOURCE_CONFIG_FIELD = "feed_source_config"; + + public static final String NO_ID = ""; + + public static final Long NO_VERSION = 1L; + public static final String VERSION_FIELD = "version"; + public static final String FEED_NAME_FIELD = "feed_name"; + public static final String FEED_FORMAT_FIELD = "feed_format"; + public static final String FEED_TYPE_FIELD = "feed_type"; + public static final String CREATED_BY_USER_FIELD = "created_by_user"; + public static final String CREATED_AT_FIELD = "created_at"; + public static final String SOURCE_FIELD = "source"; + public static final String ENABLED_TIME_FIELD = "enabled_time"; + public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String STATE_FIELD = "state"; + public static final String REFRESH_TYPE_FIELD = "refresh_type"; + public static final String LAST_REFRESHED_TIME_FIELD = "last_refreshed_time"; + public static final String LAST_REFRESHED_USER_FIELD = "last_refreshed_user"; + public static final String ENABLED_FIELD = "enabled"; + public static final String IOC_MAP_STORE_FIELD = "ioc_map_store"; + public static final String IOC_TYPES_FIELD = "ioc_types"; + + private String id; + private Long version; + private String feedName; + private String feedFormat; + private FeedType feedType; + private String createdByUser; + private Instant createdAt; + + // private Source source; TODO: create Source Object + private Instant enabledTime; + private Instant lastUpdateTime; + private IntervalSchedule schedule; + private TIFJobState state; + public RefreshType refreshType; + public Instant lastRefreshedTime; + public String lastRefreshedUser; + private Boolean isEnabled; + private Map iocMapStore; + private List iocTypes; + + public SATIFSourceConfig(String id, Long version, String feedName, String feedFormat, FeedType feedType, String createdByUser, Instant createdAt, + Instant enabledTime, Instant lastUpdateTime, IntervalSchedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, String lastRefreshedUser, + Boolean isEnabled, Map iocMapStore, List iocTypes) { + this.id = id != null ? id : NO_ID; + this.version = version != null ? version : NO_VERSION; + this.feedName = feedName; + this.feedFormat = feedFormat; + this.feedType = feedType; + this.createdByUser = createdByUser; + this.createdAt = createdAt != null ? createdAt : Instant.now(); + + if (isEnabled == null && enabledTime == null) { + this.enabledTime = Instant.now(); + } else if (isEnabled != null && !isEnabled) { + this.enabledTime = null; + } else { + this.enabledTime = enabledTime; + } + + this.lastUpdateTime = lastUpdateTime != null ? lastUpdateTime : Instant.now(); + this.schedule = schedule; + this.state = state != null ? state : TIFJobState.CREATING; + this.refreshType = refreshType != null ? refreshType : RefreshType.FULL; + this.lastRefreshedTime = lastRefreshedTime; + this.lastRefreshedUser = lastRefreshedUser; + this.isEnabled = isEnabled; + this.iocMapStore = iocMapStore != null ? iocMapStore : new HashMap<>(); + this.iocTypes = iocTypes; + } + + public SATIFSourceConfig(StreamInput sin) throws IOException { + this( + sin.readString(), // id + sin.readLong(), // version + sin.readString(), // feed name + sin.readString(), // feed format + FeedType.valueOf(sin.readString()), // feed type + sin.readOptionalString(), // created by user + sin.readInstant(), // created at + sin.readOptionalInstant(), // enabled time + sin.readInstant(), // last update time + new IntervalSchedule(sin), // schedule + TIFJobState.valueOf(sin.readString()), // state + RefreshType.valueOf(sin.readString()), // state + sin.readOptionalInstant(), // last refreshed time + sin.readOptionalString(), // last refreshed user + sin.readBoolean(), // is enabled + sin.readMap(), // ioc map store + sin.readStringList() + ); + } + + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeString(feedName); + out.writeString(feedFormat); + out.writeString(feedType.name()); + out.writeOptionalString(createdByUser); + out.writeInstant(createdAt); + out.writeOptionalInstant(enabledTime); + out.writeInstant(lastUpdateTime); + schedule.writeTo(out); + out.writeString(state.name()); + out.writeString(refreshType.name()); + out.writeOptionalInstant(lastRefreshedTime); + out.writeOptionalString(lastRefreshedUser); + out.writeBoolean(isEnabled); + out.writeMap(iocMapStore); + out.writeStringCollection(iocTypes); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject() + .startObject(FEED_SOURCE_CONFIG_FIELD) + .field(VERSION_FIELD, version) + .field(FEED_NAME_FIELD, feedName) + .field(FEED_FORMAT_FIELD, feedFormat) + .field(FEED_TYPE_FIELD, feedType.name()) + .field(CREATED_BY_USER_FIELD, createdByUser); + + if (createdAt == null) { + builder.nullField(CREATED_AT_FIELD); + } else { + builder.timeField(CREATED_AT_FIELD, String.format(Locale.getDefault(), "%s_in_millis", CREATED_AT_FIELD), createdAt.toEpochMilli()); + } + + if (enabledTime == null) { + builder.nullField(ENABLED_TIME_FIELD); + } else { + builder.timeField(ENABLED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", ENABLED_TIME_FIELD), enabledTime.toEpochMilli()); + } + + if (lastUpdateTime == null) { + builder.nullField(LAST_UPDATE_TIME_FIELD); + } else { + builder.timeField(LAST_UPDATE_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", LAST_UPDATE_TIME_FIELD), lastUpdateTime.toEpochMilli()); + } + + builder.field(SCHEDULE_FIELD, schedule); + builder.field(STATE_FIELD, state.name()); + builder.field(REFRESH_TYPE_FIELD, refreshType.name()); + if (lastRefreshedTime == null) { + builder.nullField(LAST_REFRESHED_TIME_FIELD); + } else { + builder.timeField(LAST_REFRESHED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", + LAST_REFRESHED_TIME_FIELD), lastRefreshedTime.toEpochMilli()); + } + builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); + builder.field(ENABLED_FIELD, isEnabled); + builder.field(IOC_MAP_STORE_FIELD, iocMapStore); + builder.field(IOC_TYPES_FIELD, iocTypes); + builder.endObject(); + builder.endObject(); + return builder; + } + + public static SATIFSourceConfig docParse(XContentParser xcp, String id, Long version) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, xcp.nextToken(), xcp); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + SATIFSourceConfig SaTifSourceConfig = parse(xcp, id, version); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp); + + SaTifSourceConfig.setId(id); + SaTifSourceConfig.setVersion(version); + return SaTifSourceConfig; + } + + public static SATIFSourceConfig parse(XContentParser xcp, String id, Long version) throws IOException { + if (id == null) { + id = NO_ID; + } + if (version == null) { + version = NO_VERSION; + } + + String feedName = null; + String feedFormat = null; + FeedType feedType = null; + String createdByUser = null; + Instant createdAt = null; + Instant enabledTime = null; + Instant lastUpdateTime = null; + IntervalSchedule schedule = null; + TIFJobState state = null; + RefreshType refreshType = null; + Instant lastRefreshedTime = null; + String lastRefreshedUser = null; + Boolean isEnabled = null; + Map iocMapStore = null; + List iocTypes = new ArrayList<>(); + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case FEED_SOURCE_CONFIG_FIELD: + break; + case FEED_NAME_FIELD: + feedName = xcp.text(); + break; + case FEED_FORMAT_FIELD: + feedFormat = xcp.text(); + break; + case FEED_TYPE_FIELD: + feedType = toFeedType(xcp.text()); + break; + case CREATED_BY_USER_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdByUser = null; + } else { + createdByUser = xcp.text(); + } + break; + case CREATED_AT_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdAt = null; + } else if (xcp.currentToken().isValue()) { + createdAt = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + createdAt = null; + } + break; + case ENABLED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + enabledTime = null; + } else if (xcp.currentToken().isValue()) { + enabledTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + enabledTime = null; + } + break; + case LAST_UPDATE_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastUpdateTime = null; + } else if (xcp.currentToken().isValue()) { + lastUpdateTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastUpdateTime = null; + } + break; + case SCHEDULE_FIELD: + schedule = (IntervalSchedule) ScheduleParser.parse(xcp); + break; + case STATE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + state = TIFJobState.CREATING; + } else { + state = toState(xcp.text()); + } + break; + case REFRESH_TYPE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + refreshType = null; + } else { + refreshType = toRefreshType(xcp.text()); + } + break; + case LAST_REFRESHED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedTime = null; + } else if (xcp.currentToken().isValue()) { + lastRefreshedTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastRefreshedTime = null; + } + break; + case LAST_REFRESHED_USER_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedUser = null; + } else { + lastRefreshedUser = xcp.text(); + } + break; + case ENABLED_FIELD: + isEnabled = xcp.booleanValue(); + break; + case IOC_MAP_STORE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + iocMapStore = null; + } else { + iocMapStore = xcp.map(); + } + break; + case IOC_TYPES_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + iocTypes.add(xcp.text()); + } + break; + default: + xcp.skipChildren(); + } + } + + if (isEnabled && enabledTime == null) { + enabledTime = Instant.now(); + } else if (!isEnabled) { + enabledTime = null; + } + + return new SATIFSourceConfig( + id, + version, + feedName, + feedFormat, + feedType, + createdByUser, + createdAt != null ? createdAt : Instant.now(), + enabledTime, + lastUpdateTime != null ? lastUpdateTime : Instant.now(), + schedule, + state, + refreshType, + lastRefreshedTime, + lastRefreshedUser, + isEnabled, + iocMapStore, + iocTypes + ); + } + + + public static TIFJobState toState(String stateName) { + try { + return TIFJobState.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid state, cannot be parsed.", e); + return null; + } + } + + public static FeedType toFeedType(String feedType) { + try { + return FeedType.valueOf(feedType); + } catch (IllegalArgumentException e) { + log.error("Invalid feed type, cannot be parsed.", e); + return null; + } + } + + public static RefreshType toRefreshType(String stateName) { + try { + return RefreshType.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid refresh type, cannot be parsed.", e); + return null; + } + } + + public static SATIFSourceConfig readFrom(StreamInput sin) throws IOException { + return new SATIFSourceConfig(sin); + } + + // Getters and Setters + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public Long getVersion() { + return version; + } + public void setVersion(Long version) { + this.version = version; + } + public String getName() { + return this.feedName; + } + public void setName(String name) { + this.feedName = name; + } + public String getFeedFormat() { + return feedFormat; + } + public void setFeedFormat(String feedFormat) { + this.feedFormat = feedFormat; + } + public FeedType getFeedType() { + return feedType; + } + public void setFeedType(FeedType feedType) { + this.feedType = feedType; + } + public String getCreatedByUser() { + return createdByUser; + } + public void setCreatedByUser(String createdByUser) { + this.createdByUser = createdByUser; + } + public Instant getCreatedAt() { + return createdAt; + } + public void setCreatedAt(Instant createdAt) { + this.createdAt = createdAt; + } + public Instant getEnabledTime() { + return this.enabledTime; + } + public void setEnabledTime(Instant enabledTime) { + this.enabledTime = enabledTime; + } + public Instant getLastUpdateTime() { + return this.lastUpdateTime; + } + public void setLastUpdateTime(Instant lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + public IntervalSchedule getSchedule() { + return this.schedule; + } + public void setSchedule(IntervalSchedule schedule) { + this.schedule = schedule; + } + public TIFJobState getState() { + return state; + } + public void setState(TIFJobState previousState) { + this.state = previousState; + } + public String getLastRefreshedUser() { + return lastRefreshedUser; + } + public void setLastRefreshedUser(String lastRefreshedUser) { + this.lastRefreshedUser = lastRefreshedUser; + } + public Instant getLastRefreshedTime() { + return lastRefreshedTime; + } + public void setLastRefreshedTime(Instant lastRefreshedTime) { + this.lastRefreshedTime = lastRefreshedTime; + } + public RefreshType getRefreshType() { + return refreshType; + } + public void setRefreshType(RefreshType refreshType) { + this.refreshType = refreshType; + } + public boolean isEnabled() { + return this.isEnabled; + } + public void enable() { + if (isEnabled == true) { + return; + } + enabledTime = Instant.now(); + isEnabled = true; + } + public void disable() { + enabledTime = null; + isEnabled = false; + } + public Map getIocMapStore() { + return iocMapStore; + } + public void setIocMapStore(Map iocMapStore) { + this.iocMapStore = iocMapStore; + } + + public List getIocTypes() { + return iocTypes; + } + + public void setIocTypes(List iocTypes) { + this.iocTypes = iocTypes; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java new file mode 100644 index 000000000..89dc80d17 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java @@ -0,0 +1,519 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfigDto; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Implementation of TIF Config Dto to store the feed configuration metadata as DTO object + */ +public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSourceConfigDto { + + private static final Logger log = LogManager.getLogger(SATIFSourceConfigDto.class); + + public static final String FEED_SOURCE_CONFIG_FIELD = "feed_source_config"; + + public static final String NO_ID = ""; + + public static final Long NO_VERSION = 1L; + public static final String VERSION_FIELD = "version"; + public static final String FEED_NAME_FIELD = "feed_name"; + public static final String FEED_FORMAT_FIELD = "feed_format"; + public static final String FEED_TYPE_FIELD = "feed_type"; + public static final String CREATED_BY_USER_FIELD = "created_by_user"; + public static final String CREATED_AT_FIELD = "created_at"; + public static final String SOURCE_FIELD = "source"; + public static final String ENABLED_TIME_FIELD = "enabled_time"; + public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String STATE_FIELD = "state"; + public static final String REFRESH_TYPE_FIELD = "refresh_type"; + public static final String LAST_REFRESHED_TIME_FIELD = "last_refreshed_time"; + public static final String LAST_REFRESHED_USER_FIELD = "last_refreshed_user"; + public static final String ENABLED_FIELD = "enabled"; + public static final String IOC_MAP_STORE_FIELD = "ioc_map_store"; + public static final String IOC_TYPES_FIELD = "ioc_types"; + + private String id; + private Long version; + private String feedName; + private String feedFormat; + private FeedType feedType; + private String createdByUser; + private Instant createdAt; + + // private Source source; TODO: create Source Object + private Instant enabledTime; + private Instant lastUpdateTime; + private IntervalSchedule schedule; + private TIFJobState state; + public RefreshType refreshType; + public Instant lastRefreshedTime; + public String lastRefreshedUser; + private Boolean isEnabled; + private Map iocMapStore; + private List iocTypes; + + public SATIFSourceConfigDto(SATIFSourceConfig SaTifSourceConfig) { + this.id = SaTifSourceConfig.getId(); + this.version = SaTifSourceConfig.getVersion(); + this.feedName = SaTifSourceConfig.getName(); + this.feedFormat = SaTifSourceConfig.getFeedFormat(); + this.feedType = SaTifSourceConfig.getFeedType(); + this.createdByUser = SaTifSourceConfig.getCreatedByUser(); + this.createdAt = SaTifSourceConfig.getCreatedAt(); + this.enabledTime = SaTifSourceConfig.getEnabledTime(); + this.lastUpdateTime = SaTifSourceConfig.getLastUpdateTime(); + this.schedule = SaTifSourceConfig.getSchedule(); + this.state = SaTifSourceConfig.getState();; + this.refreshType = SaTifSourceConfig.getRefreshType(); + this.lastRefreshedTime = SaTifSourceConfig.getLastRefreshedTime(); + this.lastRefreshedUser = SaTifSourceConfig.getLastRefreshedUser(); + this.isEnabled = SaTifSourceConfig.isEnabled();; + this.iocMapStore = SaTifSourceConfig.getIocMapStore(); + this.iocTypes = SaTifSourceConfig.getIocTypes(); + } + + public SATIFSourceConfigDto(String id, Long version, String feedName, String feedFormat, FeedType feedType, String createdByUser, Instant createdAt, + Instant enabledTime, Instant lastUpdateTime, IntervalSchedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, String lastRefreshedUser, + Boolean isEnabled, Map iocMapStore, List iocTypes) { + this.id = id != null ? id : NO_ID; + this.version = version != null ? version : NO_VERSION; + this.feedName = feedName; + this.feedFormat = feedFormat; + this.feedType = feedType; + this.createdByUser = createdByUser; + this.createdAt = createdAt != null ? createdAt : Instant.now(); + + if (isEnabled == null && enabledTime == null) { + this.enabledTime = Instant.now(); + } else if (isEnabled != null && !isEnabled) { + this.enabledTime = null; + } else { + this.enabledTime = enabledTime; + } + + this.lastUpdateTime = lastUpdateTime != null ? lastUpdateTime : Instant.now(); + this.schedule = schedule; + this.state = state != null ? state : TIFJobState.CREATING; + this.refreshType = refreshType != null ? refreshType : RefreshType.FULL; + this.lastRefreshedTime = lastRefreshedTime; + this.lastRefreshedUser = lastRefreshedUser; + this.isEnabled = isEnabled; + this.iocMapStore = iocMapStore != null ? iocMapStore : new HashMap<>(); + this.iocTypes = iocTypes; + } + + public SATIFSourceConfigDto(StreamInput sin) throws IOException { + this(new SATIFSourceConfig(sin)); + } + + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeString(feedName); + out.writeString(feedFormat); + out.writeString(feedType.name()); + out.writeOptionalString(createdByUser); + out.writeInstant(createdAt); + out.writeOptionalInstant(enabledTime); + out.writeInstant(lastUpdateTime); + schedule.writeTo(out); + out.writeString(state.name()); + out.writeString(refreshType.name()); + out.writeOptionalInstant(lastRefreshedTime); + out.writeOptionalString(lastRefreshedUser); + out.writeBoolean(isEnabled); + out.writeMap(iocMapStore); + out.writeStringCollection(iocTypes); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject() + .startObject(FEED_SOURCE_CONFIG_FIELD) + .field(VERSION_FIELD, version) + .field(FEED_NAME_FIELD, feedName) + .field(FEED_FORMAT_FIELD, feedFormat) + .field(FEED_TYPE_FIELD, feedType.name()) + .field(CREATED_BY_USER_FIELD, createdByUser); + + if (createdAt == null) { + builder.nullField(CREATED_AT_FIELD); + } else { + builder.timeField(CREATED_AT_FIELD, String.format(Locale.getDefault(), "%s_in_millis", CREATED_AT_FIELD), createdAt.toEpochMilli()); + } + + if (enabledTime == null) { + builder.nullField(ENABLED_TIME_FIELD); + } else { + builder.timeField(ENABLED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", ENABLED_TIME_FIELD), enabledTime.toEpochMilli()); + } + + if (lastUpdateTime == null) { + builder.nullField(LAST_UPDATE_TIME_FIELD); + } else { + builder.timeField(LAST_UPDATE_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", LAST_UPDATE_TIME_FIELD), lastUpdateTime.toEpochMilli()); + } + + builder.field(SCHEDULE_FIELD, schedule); + builder.field(STATE_FIELD, state.name()); + builder.field(REFRESH_TYPE_FIELD, refreshType.name()); + if (lastRefreshedTime == null) { + builder.nullField(LAST_REFRESHED_TIME_FIELD); + } else { + builder.timeField(LAST_REFRESHED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", + LAST_REFRESHED_TIME_FIELD), lastRefreshedTime.toEpochMilli()); + } + builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); + builder.field(ENABLED_FIELD, isEnabled); + builder.field(IOC_MAP_STORE_FIELD, iocMapStore); + builder.field(IOC_TYPES_FIELD, iocTypes); + builder.endObject(); + builder.endObject(); + return builder; + } + + public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long version) throws IOException { + if (id == null) { + id = NO_ID; + } + if (version == null) { + version = NO_VERSION; + } + + String feedName = null; + String feedFormat = null; + FeedType feedType = null; + String createdByUser = null; + Instant createdAt = null; + Instant enabledTime = null; + Instant lastUpdateTime = null; + IntervalSchedule schedule = null; + TIFJobState state = null; + RefreshType refreshType = null; + Instant lastRefreshedTime = null; + String lastRefreshedUser = null; + Boolean isEnabled = null; + Map iocMapStore = null; + List iocTypes = null; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case FEED_SOURCE_CONFIG_FIELD: + break; + case FEED_NAME_FIELD: + feedName = xcp.text(); + break; + case FEED_FORMAT_FIELD: + feedFormat = xcp.text(); + break; + case FEED_TYPE_FIELD: + feedType = toFeedType(xcp.text()); + break; + case CREATED_BY_USER_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdByUser = null; + } else { + createdByUser = xcp.text(); + } + break; + case CREATED_AT_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdAt = null; + } else if (xcp.currentToken().isValue()) { + createdAt = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + createdAt = null; + } + break; + case ENABLED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + enabledTime = null; + } else if (xcp.currentToken().isValue()) { + enabledTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + enabledTime = null; + } + break; + case LAST_UPDATE_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastUpdateTime = null; + } else if (xcp.currentToken().isValue()) { + lastUpdateTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastUpdateTime = null; + } + break; + case SCHEDULE_FIELD: + schedule = (IntervalSchedule) ScheduleParser.parse(xcp); + break; + case STATE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + state = TIFJobState.CREATING; + } else { + state = toState(xcp.text()); + } + break; + case REFRESH_TYPE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + refreshType = null; + } else { + refreshType = toRefreshType(xcp.text()); + } + break; + case LAST_REFRESHED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedTime = null; + } else if (xcp.currentToken().isValue()) { + lastRefreshedTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastRefreshedTime = null; + } + break; + case LAST_REFRESHED_USER_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedUser = null; + } else { + lastRefreshedUser = xcp.text(); + } + break; + case ENABLED_FIELD: + isEnabled = xcp.booleanValue(); + break; + case IOC_MAP_STORE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + iocMapStore = null; + } else { + iocMapStore = xcp.map(); + } + break; + case IOC_TYPES_FIELD: + iocTypes = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + iocTypes.add(xcp.text()); + } + break; + default: + xcp.skipChildren(); + } + } + + if (isEnabled && enabledTime == null) { + enabledTime = Instant.now(); + } else if (!isEnabled) { + enabledTime = null; + } + + return new SATIFSourceConfigDto( + id, + version, + feedName, + feedFormat, + feedType, + createdByUser, + createdAt != null ? createdAt : Instant.now(), + enabledTime, + lastUpdateTime != null ? lastUpdateTime : Instant.now(), + schedule, + state, + refreshType, + lastRefreshedTime, + lastRefreshedUser, + isEnabled, + iocMapStore, + iocTypes + ); + } + + // TODO: refactor out to sa commons + public static TIFJobState toState(String stateName) { + try { + return TIFJobState.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid state, cannot be parsed.", e); + return null; + } + } + + public static FeedType toFeedType(String feedType) { + try { + return FeedType.valueOf(feedType); + } catch (IllegalArgumentException e) { + log.error("Invalid feed type, cannot be parsed.", e); + return null; + } + } + + public static RefreshType toRefreshType(String stateName) { + try { + return RefreshType.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid refresh type, cannot be parsed.", e); + return null; + } + } + + + // Getters and Setters + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public Long getVersion() { + return version; + } + public void setVersion(Long version) { + this.version = version; + } + public String getName() { + return this.feedName; + } + public void setName(String name) { + this.feedName = name; + } + public String getFeedFormat() { + return feedFormat; + } + public void setFeedFormat(String feedFormat) { + this.feedFormat = feedFormat; + } + public FeedType getFeedType() { + return feedType; + } + public void setFeedType(FeedType feedType) { + this.feedType = feedType; + } + public String getCreatedByUser() { + return createdByUser; + } + public void setCreatedByUser(String createdByUser) { + this.createdByUser = createdByUser; + } + public Instant getCreatedAt() { + return createdAt; + } + public void setCreatedAt(Instant createdAt) { + this.createdAt = createdAt; + } + public Instant getEnabledTime() { + return this.enabledTime; + } + public void setEnabledTime(Instant enabledTime) { + this.enabledTime = enabledTime; + } + public Instant getLastUpdateTime() { + return this.lastUpdateTime; + } + public void setLastUpdateTime(Instant lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + public IntervalSchedule getSchedule() { + return this.schedule; + } + public void setSchedule(IntervalSchedule schedule) { + this.schedule = schedule; + } + public TIFJobState getState() { + return state; + } + public void setState(TIFJobState previousState) { + this.state = previousState; + } + public String getLastRefreshedUser() { + return lastRefreshedUser; + } + public void setLastRefreshedUser(String lastRefreshedUser) { + this.lastRefreshedUser = lastRefreshedUser; + } + public Instant getLastRefreshedTime() { + return lastRefreshedTime; + } + public void setLastRefreshedTime(Instant lastRefreshedTime) { + this.lastRefreshedTime = lastRefreshedTime; + } + public RefreshType getRefreshType() { + return refreshType; + } + public void setRefreshType(RefreshType refreshType) { + this.refreshType = refreshType; + } + public boolean isEnabled() { + return this.isEnabled; + } + + /** + * Enable auto update of threat intel feed data + */ + public void enable() { + if (isEnabled == true) { + return; + } + enabledTime = Instant.now(); + isEnabled = true; + } + + /** + * Disable auto update of threat intel feed data + */ + public void disable() { + enabledTime = null; + isEnabled = false; + } + public Map getIocMapStore() { + return iocMapStore; + } + public void setIocMapStore(Map iocMapStore) { + this.iocMapStore = iocMapStore; + } + + public List getIocTypes() { + return iocTypes; + } + + public void setIocTypes(List iocTypes) { + this.iocTypes = iocTypes; + } + + public static SATIFSourceConfigDto readFrom(StreamInput sin) throws IOException { + return new SATIFSourceConfigDto(sin); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java similarity index 99% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java index bcbb84c1c..2fa5cb199 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java @@ -6,7 +6,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.model; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; @@ -23,7 +23,6 @@ import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; import java.io.IOException; import java.time.Instant; @@ -563,7 +562,6 @@ public static TIFJobParameter build(final PutTIFJobRequest request) { ChronoUnit.MINUTES ); return new TIFJobParameter(name, schedule); - } } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java similarity index 99% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java index 04486fb7a..20035dcb8 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.common; +package org.opensearch.securityanalytics.threatIntel.model; import java.io.IOException; import java.util.Map; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java new file mode 100644 index 000000000..a6077679c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java @@ -0,0 +1,97 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.GetFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; + +import java.io.IOException; +import java.time.DateTimeException; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.opensearch.rest.RestRequest.Method.GET; + +public class RestGetIocFindingsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_ioc_findings_action_sa"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String sortString = request.param("sortString", "timestamp"); + String sortOrder = request.param("sortOrder", "asc"); + String missing = request.param("missing"); + int size = request.paramAsInt("size", 20); + int startIndex = request.paramAsInt("startIndex", 0); + String searchString = request.param("searchString", ""); + + List findingIds = null; + if (request.param("findingIds") != null) { + findingIds = Arrays.asList(request.param("findingIds").split(",")); + } + Instant startTime = null; + String startTimeParam = request.param("startTime"); + if (startTimeParam != null && !startTimeParam.isEmpty()) { + try { + startTime = Instant.ofEpochMilli(Long.parseLong(startTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + // Handle the parsing error + // For example, log the error or provide a default value + startTime = Instant.now(); // Default value or fallback + } + } + + Instant endTime = null; + String endTimeParam = request.param("endTime"); + if (endTimeParam != null && !endTimeParam.isEmpty()) { + try { + endTime = Instant.ofEpochMilli(Long.parseLong(endTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + // Handle the parsing error + // For example, log the error or provide a default value + endTime = Instant.now(); // Default value or fallback + } + } + + Table table = new Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ); + + GetIocFindingsRequest getIocFindingsRequest = new GetIocFindingsRequest( + findingIds, + startTime, + endTime, + table + ); + return channel -> client.execute( + GetIocFindingsAction.INSTANCE, + getIocFindingsRequest, + new RestToXContentListener<>(channel) + ); + } + + @Override + public List routes() { + return singletonList(new Route(GET, SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings" + "/_search")); + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetTIFSourceConfigAction.java new file mode 100644 index 000000000..6eb669c92 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetTIFSourceConfigAction.java @@ -0,0 +1,51 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestActions; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.rest.RestRequest.Method.GET; + +public class RestGetTIFSourceConfigAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestGetTIFSourceConfigAction.class); + + @Override + public String getName() { + return "get_tif_config_action"; + } + + @Override + public List routes() { + return List.of(new Route(GET, String.format(Locale.getDefault(), "%s/{%s}", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, SAGetTIFSourceConfigRequest.TIF_SOURCE_CONFIG_ID))); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String SaTifSourceConfigId = request.param(SAGetTIFSourceConfigRequest.TIF_SOURCE_CONFIG_ID, SATIFSourceConfigDto.NO_ID); + + if (SaTifSourceConfigId == null || SaTifSourceConfigId.isEmpty()) { + throw new IllegalArgumentException("missing id"); + } + + SAGetTIFSourceConfigRequest req = new SAGetTIFSourceConfigRequest(SaTifSourceConfigId, RestActions.parseVersion(request)); + + return channel -> client.execute( + SAGetTIFSourceConfigAction.INSTANCE, + req, + new RestToXContentListener<>(channel) + ); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java new file mode 100644 index 000000000..4e5d15d5c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java @@ -0,0 +1,90 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.util.RestHandlerUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +public class RestIndexTIFSourceConfigAction extends BaseRestHandler { + private static final Logger log = LogManager.getLogger(RestIndexTIFSourceConfigAction.class); + @Override + public String getName() { + return "index_tif_config_action"; + } + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI), + new Route(RestRequest.Method.PUT, String.format(Locale.getDefault(), "%s/{%s}", + SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, SAGetTIFSourceConfigRequest.TIF_SOURCE_CONFIG_ID)) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI)); + + WriteRequest.RefreshPolicy refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE; + if (request.hasParam(RestHandlerUtils.REFRESH)) { + refreshPolicy = WriteRequest.RefreshPolicy.parse(request.param(RestHandlerUtils.REFRESH)); + } + + String id = request.param("feed_id", null); + + XContentParser xcp = request.contentParser(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + + SATIFSourceConfigDto tifConfig = SATIFSourceConfigDto.parse(xcp, id, null); + tifConfig.setLastUpdateTime(Instant.now()); + + SAIndexTIFSourceConfigRequest indexTIFConfigRequest = new SAIndexTIFSourceConfigRequest(id, refreshPolicy, request.method(), tifConfig); + return channel -> client.execute(SAIndexTIFSourceConfigAction.INSTANCE, indexTIFConfigRequest, indexTIFConfigResponse(channel, request.method())); + } + + private RestResponseListener indexTIFConfigResponse(RestChannel channel, RestRequest.Method restMethod) { + return new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(SAIndexTIFSourceConfigResponse response) throws Exception { + RestStatus returnStatus = RestStatus.CREATED; + if (restMethod == RestRequest.Method.PUT) { + returnStatus = RestStatus.OK; + } + + BytesRestResponse restResponse = new BytesRestResponse(returnStatus, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + + if (restMethod == RestRequest.Method.POST) { + String location = String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, response.getTIFConfigId()); + restResponse.addHeader("Location", location); + } + + return restResponse; + } + }; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java new file mode 100644 index 000000000..a4f196ea1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.sacommons; + +public class IndexTIFSourceConfigAction { + public static final String INDEX_TIF_SOURCE_CONFIG_ACTION_NAME = "cluster:admin/security_analytics/tifSource/write"; + public static final String GET_TIF_SOURCE_CONFIG_ACTION_NAME = "cluster:admin/security_analytics/tifSource/get"; +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java new file mode 100644 index 000000000..db33575eb --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.sacommons; + +/** + * Threat intel feed config creation request interface + */ +public interface IndexTIFSourceConfigRequest { + String getTIFConfigId(); + TIFSourceConfigDto getTIFConfigDto(); +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java new file mode 100644 index 000000000..5a9e4daa6 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +public interface IndexTIFSourceConfigResponse { + String getTIFConfigId(); + Long getVersion(); + TIFSourceConfigDto getTIFConfigDto(); +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java new file mode 100644 index 000000000..822dcd4d4 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java @@ -0,0 +1,72 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; + +import java.time.Instant; +import java.util.List; +import java.util.Map; + +/** + * Threat intel config interface + */ +public interface TIFSourceConfig { + + public String getId(); + + public void setId(String id); + + Long getVersion(); + + void setVersion(Long version); + + String getName(); + + void setName(String feedName); + + String getFeedFormat(); + + void setFeedFormat(String feedFormat); + + FeedType getFeedType(); + + void setFeedType(FeedType feedType); + + String getCreatedByUser(); + + void setCreatedByUser(String createdByUser); + + Instant getCreatedAt(); + + void setCreatedAt(Instant createdAt); + + Instant getEnabledTime(); + + void setEnabledTime(Instant enabledTime); + + Instant getLastUpdateTime(); + + void setLastUpdateTime(Instant lastUpdateTime); + + IntervalSchedule getSchedule(); + + void setSchedule(IntervalSchedule schedule); + + TIFJobState getState(); + + void setState(TIFJobState previousState); + + void enable(); + + void disable(); + + Map getIocMapStore(); + + void setIocMapStore(Map iocMapStore); + + public List getIocTypes(); + + public void setIocTypes(List iocTypes); + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java new file mode 100644 index 000000000..1899c3af6 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java @@ -0,0 +1,71 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; + +import java.time.Instant; +import java.util.List; +import java.util.Map; + +/** + * Threat intel config dto interface + */ +public interface TIFSourceConfigDto { + + public String getId(); + + public void setId(String id); + + Long getVersion(); + + void setVersion(Long version); + + String getName(); + + void setName(String feedName); + + String getFeedFormat(); + + void setFeedFormat(String feedFormat); + + FeedType getFeedType(); + + void setFeedType(FeedType feedType); + + String getCreatedByUser(); + + void setCreatedByUser(String createdByUser); + + Instant getCreatedAt(); + + void setCreatedAt(Instant createdAt); + + Instant getEnabledTime(); + + void setEnabledTime(Instant enabledTime); + + Instant getLastUpdateTime(); + + void setLastUpdateTime(Instant lastUpdateTime); + + IntervalSchedule getSchedule(); + + void setSchedule(IntervalSchedule schedule); + + TIFJobState getState(); + + void setState(TIFJobState previousState); + + void enable(); + + void disable(); + + Map getIocMapStore(); + + void setIocMapStore(Map iocMapStore); + + public List getIocTypes(); + + public void setIocTypes(List iocTypes); +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java new file mode 100644 index 000000000..9f5438a6e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java @@ -0,0 +1,13 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +import org.opensearch.core.action.ActionListener; +public abstract class TIFSourceConfigService { + IndexTIFSourceConfigResponse indexTIFConfig(IndexTIFSourceConfigRequest request, ActionListener listener){ + return null; + } + + // TODO: + // update + // delete + // get +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java similarity index 99% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java index e541ee36c..6619b33f5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java new file mode 100644 index 000000000..ec0dfb104 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -0,0 +1,121 @@ +package org.opensearch.securityanalytics.threatIntel.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.StepListener; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +/** + * Service class for threat intel feed source config object + */ +public class SATIFSourceConfigService { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigService.class); + private final SATIFSourceConfigDao SaTifSourceConfigDao; + private final TIFLockService lockService; + + /** + * Default constructor + * @param SaTifSourceConfigDao the tif source config dao + * @param lockService the lock service + */ + @Inject + public SATIFSourceConfigService( + final SATIFSourceConfigDao SaTifSourceConfigDao, + final TIFLockService lockService + ) { + this.SaTifSourceConfigDao = SaTifSourceConfigDao; + this.lockService = lockService; + } + + /** + * + * Creates the job index if it doesn't exist and indexes the tif source config object + * + * @param SaTifSourceConfigDto the tif source config dto + * @param lock the lock object + * @param indexTimeout the index time out + * @param listener listener that accepts a tif source config if successful + */ + public void createIndexAndSaveTIFSourceConfig( + final SATIFSourceConfigDto SaTifSourceConfigDto, + final LockModel lock, + final TimeValue indexTimeout, + final ActionListener listener + ) { + try { + SATIFSourceConfig SaTifSourceConfig = convertToSATIFConfig(SaTifSourceConfigDto); + SaTifSourceConfig.setState(TIFJobState.AVAILABLE); + SaTifSourceConfigDao.indexTIFSourceConfig(SaTifSourceConfig, indexTimeout, lock, new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig response) { + SaTifSourceConfig.setId(response.getId()); + SaTifSourceConfig.setVersion(response.getVersion()); + listener.onResponse(SaTifSourceConfig); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public void getTIFSourceConfig( + final String SaTifSourceConfigId, + final Long version, + final ActionListener listener + ) { + try { + SaTifSourceConfigDao.getTIFSourceConfig(SaTifSourceConfigId, version, new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig SaTifSourceConfig) { + listener.onResponse(SaTifSourceConfig); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Converts the DTO to entity + * @param SaTifSourceConfigDto + * @return SaTifSourceConfig + */ + public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto SaTifSourceConfigDto) { + return new SATIFSourceConfig( + SaTifSourceConfigDto.getId(), + SaTifSourceConfigDto.getVersion(), + SaTifSourceConfigDto.getName(), + SaTifSourceConfigDto.getFeedFormat(), + SaTifSourceConfigDto.getFeedType(), + SaTifSourceConfigDto.getCreatedByUser(), + SaTifSourceConfigDto.getCreatedAt(), + SaTifSourceConfigDto.getEnabledTime(), + SaTifSourceConfigDto.getLastUpdateTime(), + SaTifSourceConfigDto.getSchedule(), + SaTifSourceConfigDto.getState(), + SaTifSourceConfigDto.getRefreshType(), + SaTifSourceConfigDto.getLastRefreshedTime(), + SaTifSourceConfigDto.getLastRefreshedUser(), + SaTifSourceConfigDto.isEnabled(), + SaTifSourceConfigDto.getIocMapStore(), + SaTifSourceConfigDto.getIocTypes() + ); + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java index 55387cb35..c7fa5566e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -15,10 +15,8 @@ import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.get.GetRequest; -import org.opensearch.action.get.GetResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; @@ -30,11 +28,10 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.IndexNotFoundException; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.io.BufferedReader; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java similarity index 97% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java index 5c48ed8aa..eb90415b4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -15,16 +15,15 @@ import org.opensearch.OpenSearchStatusException; import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.support.GroupedActionListener; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedParser; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedParser; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java similarity index 97% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java index b9d8aa3ea..61ea2374d 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.StringUtils; @@ -20,7 +20,6 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; @@ -39,8 +38,8 @@ import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils; import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -51,7 +50,6 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; -import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -60,7 +58,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; /** * Service to handle CRUD operations on Threat Intel Feed Data diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java new file mode 100644 index 000000000..debde4655 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java @@ -0,0 +1,143 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.lucene.search.join.ScoreMode; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.Operator; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.time.Instant; +import java.util.List; + +public class TransportGetIocFindingsAction extends HandledTransportAction implements SecureTransportAction { + + private final IocFindingService iocFindingService; + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private volatile Boolean filterByEnabled; + + @Inject + public TransportGetIocFindingsAction( + TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + Settings settings, + NamedXContentRegistry xContentRegistry, + Client client + ) { + super(GetIocFindingsAction.NAME, transportService, actionFilters, GetIocFindingsRequest::new); + this.settings = settings; + this.clusterService = clusterService; + this.threadPool = client.threadPool(); + this.iocFindingService = new IocFindingService(client, this.clusterService, xContentRegistry); + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + @Override + protected void doExecute(Task task, GetIocFindingsRequest request, ActionListener actionListener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + Table tableProp = request.getTable(); + FieldSortBuilder sortBuilder = SortBuilders + .fieldSort(tableProp.getSortString()) + .order(SortOrder.fromString(tableProp.getSortOrder())); + if (tableProp.getMissing() != null && !tableProp.getMissing().isBlank()) { + sortBuilder.missing(tableProp.getMissing()); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .sort(sortBuilder) + .size(tableProp.getSize()) + .from(tableProp.getStartIndex()) + .fetchSource(new FetchSourceContext(true, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY)) + .seqNoAndPrimaryTerm(true) + .version(true); + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + List findingIds = request.getFindingIds(); + + if (findingIds != null && !findingIds.isEmpty()) { + queryBuilder.filter(QueryBuilders.termsQuery("id", findingIds)); + } + + Instant startTime = request.getStartTime(); + Instant endTime = request.getEndTime(); + if (startTime != null && endTime != null) { + long startTimeMillis = startTime.toEpochMilli(); + long endTimeMillis = endTime.toEpochMilli(); + QueryBuilder timeRangeQuery = QueryBuilders.rangeQuery("timestamp") + .from(startTimeMillis) // Greater than or equal to start time + .to(endTimeMillis); // Less than or equal to end time + queryBuilder.filter(timeRangeQuery); + } + + if (tableProp.getSearchString() != null && !tableProp.getSearchString().isBlank()) { + queryBuilder.should(QueryBuilders + .queryStringQuery(tableProp.getSearchString()) + ).should( + QueryBuilders.nestedQuery( + "queries", + QueryBuilders.boolQuery() + .must( + QueryBuilders + .queryStringQuery(tableProp.getSearchString()) + .defaultOperator(Operator.AND) + .field("queries.tags") + .field("queries.name") + ), + ScoreMode.Avg + ) + ); + } + searchSourceBuilder.query(queryBuilder).trackTotalHits(true); + + this.threadPool.getThreadContext().stashContext(); + iocFindingService.searchIocMatches(searchSourceBuilder, actionListener); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java new file mode 100644 index 000000000..93dd34ebc --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java @@ -0,0 +1,84 @@ +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class TransportGetTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { + + private static final Logger log = LogManager.getLogger(TransportGetTIFSourceConfigAction.class); + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private volatile Boolean filterByEnabled; + + private final SATIFSourceConfigService SaTifConfigService; + + @Inject + public TransportGetTIFSourceConfigAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + final ThreadPool threadPool, + Settings settings, + final SATIFSourceConfigService SaTifConfigService) { + super(SAGetTIFSourceConfigAction.NAME, transportService, actionFilters, SAGetTIFSourceConfigRequest::new); + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + this.SaTifConfigService = SaTifConfigService; + } + + @Override + protected void doExecute(Task task, SAGetTIFSourceConfigRequest request, ActionListener actionListener) { + // validate user + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + + SaTifConfigService.getTIFSourceConfig(request.getId(), request.getVersion(), new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig SaTifSourceConfig) { + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto(SaTifSourceConfig); + actionListener.onResponse(new SAGetTIFSourceConfigResponse(SaTifSourceConfigDto.getId(), SaTifSourceConfigDto.getVersion(), RestStatus.OK, SaTifSourceConfigDto)); + } + + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java new file mode 100644 index 000000000..e5a475eea --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java @@ -0,0 +1,144 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.util.ConcurrentModificationException; + +import static org.opensearch.securityanalytics.threatIntel.common.TIFLockService.LOCK_DURATION_IN_SECONDS; +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Transport action to create threat intel feeds source config object and save IoCs + */ +public class TransportIndexTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { + private static final Logger log = LogManager.getLogger(TransportIndexTIFSourceConfigAction.class); + private final SATIFSourceConfigService SaTifSourceConfigService; + private final TIFLockService lockService; + private final ThreadPool threadPool; + private final Settings settings; + private volatile Boolean filterByEnabled; + private final TimeValue indexTimeout; + + + /** + * Default constructor + * @param transportService the transport service + * @param actionFilters the action filters + * @param threadPool the thread pool + * @param lockService the lock service + */ + @Inject + public TransportIndexTIFSourceConfigAction( + final TransportService transportService, + final ActionFilters actionFilters, + final ThreadPool threadPool, + final SATIFSourceConfigService SaTifSourceConfigService, + final TIFLockService lockService, + final Settings settings + ) { + super(SAIndexTIFSourceConfigAction.NAME, transportService, actionFilters, SAIndexTIFSourceConfigRequest::new); + this.threadPool = threadPool; + this.SaTifSourceConfigService = SaTifSourceConfigService; + this.lockService = lockService; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings); + } + + + @Override + protected void doExecute(final Task task, final SAIndexTIFSourceConfigRequest request, final ActionListener listener) { + // validate user + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); + return; + } + + retrieveLockAndCreateTIFConfig(request, listener, user); + } + + private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest request, ActionListener listener, User user) { + try { + lockService.acquireLock(request.getTIFConfigDto().getId(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { + if (lock == null) { + listener.onFailure( + new ConcurrentModificationException("another processor is holding a lock on the resource. Try again later") + ); + log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST); + return; + } + try { + SATIFSourceConfigDto SaTifSourceConfigDto = request.getTIFConfigDto(); + if (user != null) { + SaTifSourceConfigDto.setCreatedByUser(user.getName()); + } + try { + SaTifSourceConfigService.createIndexAndSaveTIFSourceConfig(SaTifSourceConfigDto, + lock, + indexTimeout, + new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig SaTifSourceConfig) { + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto(SaTifSourceConfig); + listener.onResponse(new SAIndexTIFSourceConfigResponse(SaTifSourceConfigDto.getId(), SaTifSourceConfigDto.getVersion(), RestStatus.OK, SaTifSourceConfigDto)); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + + } catch (Exception e) { + lockService.releaseLock(lock); + listener.onFailure(e); + log.error("listener failed when executing", e); + } + + } catch (Exception e) { + lockService.releaseLock(lock); + listener.onFailure(e); + log.error("listener failed when executing", e); + } + }, exception -> { + listener.onFailure(exception); + log.error("execution failed", exception); + })); + } catch (Exception e) { + log.error("Failed to acquire lock for job", e); + listener.onFailure(e); + } + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java similarity index 93% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java index a50beda35..2c756b3d3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.action; +package org.opensearch.securityanalytics.threatIntel.transport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -19,11 +19,14 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; +import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -120,7 +123,6 @@ protected void internalDoExecute( listener.onFailure(exception); }); tifJobParameterService.createJobIndexIfNotExists(createIndexStepListener); - } /** diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java index a96558b50..20539695b 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.util; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java similarity index 93% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java index 92a66ed12..bfbb9dbde 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.util; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -12,7 +12,7 @@ import org.opensearch.SpecialPermission; import org.opensearch.common.SuppressForbidden; import org.opensearch.securityanalytics.threatIntel.common.Constants; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import java.io.BufferedReader; import java.io.IOException; diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index ebe7d022d..ca3b39bf2 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -97,7 +97,7 @@ import org.opensearch.securityanalytics.rules.backend.QueryBackend; import org.opensearch.securityanalytics.rules.exceptions.SigmaConditionError; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.securityanalytics.util.ExceptionChecker; import org.opensearch.securityanalytics.util.IndexUtils; @@ -785,7 +785,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List } Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), false, detector.getSchedule(), detector.getLastUpdateTime(), null, - Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), new DataSources(detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), @@ -886,7 +886,7 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( } Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null, - Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), new DataSources(detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), @@ -1060,7 +1060,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { } **/ Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), false, detector.getSchedule(), detector.getLastUpdateTime(), null, - MonitorType.BUCKET_LEVEL_MONITOR, detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), + MonitorType.BUCKET_LEVEL_MONITOR.getValue(), detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), new DataSources(detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), @@ -1782,7 +1782,7 @@ private Map mapMonitorIds(List monitorResp Collectors.toMap( // In the case of bucket level monitors rule id is trigger id it -> { - if (MonitorType.BUCKET_LEVEL_MONITOR == it.getMonitor().getMonitorType()) { + if (MonitorType.BUCKET_LEVEL_MONITOR.getValue() == it.getMonitor().getMonitorType()) { return it.getMonitor().getTriggers().get(0).getId(); } else { if (it.getMonitor().getName().contains("_chained_findings")) { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java index 3b7b36503..5937769fe 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java @@ -19,7 +19,7 @@ import org.opensearch.securityanalytics.action.SearchDetectorAction; import org.opensearch.securityanalytics.action.SearchDetectorRequest; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.action.TransportPutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java b/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java index 14c241f83..bb76afd8d 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java @@ -109,7 +109,7 @@ public static List getBucketLevelMonitorIds( ) { return monitorResponses.stream().filter( // In the case of bucket level monitors rule id is trigger id - it -> Monitor.MonitorType.BUCKET_LEVEL_MONITOR == it.getMonitor().getMonitorType() + it -> Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue() == it.getMonitor().getMonitorType() ).map(IndexMonitorResponse::getId).collect(Collectors.toList()); } public static List getAggRuleIdsConfiguredToTrigger(Detector detector, List> rulesById) { diff --git a/src/main/resources/mappings/ioc_match_mapping.json b/src/main/resources/mappings/ioc_match_mapping.json new file mode 100644 index 000000000..f4573190e --- /dev/null +++ b/src/main/resources/mappings/ioc_match_mapping.json @@ -0,0 +1,38 @@ +{ + "dynamic": "strict", + "_meta" : { + "schema_version": 1 + }, + "properties": { + "schema_version": { + "type": "integer" + }, + "feed_ids" : { + "type": "keyword" + }, + "related_doc_ids": { + "type": "keyword" + }, + "ioc_scan_job_id": { + "type": "keyword" + }, + "ioc_scan_job_name": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "ioc_value" : { + "type": "keyword" + }, + "ioc_type" : { + "type": "keyword" + }, + "timestamp": { + "type": "long" + }, + "execution_id": { + "type": "keyword" + } + } +} diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json index ffd165ae5..ee258a11d 100644 --- a/src/main/resources/mappings/threat_intel_job_mapping.json +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -1,9 +1,112 @@ { - "dynamic": "strict", "_meta" : { - "schema_version": 1 + "schema_version": 2 }, + "dynamic": "strict", "properties": { + "feed_source_config": { + "properties": { + "version": { + "type": "long" + }, + "feed_name": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "feed_format": { + "type": "keyword" + }, + "feed_type": { + "type": "text" + }, + "created_by_user": { + "type": "keyword" + }, + "created_at": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "source" : { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "url": { + "type": "keyword" + }, + "path": { + "type": "text" + }, + "security": { + "type": "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + } + } + }, + "enabled_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "last_update_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "schedule": { + "properties": { + "interval": { + "properties": { + "period": { + "type": "integer" + }, + "start_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "unit": { + "type": "keyword" + } + } + } + } + }, + "state": { + "type": "text" + }, + "refresh_type": { + "type": "keyword" + }, + "last_refreshed_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "last_refreshed_user": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "ioc_map_store": { + "type": "object" + }, + "ioc_types": { + "type": "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + } + } + }, "schema_version": { "type": "integer" }, diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsIntegTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsIntegTestCase.java new file mode 100644 index 000000000..a85196663 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsIntegTestCase.java @@ -0,0 +1,23 @@ +package org.opensearch.securityanalytics; + +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +public class SecurityAnalyticsIntegTestCase extends OpenSearchIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + SecurityAnalyticsPlugin.class + ); + } + + @Override + protected boolean ignoreExternalCluster() { + return true; + } + +} diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index e7da36705..d06237e57 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -67,6 +67,8 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -101,7 +103,7 @@ import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; -import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList; +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils.getTifdList; import static org.opensearch.securityanalytics.util.RuleTopicIndices.ruleTopicIndexSettings; public class SecurityAnalyticsRestTestCase extends OpenSearchRestTestCase { @@ -662,6 +664,13 @@ protected HttpEntity toHttpEntity(UpdateIndexMappingsRequest request) throws IOE protected HttpEntity toHttpEntity(CorrelationRule rule) throws IOException { return new StringEntity(toJsonString(rule), ContentType.APPLICATION_JSON); } + protected HttpEntity toHttpEntity(SATIFSourceConfigDto SaTifSourceConfigDto) throws IOException { + return new StringEntity(toJsonString(SaTifSourceConfigDto), ContentType.APPLICATION_JSON); + } + + protected HttpEntity toHttpEntity(IocFinding iocFinding) throws IOException { + return new StringEntity(toJsonString(iocFinding), ContentType.APPLICATION_JSON); + } protected RestStatus restStatus(Response response) { return RestStatus.fromCode(response.getStatusLine().getStatusCode()); @@ -706,6 +715,16 @@ protected String toJsonString(ThreatIntelFeedData tifd) throws IOException { return IndexUtilsKt.string(shuffleXContent(tifd.toXContent(builder, ToXContent.EMPTY_PARAMS))); } + private String toJsonString(SATIFSourceConfigDto SaTifSourceConfigDto) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(SaTifSourceConfigDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + + private String toJsonString(IocFinding iocFinding) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + private String alertingScheduledJobMappings() { return " \"_meta\" : {\n" + " \"schema_version\": 5\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index a1987138d..ba2a15d63 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -6,19 +6,19 @@ import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import org.apache.lucene.tests.util.LuceneTestCase; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.ToXContent; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Schedule; import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.commons.alerting.model.action.Throttle; import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.securityanalytics.model.CorrelationQuery; @@ -28,7 +28,14 @@ import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.model.IocDao; +import org.opensearch.securityanalytics.model.IocDto; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -42,6 +49,7 @@ import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.opensearch.test.OpenSearchTestCase.randomInt; @@ -799,6 +807,12 @@ public static String toJsonStringWithUser(Detector detector) throws IOException return BytesReference.bytes(builder).utf8ToString(); } + public static String toJsonString(IocFinding iocFinding) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + public static String toJsonString(ThreatIntelFeedData threatIntelFeedData) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); builder = threatIntelFeedData.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -2705,4 +2719,200 @@ public static NamedXContentRegistry xContentRegistry() { public static XContentBuilder builder() throws IOException { return XContentBuilder.builder(XContentType.JSON.xContent()); } + + public static IocDao randomIocDao() { + return randomIocDao( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); + } + + public static IocDao randomIocDao( + String id, + String name, + IocDao.IocType type, + String value, + String severity, + String specVersion, + Instant created, + Instant modified, + String description, + List labels, + String feedId + ) { + if (id == null) { + id = randomString(); + } + if (name == null) { + name = randomString(); + } + if (type == null) { + type = IocDao.IocType.values()[randomInt(IocDao.IocType.values().length - 1)]; + } + if (value == null) { + value = randomString(); + } + if (severity == null) { + severity = randomString(); + } + if (specVersion == null) { + specVersion = randomString(); + } + if (created == null) { + created = Instant.now(); + } + if (modified == null) { + modified = Instant.now().plusSeconds(3600); // 1 hour + } + if (description == null) { + description = randomString(); + } + if (labels == null) { + labels = IntStream.range(0, randomInt()) + .mapToObj(i -> randomString()) + .collect(Collectors.toList()); + } + if (feedId == null) { + feedId = randomString(); + } + return new IocDao( + id, + name, + type, + value, + severity, + specVersion, + created, + modified, + description, + labels, + feedId + ); + } + + public static IocDto randomIocDto() { + return new IocDto(randomIocDao()); + } + + public static IocDto randomIocDto( + String id, + String name, + IocDao.IocType type, + String value, + String severity, + String specVersion, + Instant created, + Instant modified, + String description, + List labels, + String feedId + ) { + return new IocDto(randomIocDao( + id, + name, + type, + value, + severity, + specVersion, + created, + modified, + description, + labels, + feedId + )); + } + + public static SATIFSourceConfigDto randomSATIFSourceConfigDto() { + return randomSATIFSourceConfigDto( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); + } + + public static SATIFSourceConfigDto randomSATIFSourceConfigDto( + String feedName, + String feedFormat, + FeedType feedType, + String createdByUser, + Instant createdAt, + Instant enabledTime, + Instant lastUpdateTime, + org.opensearch.jobscheduler.spi.schedule.IntervalSchedule schedule, + TIFJobState state, + RefreshType refreshType, + Instant lastRefreshedTime, + String lastRefreshedUser, + Boolean isEnabled, + Map iocMapStore, + List iocTypes + ) { + if (feedName == null) { + feedName = randomString(); + } + if (feedFormat == null) { + feedFormat = "STIX"; + } + if (feedType == null) { + feedType = FeedType.INTERNAL; + } + if (isEnabled == null) { + isEnabled = true; + } + if (schedule == null) { + schedule = new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + } + if (iocTypes == null) { + iocTypes = List.of("ip"); + } + + return new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + feedType, + createdByUser, + createdAt, + enabledTime, + lastUpdateTime, + schedule, + state, + refreshType, + lastRefreshedTime, + lastRefreshedUser, + isEnabled, + iocMapStore, + iocTypes + ); + } + + public static XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent() + .createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigActionTests.java b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigActionTests.java new file mode 100644 index 000000000..f0b932472 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigActionTests.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.junit.Assert; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.test.OpenSearchTestCase; + +public class GetTIFSourceConfigActionTests extends OpenSearchTestCase { + public void testGetTIFSourceConfigActionName() { + Assert.assertNotNull(SAGetTIFSourceConfigAction.INSTANCE.name()); + Assert.assertEquals(SAGetTIFSourceConfigAction.INSTANCE.name(), SAGetTIFSourceConfigAction.NAME); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigRequestTests.java b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigRequestTests.java new file mode 100644 index 000000000..376d10b01 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigRequestTests.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.UUIDs; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigRequest; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class GetTIFSourceConfigRequestTests extends OpenSearchTestCase { + public void testStreamInOut() throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + String id = UUIDs.base64UUID(); + Long version = 1L; + + SAGetTIFSourceConfigRequest request = new SAGetTIFSourceConfigRequest(id, version); + request.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAGetTIFSourceConfigRequest newReq = new SAGetTIFSourceConfigRequest(sin); + + assertEquals(id, newReq.getId()); + assertEquals(version, newReq.getVersion()); + } + + public void testValidate() { + String id = UUIDs.base64UUID(); + Long version = 1L; + + SAGetTIFSourceConfigRequest request = new SAGetTIFSourceConfigRequest(id, version); + ActionRequestValidationException e = request.validate(); + assertNull(e); + + request = new SAGetTIFSourceConfigRequest("", 0L); + e = request.validate(); + assertNotNull(e); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java new file mode 100644 index 000000000..c6e5b08e3 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +public class GetTIFSourceConfigResponseTests extends OpenSearchTestCase { + private static final Logger log = LogManager.getLogger(GetTIFSourceConfigResponseTests.class); + + public void testStreamInOut() throws IOException { + String feedName = "test_feed_name"; + String feedFormat = "STIX"; + FeedType feedType = FeedType.INTERNAL; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + List iocTypes = List.of("ip", "dns"); + + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + feedType, + null, + Instant.now(), + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + false, + null, + iocTypes + ); + + SAGetTIFSourceConfigResponse response = new SAGetTIFSourceConfigResponse(SaTifSourceConfigDto.getId(), SaTifSourceConfigDto.getVersion(), RestStatus.OK, SaTifSourceConfigDto); + log.error(SaTifSourceConfigDto.getLastUpdateTime()); + Assert.assertNotNull(response); + + BytesStreamOutput out = new BytesStreamOutput(); + response.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAGetTIFSourceConfigResponse newResponse = new SAGetTIFSourceConfigResponse(sin); + + Assert.assertEquals(SaTifSourceConfigDto.getId(), newResponse.getId()); + Assert.assertEquals(SaTifSourceConfigDto.getVersion(), newResponse.getVersion()); + Assert.assertEquals(RestStatus.OK, newResponse.getStatus()); + Assert.assertNotNull(newResponse.getSaTifSourceConfigDto()); + Assert.assertEquals(feedName, newResponse.getSaTifSourceConfigDto().getName()); + Assert.assertEquals(feedFormat, newResponse.getSaTifSourceConfigDto().getFeedFormat()); + Assert.assertEquals(feedType, newResponse.getSaTifSourceConfigDto().getFeedType()); + Assert.assertEquals(SaTifSourceConfigDto.getState(), newResponse.getSaTifSourceConfigDto().getState()); + Assert.assertEquals(SaTifSourceConfigDto.getEnabledTime(), newResponse.getSaTifSourceConfigDto().getEnabledTime()); + Assert.assertEquals(SaTifSourceConfigDto.getCreatedAt(), newResponse.getSaTifSourceConfigDto().getCreatedAt()); + Assert.assertEquals(SaTifSourceConfigDto.getLastUpdateTime(), newResponse.getSaTifSourceConfigDto().getLastUpdateTime()); + Assert.assertEquals(SaTifSourceConfigDto.isEnabled(), newResponse.getSaTifSourceConfigDto().isEnabled()); + Assert.assertEquals(SaTifSourceConfigDto.getLastRefreshedTime(), newResponse.getSaTifSourceConfigDto().getLastRefreshedTime()); + Assert.assertEquals(SaTifSourceConfigDto.getLastRefreshedUser(), newResponse.getSaTifSourceConfigDto().getLastRefreshedUser()); + Assert.assertEquals(schedule, newResponse.getSaTifSourceConfigDto().getSchedule()); + Assert.assertEquals(SaTifSourceConfigDto.getCreatedByUser(), newResponse.getSaTifSourceConfigDto().getCreatedByUser()); + Assert.assertEquals(SaTifSourceConfigDto.getIocMapStore(), newResponse.getSaTifSourceConfigDto().getIocMapStore()); + Assert.assertTrue(iocTypes.containsAll(newResponse.getSaTifSourceConfigDto().getIocTypes()) && + newResponse.getSaTifSourceConfigDto().getIocTypes().containsAll(iocTypes)); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigActionTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigActionTests.java new file mode 100644 index 000000000..c8b8b29bd --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigActionTests.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.junit.Assert; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.test.OpenSearchTestCase; + +public class IndexTIFSourceConfigActionTests extends OpenSearchTestCase { + public void testIndexTIFSourceConfigActionName() { + Assert.assertNotNull(SAIndexTIFSourceConfigAction.INSTANCE.name()); + Assert.assertEquals(SAIndexTIFSourceConfigAction.INSTANCE.name(), SAIndexTIFSourceConfigAction.NAME); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java new file mode 100644 index 000000000..21ca175fe --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.junit.Assert; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.TestHelpers.randomSATIFSourceConfigDto; + +public class IndexTIFSourceConfigRequestTests extends OpenSearchTestCase { + + public void testTIFSourceConfigPostRequest() throws IOException { + SATIFSourceConfigDto SaTifSourceConfigDto = randomSATIFSourceConfigDto(); + String id = SaTifSourceConfigDto.getId(); + SAIndexTIFSourceConfigRequest request = new SAIndexTIFSourceConfigRequest(id, WriteRequest.RefreshPolicy.IMMEDIATE, RestRequest.Method.POST, SaTifSourceConfigDto); + Assert.assertNotNull(request); + + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAIndexTIFSourceConfigRequest newRequest = new SAIndexTIFSourceConfigRequest(sin); + Assert.assertEquals(id, request.getTIFConfigId()); + Assert.assertEquals(RestRequest.Method.POST, newRequest.getMethod()); + Assert.assertNotNull(newRequest.getTIFConfigDto()); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java new file mode 100644 index 000000000..cce168ae8 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java @@ -0,0 +1,67 @@ +package org.opensearch.securityanalytics.action; + +import org.junit.Assert; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +public class IndexTIFSourceConfigResponseTests extends OpenSearchTestCase { + + public void testIndexTIFSourceConfigPostResponse() throws IOException { + String feedName = "test_feed_name"; + String feedFormat = "STIX"; + FeedType feedType = FeedType.INTERNAL; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + List iocTypes = List.of("ip", "dns"); + + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + feedType, + null, + null, + null, + null, + schedule, + null, + null, + null, + null, + true, + null, + iocTypes + ); + + SAIndexTIFSourceConfigResponse response = new SAIndexTIFSourceConfigResponse(SaTifSourceConfigDto.getId(), SaTifSourceConfigDto.getVersion(), RestStatus.OK, SaTifSourceConfigDto); + Assert.assertNotNull(response); + + BytesStreamOutput out = new BytesStreamOutput(); + response.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAIndexTIFSourceConfigResponse newResponse = new SAIndexTIFSourceConfigResponse(sin); + + Assert.assertEquals(SaTifSourceConfigDto.getId(), newResponse.getTIFConfigId()); + Assert.assertEquals(SaTifSourceConfigDto.getVersion(), newResponse.getVersion()); + Assert.assertEquals(RestStatus.OK, newResponse.getStatus()); + Assert.assertNotNull(newResponse.getTIFConfigDto()); + Assert.assertEquals(feedName, newResponse.getTIFConfigDto().getName()); + Assert.assertEquals(feedFormat, newResponse.getTIFConfigDto().getFeedFormat()); + Assert.assertEquals(feedType, newResponse.getTIFConfigDto().getFeedType()); + Assert.assertEquals(schedule, newResponse.getTIFConfigDto().getSchedule()); + Assert.assertTrue(iocTypes.containsAll(newResponse.getTIFConfigDto().getIocTypes()) && + newResponse.getTIFConfigDto().getIocTypes().containsAll(iocTypes)); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index 3542ee309..e5c1172cf 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -88,7 +88,7 @@ public void testGetAlerts_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Monitor.MonitorType.DOC_LEVEL_MONITOR, + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), null, 1, List.of(), @@ -122,7 +122,7 @@ public void testGetAlerts_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Monitor.MonitorType.DOC_LEVEL_MONITOR, + Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), null, 1, List.of(), diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocDaoTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocDaoTests.java new file mode 100644 index 000000000..4fda1a1b4 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/IocDaoTests.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.TestHelpers.getParser; +import static org.opensearch.securityanalytics.TestHelpers.randomIocDao; + +public class IocDaoTests extends OpenSearchTestCase { + public void testAsStream() throws IOException { + IocDao ioc = randomIocDao(); + BytesStreamOutput out = new BytesStreamOutput(); + ioc.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocDao newIoc = new IocDao(sin); + assertEqualIocDaos(ioc, newIoc); + } + + public void testParseFunction() throws IOException { + IocDao ioc = randomIocDao(); + String json = toJsonString(ioc); + IocDao newIoc = IocDao.parse(getParser(json), ioc.getId()); + assertEqualIocDaos(ioc, newIoc); + } + + private String toJsonString(IocDao ioc) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = ioc.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + private void assertEqualIocDaos(IocDao ioc, IocDao newIoc) { + assertEquals(ioc.getId(), newIoc.getId()); + assertEquals(ioc.getName(), newIoc.getName()); + assertEquals(ioc.getValue(), newIoc.getValue()); + assertEquals(ioc.getSeverity(), newIoc.getSeverity()); + assertEquals(ioc.getSpecVersion(), newIoc.getSpecVersion()); + assertEquals(ioc.getCreated(), newIoc.getCreated()); + assertEquals(ioc.getModified(), newIoc.getModified()); + assertEquals(ioc.getDescription(), newIoc.getDescription()); + assertEquals(ioc.getLabels(), newIoc.getLabels()); + assertEquals(ioc.getFeedId(), newIoc.getFeedId()); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocDtoTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocDtoTests.java new file mode 100644 index 000000000..c1af99dfd --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/IocDtoTests.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.TestHelpers.getParser; +import static org.opensearch.securityanalytics.TestHelpers.randomIocDto; + +public class IocDtoTests extends OpenSearchTestCase { + public void testAsStream() throws IOException { + IocDto ioc = randomIocDto(); + BytesStreamOutput out = new BytesStreamOutput(); + ioc.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocDto newIoc = new IocDto(sin); + assertEqualIocDtos(ioc, newIoc); + } + + public void testParseFunction() throws IOException { + IocDto ioc = randomIocDto(); + String json = toJsonString(ioc); + IocDto newIoc = IocDto.parse(getParser(json), ioc.getId()); + assertEqualIocDtos(ioc, newIoc); + } + + private String toJsonString(IocDto ioc) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = ioc.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + private void assertEqualIocDtos(IocDto ioc, IocDto newIoc) { + assertEquals(ioc.getId(), newIoc.getId()); + assertEquals(ioc.getName(), newIoc.getName()); + assertEquals(ioc.getValue(), newIoc.getValue()); + assertEquals(ioc.getSeverity(), newIoc.getSeverity()); + assertEquals(ioc.getSpecVersion(), newIoc.getSpecVersion()); + assertEquals(ioc.getCreated(), newIoc.getCreated()); + assertEquals(ioc.getModified(), newIoc.getModified()); + assertEquals(ioc.getDescription(), newIoc.getDescription()); + assertEquals(ioc.getLabels(), newIoc.getLabels()); + assertEquals(ioc.getFeedId(), newIoc.getFeedId()); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java new file mode 100644 index 000000000..cbad9f5d2 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java @@ -0,0 +1,79 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; + +import static org.opensearch.securityanalytics.TestHelpers.toJsonString; + +public class IocFindingTests extends OpenSearchTestCase { + + public void testIoCMatchAsAStream() throws IOException { + IocFinding iocFinding = getRandomIoCMatch(); + String jsonString = toJsonString(iocFinding); + BytesStreamOutput out = new BytesStreamOutput(); + iocFinding.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getIocScanJobId(), newIocFinding.getIocScanJobId()); + assertEquals(iocFinding.getIocScanJobName(), newIocFinding.getIocScanJobName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); + } + + public void testIoCMatchParse() throws IOException { + String iocMatchString = "{ \"id\": \"exampleId123\", \"related_doc_ids\": [\"relatedDocId1\", " + + "\"relatedDocId2\"], \"feed_ids\": [\"feedId1\", \"feedId2\"], \"ioc_scan_job_id\":" + + " \"scanJob123\", \"ioc_scan_job_name\": \"Example Scan Job\", \"ioc_value\": \"exampleIocValue\", " + + "\"ioc_type\": \"exampleIocType\", \"timestamp\": 1620912896000, \"execution_id\": \"execution123\" }"; + IocFinding iocFinding = IocFinding.parse((getParser(iocMatchString))); + BytesStreamOutput out = new BytesStreamOutput(); + iocFinding.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getIocScanJobId(), newIocFinding.getIocScanJobId()); + assertEquals(iocFinding.getIocScanJobName(), newIocFinding.getIocScanJobName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); + } + + public XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } + + private static IocFinding getRandomIoCMatch() { + return new IocFinding( + randomAlphaOfLength(10), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Instant.now(), + randomAlphaOfLength(10)); + } + + +} diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java new file mode 100644 index 000000000..da9fe8ca2 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java @@ -0,0 +1,136 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; + +public class SATIFSourceConfigRestApiIT extends SecurityAnalyticsRestTestCase { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigRestApiIT.class); + public void testCreateSATIFSourceConfig() throws IOException { + String feedName = "test_feed_name"; + String feedFormat = "STIX"; + FeedType feedType = FeedType.INTERNAL; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + List iocTypes = List.of("ip", "dns"); + + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + feedType, + null, + null, + null, + null, + schedule, + null, + null, + null, + null, + true, + null, + iocTypes + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(SaTifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + } + + public void testGetSATIFSourceConfigById() throws IOException { + String feedName = "test_feed_name"; + String feedFormat = "STIX"; + FeedType feedType = FeedType.INTERNAL; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + List iocTypes = List.of("ip", "dns"); + + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + feedType, + null, + null, + null, + null, + schedule, + null, + null, + null, + null, + true, + null, + iocTypes + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(SaTifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId, Collections.emptyMap(), null); + Map getResponse = entityAsMap(response); + + String responseId = responseBody.get("_id").toString(); + Assert.assertEquals("Created Id and returned Id do not match", createdId, responseId); + + int responseVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("Incorrect version", responseVersion > 0); + + String returnedFeedName = (String) ((Map)responseBody.get("tif_config")).get("feed_name"); + Assert.assertEquals("Created feed name and returned feed name do not match", feedName, returnedFeedName); + + String returnedFeedFormat = (String) ((Map)responseBody.get("tif_config")).get("feed_format"); + Assert.assertEquals("Created feed format and returned feed format do not match", feedFormat, returnedFeedFormat); + + String returnedFeedType = (String) ((Map)responseBody.get("tif_config")).get("feed_type"); + Assert.assertEquals("Created feed type and returned feed type do not match", feedType, SATIFSourceConfigDto.toFeedType(returnedFeedType)); + + List returnedIocTypes = (List) ((Map)responseBody.get("tif_config")).get("ioc_types"); + Assert.assertTrue("Created ioc types and returned ioc types do not match", iocTypes.containsAll(returnedIocTypes) && returnedIocTypes.containsAll(iocTypes)); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java index 20d36ab2d..d62ea5888 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java @@ -31,9 +31,11 @@ import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskListener; import org.opensearch.test.client.NoOpNodeClient; diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java index 27a01f5c0..f8c6ecadc 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java @@ -5,25 +5,6 @@ package org.opensearch.securityanalytics.threatIntel.action; -import org.junit.Before; -import org.mockito.ArgumentCaptor; -import org.opensearch.action.StepListener; -import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.core.action.ActionListener; -import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.tasks.Task; -import org.opensearch.securityanalytics.TestHelpers; - -import java.io.IOException; -import java.util.ConcurrentModificationException; - -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - /*public class TransportPutTIFJobActionTests extends ThreatIntelTestCase { private TransportPutTIFJobAction action; diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java index 1bf2025cd..03769ac43 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java @@ -28,7 +28,7 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import java.io.IOException; import java.time.Instant; @@ -45,7 +45,7 @@ import static org.opensearch.securityanalytics.TestHelpers.*; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL; -import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList; +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils.getTifdList; public class ThreatIntelJobRunnerIT extends SecurityAnalyticsRestTestCase { private static final Logger log = LogManager.getLogger(ThreatIntelJobRunnerIT.class); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceIT.java new file mode 100644 index 000000000..7f3dfcba5 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceIT.java @@ -0,0 +1,64 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.common.util.concurrent.CountDown; +import org.opensearch.core.action.ActionListener; +import org.opensearch.securityanalytics.SecurityAnalyticsIntegTestCase; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + + +public class IocFindingServiceIT extends SecurityAnalyticsIntegTestCase { + + public void test_indexIocMatches() throws InterruptedException { + IocFindingService service = new IocFindingService(client(), clusterService(), xContentRegistry()); + List iocFindings = generateIocMatches(10); + CountDown countdown = new CountDown(1); + service.indexIocMatches(iocFindings, ActionListener.wrap(r -> { + countdown.countDown(); + }, e -> { + logger.error("failed to index ioc matches", e); + fail(); + countdown.countDown(); + })); + SearchRequest request = new SearchRequest(IocFindingService.INDEX_NAME); + request.source().size(10); + CountDown countDownLatch1 = new CountDown(1); + client().search(request, ActionListener.wrap( + response -> { + assertEquals(response.getHits().getHits().length, 10); + countDownLatch1.countDown(); + }, + e -> { + logger.error("failed to search indexed ioc matches", e); + fail(); + countDownLatch1.countDown(); + } + + )); + countDownLatch1.isCountedDown(); + } + + private List generateIocMatches(int i) { + List iocFindings = new ArrayList<>(); + String monitorId = randomAlphaOfLength(10); + String monitorName = randomAlphaOfLength(10); + for (int i1 = 0; i1 < i; i1++) { + iocFindings.add(new IocFinding( + randomAlphaOfLength(10), + randomList(1, 10, () -> randomAlphaOfLength(10)),//docids + randomList(1, 10, () -> randomAlphaOfLength(10)), //feedids + monitorId, + monitorName, + randomAlphaOfLength(10), + "IP", + Instant.now(), + randomAlphaOfLength(10) + )); + } + return iocFindings; + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java new file mode 100644 index 000000000..22bd9536f --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class IocFindingServiceRestApiIT extends SecurityAnalyticsRestTestCase { + + @SuppressWarnings("unchecked") + public void testGetIocFindings() throws IOException { + List iocFindings = generateIocMatches(10); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.INDEX_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + + Response response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + Map responseAsMap = responseAsMap(response); + Assert.assertEquals(5, ((List>) responseAsMap.get("ioc_findings")).size()); + } + + private List generateIocMatches(int i) { + List iocFindings = new ArrayList<>(); + String monitorId = randomAlphaOfLength(10); + String monitorName = randomAlphaOfLength(10); + for (int i1 = 0; i1 < i; i1++) { + iocFindings.add(new IocFinding( + randomAlphaOfLength(10), + randomList(1, 10, () -> randomAlphaOfLength(10)),//docids + randomList(1, 10, () -> randomAlphaOfLength(10)), //feedids + monitorId, + monitorName, + randomAlphaOfLength(10), + "IP", + Instant.now(), + randomAlphaOfLength(10) + )); + } + return iocFindings; + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java index f7b7ff8d1..1d7f1706c 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java @@ -10,17 +10,16 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.securityanalytics.TestHelpers; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.List; import java.util.Locale; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; public class TIFJobParameterTests extends ThreatIntelTestCase { private static final Logger log = LogManager.getLogger(TIFJobParameterTests.class); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java index 71bd68c61..ec13b7635 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java @@ -6,23 +6,7 @@ package org.opensearch.securityanalytics.threatIntel.jobscheduler; -import org.junit.Before; -import org.opensearch.jobscheduler.spi.JobDocVersion; -import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.jobscheduler.spi.ScheduledJobParameter; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.TestHelpers; - -import java.io.IOException; -import java.time.Instant; -import java.util.Optional; - import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; /*public class TIFJobRunnerTests extends ThreatIntelTestCase { @Before