diff --git a/security-analytics-commons-1.0.0.jar b/security-analytics-commons-1.0.0.jar index dd271f4eb..9ae59c9c2 100644 Binary files a/security-analytics-commons-1.0.0.jar and b/security-analytics-commons-1.0.0.jar differ diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 9349bc020..f4a84c8f1 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -157,6 +157,7 @@ import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestIndexThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestUpdateThreatIntelAlertsStatusAction; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; @@ -326,12 +327,13 @@ public Collection createComponents(Client client, IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry); ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry); SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService); + DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService); return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, threatIntelAlertService, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, correlationAlertService, notificationService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, saTifSourceConfigService, saTifSourceConfigManagementService, stix2IOCFetchService, - ioCScanService); + ioCScanService, defaultTifSourceConfigLoaderService); } @Override diff --git a/src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java b/src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java index 5ae864dd2..cdcca8368 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java +++ b/src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java @@ -75,7 +75,6 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers xcp.nextToken(); switch (fieldName) { - // synced up with @hurneyt, parsing the id and version but may need to change ioc id/version logic case STIX2.ID_FIELD: if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { id = xcp.text(); @@ -90,7 +89,7 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers name = xcp.text(); break; case STIX2.TYPE_FIELD: - type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT)); + type = new IOCType(xcp.text().toLowerCase(Locale.ROOT)); break; case STIX2.VALUE_FIELD: value = xcp.text(); @@ -177,7 +176,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.startObject() .field(STIX2IOC.ID_FIELD, ioc.getId()) .field(STIX2IOC.NAME_FIELD, ioc.getName()) - .field(STIX2IOC.TYPE_FIELD, ioc.getType()) + .field(STIX2IOC.TYPE_FIELD, ioc.getType().toString()) .field(STIX2IOC.VALUE_FIELD, ioc.getValue()) .field(STIX2IOC.SEVERITY_FIELD, ioc.getSeverity()) .timeField(STIX2IOC.CREATED_FIELD, ioc.getCreated()) diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java index 7b2d473ec..7769203b2 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java +++ b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java @@ -84,7 +84,7 @@ public STIX2IOC(StreamInput sin) throws IOException { this( sin.readString(), // id sin.readString(), // name - sin.readEnum(IOCType.class), // type + new IOCType(sin.readString()), // type sin.readString(), // value sin.readString(), // severity sin.readInstant(), // created @@ -142,7 +142,7 @@ public static STIX2IOC readFrom(StreamInput sin) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeString(super.getId()); out.writeString(super.getName()); - out.writeEnum(super.getType()); + out.writeString(super.getType().toString()); out.writeString(super.getValue()); out.writeString(super.getSeverity()); out.writeInstant(super.getCreated()); @@ -160,7 +160,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject() .field(ID_FIELD, super.getId()) .field(NAME_FIELD, super.getName()) - .field(TYPE_FIELD, super.getType()) + .field(TYPE_FIELD, super.getType().toString()) .field(VALUE_FIELD, super.getValue()) .field(SEVERITY_FIELD, super.getSeverity()); XContentUtils.buildInstantAsField(builder, super.getCreated(), CREATED_FIELD); @@ -205,7 +205,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws name = xcp.text(); break; case TYPE_FIELD: - type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT)); + type = new IOCType(xcp.text()); break; case VALUE_FIELD: value = xcp.text(); @@ -292,8 +292,8 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws public void validate() throws IllegalArgumentException { if (super.getType() == null) { throw new IllegalArgumentException(String.format("[%s] is required.", TYPE_FIELD)); - } else if (!Arrays.asList(IOCType.values()).contains(super.getType())) { - logger.debug("Unsupported IOCType: {}", super.getType()); + } else if (!IOCType.supportedType(super.getType().toString())) { + logger.debug("Unsupported IOCType: {}", super.getType().toString()); throw new IllegalArgumentException(String.format("[%s] is not supported.", TYPE_FIELD)); } diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java index 19db0a4f5..2145c55cb 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java +++ b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java @@ -102,7 +102,7 @@ public static STIX2IOCDto readFrom(StreamInput sin) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeString(name); - out.writeEnum(type); + out.writeString(type.toString()); out.writeString(value); out.writeString(severity); out.writeInstant(created); @@ -120,7 +120,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.startObject() .field(STIX2IOC.ID_FIELD, id) .field(STIX2IOC.NAME_FIELD, name) - .field(STIX2IOC.TYPE_FIELD, type) + .field(STIX2IOC.TYPE_FIELD, type.toString()) .field(STIX2IOC.VALUE_FIELD, value) .field(STIX2IOC.SEVERITY_FIELD, severity) .timeField(STIX2IOC.CREATED_FIELD, created) @@ -161,7 +161,6 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr xcp.nextToken(); switch (fieldName) { - // synced up with @hurneyt, parsing the id and version but may need to change ioc id/version logic case STIX2.ID_FIELD: if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { id = xcp.text(); @@ -176,7 +175,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr name = xcp.text(); break; case STIX2.TYPE_FIELD: - type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT)); + type = new IOCType(xcp.text()); break; case STIX2.VALUE_FIELD: value = xcp.text(); diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConnectorFactory.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConnectorFactory.java index 4045f0ded..7c05f0b57 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConnectorFactory.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConnectorFactory.java @@ -6,6 +6,8 @@ package org.opensearch.securityanalytics.services; import com.amazonaws.services.s3.AmazonS3; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.securityanalytics.commons.connector.Connector; import org.opensearch.securityanalytics.commons.connector.S3Connector; import org.opensearch.securityanalytics.commons.connector.codec.InputCodec; @@ -21,6 +23,7 @@ import java.util.List; public class STIX2IOCConnectorFactory extends UnaryParameterCachingFactory> { + private static final Logger logger = LogManager.getLogger(STIX2IOCConnectorFactory.class); private final InputCodecFactory inputCodecFactory; private final S3ClientFactory s3ClientFactory; @@ -31,7 +34,7 @@ public STIX2IOCConnectorFactory(final InputCodecFactory inputCodecFactory, final protected Connector doCreate(FeedConfiguration feedConfiguration) { final FeedLocation feedLocation = FeedLocation.fromFeedConfiguration(feedConfiguration); - // TODO hurneyt add debug log for which location gets used + logger.debug("FeedLocation: {}", feedLocation); switch(feedLocation) { case S3: return createS3Connector(feedConfiguration); default: throw new IllegalArgumentException("Unsupported feedLocation: " + feedLocation); diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConsumer.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConsumer.java index 44d3e169e..9808b4387 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConsumer.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConsumer.java @@ -42,11 +42,10 @@ public void accept(final STIX2 ioc) { feedStore.getSaTifSourceConfig().getName() ); - // TODO hurneyt refactor once the enum values are updated // If the IOC received is not a type listed for the config, do not add it to the queue - if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().name())) { + if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().toString())) { log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}", - stix2IOC.getType().name(), feedStore.getSaTifSourceConfig().getId(), + stix2IOC.getType().toString(), feedStore.getSaTifSourceConfig().getId(), stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue() ); return; diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java index 8c5769309..2998e6934 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java @@ -57,16 +57,13 @@ public class STIX2IOCFeedStore implements FeedStore { public static final String IOC_INDEX_PATTERN = IOC_INDEX_NAME_TEMPLATE + "-" + IOC_TIME_PLACEHOLDER; private final Logger log = LogManager.getLogger(STIX2IOCFeedStore.class); + Instant startTime = Instant.now(); private Client client; private ClusterService clusterService; private SATIFSourceConfig saTifSourceConfig; - - // TODO hurneyt FetchIocsActionResponse is just a placeholder response type for now private ActionListener baseListener; - - // TODO hurneyt this is using TIF batch size setting. Consider adding IOC-specific setting private Integer batchSize; public STIX2IOCFeedStore( @@ -97,7 +94,6 @@ public void storeIOCs(Map actionToIOCs) { for (Map.Entry> entry : iocsSortedByAction.entrySet()) { switch (entry.getKey()) { case DELETE: - // TODO hurneyt consider whether DELETE actions should be handled elsewhere break; case UPSERT: try { @@ -119,7 +115,7 @@ public void indexIocs(List iocs) throws IOException { initFeedIndex(newActiveIndex, ActionListener.wrap( r -> { saTifSourceConfig.getIocTypes().forEach(type -> { - IOCType iocType = IOCType.fromString(type); + IOCType iocType = new IOCType(type); if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { List listOfIocToIndexDetails = ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails(); diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java index 867958a84..0b7da3d70 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java @@ -7,8 +7,11 @@ import com.amazonaws.AmazonServiceException; import com.amazonaws.SdkClientException; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.bulk.BulkRequest; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.action.ActionListener; @@ -29,6 +32,7 @@ import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig; import org.opensearch.securityanalytics.commons.model.FeedConfiguration; import org.opensearch.securityanalytics.commons.model.IOCSchema; +import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.commons.model.STIX2; import org.opensearch.securityanalytics.commons.model.UpdateType; import org.opensearch.securityanalytics.model.STIX2IOC; @@ -36,7 +40,9 @@ import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedParser; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; @@ -49,10 +55,16 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; +import static org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService.isValidIp; + /** * IOC Service implements operations that interact with retrieving IOCs from data sources, * parsing them into threat intel data models (i.e., [IOC]), and ingesting them to system indexes. @@ -66,7 +78,6 @@ public class STIX2IOCFetchService { private STIX2IOCConnectorFactory connectorFactory; private S3ClientFactory s3ClientFactory; - // TODO hurneyt this is using TIF batch size setting. Consider adding IOC-specific setting private Integer batchSize; private String internalAuthEndpoint = ""; @@ -84,14 +95,14 @@ public STIX2IOCFetchService(Client client, ClusterService clusterService) { /** * Method takes in and calls method to rollover and bulk index a list of STIX2IOCs + * * @param saTifSourceConfig * @param stix2IOCList * @param listener */ public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig, List stix2IOCList, - ActionListener listener) - { + ActionListener listener) { STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener); try { feedStore.indexIocs(stix2IOCList); @@ -100,6 +111,7 @@ public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig, listener.onFailure(e); } } + public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener listener) { S3ConnectorConfig s3ConnectorConfig = constructS3ConnectorConfig(saTifSourceConfig); Connector s3Connector = constructS3Connector(s3ConnectorConfig); @@ -113,7 +125,6 @@ public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionList listener.onFailure(e); } - // TODO consider passing listener into the flush IOC function try { consumer.flushIOCs(); } catch (Exception e) { @@ -144,7 +155,7 @@ private void testS3ClientConnection(S3ConnectorConfig s3ConnectorConfig, ActionL } catch (StsException stsException) { log.warn("S3Client connection test failed with StsException: ", stsException); listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(stsException.statusCode()), stsException.awsErrorDetails().errorMessage())); - } catch (SdkException sdkException ) { + } catch (SdkException sdkException) { // SdkException is a RunTimeException that doesn't have a status code. // Logging the full exception, and providing generic response as output. log.warn("S3Client connection test failed with SdkException: ", sdkException); @@ -227,6 +238,77 @@ private String getEndpoint() { return ""; } + public void downloadFromUrlAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener listener) { + UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource(); + switch (source.getFeedFormat()) { // todo add check to stop user from creating url type config from rest api. only internal allowed + case "csv": + try (CSVParser reader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl())) { + CSVParser noHeaderReader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl()); + boolean notFound = true; + + while (notFound) { + CSVRecord hasHeaderRecord = reader.iterator().next(); + + //if we want to skip this line and keep iterating + if ((hasHeaderRecord.values().length == 1 && "".equals(hasHeaderRecord.values()[0])) || hasHeaderRecord.get(0).charAt(0) == '#' || hasHeaderRecord.get(0).charAt(0) == ' ') { + noHeaderReader.iterator().next(); + } else { // we found the first line that contains information + notFound = false; + } + } + if (source.hasCsvHeader()) { + parseAndSaveThreatIntelFeedDataCSV(reader.iterator(), saTifSourceConfig, listener); + } else { + parseAndSaveThreatIntelFeedDataCSV(noHeaderReader.iterator(), saTifSourceConfig, listener); + } + } catch (Exception e) { + log.error("Failed to download the IoCs in CSV format for source " + saTifSourceConfig.getId()); + listener.onFailure(e); + return; + } + break; + default: + log.error("unsupported feed format for url download:" + source.getFeedFormat()); + listener.onFailure(new UnsupportedOperationException("unsupported feed format for url download:" + source.getFeedFormat())); + } + } + + private void parseAndSaveThreatIntelFeedDataCSV(Iterator iterator, SATIFSourceConfig saTifSourceConfig, ActionListener listener) throws IOException { + List bulkRequestList = new ArrayList<>(); + + UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource(); + List iocs = new ArrayList<>(); + while (iterator.hasNext()) { + CSVRecord record = iterator.next(); + String iocType = saTifSourceConfig.getIocTypes().stream().findFirst().orElse(null); + Integer colNum = source.getCsvIocValueColumnNo(); + String iocValue = record.values()[colNum].split(" ")[0]; + if (iocType.equalsIgnoreCase(IOCType.IPV4_TYPE) && !isValidIp(iocValue)) { + log.info("Invalid IP address, skipping this ioc record: {}", iocValue); + continue; + } + Instant now = Instant.now(); + STIX2IOC stix2IOC = new STIX2IOC( + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + iocType == null ? new IOCType(IOCType.IPV4_TYPE) : new IOCType(iocType), + iocValue, + "high", + now, + now, + "", + Collections.emptyList(), + "", + saTifSourceConfig.getId(), + saTifSourceConfig.getName(), + STIX2IOC.NO_VERSION + ); + iocs.add(stix2IOC); + } + STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener); + feedStore.indexIocs(iocs); + } + public static class STIX2IOCFetchResponse extends ActionResponse implements ToXContentObject { public static String IOCS_FIELD = "iocs"; public static String TOTAL_FIELD = "total"; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java index c08b74eea..8001f37ea 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java @@ -5,10 +5,12 @@ package org.opensearch.securityanalytics.threatIntel.common; +import org.opensearch.securityanalytics.commons.model.IOC; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import java.util.ArrayList; import java.util.Arrays; @@ -21,15 +23,12 @@ public class SourceConfigDtoValidator { public List validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto) { List errorMsgs = new ArrayList<>(); - List iocTypeEnumNames = Arrays.stream(IOCType.values()) - .map(Enum::name) - .collect(Collectors.toList()); if (sourceConfigDto.getIocTypes().isEmpty()) { errorMsgs.add("Must specify at least one IOC type"); } else { for (String s: sourceConfigDto.getIocTypes()) { - if (false == iocTypeEnumNames.contains(s)) { + if (!IOCType.supportedType(s)) { errorMsgs.add("Invalid IOC type: " + s); } } @@ -55,6 +54,14 @@ public List validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto errorMsgs.add("Source must be S3_CUSTOM type"); } break; + case URL_DOWNLOAD: + if (sourceConfigDto.getSchedule() == null) { + errorMsgs.add("Must pass in schedule for URL_DOWNLOAD source type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof UrlDownloadSource == false) { + errorMsgs.add("Source must be URL_DOWNLOAD source type"); + } + break; } return errorMsgs; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java index 04f7e8034..8efa5cfa5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java @@ -7,11 +7,11 @@ /** * Types of feeds threat intel can support - * Feed types include: S3_CUSTOM */ public enum SourceConfigType { S3_CUSTOM, - IOC_UPLOAD + IOC_UPLOAD, + URL_DOWNLOAD // LICENSED, // diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java index 81a814915..47dfeed09 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java @@ -41,6 +41,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -375,9 +376,9 @@ private static SearchRequest getSearchRequestForIocType(List indices, St SearchRequest searchRequest = new SearchRequest(indices.toArray(new String[0])); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); // add the iocs sublist - boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD, iocsSublist)); + boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD + ".keyword", iocsSublist)); // add ioc type filter - boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD, iocType.toLowerCase())); + boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD + ".keyword", iocType.toLowerCase(Locale.ROOT))); searchRequest.source().query(boolQueryBuilder); return searchRequest; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java index 60c749ca4..a63bc99d3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java @@ -101,13 +101,15 @@ public IocToIndexDetails(IOCType iocType, String indexPattern, String activeInde } public IocToIndexDetails(StreamInput sin) throws IOException { - this(sin.readEnum(IOCType.class), + this( + new IOCType(sin.readString()), sin.readString(), - sin.readString()); + sin.readString() + ); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeEnum(iocType); + out.writeString(iocType.toString()); out.writeString(indexPattern); out.writeString(activeIndex); } @@ -115,7 +117,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.startObject() - .field(IOC_TYPE_FIELD, iocType) + .field(IOC_TYPE_FIELD, iocType.toString()) .field(INDEX_PATTERN_FIELD, indexPattern) .field(ACTIVE_INDEX_FIELD, activeIndex) .endObject(); @@ -150,7 +152,7 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException { public static IOCType toIocType(String name) { try { - return IOCType.fromString(name); + return new IOCType(name); } catch (IllegalArgumentException e) { log.error("Invalid Ioc type, cannot be parsed.", e); return null; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java index 8f79143e3..865120fac 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java @@ -96,8 +96,4 @@ public void setIocs(List iocs) { public String getFileName() { return fileName; } - - public void setFileName(String fileName) { - this.fileName = fileName; - } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java index a9d75c646..61444fe67 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java @@ -20,6 +20,7 @@ public abstract class Source { abstract String name(); public static final String S3_FIELD = "s3"; public static final String IOC_UPLOAD_FIELD = "ioc_upload"; + public static final String URL_DOWNLOAD_FIELD = "url_download"; static Source readFrom(StreamInput sin) throws IOException { Type type = sin.readEnum(Type.class); @@ -28,6 +29,8 @@ static Source readFrom(StreamInput sin) throws IOException { return new S3Source(sin); case IOC_UPLOAD: return new IocUploadSource(sin); + case URL_DOWNLOAD: + return new UrlDownloadSource(sin); default: throw new IllegalStateException("Unexpected input ["+ type + "] when reading ioc store config"); } @@ -47,6 +50,9 @@ static Source parse(XContentParser xcp) throws IOException { case IOC_UPLOAD_FIELD: source = IocUploadSource.parse(xcp); break; + case URL_DOWNLOAD_FIELD: + source = UrlDownloadSource.parse(xcp); + break; } } return source; @@ -57,7 +63,9 @@ public void writeTo(StreamOutput out) throws IOException {} enum Type { S3(), - IOC_UPLOAD(); + IOC_UPLOAD(), + + URL_DOWNLOAD(); @Override public String toString() { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java new file mode 100644 index 000000000..fdc2d9756 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java @@ -0,0 +1,117 @@ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.net.URL; + +/** + * This is a Threat Intel Source config where the iocs are downloaded from the URL + */ +public class UrlDownloadSource extends Source implements Writeable, ToXContent { + public static final String URL_FIELD = "url"; + public static final String FEED_FORMAT_FIELD = "feed_format"; + public static final String HAS_CSV_HEADER_FIELD = "has_csv_header_field"; + public static final String CSV_IOC_VALUE_COLUMN_NUM_FIELD = "csv_ioc_value_colum_num"; + public static final String SOURCE_NAME = "URL_DOWNLOAD"; + + private final URL url; + private final String feedFormat; + private final Boolean hasCsvHeader; + private final Integer csvIocValueColumnNo; + + public UrlDownloadSource(URL url, String feedFormat, Boolean hasCsvHeader, Integer csvIocValueColumnNo) { + this.url = url; + this.feedFormat = feedFormat; + this.hasCsvHeader = hasCsvHeader; + this.csvIocValueColumnNo = csvIocValueColumnNo; + + } + + public UrlDownloadSource(StreamInput sin) throws IOException { + this( + new URL(sin.readString()), + sin.readString(), + sin.readOptionalBoolean(), + sin.readOptionalInt() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(url.toString()); + out.writeString(feedFormat); + out.writeOptionalBoolean(hasCsvHeader); + out.writeOptionalInt(csvIocValueColumnNo); + } + + @Override + String name() { + return SOURCE_NAME; + } + + public URL getUrl() { + return url; + } + + public static UrlDownloadSource parse(XContentParser xcp) throws IOException { + URL url = null; + String feedFormat = null; + Boolean hasCsvHeader = false; + Integer csvIocValueColumnNo = null; + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case URL_FIELD: + String urlString = xcp.text(); + url = new URL(urlString); + break; + case FEED_FORMAT_FIELD: + feedFormat = xcp.text(); + break; + case HAS_CSV_HEADER_FIELD: + hasCsvHeader = xcp.booleanValue(); + break; + case CSV_IOC_VALUE_COLUMN_NUM_FIELD: + if (xcp.currentToken() == null) + xcp.skipChildren(); + else + csvIocValueColumnNo = xcp.intValue(); + break; + default: + xcp.skipChildren(); + } + } + return new UrlDownloadSource(url, feedFormat, hasCsvHeader, csvIocValueColumnNo); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .startObject(URL_DOWNLOAD_FIELD) + .field(URL_FIELD, url.toString()) + .field(FEED_FORMAT_FIELD, feedFormat) + .field(HAS_CSV_HEADER_FIELD, hasCsvHeader) + .field(CSV_IOC_VALUE_COLUMN_NUM_FIELD, csvIocValueColumnNo) + .endObject() + .endObject(); + } + + public String getFeedFormat() { + return feedFormat; + } + + public boolean hasCsvHeader() { + return hasCsvHeader; + } + + public Integer getCsvIocValueColumnNo() { + return csvIocValueColumnNo; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java new file mode 100644 index 000000000..ed160bbf1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java @@ -0,0 +1,180 @@ +package org.opensearch.securityanalytics.threatIntel.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.rest.RestRequest; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; + +import java.net.URL; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +//todo handle refresh, update tif config +// todo block creation of url based config in transport layer +public class DefaultTifSourceConfigLoaderService { + private static final Logger log = LogManager.getLogger(DefaultTifSourceConfigLoaderService.class); + private final BuiltInTIFMetadataLoader tifMetadataLoader; + private final Client client; + private final SATIFSourceConfigManagementService satifSourceConfigManagementService; + + public DefaultTifSourceConfigLoaderService(BuiltInTIFMetadataLoader tifMetadataLoader, Client client, SATIFSourceConfigManagementService satifSourceConfigManagementService) { + this.tifMetadataLoader = tifMetadataLoader; + this.client = client; + this.satifSourceConfigManagementService = satifSourceConfigManagementService; + } + + /** + * check if the default tif source configs are loaded. if not, try create them from the feedMetadata.json file. + */ + public void createDefaultTifConfigsIfNotExists(ActionListener listener) { + List tifMetadataList = tifMetadataLoader.getTifMetadataList(); + if (tifMetadataList.isEmpty()) { + log.error("No built-in TIF Configs found"); + listener.onResponse(null); + return; + } + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + for (TIFMetadata tifMetadata : tifMetadataList) { + boolQueryBuilder.should(new MatchQueryBuilder("_id", tifMetadata.getFeedId())); + } + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder).size(9999); + satifSourceConfigManagementService.searchTIFSourceConfigs(searchSourceBuilder, + ActionListener.wrap(searchResponse -> { + createTifConfigsThatDontExist(searchResponse, tifMetadataList, listener); + }, e -> { + log.error("Failed to search tif config index for default tif configs", e); + listener.onFailure(e); + })); + } + + private void createTifConfigsThatDontExist(SearchResponse searchResponse, List tifMetadataList, ActionListener listener) { + Map feedsToCreate = tifMetadataList.stream() + .collect(Collectors.toMap( + TIFMetadata::getFeedId, + Function.identity() + )); + if (searchResponse.getHits() != null && searchResponse.getHits().getHits() != null) { + for (SearchHit hit : searchResponse.getHits().getHits()) { + feedsToCreate.remove(hit.getId()); + } + } + if (feedsToCreate.isEmpty()) { + listener.onResponse(null); + return; + } + GroupedActionListener> groupedActionListener = new GroupedActionListener<>( + new ActionListener<>() { + @Override + public void onResponse(Collection> responseOrExceptions) { + if (responseOrExceptions.stream().allMatch(it -> it.getException() != null)) { // all configs returned error + Exception e = responseOrExceptions.stream().findFirst().get().getException(); + log.error("Failed to create default tif configs", e); + listener.onFailure(e); + return; + } + listener.onResponse(null); + return; + } + + @Override + public void onFailure(Exception e) { + log.error("Unexpected failure while creating Default Threat intel source configs", e); + listener.onFailure(e); + return; + } + }, feedsToCreate.size() + ); + for (TIFMetadata tifMetadata : feedsToCreate.values()) { + if (tifMetadata == null) { + continue; + } + try { + Instant now = Instant.now(); + String iocType = null; + if (tifMetadata.getIocType().equalsIgnoreCase("ip")) { + iocType = IOCType.IPV4_TYPE; + } + satifSourceConfigManagementService.createOrUpdateTifSourceConfig( + new SATIFSourceConfigDto( + tifMetadata.getFeedId(), + SATIFSourceConfigDto.NO_VERSION, + tifMetadata.getName(), + "STIX2", + SourceConfigType.URL_DOWNLOAD, + tifMetadata.getDescription(), + null, + now, + new UrlDownloadSource(new URL(tifMetadata.getUrl()), tifMetadata.getFeedType(), tifMetadata.hasHeader(), tifMetadata.getIocCol()), + now, + now, + new IntervalSchedule(now, 1, ChronoUnit.DAYS), + TIFJobState.CREATING, + RefreshType.FULL, + null, + null, + true, + List.of(iocType), + true + ), + null, + RestRequest.Method.POST, + null, + ActionListener.wrap( + r -> { + groupedActionListener.onResponse(new ResponseOrException<>(r, null)); + }, + e -> { + log.error("failed to create default tif source config " + tifMetadata.getFeedId(), e); + groupedActionListener.onResponse(new ResponseOrException<>(null, e)); + }) + ); + continue; + } catch (Exception ex) { + log.error("Unexpected failure while creating Default Threat intel source configs " + tifMetadata.getFeedId(), ex); + groupedActionListener.onResponse(new ResponseOrException<>(null, ex)); + continue; + } + } + } + + private static class ResponseOrException { + private final R response; + private final Exception exception; + + private ResponseOrException(R response, Exception exception) { + this.response = response; + this.exception = exception; + } + + public R getResponse() { + return response; + } + + public Exception getException() { + return exception; + } + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index 1d9a71a2b..65514f459 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -27,6 +27,7 @@ import org.opensearch.rest.RestRequest; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.services.STIX2IOCFetchService; @@ -48,10 +49,10 @@ import java.util.Map; import java.util.Set; import java.util.SortedMap; - import java.util.stream.Collectors; import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.IOC_UPLOAD; +import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.URL_DOWNLOAD; /** * Service class for threat intel feed source config object @@ -193,16 +194,19 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig, case S3_CUSTOM: stix2IOCFetchService.downloadAndIndexIOCs(saTifSourceConfig, actionListener); break; + case URL_DOWNLOAD: + stix2IOCFetchService.downloadFromUrlAndIndexIOCs(saTifSourceConfig, actionListener); + break; case IOC_UPLOAD: List validStix2IocList = new ArrayList<>(); // If the IOC received is not a type listed for the config, do not add it to the queue for (STIX2IOC stix2IOC : stix2IOCList) { - if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().name())) { + if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().toString())) { validStix2IocList.add(stix2IOC); } else { log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}", - stix2IOC.getType().name(), saTifSourceConfig.getId(), - stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue() + stix2IOC.getType().toString(), saTifSourceConfig.getId(), + stix2IOC.getId(), stix2IOC.getType().toString(), stix2IOC.getValue() ); } } @@ -351,7 +355,7 @@ private void storeAndDeleteIocIndices(List stix2IOCList, ActionListene Set concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse); // remove ioc types not specified in list - defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> false == iocTypes.contains(iocToIndexDetails.getIocType().name())); + defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString())); // get the active indices defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> activeIndices.add(e.getActiveIndex())); @@ -464,7 +468,7 @@ private void downloadAndSaveIocsToRefresh(ActionListener l if (newIocStoreConfig instanceof DefaultIocStoreConfig) { DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) newIocStoreConfig; // remove ioc types not specified in list - defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> false == iocTypes.contains(iocToIndexDetails.getIocType().name())); + defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString())); updatedSourceConfig.setIocStoreConfig(defaultIocStoreConfig); } // Update source config as succeeded, change state back to available @@ -509,6 +513,11 @@ public void deleteTIFSourceConfig( ) { saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigId, ActionListener.wrap( saTifSourceConfig -> { + if (URL_DOWNLOAD.equals(saTifSourceConfig.getType())) { + log.error("Cannot delete tif source config {} as it's a built-in config and not user-defined.", saTifSourceConfigId); + listener.onFailure(new IllegalArgumentException("Cannot delete built-in tif source config " + saTifSourceConfigId)); + return; + } // Check if all threat intel monitors are deleted saTifSourceConfigService.checkAndEnsureThreatIntelMonitorsDeleted(ActionListener.wrap( isDeleted -> { @@ -768,15 +777,42 @@ public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto saTifSourceCo } private SATIFSourceConfig updateSaTifSourceConfig(SATIFSourceConfigDto saTifSourceConfigDto, SATIFSourceConfig saTifSourceConfig) { + // currently url download is only for default tif configs and supports only activate/deactivate. Ideally should be via an activate API + if (URL_DOWNLOAD.equals(saTifSourceConfig.getType())) { + return new SATIFSourceConfig( + saTifSourceConfig.getId(), + saTifSourceConfig.getVersion(), + saTifSourceConfig.getName(), + saTifSourceConfig.getFormat(), + saTifSourceConfig.getType(), + saTifSourceConfig.getDescription(), + saTifSourceConfig.getCreatedByUser(), + saTifSourceConfig.getCreatedAt(), + saTifSourceConfig.getSource(), + saTifSourceConfig.getEnabledTime(), + saTifSourceConfig.getLastUpdateTime(), + saTifSourceConfig.getSchedule(), + saTifSourceConfig.getState(), + saTifSourceConfig.getRefreshType(), + saTifSourceConfig.getLastRefreshedTime(), + saTifSourceConfig.getLastRefreshedUser(), + saTifSourceConfig.isEnabled(), + saTifSourceConfig.getIocStoreConfig(), + saTifSourceConfig.getIocTypes(), + saTifSourceConfigDto.isEnabledForScan() + ); + } + if (false == saTifSourceConfig.getSource().getClass().equals(saTifSourceConfigDto.getSource().getClass())) { + throw new IllegalArgumentException(""); + } // remove duplicates from iocTypes Set iocTypes = new LinkedHashSet<>(saTifSourceConfigDto.getIocTypes()); - return new SATIFSourceConfig( saTifSourceConfig.getId(), saTifSourceConfig.getVersion(), saTifSourceConfigDto.getName(), saTifSourceConfigDto.getFormat(), - saTifSourceConfigDto.getType(), + saTifSourceConfig.getType(), saTifSourceConfigDto.getDescription(), saTifSourceConfig.getCreatedByUser(), saTifSourceConfig.getCreatedAt(), diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java index eb90415b4..ab6ed9915 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java @@ -187,7 +187,7 @@ public void onFailure(Exception e) { } break; default: - // if the feed type doesn't match any of the supporting feed types, throw an exception + onFailure(new UnsupportedOperationException("Not a supported feed format : " + tifMetadata.getFeedType())); } } } catch (IOException ex) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java index 61ea2374d..1cb9e7428 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java @@ -229,6 +229,8 @@ public void parseAndSaveThreatIntelFeedDataCSV( } public static boolean isValidIp(String ip) { + if (StringUtils.isBlank(ip)) + return false; String ipPattern = "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$"; Pattern pattern = Pattern.compile(ipPattern); Matcher matcher = pattern.matcher(ip); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java index ae06d7724..9b6378cf9 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java @@ -15,12 +15,15 @@ import org.opensearch.commons.authuser.User; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.RestRequest; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -94,6 +97,11 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques } try { SATIFSourceConfigDto saTifSourceConfigDto = request.getTIFConfigDto(); + if (SourceConfigType.URL_DOWNLOAD.equals(saTifSourceConfigDto.getType()) || saTifSourceConfigDto.getSource() instanceof UrlDownloadSource + && request.getMethod().equals(RestRequest.Method.POST)) { + listener.onFailure(new UnsupportedOperationException("Unsupported Threat intel Source Config Type passed - " + saTifSourceConfigDto.getType())); + return; + } saTifSourceConfigManagementService.createOrUpdateTifSourceConfig( saTifSourceConfigDto, lock, diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java index d046a35e5..9eb47f0a3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java @@ -3,6 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.StepListener; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; @@ -15,6 +16,7 @@ import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsAction; import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsRequest; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.tasks.Task; @@ -28,6 +30,7 @@ public class TransportSearchTIFSourceConfigsAction extends HandledTransportActio private final ClusterService clusterService; private final Settings settings; + private final DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService; private final ThreadPool threadPool; @@ -41,11 +44,13 @@ public TransportSearchTIFSourceConfigsAction(TransportService transportService, ClusterService clusterService, final ThreadPool threadPool, Settings settings, + DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService, final SATIFSourceConfigManagementService saTifConfigService) { super(SASearchTIFSourceConfigsAction.NAME, transportService, actionFilters, SASearchTIFSourceConfigsRequest::new); this.clusterService = clusterService; this.threadPool = threadPool; this.settings = settings; + this.defaultTifSourceConfigLoaderService = defaultTifSourceConfigLoaderService; this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); this.saTifConfigService = saTifConfigService; @@ -63,16 +68,38 @@ protected void doExecute(Task task, SASearchTIFSourceConfigsRequest request, Act } this.threadPool.getThreadContext().stashContext(); // stash context to make calls as admin client - - saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( - r -> { - log.debug("Successfully listed all threat intel source configs"); - actionListener.onResponse(r); - }, e -> { - log.error("Failed to list all threat intel source configs"); - actionListener.onFailure(e); - } - )); + StepListener defaultTifConfigsLoadedListener; + try { + defaultTifConfigsLoadedListener = new StepListener<>(); + defaultTifSourceConfigLoaderService.createDefaultTifConfigsIfNotExists(defaultTifConfigsLoadedListener); + defaultTifConfigsLoadedListener.whenComplete(res -> saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, e -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + } + )), ex -> saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, e -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + } + ))); + } catch (Exception e) { + log.error("Failed to load default tif source configs. Moving on to list iocs", e); + saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, ex -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + })); + } } private void setFilterByEnabled(boolean filterByEnabled) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java index bfbb9dbde..3cbf31086 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java @@ -42,9 +42,27 @@ public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadat connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); return new CSVParser(new BufferedReader(new InputStreamReader(connection.getInputStream())), CSVFormat.RFC4180); } catch (IOException e) { - log.error("Exception: failed to read threat intel feed data from {}",tifMetadata.getUrl(), e); + log.error("Exception: failed to read threat intel feed data from {}", tifMetadata.getUrl(), e); throw new OpenSearchException("failed to read threat intel feed data from {}", tifMetadata.getUrl(), e); } }); } + + /** + * Create CSVParser of a threat intel feed + */ + @SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file") + public static CSVParser getThreatIntelFeedReaderCSV(URL url) { + SpecialPermission.check(); + return AccessController.doPrivileged((PrivilegedAction) () -> { + try { + URLConnection connection = url.openConnection(); + connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); + return new CSVParser(new BufferedReader(new InputStreamReader(connection.getInputStream())), CSVFormat.RFC4180); + } catch (IOException e) { + log.error("Exception: failed to read threat intel feed data from {}", url, e); + throw new OpenSearchException("failed to read threat intel feed data from {}", url, e); + } + }); + } } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java index 132725d71..2e1954ce7 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchStatusException; import org.opensearch.action.ActionRunnable; +import org.opensearch.action.StepListener; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; @@ -43,6 +44,7 @@ import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; import org.opensearch.securityanalytics.threatIntel.transport.TransportSearchTIFSourceConfigsAction; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -68,6 +70,7 @@ public class TransportListIOCsAction extends HandledTransportAction defaultTifConfigsLoadedListener = null; + try { + defaultTifConfigsLoadedListener = new StepListener<>(); + defaultTifSourceConfigLoaderService.createDefaultTifConfigsIfNotExists(defaultTifConfigsLoadedListener); + defaultTifConfigsLoadedListener.whenComplete(r -> searchIocs(), e -> searchIocs()); + } catch (Exception e) { + log.error("Failed to load default tif source configs. Moving on to list iocs", e); + searchIocs(); + } + } + + private void searchIocs() { /** get all match threat intel source configs. fetch write index of each config if no iocs provided else fetch just index alias */ List configIds = request.getFeedIds() == null ? Collections.emptyList() : request.getFeedIds(); saTifSourceConfigService.searchTIFSourceConfigs(getFeedsSearchSourceBuilder(configIds), @@ -128,7 +145,7 @@ void start() { SATIFSourceConfig config = SATIFSourceConfig.docParse(xcp, hit.getId(), hit.getVersion()); if (config.getIocStoreConfig() instanceof DefaultIocStoreConfig) { DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) config.getIocStoreConfig(); - for (DefaultIocStoreConfig.IocToIndexDetails iocToindexDetails: iocStoreConfig.getIocToIndexDetails()) { + for (DefaultIocStoreConfig.IocToIndexDetails iocToindexDetails : iocStoreConfig.getIocToIndexDetails()) { String writeIndex = iocToindexDetails.getActiveIndex(); if (writeIndex != null) { iocIndices.add(writeIndex); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 2a32098f2..75ee7cd89 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -2889,7 +2889,7 @@ public static SATIFSourceConfig randomSATIFSourceConfig( schedule = new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); } if (iocStoreConfig == null) { - iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(IOCType.domain_name, "indexPattern", "writeIndex"))); + iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.DOMAIN_NAME_TYPE), "indexPattern", "writeIndex"))); } if (iocTypes == null) { iocTypes = List.of("ip"); diff --git a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java index 61f7ecf07..2687907d1 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java @@ -75,7 +75,7 @@ private void assertEqualsSaTifSourceConfigs(SATIFSourceConfig saTifSourceConfig, assertEquals(saTifSourceConfig.isEnabled(), newSaTifSourceConfig.isEnabled()); DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig(); DefaultIocStoreConfig newIocStoreConfig = (DefaultIocStoreConfig) newSaTifSourceConfig.getIocStoreConfig(); - assertEquals(iocStoreConfig.getIocToIndexDetails().get(0).getIocType(), newIocStoreConfig.getIocToIndexDetails().get(0).getIocType()); + assertEquals(iocStoreConfig.getIocToIndexDetails().get(0).getIocType().toString(), newIocStoreConfig.getIocToIndexDetails().get(0).getIocType().toString()); assertEquals(iocStoreConfig.getIocToIndexDetails().get(0).getIndexPattern(), newIocStoreConfig.getIocToIndexDetails().get(0).getIndexPattern()); assertEquals(iocStoreConfig.getIocToIndexDetails().get(0).getActiveIndex(), newIocStoreConfig.getIocToIndexDetails().get(0).getActiveIndex()); assertEquals(saTifSourceConfig.getIocTypes(), newSaTifSourceConfig.getIocTypes()); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java index 7624a746a..2725d71ee 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java @@ -8,12 +8,12 @@ package org.opensearch.securityanalytics.resthandler; +import com.google.common.collect.ImmutableList; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; -import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; @@ -22,23 +22,31 @@ import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.commons.utils.testUtils.S3ObjectGenerator; import org.opensearch.securityanalytics.model.STIX2IOC; -import org.opensearch.securityanalytics.services.STIX2IOCFeedStore; import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.threatIntel.model.Source; import org.opensearch.securityanalytics.util.STIX2IOCGenerator; +import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -62,8 +70,13 @@ * -Dtests.SATIFSourceConfigRestApiIT.bucketName= \ * -Dtests.SATIFSourceConfigRestApiIT.region= \ * -Dtests.SATIFSourceConfigRestApiIT.roleArn= + * + * Optionally, the following system parameter can be supplied to PREVENT the tests from cleaning up the bucket objects. + * This could be helpful when troubleshooting failing tests by investigating the data generated during execution. + * By default, the bucket objects (not the bucket) will be cleaned up after the tests. + * To disable cleanup, add the following system parameter. + * -Dtests.SATIFSourceConfigRestApiIT.cleanup=false */ -@EnabledIfSystemProperty(named = "tests.SATIFSourceConfigRestApiIT.bucketName", matches = ".+") public class SATIFSourceConfigRestApiIT extends SecurityAnalyticsRestTestCase { private String bucketName; @@ -75,17 +88,44 @@ public class SATIFSourceConfigRestApiIT extends SecurityAnalyticsRestTestCase { private S3ObjectGenerator s3ObjectGenerator; private STIX2IOCGenerator stix2IOCGenerator; + /** + * Is reassigned in the initSource function. + * Will only be TRUE if 'bucketName', 'region', and 'roleArn' are supplied through system params. + * Disables tests when FALSE. + */ + private boolean canRunTests; + + /** + * List of invalid type patterns for easy test execution + */ + private final List invalidTypes = ImmutableList.of( + "ip", // "ip" is not currently a supported IOCType + "ipv4_addr" // Currently, the supported IOCTypes do not contain underscores + ); + @Before public void initSource() { - // Retrieve system parameters needed to run the tests + // Retrieve system parameters needed to run the tests. Only retrieve once if (bucketName == null) { bucketName = System.getProperty("tests.SATIFSourceConfigRestApiIT.bucketName"); region = System.getProperty("tests.SATIFSourceConfigRestApiIT.region"); roleArn = System.getProperty("tests.SATIFSourceConfigRestApiIT.roleArn"); } + // Confirm necessary system params are provided + canRunTests = bucketName != null && !bucketName.isBlank() && + region != null && !region.isBlank() && + roleArn != null && !roleArn.isBlank(); + + // Exit test setup if necessary system params are not provided + if (!canRunTests) { + logger.info(getClass().getName() + " tests disabled."); + System.out.println(getClass().getName() + " tests disabled."); + return; + } + // Only create the s3Client once - if (bucketName != null && s3Client == null) { + if (s3Client == null) { s3Client = S3Client.builder() .region(Region.of(region)) .build(); @@ -99,15 +139,36 @@ public void initSource() { @After public void afterTest() { + // Exit test cleanup if necessary system params are not provided + if (!canRunTests) return; + + // Delete the bucket object unless cleanup is disabled + if (!Objects.equals(System.getProperty("tests.SATIFSourceConfigRestApiIT.cleanup"), "false")) { + DeleteObjectResponse response = s3Client.deleteObject( + DeleteObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .build() + ); + + // Confirm bucket object was deleted successfully + assertTrue( + String.format("Failed to delete object with key %s in bucket %s", objectKey, bucketName), + response.sdkHttpResponse().isSuccessful() + ); + } + + // Close the client s3Client.close(); } - @Ignore public void testCreateSATIFSourceConfigAndVerifyJobRan() throws IOException, InterruptedException { + // Only run tests when required system params are provided + if (!canRunTests) return; + // Generate test IOCs, and upload them to S3 to create the bucket object. Feed creation fails if the bucket object doesn't exist. int numOfIOCs = 1; - stix2IOCGenerator = new STIX2IOCGenerator(); - stix2IOCGenerator.setType(IOCType.ipv4_addr); + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE))); s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size()); @@ -116,7 +177,7 @@ public void testCreateSATIFSourceConfigAndVerifyJobRan() throws IOException, Int String feedFormat = "STIX2"; SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); - List iocTypes = List.of("ip", "domain-name"); + List iocTypes = List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE); SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( null, @@ -136,7 +197,8 @@ public void testCreateSATIFSourceConfigAndVerifyJobRan() throws IOException, Int Instant.now(), null, true, - iocTypes, true + iocTypes, + true ); Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); Assert.assertEquals(201, response.getStatusLine().getStatusCode()); @@ -173,35 +235,13 @@ public void testCreateSATIFSourceConfigAndVerifyJobRan() throws IOException, Int }, 240, TimeUnit.SECONDS); } - /** - * Calls the get source config api and checks if the last updated time is different from the time that was passed in - * @param createdId - * @param firstUpdatedTime - * @return - * @throws IOException - */ - protected boolean verifyJobRan(String createdId, String firstUpdatedTime) throws IOException { - Response response; - Map responseBody; - - // call get API to get the latest source config by ID - response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId, Collections.emptyMap(), null); - responseBody = asMap(response); - - String returnedLastUpdatedTime = (String) ((Map)responseBody.get("source_config")).get("last_update_time"); - - if(firstUpdatedTime.equals(returnedLastUpdatedTime.toString()) == false) { - return true; - } - return false; - } - - @Ignore public void testGetSATIFSourceConfigById() throws IOException { + // Only run tests when required system params are provided + if (!canRunTests) return; + // Generate test IOCs, and upload them to S3 to create the bucket object. Feed creation fails if the bucket object doesn't exist. int numOfIOCs = 1; - stix2IOCGenerator = new STIX2IOCGenerator(); - stix2IOCGenerator.setType(IOCType.hashes); + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.HASHES_TYPE))); s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size()); @@ -210,7 +250,7 @@ public void testGetSATIFSourceConfigById() throws IOException { String feedFormat = "STIX2"; SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); - List iocTypes = List.of("hashes"); + List iocTypes = List.of(IOCType.HASHES_TYPE); SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( null, @@ -230,7 +270,8 @@ public void testGetSATIFSourceConfigById() throws IOException { Instant.now(), null, true, - iocTypes, true + iocTypes, + true ); Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); @@ -262,12 +303,13 @@ public void testGetSATIFSourceConfigById() throws IOException { Assert.assertTrue("Created ioc types and returned ioc types do not match", iocTypes.containsAll(returnedIocTypes) && returnedIocTypes.containsAll(iocTypes)); } - @Ignore public void testDeleteSATIFSourceConfig() throws IOException { + // Only run tests when required system params are provided + if (!canRunTests) return; + // Generate test IOCs, and upload them to S3 to create the bucket object. Feed creation fails if the bucket object doesn't exist. int numOfIOCs = 1; - stix2IOCGenerator = new STIX2IOCGenerator(); - stix2IOCGenerator.setType(IOCType.ipv4_addr); + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE))); s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size()); @@ -276,7 +318,7 @@ public void testDeleteSATIFSourceConfig() throws IOException { String feedFormat = "STIX2"; SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); - List iocTypes = List.of("ip", "hashes"); + List iocTypes = List.of(IOCType.IPV4_TYPE, IOCType.HASHES_TYPE); SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( null, @@ -296,7 +338,8 @@ public void testDeleteSATIFSourceConfig() throws IOException { Instant.now(), null, true, - iocTypes, true + iocTypes, + true ); Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); @@ -331,21 +374,121 @@ public void testDeleteSATIFSourceConfig() throws IOException { Assert.assertEquals(0, hits.size()); } - @Ignore public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedException { - // Generate test IOCs, and upload them to S3 + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Execute test for each IOCType + for (String type : IOCType.types) { + // Generate test IOCs, and upload them to S3 + int numOfIOCs = 5; + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(type))); + s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); + assertEquals("Incorrect number of test IOCs generated for type: " + type, numOfIOCs, stix2IOCGenerator.getIocs().size()); + + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + List iocTypes = List.of(type); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + // Confirm test feed was created successfully + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("Response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + + // Wait for feed to execute + String firstUpdatedTime = (String) ((Map)responseBody.get("source_config")).get("last_refreshed_time"); + waitUntil(() -> { + try { + return verifyJobRan(createdId, firstUpdatedTime); + } catch (IOException e) { + throw new RuntimeException("failed to verify that job ran"); + } + }, 240, TimeUnit.SECONDS); + + // Confirm IOCs were ingested to system index for the feed + String indexName = getAllIocIndexPatternById(createdId); + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(indexName, request); + + // Confirm expected number of results are returned + assertEquals(numOfIOCs, hits.size()); + List> iocs = hits.stream() + .map(SearchHit::getSourceAsMap) + .collect(Collectors.toList()); + + // Sort IOC lists for easy comparison + stix2IOCGenerator.getIocs().sort(Comparator.comparing(STIX2IOC::getName)); + iocs.sort(Comparator.comparing(ioc -> (String) ioc.get(STIX2IOC.NAME_FIELD))); + + // Confirm expected IOCs have been ingested + for (int i = 0; i < numOfIOCs; i++) { + assertEquals(stix2IOCGenerator.getIocs().get(i).getName(), iocs.get(i).get(STIX2IOC.NAME_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getType().toString(), IOCType.fromString((String) iocs.get(i).get(STIX2IOC.TYPE_FIELD))); + assertEquals(stix2IOCGenerator.getIocs().get(i).getValue(), iocs.get(i).get(STIX2IOC.VALUE_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getSeverity(), iocs.get(i).get(STIX2IOC.SEVERITY_FIELD)); + + // TODO troubleshoot instant assertions +// assertEquals(stix2IOCGenerator.getIocs().get(i).getCreated().toString(), iocs.get(i).get(STIX2IOC.CREATED_FIELD)); +// assertEquals(stix2IOCGenerator.getIocs().get(i).getModified().toString(), iocs.get(i).get(STIX2IOC.MODIFIED_FIELD)); + + assertEquals(stix2IOCGenerator.getIocs().get(i).getDescription(), iocs.get(i).get(STIX2IOC.DESCRIPTION_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getLabels(), iocs.get(i).get(STIX2IOC.LABELS_FIELD)); + assertEquals(createdId, iocs.get(i).get(STIX2IOC.FEED_ID_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getSpecVersion(), iocs.get(i).get(STIX2IOC.SPEC_VERSION_FIELD)); + } + } + } + + public void testRetrieveMultipleIOCTypesSuccessfully() throws IOException, InterruptedException { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Generate test IOCs for each type, and upload them to S3 int numOfIOCs = 5; stix2IOCGenerator = new STIX2IOCGenerator(); - stix2IOCGenerator.setType(IOCType.ipv4_addr); s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); - assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size()); + List allIocs = stix2IOCGenerator.getIocs(); + assertEquals("Incorrect total number of test IOCs generated.", IOCType.types.size() * numOfIOCs, allIocs.size()); // Create test feed String feedName = "download_test_feed_name"; String feedFormat = "STIX2"; SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); - List iocTypes = List.of(IOCType.ipv4_addr.toString()); SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( null, @@ -365,7 +508,8 @@ public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedExcept Instant.now(), null, true, - iocTypes, true + IOCType.types, + true ); // Confirm test feed was created successfully @@ -389,7 +533,9 @@ public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedExcept // Confirm IOCs were ingested to system index for the feed String indexName = getAllIocIndexPatternById(createdId); + String request = "{\n" + + " \"size\" : 10000,\n" + " \"query\" : {\n" + " \"match_all\":{\n" + " }\n" + @@ -398,30 +544,264 @@ public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedExcept List hits = executeSearch(indexName, request); // Confirm expected number of results are returned - assertEquals(numOfIOCs, hits.size()); - List> iocs = hits.stream() + assertEquals(allIocs.size(), hits.size()); + List> iocHits = hits.stream() .map(SearchHit::getSourceAsMap) .collect(Collectors.toList()); // Sort IOC lists for easy comparison - stix2IOCGenerator.getIocs().sort(Comparator.comparing(STIX2IOC::getName)); - iocs.sort(Comparator.comparing(ioc -> (String) ioc.get(STIX2IOC.NAME_FIELD))); + allIocs.sort(Comparator.comparing(STIX2IOC::getName)); + iocHits.sort(Comparator.comparing(ioc -> (String) ioc.get(STIX2IOC.NAME_FIELD))); // Confirm expected IOCs have been ingested - for (int i = 0; i < numOfIOCs; i++) { - assertEquals(stix2IOCGenerator.getIocs().get(i).getName(), iocs.get(i).get(STIX2IOC.NAME_FIELD)); - assertEquals(stix2IOCGenerator.getIocs().get(i).getType(), IOCType.fromString((String) iocs.get(i).get(STIX2IOC.TYPE_FIELD))); - assertEquals(stix2IOCGenerator.getIocs().get(i).getValue(), iocs.get(i).get(STIX2IOC.VALUE_FIELD)); - assertEquals(stix2IOCGenerator.getIocs().get(i).getSeverity(), iocs.get(i).get(STIX2IOC.SEVERITY_FIELD)); + for (int i = 0; i < allIocs.size(); i++) { + assertEquals(stix2IOCGenerator.getIocs().get(i).getName(), iocHits.get(i).get(STIX2IOC.NAME_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getType().toString(), IOCType.fromString((String) iocHits.get(i).get(STIX2IOC.TYPE_FIELD))); + assertEquals(stix2IOCGenerator.getIocs().get(i).getValue(), iocHits.get(i).get(STIX2IOC.VALUE_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getSeverity(), iocHits.get(i).get(STIX2IOC.SEVERITY_FIELD)); // TODO troubleshoot instant assertions -// assertEquals(stix2IOCGenerator.getIocs().get(i).getCreated().toString(), iocs.get(i).get(STIX2IOC.CREATED_FIELD)); -// assertEquals(stix2IOCGenerator.getIocs().get(i).getModified().toString(), iocs.get(i).get(STIX2IOC.MODIFIED_FIELD)); +// assertEquals(stix2IOCGenerator.getIocs().get(i).getCreated().toString(), iocHits.get(i).get(STIX2IOC.CREATED_FIELD)); +// assertEquals(stix2IOCGenerator.getIocs().get(i).getModified().toString(), iocHits.get(i).get(STIX2IOC.MODIFIED_FIELD)); - assertEquals(stix2IOCGenerator.getIocs().get(i).getDescription(), iocs.get(i).get(STIX2IOC.DESCRIPTION_FIELD)); - assertEquals(stix2IOCGenerator.getIocs().get(i).getLabels(), iocs.get(i).get(STIX2IOC.LABELS_FIELD)); - assertEquals(createdId, iocs.get(i).get(STIX2IOC.FEED_ID_FIELD)); - assertEquals(stix2IOCGenerator.getIocs().get(i).getSpecVersion(), iocs.get(i).get(STIX2IOC.SPEC_VERSION_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getDescription(), iocHits.get(i).get(STIX2IOC.DESCRIPTION_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getLabels(), iocHits.get(i).get(STIX2IOC.LABELS_FIELD)); + assertEquals(createdId, iocHits.get(i).get(STIX2IOC.FEED_ID_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getSpecVersion(), iocHits.get(i).get(STIX2IOC.SPEC_VERSION_FIELD)); } } + + public void testWithValidAndInvalidIOCTypes() throws IOException { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Generate test IOCs, and upload them to S3 + int numOfIOCs = 5; + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE))); + s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); + assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size()); + + List types = new ArrayList<>(invalidTypes); + types.addAll(IOCType.types); + + // Execute the test for each invalid type + for (String type : invalidTypes) { + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + + List iocTypes = List.of(type); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + Exception exception = assertThrows(ResponseException.class, () -> + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)) + ); + + String expectedError = "{\"error\":{\"root_cause\":[{\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"}],\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"},\"status\":400}"; + assertTrue(exception.getMessage().contains(expectedError)); + } + } + + public void testWithInvalidIOCTypes() throws IOException { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Generate test IOCs, and upload them to S3 + int numOfIOCs = 5; + stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE))); + s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); + assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size()); + + // Execute the test for each invalid type + for (String type : invalidTypes) { + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + + List iocTypes = List.of(type); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + Exception exception = assertThrows(ResponseException.class, () -> + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)) + ); + + String expectedError = "{\"error\":{\"root_cause\":[{\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"}],\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"},\"status\":400}"; + assertTrue(exception.getMessage().contains(expectedError)); + } + } + + public void testWithNoIOCsToDownload() { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Create the bucket object without any IOCs + PutObjectRequest putObjectRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .build(); + PutObjectResponse putObjectResponse = s3Client.putObject(putObjectRequest, RequestBody.empty()); + assertTrue("Failed to create empty bucket object for type.", putObjectResponse.sdkHttpResponse().isSuccessful()); + + // Execute the test case for each IOC type + for (String type : IOCType.types) { + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + List iocTypes = List.of(type); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + Exception exception = assertThrows(ResponseException.class, () -> + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)) + ); + + String expectedError = "{\"error\":{\"root_cause\":[{\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"}],\"type\":\"status_exception\",\"reason\":\"No compatible Iocs were downloaded for config download_test_feed_name\"},\"status\":400}"; + assertTrue(exception.getMessage().contains(expectedError)); + } + } + + public void testWhenBucketObjectDoesNotExist() { + // Only run tests when required system params are provided + if (!canRunTests) return; + + // Confirm bucket object does not exist + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder() + .bucket(bucketName) + .key(objectKey) + .build(); + assertThrows( + String.format("Object %s in bucket %s should not exist.", objectKey, bucketName), + NoSuchKeyException.class, () -> s3Client.headObject(headObjectRequest) + ); + + // Execute the test case for each IOC type + for (String type : IOCType.types) { + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + List iocTypes = List.of(type); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + Exception exception = assertThrows(ResponseException.class, () -> + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)) + ); + + String expectedError = "{\"error\":{\"root_cause\":[{\"type\":\"no_such_key_exception\",\"reason\":\"The specified key does not exist."; + assertTrue("Exception contains unexpected message: " + exception.getMessage(), exception.getMessage().contains(expectedError)); + } + } + + /** + * Calls the get source config api and checks if the last updated time is different from the time that was passed in + * @param createdId + * @param firstUpdatedTime + * @return + * @throws IOException + */ + protected boolean verifyJobRan(String createdId, String firstUpdatedTime) throws IOException { + Response response; + Map responseBody; + + // call get API to get the latest source config by ID + response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId, Collections.emptyMap(), null); + responseBody = asMap(response); + + String returnedLastUpdatedTime = (String) ((Map) responseBody.get("source_config")).get("last_update_time"); + + if(firstUpdatedTime.equals(returnedLastUpdatedTime.toString()) == false) { + return true; + } + return false; + } } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java index cd651a012..3aa5f739a 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java @@ -45,7 +45,7 @@ public void testCreateIocUploadSourceConfig() throws IOException { List iocs = List.of(new STIX2IOCDto( "id", "name", - IOCType.ipv4_addr, + new IOCType(IOCType.IPV4_TYPE), "value", "severity", null, @@ -59,7 +59,7 @@ public void testCreateIocUploadSourceConfig() throws IOException { IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); Boolean enabled = false; - List iocTypes = List.of("ipv4_addr"); + List iocTypes = List.of(IOCType.IPV4_TYPE); SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( null, null, @@ -113,10 +113,10 @@ public void testCreateIocUploadSourceConfig() throws IOException { // Evaluate response int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); - assertEquals(iocs.size(), totalHits); + assertTrue(iocs.size() < totalHits); //due to default feed leading to more iocs List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); - assertEquals(iocs.size(), iocHits.size()); + assertTrue(iocs.size() < iocHits.size()); // Retrieve all IOCs by feed Ids iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); Assert.assertEquals(200, iocResponse.getStatusLine().getStatusCode()); @@ -129,16 +129,20 @@ public void testCreateIocUploadSourceConfig() throws IOException { iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); assertEquals(iocs.size(), iocHits.size()); // Retrieve all IOCs by ip types - iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of(ListIOCsActionRequest.TYPE_FIELD, "ipv4_addr,domain_name"), null); + Map params = Map.of( + ListIOCsActionRequest.TYPE_FIELD, + String.format("%s,%s", IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE) + ); + iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), params, null); Assert.assertEquals(200, iocResponse.getStatusLine().getStatusCode()); respMap = asMap(iocResponse); // Evaluate response totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); - assertEquals(iocs.size(), totalHits); + assertTrue(iocs.size() < totalHits); iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); - assertEquals(iocs.size(), iocHits.size()); + assertTrue(iocs.size() < iocHits.size()); } } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index bec504072..26ee00495 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -9,14 +9,12 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Monitor; -import org.opensearch.commons.alerting.model.Schedule; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.model.STIX2IOC; -import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; import org.opensearch.securityanalytics.threatIntel.common.RefreshType; import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; @@ -61,7 +59,7 @@ private void indexIocs(List iocVals, String iocIndexName, int i1, String STIX2IOC stix2IOC = new STIX2IOC( iocId, "random", - IOCType.ipv4_addr, + new IOCType(IOCType.IPV4_TYPE), iocVals.get(i1), "", Instant.now(), @@ -97,8 +95,8 @@ private void indexTifSourceConfig(int num, String configId, String indexPattern, null, null, false, - new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(IOCType.ipv4_addr, indexPattern, iocActiveIndex))), - List.of("ipv4_addr"), + new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.IPV4_TYPE), indexPattern, iocActiveIndex))), + List.of(IOCType.IPV4_TYPE), true ); String indexName = SecurityAnalyticsPlugin.JOB_INDEX_NAME; @@ -232,15 +230,15 @@ public static String getMatchAllRequest() { } public static ThreatIntelMonitorDto randomIocScanMonitorDto(String index) { - ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(index, "randomIndex"), List.of("ipv4_addr", "domain_name"), emptyList(), "match", null, "severity"); - ThreatIntelTriggerDto t2 = new ThreatIntelTriggerDto(List.of("randomIndex"), List.of("domain_name"), emptyList(), "nomatch", null, "severity"); - ThreatIntelTriggerDto t3 = new ThreatIntelTriggerDto(emptyList(), List.of("domain_name"), emptyList(), "domainmatchsonomatch", null, "severity"); + ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(index, "randomIndex"), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), emptyList(), "match", null, "severity"); + ThreatIntelTriggerDto t2 = new ThreatIntelTriggerDto(List.of("randomIndex"), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "nomatch", null, "severity"); + ThreatIntelTriggerDto t3 = new ThreatIntelTriggerDto(emptyList(), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "domainmatchsonomatch", null, "severity"); ThreatIntelTriggerDto t4 = new ThreatIntelTriggerDto(List.of(index), emptyList(), emptyList(), "indexmatch", null, "severity"); return new ThreatIntelMonitorDto( Monitor.NO_ID, randomAlphaOfLength(10), - List.of(new PerIocTypeScanInputDto("ipv4_addr", Map.of(index, List.of("ip")))), + List.of(new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(index, List.of("ip")))), new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), false, null, diff --git a/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java b/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java index a7c39bd72..59302beac 100644 --- a/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java +++ b/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java @@ -7,12 +7,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.commons.alerting.model.Table; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.action.ListIOCsActionRequest; import org.opensearch.securityanalytics.commons.model.IOC; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.commons.utils.testUtils.PojoGenerator; @@ -24,6 +22,8 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -35,10 +35,8 @@ import static org.opensearch.test.OpenSearchTestCase.randomLong; public class STIX2IOCGenerator implements PojoGenerator { - List iocs; - - // Optional value. When not null, all IOCs generated will use this type. - IOCType type; + private List iocs; + private List types = IOCType.types.stream().map(IOCType::new).collect(Collectors.toList()); private final ObjectMapper objectMapper; @@ -46,6 +44,11 @@ public STIX2IOCGenerator() { this.objectMapper = new ObjectMapper(); } + public STIX2IOCGenerator(List types) { + this(); + this.types = types; + } + @Override public void write(final int numberOfIOCs, final OutputStream outputStream) { try (final PrintWriter printWriter = new PrintWriter(outputStream)) { @@ -53,10 +56,20 @@ public void write(final int numberOfIOCs, final OutputStream outputStream) { } } + /** + * For each IOCType in 'types', 'numberOfIOCs' will be generated in the bucket object. + * Defaults to generating 'numberOfIOCs' of each IOCType. + * @param numberOfIOCs the number of each IOCType to generate in the bucket object. + * @param printWriter prints formatted representations of objects to a text-output stream. + */ private void writeLines(final int numberOfIOCs, final PrintWriter printWriter) { - final List iocs = IntStream.range(0, numberOfIOCs) - .mapToObj(i -> randomIOC(type)) - .collect(Collectors.toList()); + final List iocs = new ArrayList<>(); + for (IOCType type : types) { + final List newIocs = IntStream.range(0, numberOfIOCs) + .mapToObj(i -> randomIOC(type)) + .collect(Collectors.toList()); + iocs.addAll(newIocs); + } this.iocs = iocs; iocs.forEach(ioc -> writeLine(ioc, printWriter)); } @@ -101,12 +114,8 @@ public List getIocs() { return iocs; } - public IOCType getType() { - return type; - } - - public void setType(IOCType type) { - this.type = type; + public List getTypes() { + return types; } public static STIX2IOC randomIOC( @@ -128,7 +137,7 @@ public static STIX2IOC randomIOC( name = randomLowerCaseString(); } if (type == null) { - type = IOCType.values()[randomInt(IOCType.values().length - 1)]; + type = new IOCType(IOCType.types.get(randomInt(IOCType.types.size() - 1))); } if (value == null) { value = randomLowerCaseString(); @@ -242,6 +251,7 @@ public static void assertIOCEqualsDTO(STIX2IOC ioc, STIX2IOCDto iocDto) { public static void assertEqualIOCs(STIX2IOC ioc, STIX2IOC newIoc) { assertNotNull(newIoc.getId()); assertEquals(ioc.getName(), newIoc.getName()); + assertEquals(ioc.getType().toString(), newIoc.getType().toString()); assertEquals(ioc.getValue(), newIoc.getValue()); assertEquals(ioc.getSeverity(), newIoc.getSeverity()); // assertEquals(ioc.getCreated(), newIoc.getCreated()); @@ -256,6 +266,7 @@ public static void assertEqualIOCs(STIX2IOC ioc, STIX2IOC newIoc) { public static void assertEqualIocDtos(STIX2IOCDto ioc, STIX2IOCDto newIoc) { assertNotNull(newIoc.getId()); assertEquals(ioc.getName(), newIoc.getName()); + assertEquals(ioc.getType().toString(), newIoc.getType().toString()); assertEquals(ioc.getValue(), newIoc.getValue()); assertEquals(ioc.getSeverity(), newIoc.getSeverity()); // assertEquals(ioc.getCreated(), newIoc.getCreated());