diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 4e5932d04..8b09b5405 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -152,6 +152,7 @@ import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestIndexThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestUpdateThreatIntelAlertsStatusAction; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; @@ -317,11 +318,12 @@ public Collection createComponents(Client client, IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry); ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry); SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService); + DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService); return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices,threatIntelAlertService, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, notificationService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, saTifSourceConfigService, saTifSourceConfigManagementService, stix2IOCFetchService, - ioCScanService); + ioCScanService, defaultTifSourceConfigLoaderService); } @Override diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java index 5c81feb8c..8a95bf3c1 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java @@ -7,8 +7,11 @@ import com.amazonaws.AmazonServiceException; import com.amazonaws.SdkClientException; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.bulk.BulkRequest; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.action.ActionListener; @@ -29,6 +32,7 @@ import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig; import org.opensearch.securityanalytics.commons.model.FeedConfiguration; import org.opensearch.securityanalytics.commons.model.IOCSchema; +import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.commons.model.STIX2; import org.opensearch.securityanalytics.commons.model.UpdateType; import org.opensearch.securityanalytics.model.STIX2IOC; @@ -36,7 +40,9 @@ import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedParser; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; @@ -49,10 +55,16 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; +import static org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService.isValidIp; + /** * IOC Service implements operations that interact with retrieving IOCs from data sources, * parsing them into threat intel data models (i.e., [IOC]), and ingesting them to system indexes. @@ -84,14 +96,14 @@ public STIX2IOCFetchService(Client client, ClusterService clusterService) { /** * Method takes in and calls method to rollover and bulk index a list of STIX2IOCs + * * @param saTifSourceConfig * @param stix2IOCList * @param listener */ public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig, List stix2IOCList, - ActionListener listener) - { + ActionListener listener) { STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener); try { feedStore.indexIocs(stix2IOCList); @@ -100,6 +112,7 @@ public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig, listener.onFailure(e); } } + public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener listener) { S3ConnectorConfig s3ConnectorConfig = constructS3ConnectorConfig(saTifSourceConfig); Connector s3Connector = constructS3Connector(s3ConnectorConfig); @@ -145,7 +158,7 @@ private void testS3ClientConnection(S3ConnectorConfig s3ConnectorConfig, ActionL } catch (StsException stsException) { log.warn("S3Client connection test failed with StsException: ", stsException); listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(stsException.statusCode()), stsException.awsErrorDetails().errorMessage())); - } catch (SdkException sdkException ) { + } catch (SdkException sdkException) { // SdkException is a RunTimeException that doesn't have a status code. // Logging the full exception, and providing generic response as output. log.warn("S3Client connection test failed with SdkException: ", sdkException); @@ -228,6 +241,77 @@ private String getEndpoint() { return ""; } + public void downloadFromUrlAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener listener) { + UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource(); + switch (source.getFeedFormat()) { // todo add check to stop user from creating url type config from rest api. only internal allowed + case "csv": + try (CSVParser reader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl())) { + CSVParser noHeaderReader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl()); + boolean notFound = true; + + while (notFound) { + CSVRecord hasHeaderRecord = reader.iterator().next(); + + //if we want to skip this line and keep iterating + if ((hasHeaderRecord.values().length == 1 && "".equals(hasHeaderRecord.values()[0])) || hasHeaderRecord.get(0).charAt(0) == '#' || hasHeaderRecord.get(0).charAt(0) == ' ') { + noHeaderReader.iterator().next(); + } else { // we found the first line that contains information + notFound = false; + } + } + if (source.hasCsvHeader()) { + parseAndSaveThreatIntelFeedDataCSV(reader.iterator(), saTifSourceConfig, listener); + } else { + parseAndSaveThreatIntelFeedDataCSV(noHeaderReader.iterator(), saTifSourceConfig, listener); + } + } catch (Exception e) { + log.error("Failed to download the IoCs in CSV format for source " + saTifSourceConfig.getId()); + listener.onFailure(e); + return; + } + break; + default: + log.error("unsupported feed format for url download:" + source.getFeedFormat()); + listener.onFailure(new UnsupportedOperationException("unsupported feed format for url download:" + source.getFeedFormat())); + } + } + + private void parseAndSaveThreatIntelFeedDataCSV(Iterator iterator, SATIFSourceConfig saTifSourceConfig, ActionListener listener) throws IOException { + List bulkRequestList = new ArrayList<>(); + + UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource(); + List iocs = new ArrayList<>(); + while (iterator.hasNext()) { + CSVRecord record = iterator.next(); + String iocType = saTifSourceConfig.getIocTypes().stream().findFirst().orElse(null); + Integer colNum = source.getCsvIocValueColumnNo(); + String iocValue = record.values()[colNum].split(" ")[0]; + if (iocType.equalsIgnoreCase(IOCType.ipv6_addr.toString()) && !isValidIp(iocValue)) { + log.info("Invalid IP address, skipping this ioc record: {}", iocValue); + continue; + } + Instant now = Instant.now(); + STIX2IOC stix2IOC = new STIX2IOC( + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + iocType == null ? IOCType.ipv4_addr : IOCType.valueOf(iocType), + iocValue, + "high", + now, + now, + "", + Collections.emptyList(), + "", + saTifSourceConfig.getId(), + saTifSourceConfig.getName(), + STIX2IOC.NO_VERSION + ); + iocs.add(stix2IOC); + } + STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener); + feedStore.indexIocs(iocs); + } + public static class STIX2IOCFetchResponse extends ActionResponse implements ToXContentObject { public static String IOCS_FIELD = "iocs"; public static String TOTAL_FIELD = "total"; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java index c08b74eea..6cb4450e4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java @@ -9,6 +9,7 @@ import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import java.util.ArrayList; import java.util.Arrays; @@ -55,6 +56,14 @@ public List validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto errorMsgs.add("Source must be S3_CUSTOM type"); } break; + case URL_DOWNLOAD: + if (sourceConfigDto.getSchedule() == null) { + errorMsgs.add("Must pass in schedule for URL_DOWNLOAD source type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof UrlDownloadSource == false) { + errorMsgs.add("Source must be URL_DOWNLOAD source type"); + } + break; } return errorMsgs; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java index 04f7e8034..8efa5cfa5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java @@ -7,11 +7,11 @@ /** * Types of feeds threat intel can support - * Feed types include: S3_CUSTOM */ public enum SourceConfigType { S3_CUSTOM, - IOC_UPLOAD + IOC_UPLOAD, + URL_DOWNLOAD // LICENSED, // diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java index 8f79143e3..865120fac 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java @@ -96,8 +96,4 @@ public void setIocs(List iocs) { public String getFileName() { return fileName; } - - public void setFileName(String fileName) { - this.fileName = fileName; - } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java index a9d75c646..61444fe67 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java @@ -20,6 +20,7 @@ public abstract class Source { abstract String name(); public static final String S3_FIELD = "s3"; public static final String IOC_UPLOAD_FIELD = "ioc_upload"; + public static final String URL_DOWNLOAD_FIELD = "url_download"; static Source readFrom(StreamInput sin) throws IOException { Type type = sin.readEnum(Type.class); @@ -28,6 +29,8 @@ static Source readFrom(StreamInput sin) throws IOException { return new S3Source(sin); case IOC_UPLOAD: return new IocUploadSource(sin); + case URL_DOWNLOAD: + return new UrlDownloadSource(sin); default: throw new IllegalStateException("Unexpected input ["+ type + "] when reading ioc store config"); } @@ -47,6 +50,9 @@ static Source parse(XContentParser xcp) throws IOException { case IOC_UPLOAD_FIELD: source = IocUploadSource.parse(xcp); break; + case URL_DOWNLOAD_FIELD: + source = UrlDownloadSource.parse(xcp); + break; } } return source; @@ -57,7 +63,9 @@ public void writeTo(StreamOutput out) throws IOException {} enum Type { S3(), - IOC_UPLOAD(); + IOC_UPLOAD(), + + URL_DOWNLOAD(); @Override public String toString() { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java new file mode 100644 index 000000000..fdc2d9756 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java @@ -0,0 +1,117 @@ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.net.URL; + +/** + * This is a Threat Intel Source config where the iocs are downloaded from the URL + */ +public class UrlDownloadSource extends Source implements Writeable, ToXContent { + public static final String URL_FIELD = "url"; + public static final String FEED_FORMAT_FIELD = "feed_format"; + public static final String HAS_CSV_HEADER_FIELD = "has_csv_header_field"; + public static final String CSV_IOC_VALUE_COLUMN_NUM_FIELD = "csv_ioc_value_colum_num"; + public static final String SOURCE_NAME = "URL_DOWNLOAD"; + + private final URL url; + private final String feedFormat; + private final Boolean hasCsvHeader; + private final Integer csvIocValueColumnNo; + + public UrlDownloadSource(URL url, String feedFormat, Boolean hasCsvHeader, Integer csvIocValueColumnNo) { + this.url = url; + this.feedFormat = feedFormat; + this.hasCsvHeader = hasCsvHeader; + this.csvIocValueColumnNo = csvIocValueColumnNo; + + } + + public UrlDownloadSource(StreamInput sin) throws IOException { + this( + new URL(sin.readString()), + sin.readString(), + sin.readOptionalBoolean(), + sin.readOptionalInt() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(url.toString()); + out.writeString(feedFormat); + out.writeOptionalBoolean(hasCsvHeader); + out.writeOptionalInt(csvIocValueColumnNo); + } + + @Override + String name() { + return SOURCE_NAME; + } + + public URL getUrl() { + return url; + } + + public static UrlDownloadSource parse(XContentParser xcp) throws IOException { + URL url = null; + String feedFormat = null; + Boolean hasCsvHeader = false; + Integer csvIocValueColumnNo = null; + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case URL_FIELD: + String urlString = xcp.text(); + url = new URL(urlString); + break; + case FEED_FORMAT_FIELD: + feedFormat = xcp.text(); + break; + case HAS_CSV_HEADER_FIELD: + hasCsvHeader = xcp.booleanValue(); + break; + case CSV_IOC_VALUE_COLUMN_NUM_FIELD: + if (xcp.currentToken() == null) + xcp.skipChildren(); + else + csvIocValueColumnNo = xcp.intValue(); + break; + default: + xcp.skipChildren(); + } + } + return new UrlDownloadSource(url, feedFormat, hasCsvHeader, csvIocValueColumnNo); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .startObject(URL_DOWNLOAD_FIELD) + .field(URL_FIELD, url.toString()) + .field(FEED_FORMAT_FIELD, feedFormat) + .field(HAS_CSV_HEADER_FIELD, hasCsvHeader) + .field(CSV_IOC_VALUE_COLUMN_NUM_FIELD, csvIocValueColumnNo) + .endObject() + .endObject(); + } + + public String getFeedFormat() { + return feedFormat; + } + + public boolean hasCsvHeader() { + return hasCsvHeader; + } + + public Integer getCsvIocValueColumnNo() { + return csvIocValueColumnNo; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java new file mode 100644 index 000000000..4114a9526 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java @@ -0,0 +1,180 @@ +package org.opensearch.securityanalytics.threatIntel.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.rest.RestRequest; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; + +import java.net.URL; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +//todo handle refresh, update tif config +// todo block creation of url based config in transport layer +public class DefaultTifSourceConfigLoaderService { + private static final Logger log = LogManager.getLogger(DefaultTifSourceConfigLoaderService.class); + private final BuiltInTIFMetadataLoader tifMetadataLoader; + private final Client client; + private final SATIFSourceConfigManagementService satifSourceConfigManagementService; + + public DefaultTifSourceConfigLoaderService(BuiltInTIFMetadataLoader tifMetadataLoader, Client client, SATIFSourceConfigManagementService satifSourceConfigManagementService) { + this.tifMetadataLoader = tifMetadataLoader; + this.client = client; + this.satifSourceConfigManagementService = satifSourceConfigManagementService; + } + + /** + * check if the default tif source configs are loaded. if not, try create them from the feedMetadata.json file. + */ + public void createDefaultTifConfigsIfNotExists(ActionListener listener) { + List tifMetadataList = tifMetadataLoader.getTifMetadataList(); + if (tifMetadataList.isEmpty()) { + log.error("No built-in TIF Configs found"); + listener.onResponse(null); + return; + } + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + for (TIFMetadata tifMetadata : tifMetadataList) { + boolQueryBuilder.should(new MatchQueryBuilder("_id", tifMetadata.getFeedId())); + } + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder).size(9999); + satifSourceConfigManagementService.searchTIFSourceConfigs(searchSourceBuilder, + ActionListener.wrap(searchResponse -> { + createTifConfigsThatDontExist(searchResponse, tifMetadataList, listener); + }, e -> { + log.error("Failed to search tif config index for default tif configs", e); + listener.onFailure(e); + })); + } + + private void createTifConfigsThatDontExist(SearchResponse searchResponse, List tifMetadataList, ActionListener listener) { + Map feedsToCreate = tifMetadataList.stream() + .collect(Collectors.toMap( + TIFMetadata::getFeedId, + Function.identity() + )); + if (searchResponse.getHits() != null && searchResponse.getHits().getHits() != null) { + for (SearchHit hit : searchResponse.getHits().getHits()) { + feedsToCreate.remove(hit.getId()); + } + } + if (feedsToCreate.isEmpty()) { + listener.onResponse(null); + return; + } + GroupedActionListener> groupedActionListener = new GroupedActionListener<>( + new ActionListener<>() { + @Override + public void onResponse(Collection> responseOrExceptions) { + if (responseOrExceptions.stream().allMatch(it -> it.getException() != null)) { // all configs returned error + Exception e = responseOrExceptions.stream().findFirst().get().getException(); + log.error("Failed to create default tif configs", e); + listener.onFailure(e); + return; + } + listener.onResponse(null); + return; + } + + @Override + public void onFailure(Exception e) { + log.error("Unexpected failure while creating Default Threat intel source configs", e); + listener.onFailure(e); + return; + } + }, feedsToCreate.size() + ); + for (TIFMetadata tifMetadata : feedsToCreate.values()) { + if (tifMetadata == null) { + continue; + } + try { + Instant now = Instant.now(); + String iocType = null; + if (tifMetadata.getIocType().equalsIgnoreCase("ip")) { + iocType = IOCType.ipv4_addr.toString(); + } + satifSourceConfigManagementService.createOrUpdateTifSourceConfig( + new SATIFSourceConfigDto( + tifMetadata.getFeedId(), + SATIFSourceConfigDto.NO_VERSION, + tifMetadata.getName(), + "STIX2", + SourceConfigType.URL_DOWNLOAD, + tifMetadata.getDescription(), + null, + now, + new UrlDownloadSource(new URL(tifMetadata.getUrl()), tifMetadata.getFeedType(), tifMetadata.hasHeader(), tifMetadata.getIocCol()), + now, + now, + new IntervalSchedule(now, 1, ChronoUnit.DAYS), + TIFJobState.CREATING, + RefreshType.FULL, + null, + null, + true, + List.of(iocType), + true + ), + null, + RestRequest.Method.POST, + null, + ActionListener.wrap( + r -> { + groupedActionListener.onResponse(new ResponseOrException<>(r, null)); + }, + e -> { + log.error("failed to create default tif source config " + tifMetadata.getFeedId(), e); + groupedActionListener.onResponse(new ResponseOrException<>(null, e)); + }) + ); + continue; + } catch (Exception ex) { + log.error("Unexpected failure while creating Default Threat intel source configs " + tifMetadata.getFeedId(), ex); + groupedActionListener.onResponse(new ResponseOrException<>(null, ex)); + continue; + } + } + } + + private static class ResponseOrException { + private final R response; + private final Exception exception; + + private ResponseOrException(R response, Exception exception) { + this.response = response; + this.exception = exception; + } + + public R getResponse() { + return response; + } + + public Exception getException() { + return exception; + } + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index 60ebc9e8f..bdcd5713c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -48,10 +48,10 @@ import java.util.Map; import java.util.Set; import java.util.SortedMap; - import java.util.stream.Collectors; import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.IOC_UPLOAD; +import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.URL_DOWNLOAD; /** * Service class for threat intel feed source config object @@ -193,6 +193,9 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig, case S3_CUSTOM: stix2IOCFetchService.downloadAndIndexIOCs(saTifSourceConfig, actionListener); break; + case URL_DOWNLOAD: + stix2IOCFetchService.downloadFromUrlAndIndexIOCs(saTifSourceConfig, actionListener); + break; case IOC_UPLOAD: List validStix2IocList = new ArrayList<>(); // If the IOC received is not a type listed for the config, do not add it to the queue @@ -509,6 +512,11 @@ public void deleteTIFSourceConfig( ) { saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigId, ActionListener.wrap( saTifSourceConfig -> { + if (URL_DOWNLOAD.equals(saTifSourceConfig.getType())) { + log.error("Cannot delete tif source config {} as it's a built-in config and not user-defined.", saTifSourceConfigId); + listener.onFailure(new IllegalArgumentException("Cannot delete built-in tif source config " + saTifSourceConfigId)); + return; + } // Check if all threat intel monitors are deleted saTifSourceConfigService.checkAndEnsureThreatIntelMonitorsDeleted(ActionListener.wrap( isDeleted -> { @@ -768,15 +776,42 @@ public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto saTifSourceCo } private SATIFSourceConfig updateSaTifSourceConfig(SATIFSourceConfigDto saTifSourceConfigDto, SATIFSourceConfig saTifSourceConfig) { + // currently url download is only for default tif configs and supports only activate/deactivate. Ideally should be via an activate API + if (URL_DOWNLOAD.equals(saTifSourceConfig.getType())) { + return new SATIFSourceConfig( + saTifSourceConfig.getId(), + saTifSourceConfig.getVersion(), + saTifSourceConfig.getName(), + saTifSourceConfig.getFormat(), + saTifSourceConfig.getType(), + saTifSourceConfig.getDescription(), + saTifSourceConfig.getCreatedByUser(), + saTifSourceConfig.getCreatedAt(), + saTifSourceConfig.getSource(), + saTifSourceConfig.getEnabledTime(), + saTifSourceConfig.getLastUpdateTime(), + saTifSourceConfig.getSchedule(), + saTifSourceConfig.getState(), + saTifSourceConfig.getRefreshType(), + saTifSourceConfig.getLastRefreshedTime(), + saTifSourceConfig.getLastRefreshedUser(), + saTifSourceConfig.isEnabled(), + saTifSourceConfig.getIocStoreConfig(), + saTifSourceConfig.getIocTypes(), + saTifSourceConfigDto.isEnabledForScan() + ); + } + if (false == saTifSourceConfig.getSource().getClass().equals(saTifSourceConfigDto.getSource().getClass())) { + throw new IllegalArgumentException(""); + } // remove duplicates from iocTypes Set iocTypes = new LinkedHashSet<>(saTifSourceConfigDto.getIocTypes()); - return new SATIFSourceConfig( saTifSourceConfig.getId(), saTifSourceConfig.getVersion(), saTifSourceConfigDto.getName(), saTifSourceConfigDto.getFormat(), - saTifSourceConfigDto.getType(), + saTifSourceConfig.getType(), saTifSourceConfigDto.getDescription(), saTifSourceConfig.getCreatedByUser(), saTifSourceConfig.getCreatedAt(), diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java index eb90415b4..ab6ed9915 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java @@ -187,7 +187,7 @@ public void onFailure(Exception e) { } break; default: - // if the feed type doesn't match any of the supporting feed types, throw an exception + onFailure(new UnsupportedOperationException("Not a supported feed format : " + tifMetadata.getFeedType())); } } } catch (IOException ex) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java index 61ea2374d..1cb9e7428 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java @@ -229,6 +229,8 @@ public void parseAndSaveThreatIntelFeedDataCSV( } public static boolean isValidIp(String ip) { + if (StringUtils.isBlank(ip)) + return false; String ipPattern = "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$"; Pattern pattern = Pattern.compile(ipPattern); Matcher matcher = pattern.matcher(ip); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java index ae06d7724..9b6378cf9 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java @@ -15,12 +15,15 @@ import org.opensearch.commons.authuser.User; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.RestRequest; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -94,6 +97,11 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques } try { SATIFSourceConfigDto saTifSourceConfigDto = request.getTIFConfigDto(); + if (SourceConfigType.URL_DOWNLOAD.equals(saTifSourceConfigDto.getType()) || saTifSourceConfigDto.getSource() instanceof UrlDownloadSource + && request.getMethod().equals(RestRequest.Method.POST)) { + listener.onFailure(new UnsupportedOperationException("Unsupported Threat intel Source Config Type passed - " + saTifSourceConfigDto.getType())); + return; + } saTifSourceConfigManagementService.createOrUpdateTifSourceConfig( saTifSourceConfigDto, lock, diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java index d046a35e5..9eb47f0a3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java @@ -3,6 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.StepListener; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; @@ -15,6 +16,7 @@ import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsAction; import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsRequest; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.tasks.Task; @@ -28,6 +30,7 @@ public class TransportSearchTIFSourceConfigsAction extends HandledTransportActio private final ClusterService clusterService; private final Settings settings; + private final DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService; private final ThreadPool threadPool; @@ -41,11 +44,13 @@ public TransportSearchTIFSourceConfigsAction(TransportService transportService, ClusterService clusterService, final ThreadPool threadPool, Settings settings, + DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService, final SATIFSourceConfigManagementService saTifConfigService) { super(SASearchTIFSourceConfigsAction.NAME, transportService, actionFilters, SASearchTIFSourceConfigsRequest::new); this.clusterService = clusterService; this.threadPool = threadPool; this.settings = settings; + this.defaultTifSourceConfigLoaderService = defaultTifSourceConfigLoaderService; this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); this.saTifConfigService = saTifConfigService; @@ -63,16 +68,38 @@ protected void doExecute(Task task, SASearchTIFSourceConfigsRequest request, Act } this.threadPool.getThreadContext().stashContext(); // stash context to make calls as admin client - - saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( - r -> { - log.debug("Successfully listed all threat intel source configs"); - actionListener.onResponse(r); - }, e -> { - log.error("Failed to list all threat intel source configs"); - actionListener.onFailure(e); - } - )); + StepListener defaultTifConfigsLoadedListener; + try { + defaultTifConfigsLoadedListener = new StepListener<>(); + defaultTifSourceConfigLoaderService.createDefaultTifConfigsIfNotExists(defaultTifConfigsLoadedListener); + defaultTifConfigsLoadedListener.whenComplete(res -> saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, e -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + } + )), ex -> saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, e -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + } + ))); + } catch (Exception e) { + log.error("Failed to load default tif source configs. Moving on to list iocs", e); + saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, ex -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + })); + } } private void setFilterByEnabled(boolean filterByEnabled) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java index bfbb9dbde..3cbf31086 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java @@ -42,9 +42,27 @@ public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadat connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); return new CSVParser(new BufferedReader(new InputStreamReader(connection.getInputStream())), CSVFormat.RFC4180); } catch (IOException e) { - log.error("Exception: failed to read threat intel feed data from {}",tifMetadata.getUrl(), e); + log.error("Exception: failed to read threat intel feed data from {}", tifMetadata.getUrl(), e); throw new OpenSearchException("failed to read threat intel feed data from {}", tifMetadata.getUrl(), e); } }); } + + /** + * Create CSVParser of a threat intel feed + */ + @SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file") + public static CSVParser getThreatIntelFeedReaderCSV(URL url) { + SpecialPermission.check(); + return AccessController.doPrivileged((PrivilegedAction) () -> { + try { + URLConnection connection = url.openConnection(); + connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); + return new CSVParser(new BufferedReader(new InputStreamReader(connection.getInputStream())), CSVFormat.RFC4180); + } catch (IOException e) { + log.error("Exception: failed to read threat intel feed data from {}", url, e); + throw new OpenSearchException("failed to read threat intel feed data from {}", url, e); + } + }); + } } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java index 132725d71..2e1954ce7 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchStatusException; import org.opensearch.action.ActionRunnable; +import org.opensearch.action.StepListener; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; @@ -43,6 +44,7 @@ import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; import org.opensearch.securityanalytics.threatIntel.transport.TransportSearchTIFSourceConfigsAction; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -68,6 +70,7 @@ public class TransportListIOCsAction extends HandledTransportAction defaultTifConfigsLoadedListener = null; + try { + defaultTifConfigsLoadedListener = new StepListener<>(); + defaultTifSourceConfigLoaderService.createDefaultTifConfigsIfNotExists(defaultTifConfigsLoadedListener); + defaultTifConfigsLoadedListener.whenComplete(r -> searchIocs(), e -> searchIocs()); + } catch (Exception e) { + log.error("Failed to load default tif source configs. Moving on to list iocs", e); + searchIocs(); + } + } + + private void searchIocs() { /** get all match threat intel source configs. fetch write index of each config if no iocs provided else fetch just index alias */ List configIds = request.getFeedIds() == null ? Collections.emptyList() : request.getFeedIds(); saTifSourceConfigService.searchTIFSourceConfigs(getFeedsSearchSourceBuilder(configIds), @@ -128,7 +145,7 @@ void start() { SATIFSourceConfig config = SATIFSourceConfig.docParse(xcp, hit.getId(), hit.getVersion()); if (config.getIocStoreConfig() instanceof DefaultIocStoreConfig) { DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) config.getIocStoreConfig(); - for (DefaultIocStoreConfig.IocToIndexDetails iocToindexDetails: iocStoreConfig.getIocToIndexDetails()) { + for (DefaultIocStoreConfig.IocToIndexDetails iocToindexDetails : iocStoreConfig.getIocToIndexDetails()) { String writeIndex = iocToindexDetails.getActiveIndex(); if (writeIndex != null) { iocIndices.add(writeIndex); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java index cd651a012..5da58f79a 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java @@ -113,10 +113,10 @@ public void testCreateIocUploadSourceConfig() throws IOException { // Evaluate response int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); - assertEquals(iocs.size(), totalHits); + assertTrue(iocs.size() < totalHits); //due to default feed leading to more iocs List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); - assertEquals(iocs.size(), iocHits.size()); + assertTrue(iocs.size() < iocHits.size()); // Retrieve all IOCs by feed Ids iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); Assert.assertEquals(200, iocResponse.getStatusLine().getStatusCode()); @@ -135,10 +135,10 @@ public void testCreateIocUploadSourceConfig() throws IOException { // Evaluate response totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); - assertEquals(iocs.size(), totalHits); + assertTrue(iocs.size() < totalHits); iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); - assertEquals(iocs.size(), iocHits.size()); + assertTrue(iocs.size() < iocHits.size()); } }