diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 36fa71a2a..8ceb790c7 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -157,6 +157,7 @@ import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestIndexThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestUpdateThreatIntelAlertsStatusAction; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; @@ -326,12 +327,13 @@ public Collection createComponents(Client client, IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry); ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry); SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService); + DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService); return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, threatIntelAlertService, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, correlationAlertService, notificationService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, saTifSourceConfigService, saTifSourceConfigManagementService, stix2IOCFetchService, - ioCScanService); + ioCScanService, defaultTifSourceConfigLoaderService); } @Override diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java index 867958a84..17995a571 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java @@ -7,8 +7,11 @@ import com.amazonaws.AmazonServiceException; import com.amazonaws.SdkClientException; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.bulk.BulkRequest; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.action.ActionListener; @@ -29,6 +32,7 @@ import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig; import org.opensearch.securityanalytics.commons.model.FeedConfiguration; import org.opensearch.securityanalytics.commons.model.IOCSchema; +import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.commons.model.STIX2; import org.opensearch.securityanalytics.commons.model.UpdateType; import org.opensearch.securityanalytics.model.STIX2IOC; @@ -36,7 +40,9 @@ import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedParser; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; @@ -49,10 +55,16 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; +import static org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService.isValidIp; + /** * IOC Service implements operations that interact with retrieving IOCs from data sources, * parsing them into threat intel data models (i.e., [IOC]), and ingesting them to system indexes. @@ -84,14 +96,14 @@ public STIX2IOCFetchService(Client client, ClusterService clusterService) { /** * Method takes in and calls method to rollover and bulk index a list of STIX2IOCs + * * @param saTifSourceConfig * @param stix2IOCList * @param listener */ public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig, List stix2IOCList, - ActionListener listener) - { + ActionListener listener) { STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener); try { feedStore.indexIocs(stix2IOCList); @@ -100,6 +112,7 @@ public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig, listener.onFailure(e); } } + public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener listener) { S3ConnectorConfig s3ConnectorConfig = constructS3ConnectorConfig(saTifSourceConfig); Connector s3Connector = constructS3Connector(s3ConnectorConfig); @@ -144,7 +157,7 @@ private void testS3ClientConnection(S3ConnectorConfig s3ConnectorConfig, ActionL } catch (StsException stsException) { log.warn("S3Client connection test failed with StsException: ", stsException); listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(stsException.statusCode()), stsException.awsErrorDetails().errorMessage())); - } catch (SdkException sdkException ) { + } catch (SdkException sdkException) { // SdkException is a RunTimeException that doesn't have a status code. // Logging the full exception, and providing generic response as output. log.warn("S3Client connection test failed with SdkException: ", sdkException); @@ -227,6 +240,76 @@ private String getEndpoint() { return ""; } + public void downloadFromUrlAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener listener) { + UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource(); + switch (source.getFeedFormat()) { // todo add check to stop user from creating url type config from rest api. only internal allowed + case "csv": + try (CSVParser reader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl())) { + CSVParser noHeaderReader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl()); + boolean notFound = true; + + while (notFound) { + CSVRecord hasHeaderRecord = reader.iterator().next(); + + //if we want to skip this line and keep iterating + if ((hasHeaderRecord.values().length == 1 && "".equals(hasHeaderRecord.values()[0])) || hasHeaderRecord.get(0).charAt(0) == '#' || hasHeaderRecord.get(0).charAt(0) == ' ') { + noHeaderReader.iterator().next(); + } else { // we found the first line that contains information + notFound = false; + } + } + if (source.hasCsvHeader()) { + parseAndSaveThreatIntelFeedDataCSV(reader.iterator(), saTifSourceConfig, listener); + } else { + parseAndSaveThreatIntelFeedDataCSV(noHeaderReader.iterator(), saTifSourceConfig, listener); + } + } catch (Exception e) { + log.error("Failed to download the IoCs in CSV format for source " + saTifSourceConfig.getId()); + listener.onFailure(e); + return; + } + break; + default: + // if the feed type doesn't match any of the supporting feed types, throw an exception + } + } + + private void parseAndSaveThreatIntelFeedDataCSV(Iterator iterator, SATIFSourceConfig saTifSourceConfig, ActionListener listener) throws IOException { + List bulkRequestList = new ArrayList<>(); + + UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource(); + List iocs = new ArrayList<>(); + while (iterator.hasNext()) { + CSVRecord record = iterator.next(); + String iocType = saTifSourceConfig.getIocTypes().stream().findFirst().orElse(null); + Integer colNum = source.getCsvIocValueColumnNo(); + String iocValue = record.values()[colNum].split(" ")[0]; + if (iocType.equalsIgnoreCase(IOCType.ipv6_addr.toString()) && !isValidIp(iocValue)) { + log.info("Invalid IP address, skipping this ioc record: {}", iocValue); + continue; + } + Instant now = Instant.now(); + STIX2IOC stix2IOC = new STIX2IOC( + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + iocType == null ? IOCType.ipv4_addr : IOCType.valueOf(iocType), + iocValue, + "high", + now, + now, + "", + Collections.emptyList(), + "", + saTifSourceConfig.getId(), + saTifSourceConfig.getName(), + STIX2IOC.NO_VERSION + ); + iocs.add(stix2IOC); + } + STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener); + feedStore.indexIocs(iocs); + } + public static class STIX2IOCFetchResponse extends ActionResponse implements ToXContentObject { public static String IOCS_FIELD = "iocs"; public static String TOTAL_FIELD = "total"; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java index 6bcd483fe..d6e7da47a 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java @@ -8,6 +8,7 @@ import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import java.util.ArrayList; import java.util.List; @@ -42,6 +43,14 @@ public List validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto errorMsgs.add("Source must be S3_CUSTOM type"); } break; + case URL_DOWNLOAD: + if (sourceConfigDto.getSchedule() == null) { + errorMsgs.add("Must pass in schedule for URL_DONWLOAD source type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof UrlDownloadSource == false) { + errorMsgs.add("Source must be URL_DONWLOAD source type"); + } + break; } return errorMsgs; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java index 04f7e8034..8efa5cfa5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java @@ -7,11 +7,11 @@ /** * Types of feeds threat intel can support - * Feed types include: S3_CUSTOM */ public enum SourceConfigType { S3_CUSTOM, - IOC_UPLOAD + IOC_UPLOAD, + URL_DOWNLOAD // LICENSED, // diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java index 7f607e88a..61444fe67 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java @@ -50,6 +50,9 @@ static Source parse(XContentParser xcp) throws IOException { case IOC_UPLOAD_FIELD: source = IocUploadSource.parse(xcp); break; + case URL_DOWNLOAD_FIELD: + source = UrlDownloadSource.parse(xcp); + break; } } return source; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java index 5e37dd17d..fdc2d9756 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java @@ -1,6 +1,7 @@ package org.opensearch.securityanalytics.threatIntel.model; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -14,16 +15,39 @@ */ public class UrlDownloadSource extends Source implements Writeable, ToXContent { public static final String URL_FIELD = "url"; + public static final String FEED_FORMAT_FIELD = "feed_format"; + public static final String HAS_CSV_HEADER_FIELD = "has_csv_header_field"; + public static final String CSV_IOC_VALUE_COLUMN_NUM_FIELD = "csv_ioc_value_colum_num"; public static final String SOURCE_NAME = "URL_DOWNLOAD"; private final URL url; + private final String feedFormat; + private final Boolean hasCsvHeader; + private final Integer csvIocValueColumnNo; - public UrlDownloadSource(URL url) { + public UrlDownloadSource(URL url, String feedFormat, Boolean hasCsvHeader, Integer csvIocValueColumnNo) { this.url = url; + this.feedFormat = feedFormat; + this.hasCsvHeader = hasCsvHeader; + this.csvIocValueColumnNo = csvIocValueColumnNo; + } public UrlDownloadSource(StreamInput sin) throws IOException { - this(new URL(sin.readString())); + this( + new URL(sin.readString()), + sin.readString(), + sin.readOptionalBoolean(), + sin.readOptionalInt() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(url.toString()); + out.writeString(feedFormat); + out.writeOptionalBoolean(hasCsvHeader); + out.writeOptionalInt(csvIocValueColumnNo); } @Override @@ -37,6 +61,9 @@ public URL getUrl() { public static UrlDownloadSource parse(XContentParser xcp) throws IOException { URL url = null; + String feedFormat = null; + Boolean hasCsvHeader = false; + Integer csvIocValueColumnNo = null; while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = xcp.currentName(); xcp.nextToken(); @@ -45,19 +72,46 @@ public static UrlDownloadSource parse(XContentParser xcp) throws IOException { String urlString = xcp.text(); url = new URL(urlString); break; + case FEED_FORMAT_FIELD: + feedFormat = xcp.text(); + break; + case HAS_CSV_HEADER_FIELD: + hasCsvHeader = xcp.booleanValue(); + break; + case CSV_IOC_VALUE_COLUMN_NUM_FIELD: + if (xcp.currentToken() == null) + xcp.skipChildren(); + else + csvIocValueColumnNo = xcp.intValue(); + break; default: xcp.skipChildren(); } } - return new UrlDownloadSource(url); + return new UrlDownloadSource(url, feedFormat, hasCsvHeader, csvIocValueColumnNo); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.startObject() .startObject(URL_DOWNLOAD_FIELD) - .field(URL_FIELD, url) + .field(URL_FIELD, url.toString()) + .field(FEED_FORMAT_FIELD, feedFormat) + .field(HAS_CSV_HEADER_FIELD, hasCsvHeader) + .field(CSV_IOC_VALUE_COLUMN_NUM_FIELD, csvIocValueColumnNo) .endObject() .endObject(); } + + public String getFeedFormat() { + return feedFormat; + } + + public boolean hasCsvHeader() { + return hasCsvHeader; + } + + public Integer getCsvIocValueColumnNo() { + return csvIocValueColumnNo; + } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java new file mode 100644 index 000000000..f488bbaf3 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java @@ -0,0 +1,179 @@ +package org.opensearch.securityanalytics.threatIntel.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.rest.RestRequest; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; + +import java.net.URL; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +//todo handle refresh, update tif config +// todo block creation of url based config in transport layer +public class DefaultTifSourceConfigLoaderService { + private static final Logger log = LogManager.getLogger(DefaultTifSourceConfigLoaderService.class); + private final BuiltInTIFMetadataLoader tifMetadataLoader; + private final Client client; + private final SATIFSourceConfigManagementService satifSourceConfigManagementService; + + public DefaultTifSourceConfigLoaderService(BuiltInTIFMetadataLoader tifMetadataLoader, Client client, SATIFSourceConfigManagementService satifSourceConfigManagementService) { + this.tifMetadataLoader = tifMetadataLoader; + this.client = client; + this.satifSourceConfigManagementService = satifSourceConfigManagementService; + } + + /** + * check if the default + */ + public void createDefaultTifConfigsIfNotExists(ActionListener listener) { + List tifMetadataList = tifMetadataLoader.getTifMetadataList(); + if (tifMetadataList.isEmpty()) { + log.error("No built-in TIF Configs found"); + listener.onResponse(null); + return; + } + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + for (TIFMetadata tifMetadata : tifMetadataList) { + boolQueryBuilder.should(new MatchQueryBuilder("_id", tifMetadata.getFeedId())); + } + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder).size(9999); + satifSourceConfigManagementService.searchTIFSourceConfigs(searchSourceBuilder, + ActionListener.wrap(searchResponse -> { + createTifConfigsThatDontExist(searchResponse, tifMetadataList, listener); + }, e -> { + log.error("Failed to search tif config index for default tif configs", e); + listener.onFailure(e); + })); + } + + private void createTifConfigsThatDontExist(SearchResponse searchResponse, List tifMetadataList, ActionListener listener) { + Map feedsToCreate = tifMetadataList.stream() + .collect(Collectors.toMap( + TIFMetadata::getFeedId, + Function.identity() + )); + if (searchResponse.getHits() != null && searchResponse.getHits().getHits() != null) { + for (SearchHit hit : searchResponse.getHits().getHits()) { + feedsToCreate.remove(hit.getId()); + } + } + if (feedsToCreate.isEmpty()) { + listener.onResponse(null); + return; + } + GroupedActionListener> groupedActionListener = new GroupedActionListener<>( + new ActionListener<>() { + @Override + public void onResponse(Collection> responseOrExceptions) { + if (responseOrExceptions.stream().allMatch(it -> it.getException() != null)) { // all configs returned error + Exception e = responseOrExceptions.stream().findFirst().get().getException(); + log.error("Failed to create default tif configs", e); + listener.onFailure(e); + return; + } + listener.onResponse(null); + return; + } + + @Override + public void onFailure(Exception e) { + log.error("Unexpected failure while creating Default Threat intel source configs", e); + listener.onFailure(e); + return; + } + }, feedsToCreate.size() + ); + for (TIFMetadata tifMetadata : feedsToCreate.values()) { + if (tifMetadata == null) { + continue; + } + try { + Instant now = Instant.now(); + String iocType = null; + if (tifMetadata.getIocType().equalsIgnoreCase("ip")) { + iocType = IOCType.ipv4_addr.toString(); + } + satifSourceConfigManagementService.createOrUpdateTifSourceConfig( + new SATIFSourceConfigDto( + tifMetadata.getFeedId(), + SATIFSourceConfigDto.NO_VERSION, + tifMetadata.getName(), + "STIX2", + SourceConfigType.URL_DOWNLOAD, + tifMetadata.getDescription(), + null, + now, + new UrlDownloadSource(new URL(tifMetadata.getUrl()), tifMetadata.getFeedType(), tifMetadata.hasHeader(), tifMetadata.getIocCol()), + now, + now, + new IntervalSchedule(now, 1, ChronoUnit.DAYS), + TIFJobState.CREATING, + RefreshType.FULL, + null, + null, + true, + List.of(iocType) + ), + null, + RestRequest.Method.POST, + null, + ActionListener.wrap( + r -> { + groupedActionListener.onResponse(new ResponseOrException<>(r, null)); + }, + e -> { + log.error("failed to create default tif source config " + tifMetadata.getFeedId(), e); + groupedActionListener.onResponse(new ResponseOrException<>(null, e)); + }) + ); + continue; + } catch (Exception ex) { + log.error("Unexpected failure while creating Default Threat intel source configs " + tifMetadata.getFeedId(), ex); + groupedActionListener.onResponse(new ResponseOrException<>(null, ex)); + continue; + } + } + } + + private static class ResponseOrException { + private final R response; + private final Exception exception; + + private ResponseOrException(R response, Exception exception) { + this.response = response; + this.exception = exception; + } + + public R getResponse() { + return response; + } + + public Exception getException() { + return exception; + } + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index fd164224d..7a4c82c79 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -49,11 +49,9 @@ import java.util.List; import java.util.Map; import java.util.SortedMap; - -import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getIocIndexAlias; - import java.util.stream.Collectors; +import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getIocIndexAlias; import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.IOC_UPLOAD; /** @@ -196,6 +194,9 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig, case S3_CUSTOM: stix2IOCFetchService.downloadAndIndexIOCs(saTifSourceConfig, actionListener); break; + case URL_DOWNLOAD: + stix2IOCFetchService.downloadFromUrlAndIndexIOCs(saTifSourceConfig, actionListener); + break; case IOC_UPLOAD: stix2IOCFetchService.onlyIndexIocs(saTifSourceConfig, stix2IOCList, actionListener); break; @@ -354,7 +355,7 @@ private void storeAndDeleteIocIndices(List stix2IOCList, ActionListene List indicesToDelete = new ArrayList<>(); String alias = getIocIndexAlias(updatedSaTifSourceConfig.getId()); String writeIndex = IndexUtils.getWriteIndex(alias, clusterService.state()); - for (String index: iocIndices) { + for (String index : iocIndices) { if (index.equals(writeIndex) == false && index.equals(alias) == false) { indicesToDelete.add(index); } @@ -563,7 +564,7 @@ public void deleteOldIocIndices( // return source config listener.onResponse(new DefaultIocStoreConfig(iocToAliasMap)); - }, e-> { + }, e -> { log.error("Failed to get the cluster metadata"); listener.onFailure(e); } @@ -601,6 +602,7 @@ private List getIocIndicesToDeleteByAge( /** * Helper function to retrieve a list of IOC indices to delete based on number of indices associated with alias + * * @param clusterState * @param totalNumIndicesAndAlias * @param totalNumIndicesDeleteByAge @@ -653,6 +655,7 @@ private List getIocIndicesToDeleteBySize( /** * Helper function to determine how many indices should be deleted based on setting for number of indices per alias + * * @param totalNumIndices * @param totalNumIndicesDeleteByAge * @return diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java index 61ea2374d..1cb9e7428 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java @@ -229,6 +229,8 @@ public void parseAndSaveThreatIntelFeedDataCSV( } public static boolean isValidIp(String ip) { + if (StringUtils.isBlank(ip)) + return false; String ipPattern = "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$"; Pattern pattern = Pattern.compile(ipPattern); Matcher matcher = pattern.matcher(ip); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java index ae06d7724..0528b821a 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java @@ -19,8 +19,10 @@ import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -94,6 +96,9 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques } try { SATIFSourceConfigDto saTifSourceConfigDto = request.getTIFConfigDto(); + if (SourceConfigType.URL_DOWNLOAD.equals(saTifSourceConfigDto.getType()) || saTifSourceConfigDto.getSource() instanceof UrlDownloadSource) { + listener.onFailure(new UnsupportedOperationException("Unsupported Threat intel Source Config Type passed - " + saTifSourceConfigDto.getType())); + } saTifSourceConfigManagementService.createOrUpdateTifSourceConfig( saTifSourceConfigDto, lock, diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java index 23d0b3a0d..636d4207d 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java @@ -3,6 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.StepListener; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; @@ -15,6 +16,7 @@ import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsAction; import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsRequest; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.tasks.Task; @@ -28,6 +30,7 @@ public class TransportSearchTIFSourceConfigsAction extends HandledTransportActio private final ClusterService clusterService; private final Settings settings; + private final DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService; private final ThreadPool threadPool; @@ -41,11 +44,13 @@ public TransportSearchTIFSourceConfigsAction(TransportService transportService, ClusterService clusterService, final ThreadPool threadPool, Settings settings, + DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService, final SATIFSourceConfigManagementService saTifConfigService) { super(SASearchTIFSourceConfigsAction.NAME, transportService, actionFilters, SASearchTIFSourceConfigsRequest::new); this.clusterService = clusterService; this.threadPool = threadPool; this.settings = settings; + this.defaultTifSourceConfigLoaderService = defaultTifSourceConfigLoaderService; this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); this.saTifConfigService = saTifConfigService; @@ -63,16 +68,38 @@ protected void doExecute(Task task, SASearchTIFSourceConfigsRequest request, Act } this.threadPool.getThreadContext().stashContext(); // TODO: sync up with @deysubho about thread context - - saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( - r -> { - log.debug("Successfully listed all threat intel source configs"); - actionListener.onResponse(r); - }, e -> { - log.error("Failed to list all threat intel source configs"); - actionListener.onFailure(e); - } - )); + StepListener defaultTifConfigsLoadedListener; + try { + defaultTifConfigsLoadedListener = new StepListener<>(); + defaultTifSourceConfigLoaderService.createDefaultTifConfigsIfNotExists(defaultTifConfigsLoadedListener); + defaultTifConfigsLoadedListener.whenComplete(res -> saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, e -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + } + )), ex -> saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, e -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + } + ))); + } catch (Exception e) { + log.error("Failed to load default tif source configs. Moving on to list iocs", e); + saTifConfigService.searchTIFSourceConfigs(request.getSearchSourceBuilder(), ActionListener.wrap( + r -> { + log.debug("Successfully listed all threat intel source configs"); + actionListener.onResponse(r); + }, ex -> { + log.error("Failed to list all threat intel source configs"); + actionListener.onFailure(e); + })); + } } private void setFilterByEnabled(boolean filterByEnabled) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java index bfbb9dbde..3cbf31086 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java @@ -42,9 +42,27 @@ public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadat connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); return new CSVParser(new BufferedReader(new InputStreamReader(connection.getInputStream())), CSVFormat.RFC4180); } catch (IOException e) { - log.error("Exception: failed to read threat intel feed data from {}",tifMetadata.getUrl(), e); + log.error("Exception: failed to read threat intel feed data from {}", tifMetadata.getUrl(), e); throw new OpenSearchException("failed to read threat intel feed data from {}", tifMetadata.getUrl(), e); } }); } + + /** + * Create CSVParser of a threat intel feed + */ + @SuppressForbidden(reason = "Need to connect to http endpoint to read threat intel feed database file") + public static CSVParser getThreatIntelFeedReaderCSV(URL url) { + SpecialPermission.check(); + return AccessController.doPrivileged((PrivilegedAction) () -> { + try { + URLConnection connection = url.openConnection(); + connection.addRequestProperty(Constants.USER_AGENT_KEY, Constants.USER_AGENT_VALUE); + return new CSVParser(new BufferedReader(new InputStreamReader(connection.getInputStream())), CSVFormat.RFC4180); + } catch (IOException e) { + log.error("Exception: failed to read threat intel feed data from {}", url, e); + throw new OpenSearchException("failed to read threat intel feed data from {}", url, e); + } + }); + } } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java index 4abd3750c..099641b1c 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchStatusException; import org.opensearch.action.ActionRunnable; +import org.opensearch.action.StepListener; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; @@ -42,6 +43,7 @@ import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.threatIntel.action.SASearchTIFSourceConfigsRequest; +import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; import org.opensearch.securityanalytics.threatIntel.transport.TransportSearchTIFSourceConfigsAction; import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -68,6 +70,7 @@ public class TransportListIOCsAction extends HandledTransportAction defaultTifConfigsLoadedListener = null; + try { + defaultTifConfigsLoadedListener = new StepListener<>(); + defaultTifSourceConfigLoaderService.createDefaultTifConfigsIfNotExists(defaultTifConfigsLoadedListener); + defaultTifConfigsLoadedListener.whenComplete(r -> searchIocs(), e -> searchIocs()); + } catch (Exception e) { + log.error("Failed to load default tif source configs. Moving on to list iocs", e); + searchIocs(); + } + } + + private void searchIocs() { /** get all match threat intel source configs. fetch write index of each config if no iocs provided else fetch just index alias */ List configIds = request.getFeedIds() == null ? Collections.emptyList() : request.getFeedIds(); transportSearchTIFSourceConfigsAction.execute(new SASearchTIFSourceConfigsRequest(getFeedsSearchSourceBuilder(configIds)), diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java index 7c2b50d48..cecd091d2 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java @@ -112,10 +112,10 @@ public void testCreateIocUploadSourceConfig() throws IOException { // Evaluate response int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); - assertEquals(iocs.size(), totalHits); + assertTrue(iocs.size() < totalHits); //due to default feed leading to more iocs List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); - assertEquals(iocs.size(), iocHits.size()); + assertTrue(iocs.size() < iocHits.size()); // Retrieve all IOCs by feed Ids iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); Assert.assertEquals(200, iocResponse.getStatusLine().getStatusCode()); @@ -134,10 +134,10 @@ public void testCreateIocUploadSourceConfig() throws IOException { // Evaluate response totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); - assertEquals(iocs.size(), totalHits); + assertTrue(iocs.size() < totalHits); iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); - assertEquals(iocs.size(), iocHits.size()); + assertTrue(iocs.size() < iocHits.size()); } }