Skip to content

Commit

Permalink
set up create default tif configs
Browse files Browse the repository at this point in the history
Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>
  • Loading branch information
eirsep committed Jul 9, 2024
1 parent 4573483 commit 00fcd33
Show file tree
Hide file tree
Showing 14 changed files with 429 additions and 26 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@
import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestIndexThreatIntelMonitorAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestUpdateThreatIntelAlertsStatusAction;
import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService;
import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService;
import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService;
import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService;
Expand Down Expand Up @@ -326,12 +327,13 @@ public Collection<Object> createComponents(Client client,
IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry);
ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry);
SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService);
DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService);
return List.of(
detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, threatIntelAlertService,
mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService,
correlationAlertService, notificationService,
tifJobUpdateService, tifJobParameterService, threatIntelLockService, saTifSourceConfigService, saTifSourceConfigManagementService, stix2IOCFetchService,
ioCScanService);
ioCScanService, defaultTifSourceConfigLoaderService);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@

import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.action.bulk.BulkRequest;
import org.opensearch.client.Client;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.core.action.ActionListener;
Expand All @@ -29,14 +32,17 @@
import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig;
import org.opensearch.securityanalytics.commons.model.FeedConfiguration;
import org.opensearch.securityanalytics.commons.model.IOCSchema;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.commons.model.STIX2;
import org.opensearch.securityanalytics.commons.model.UpdateType;
import org.opensearch.securityanalytics.model.STIX2IOC;
import org.opensearch.securityanalytics.model.STIX2IOCDto;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.threatIntel.model.S3Source;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource;
import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService;
import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedParser;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
Expand All @@ -49,10 +55,16 @@
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;

import static org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService.isValidIp;

/**
* IOC Service implements operations that interact with retrieving IOCs from data sources,
* parsing them into threat intel data models (i.e., [IOC]), and ingesting them to system indexes.
Expand Down Expand Up @@ -84,14 +96,14 @@ public STIX2IOCFetchService(Client client, ClusterService clusterService) {

/**
* Method takes in and calls method to rollover and bulk index a list of STIX2IOCs
*
* @param saTifSourceConfig
* @param stix2IOCList
* @param listener
*/
public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig,
List<STIX2IOC> stix2IOCList,
ActionListener<STIX2IOCFetchResponse> listener)
{
ActionListener<STIX2IOCFetchResponse> listener) {
STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener);
try {
feedStore.indexIocs(stix2IOCList);
Expand All @@ -100,6 +112,7 @@ public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig,
listener.onFailure(e);
}
}

public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener<STIX2IOCFetchResponse> listener) {
S3ConnectorConfig s3ConnectorConfig = constructS3ConnectorConfig(saTifSourceConfig);
Connector<STIX2> s3Connector = constructS3Connector(s3ConnectorConfig);
Expand Down Expand Up @@ -144,7 +157,7 @@ private void testS3ClientConnection(S3ConnectorConfig s3ConnectorConfig, ActionL
} catch (StsException stsException) {
log.warn("S3Client connection test failed with StsException: ", stsException);
listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(stsException.statusCode()), stsException.awsErrorDetails().errorMessage()));
} catch (SdkException sdkException ) {
} catch (SdkException sdkException) {
// SdkException is a RunTimeException that doesn't have a status code.
// Logging the full exception, and providing generic response as output.
log.warn("S3Client connection test failed with SdkException: ", sdkException);
Expand Down Expand Up @@ -227,6 +240,76 @@ private String getEndpoint() {
return "";
}

public void downloadFromUrlAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener<STIX2IOCFetchResponse> listener) {
UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource();
switch (source.getFeedFormat()) { // todo add check to stop user from creating url type config from rest api. only internal allowed
case "csv":
try (CSVParser reader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl())) {
CSVParser noHeaderReader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl());
boolean notFound = true;

while (notFound) {
CSVRecord hasHeaderRecord = reader.iterator().next();

//if we want to skip this line and keep iterating
if ((hasHeaderRecord.values().length == 1 && "".equals(hasHeaderRecord.values()[0])) || hasHeaderRecord.get(0).charAt(0) == '#' || hasHeaderRecord.get(0).charAt(0) == ' ') {
noHeaderReader.iterator().next();
} else { // we found the first line that contains information
notFound = false;
}
}
if (source.hasCsvHeader()) {
parseAndSaveThreatIntelFeedDataCSV(reader.iterator(), saTifSourceConfig, listener);
} else {
parseAndSaveThreatIntelFeedDataCSV(noHeaderReader.iterator(), saTifSourceConfig, listener);
}
} catch (Exception e) {
log.error("Failed to download the IoCs in CSV format for source " + saTifSourceConfig.getId());
listener.onFailure(e);
return;
}
break;
default:
// if the feed type doesn't match any of the supporting feed types, throw an exception
}
}

private void parseAndSaveThreatIntelFeedDataCSV(Iterator<CSVRecord> iterator, SATIFSourceConfig saTifSourceConfig, ActionListener<STIX2IOCFetchResponse> listener) throws IOException {
List<BulkRequest> bulkRequestList = new ArrayList<>();

UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource();
List<STIX2IOC> iocs = new ArrayList<>();
while (iterator.hasNext()) {
CSVRecord record = iterator.next();
String iocType = saTifSourceConfig.getIocTypes().stream().findFirst().orElse(null);
Integer colNum = source.getCsvIocValueColumnNo();
String iocValue = record.values()[colNum].split(" ")[0];
if (iocType.equalsIgnoreCase(IOCType.ipv6_addr.toString()) && !isValidIp(iocValue)) {
log.info("Invalid IP address, skipping this ioc record: {}", iocValue);
continue;
}
Instant now = Instant.now();
STIX2IOC stix2IOC = new STIX2IOC(
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
iocType == null ? IOCType.ipv4_addr : IOCType.valueOf(iocType),
iocValue,
"high",
now,
now,
"",
Collections.emptyList(),
"",
saTifSourceConfig.getId(),
saTifSourceConfig.getName(),
STIX2IOC.NO_VERSION
);
iocs.add(stix2IOC);
}
STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener);
feedStore.indexIocs(iocs);
}

public static class STIX2IOCFetchResponse extends ActionResponse implements ToXContentObject {
public static String IOCS_FIELD = "iocs";
public static String TOTAL_FIELD = "total";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
import org.opensearch.securityanalytics.threatIntel.model.S3Source;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;
import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource;

import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -55,6 +56,14 @@ public List<String> validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto
errorMsgs.add("Source must be S3_CUSTOM type");
}
break;
case URL_DOWNLOAD:
if (sourceConfigDto.getSchedule() == null) {
errorMsgs.add("Must pass in schedule for URL_DONWLOAD source type");
}
if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof UrlDownloadSource == false) {
errorMsgs.add("Source must be URL_DONWLOAD source type");
}
break;
}
return errorMsgs;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@

/**
* Types of feeds threat intel can support
* Feed types include: S3_CUSTOM
*/
public enum SourceConfigType {
S3_CUSTOM,
IOC_UPLOAD
IOC_UPLOAD,
URL_DOWNLOAD

// LICENSED,
//
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ static Source parse(XContentParser xcp) throws IOException {
case IOC_UPLOAD_FIELD:
source = IocUploadSource.parse(xcp);
break;
case URL_DOWNLOAD_FIELD:
source = UrlDownloadSource.parse(xcp);
break;
}
}
return source;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.opensearch.securityanalytics.threatIntel.model;

import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
Expand All @@ -14,16 +15,39 @@
*/
public class UrlDownloadSource extends Source implements Writeable, ToXContent {
public static final String URL_FIELD = "url";
public static final String FEED_FORMAT_FIELD = "feed_format";
public static final String HAS_CSV_HEADER_FIELD = "has_csv_header_field";
public static final String CSV_IOC_VALUE_COLUMN_NUM_FIELD = "csv_ioc_value_colum_num";
public static final String SOURCE_NAME = "URL_DOWNLOAD";

private final URL url;
private final String feedFormat;
private final Boolean hasCsvHeader;
private final Integer csvIocValueColumnNo;

public UrlDownloadSource(URL url) {
public UrlDownloadSource(URL url, String feedFormat, Boolean hasCsvHeader, Integer csvIocValueColumnNo) {
this.url = url;
this.feedFormat = feedFormat;
this.hasCsvHeader = hasCsvHeader;
this.csvIocValueColumnNo = csvIocValueColumnNo;

}

public UrlDownloadSource(StreamInput sin) throws IOException {
this(new URL(sin.readString()));
this(
new URL(sin.readString()),
sin.readString(),
sin.readOptionalBoolean(),
sin.readOptionalInt()
);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(url.toString());
out.writeString(feedFormat);
out.writeOptionalBoolean(hasCsvHeader);
out.writeOptionalInt(csvIocValueColumnNo);
}

@Override
Expand All @@ -37,6 +61,9 @@ public URL getUrl() {

public static UrlDownloadSource parse(XContentParser xcp) throws IOException {
URL url = null;
String feedFormat = null;
Boolean hasCsvHeader = false;
Integer csvIocValueColumnNo = null;
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
String fieldName = xcp.currentName();
xcp.nextToken();
Expand All @@ -45,19 +72,46 @@ public static UrlDownloadSource parse(XContentParser xcp) throws IOException {
String urlString = xcp.text();
url = new URL(urlString);
break;
case FEED_FORMAT_FIELD:
feedFormat = xcp.text();
break;
case HAS_CSV_HEADER_FIELD:
hasCsvHeader = xcp.booleanValue();
break;
case CSV_IOC_VALUE_COLUMN_NUM_FIELD:
if (xcp.currentToken() == null)
xcp.skipChildren();
else
csvIocValueColumnNo = xcp.intValue();
break;
default:
xcp.skipChildren();
}
}
return new UrlDownloadSource(url);
return new UrlDownloadSource(url, feedFormat, hasCsvHeader, csvIocValueColumnNo);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.startObject()
.startObject(URL_DOWNLOAD_FIELD)
.field(URL_FIELD, url)
.field(URL_FIELD, url.toString())
.field(FEED_FORMAT_FIELD, feedFormat)
.field(HAS_CSV_HEADER_FIELD, hasCsvHeader)
.field(CSV_IOC_VALUE_COLUMN_NUM_FIELD, csvIocValueColumnNo)
.endObject()
.endObject();
}

public String getFeedFormat() {
return feedFormat;
}

public boolean hasCsvHeader() {
return hasCsvHeader;
}

public Integer getCsvIocValueColumnNo() {
return csvIocValueColumnNo;
}
}
Loading

0 comments on commit 00fcd33

Please sign in to comment.