Skip to content

Commit

Permalink
[Backport 2.15] Adjusted IOCTypes usage. (#1156) (#1159)
Browse files Browse the repository at this point in the history 
* Adjusted IOCTypes usage. (#1156)

* Removed TODOs.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Refactored how STIX2IOCGenerator creates IOCs of specific types.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Added additional integration tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Changed IOCType usage as it's no longer an enum in SA commons.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Updated jar file.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed unit tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Refactored build.gradle to exclude redundant dependencies from SA commons, instead of the SA commons jar being generated withhout those dependencies.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Updated jar.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Reverted changes to build.gradle.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Updated jar.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed IOCType usage.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed log message.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Addressed PR feedback.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

---------

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

(cherry picked from commit a6eb64d)
Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed jar.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

---------

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>
  • Loading branch information
@AWSHurneyt
AWSHurneyt committed Jul 12, 2024
1 parent 8b28f2c commit 95d0099
Show file tree
Hide file tree
Showing 19 changed files with 528 additions and 139 deletions.
Binary file modified security-analytics-commons-1.0.0.jar
View file
Binary file not shown.
5 changes: 2 additions & 3 deletions src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
xcp.nextToken();

switch (fieldName) {
// synced up with @hurneyt, parsing the id and version but may need to change ioc id/version logic
case STIX2.ID_FIELD:
if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
id = xcp.text();
Expand All @@ -90,7 +89,7 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
name = xcp.text();
break;
case STIX2.TYPE_FIELD:
type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
type = new IOCType(xcp.text().toLowerCase(Locale.ROOT));
break;
case STIX2.VALUE_FIELD:
value = xcp.text();
Expand Down Expand Up @@ -177,7 +176,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
return builder.startObject()
.field(STIX2IOC.ID_FIELD, ioc.getId())
.field(STIX2IOC.NAME_FIELD, ioc.getName())
.field(STIX2IOC.TYPE_FIELD, ioc.getType())
.field(STIX2IOC.TYPE_FIELD, ioc.getType().toString())
.field(STIX2IOC.VALUE_FIELD, ioc.getValue())
.field(STIX2IOC.SEVERITY_FIELD, ioc.getSeverity())
.timeField(STIX2IOC.CREATED_FIELD, ioc.getCreated())
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public STIX2IOC(StreamInput sin) throws IOException {
this(
sin.readString(), // id
sin.readString(), // name
sin.readEnum(IOCType.class), // type
new IOCType(sin.readString()), // type
sin.readString(), // value
sin.readString(), // severity
sin.readInstant(), // created
Expand Down Expand Up @@ -142,7 +142,7 @@ public static STIX2IOC readFrom(StreamInput sin) throws IOException {
public void writeTo(StreamOutput out) throws IOException {
out.writeString(super.getId());
out.writeString(super.getName());
out.writeEnum(super.getType());
out.writeString(super.getType().toString());
out.writeString(super.getValue());
out.writeString(super.getSeverity());
out.writeInstant(super.getCreated());
Expand All @@ -160,7 +160,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject()
.field(ID_FIELD, super.getId())
.field(NAME_FIELD, super.getName())
.field(TYPE_FIELD, super.getType())
.field(TYPE_FIELD, super.getType().toString())
.field(VALUE_FIELD, super.getValue())
.field(SEVERITY_FIELD, super.getSeverity());
XContentUtils.buildInstantAsField(builder, super.getCreated(), CREATED_FIELD);
Expand Down Expand Up @@ -205,7 +205,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
name = xcp.text();
break;
case TYPE_FIELD:
type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
type = new IOCType(xcp.text());
break;
case VALUE_FIELD:
value = xcp.text();
Expand Down Expand Up @@ -292,8 +292,8 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
public void validate() throws IllegalArgumentException {
if (super.getType() == null) {
throw new IllegalArgumentException(String.format("[%s] is required.", TYPE_FIELD));
} else if (!Arrays.asList(IOCType.values()).contains(super.getType())) {
logger.debug("Unsupported IOCType: {}", super.getType());
} else if (!IOCType.supportedType(super.getType().toString())) {
logger.debug("Unsupported IOCType: {}", super.getType().toString());
throw new IllegalArgumentException(String.format("[%s] is not supported.", TYPE_FIELD));
}

Expand Down
7 changes: 3 additions & 4 deletions src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public static STIX2IOCDto readFrom(StreamInput sin) throws IOException {
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
out.writeString(name);
out.writeEnum(type);
out.writeString(type.toString());
out.writeString(value);
out.writeString(severity);
out.writeInstant(created);
Expand All @@ -120,7 +120,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
return builder.startObject()
.field(STIX2IOC.ID_FIELD, id)
.field(STIX2IOC.NAME_FIELD, name)
.field(STIX2IOC.TYPE_FIELD, type)
.field(STIX2IOC.TYPE_FIELD, type.toString())
.field(STIX2IOC.VALUE_FIELD, value)
.field(STIX2IOC.SEVERITY_FIELD, severity)
.timeField(STIX2IOC.CREATED_FIELD, created)
Expand Down Expand Up @@ -161,7 +161,6 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
xcp.nextToken();

switch (fieldName) {
// synced up with @hurneyt, parsing the id and version but may need to change ioc id/version logic
case STIX2.ID_FIELD:
if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
id = xcp.text();
Expand All @@ -176,7 +175,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
name = xcp.text();
break;
case STIX2.TYPE_FIELD:
type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
type = new IOCType(xcp.text());
break;
case STIX2.VALUE_FIELD:
value = xcp.text();
Expand Down
5 changes: 4 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConnectorFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
package org.opensearch.securityanalytics.services;

import com.amazonaws.services.s3.AmazonS3;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.securityanalytics.commons.connector.Connector;
import org.opensearch.securityanalytics.commons.connector.S3Connector;
import org.opensearch.securityanalytics.commons.connector.codec.InputCodec;
Expand All @@ -21,6 +23,7 @@
import java.util.List;

public class STIX2IOCConnectorFactory extends UnaryParameterCachingFactory<FeedConfiguration, Connector<STIX2>> {
private static final Logger logger = LogManager.getLogger(STIX2IOCConnectorFactory.class);
private final InputCodecFactory inputCodecFactory;
private final S3ClientFactory s3ClientFactory;

Expand All @@ -31,7 +34,7 @@ public STIX2IOCConnectorFactory(final InputCodecFactory inputCodecFactory, final

protected Connector<STIX2> doCreate(FeedConfiguration feedConfiguration) {
final FeedLocation feedLocation = FeedLocation.fromFeedConfiguration(feedConfiguration);
// TODO hurneyt add debug log for which location gets used
logger.debug("FeedLocation: {}", feedLocation);
switch(feedLocation) {
case S3: return createS3Connector(feedConfiguration);
default: throw new IllegalArgumentException("Unsupported feedLocation: " + feedLocation);
Expand Down
5 changes: 2 additions & 3 deletions src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConsumer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,10 @@ public void accept(final STIX2 ioc) {
feedStore.getSaTifSourceConfig().getName()
);

// TODO hurneyt refactor once the enum values are updated
// If the IOC received is not a type listed for the config, do not add it to the queue
if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().name())) {
if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().toString())) {
log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}",
stix2IOC.getType().name(), feedStore.getSaTifSourceConfig().getId(),
stix2IOC.getType().toString(), feedStore.getSaTifSourceConfig().getId(),
stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue()
);
return;
Expand Down
8 changes: 2 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,13 @@ public class STIX2IOCFeedStore implements FeedStore {
public static final String IOC_INDEX_PATTERN = IOC_INDEX_NAME_TEMPLATE + "-" + IOC_TIME_PLACEHOLDER;

private final Logger log = LogManager.getLogger(STIX2IOCFeedStore.class);

Instant startTime = Instant.now();

private Client client;
private ClusterService clusterService;
private SATIFSourceConfig saTifSourceConfig;

// TODO hurneyt FetchIocsActionResponse is just a placeholder response type for now
private ActionListener<STIX2IOCFetchService.STIX2IOCFetchResponse> baseListener;

// TODO hurneyt this is using TIF batch size setting. Consider adding IOC-specific setting
private Integer batchSize;

public STIX2IOCFeedStore(
Expand Down Expand Up @@ -97,7 +94,6 @@ public void storeIOCs(Map<IOC, UpdateAction> actionToIOCs) {
for (Map.Entry<UpdateAction, List<STIX2IOC>> entry : iocsSortedByAction.entrySet()) {
switch (entry.getKey()) {
case DELETE:
// TODO hurneyt consider whether DELETE actions should be handled elsewhere
break;
case UPSERT:
try {
Expand All @@ -119,7 +115,7 @@ public void indexIocs(List<STIX2IOC> iocs) throws IOException {
initFeedIndex(newActiveIndex, ActionListener.wrap(
r -> {
saTifSourceConfig.getIocTypes().forEach(type -> {
IOCType iocType = IOCType.fromString(type);
IOCType iocType = new IOCType(type);
if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
List<DefaultIocStoreConfig.IocToIndexDetails> listOfIocToIndexDetails =
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails();
Expand Down
6 changes: 2 additions & 4 deletions src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ public class STIX2IOCFetchService {
private STIX2IOCConnectorFactory connectorFactory;
private S3ClientFactory s3ClientFactory;

// TODO hurneyt this is using TIF batch size setting. Consider adding IOC-specific setting
private Integer batchSize;
private String internalAuthEndpoint = "";

Expand Down Expand Up @@ -126,7 +125,6 @@ public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionList
listener.onFailure(e);
}

// TODO consider passing listener into the flush IOC function
try {
consumer.flushIOCs();
} catch (Exception e) {
Expand Down Expand Up @@ -285,15 +283,15 @@ private void parseAndSaveThreatIntelFeedDataCSV(Iterator<CSVRecord> iterator, SA
String iocType = saTifSourceConfig.getIocTypes().stream().findFirst().orElse(null);
Integer colNum = source.getCsvIocValueColumnNo();
String iocValue = record.values()[colNum].split(" ")[0];
if (iocType.equalsIgnoreCase(IOCType.ipv6_addr.toString()) && !isValidIp(iocValue)) {
if (iocType.equalsIgnoreCase(IOCType.IPV4_TYPE) && !isValidIp(iocValue)) {
log.info("Invalid IP address, skipping this ioc record: {}", iocValue);
continue;
}
Instant now = Instant.now();
STIX2IOC stix2IOC = new STIX2IOC(
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
iocType == null ? IOCType.ipv4_addr : IOCType.valueOf(iocType),
iocType == null ? new IOCType(IOCType.IPV4_TYPE) : new IOCType(iocType),
iocValue,
"high",
now,
Expand Down
6 changes: 2 additions & 4 deletions ...in/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.securityanalytics.threatIntel.common;

import org.opensearch.securityanalytics.commons.model.IOC;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
import org.opensearch.securityanalytics.threatIntel.model.S3Source;
Expand All @@ -22,15 +23,12 @@
public class SourceConfigDtoValidator {
public List<String> validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto) {
List<String> errorMsgs = new ArrayList<>();
List<String> iocTypeEnumNames = Arrays.stream(IOCType.values())
.map(Enum::name)
.collect(Collectors.toList());

if (sourceConfigDto.getIocTypes().isEmpty()) {
errorMsgs.add("Must specify at least one IOC type");
} else {
for (String s: sourceConfigDto.getIocTypes()) {
if (false == iocTypeEnumNames.contains(s)) {
if (!IOCType.supportedType(s)) {
errorMsgs.add("Invalid IOC type: " + s);
}
}
Expand Down
5 changes: 3 additions & 2 deletions ...n/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
Expand Down Expand Up @@ -375,9 +376,9 @@ private static SearchRequest getSearchRequestForIocType(List<String> indices, St
SearchRequest searchRequest = new SearchRequest(indices.toArray(new String[0]));
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
// add the iocs sublist
boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD, iocsSublist));
boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD + ".keyword", iocsSublist));
// add ioc type filter
boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD, iocType.toLowerCase()));
boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD + ".keyword", iocType.toLowerCase(Locale.ROOT)));
searchRequest.source().query(boolQueryBuilder);
return searchRequest;
}
Expand Down
12 changes: 7 additions & 5 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,21 +101,23 @@ public IocToIndexDetails(IOCType iocType, String indexPattern, String activeInde
}

public IocToIndexDetails(StreamInput sin) throws IOException {
this(sin.readEnum(IOCType.class),
this(
new IOCType(sin.readString()),
sin.readString(),
sin.readString());
sin.readString()
);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeEnum(iocType);
out.writeString(iocType.toString());
out.writeString(indexPattern);
out.writeString(activeIndex);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.startObject()
.field(IOC_TYPE_FIELD, iocType)
.field(IOC_TYPE_FIELD, iocType.toString())
.field(INDEX_PATTERN_FIELD, indexPattern)
.field(ACTIVE_INDEX_FIELD, activeIndex)
.endObject();
Expand Down Expand Up @@ -150,7 +152,7 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException {

public static IOCType toIocType(String name) {
try {
return IOCType.fromString(name);
return new IOCType(name);
} catch (IllegalArgumentException e) {
log.error("Invalid Ioc type, cannot be parsed.", e);
return null;
Expand Down
2 changes: 1 addition & 1 deletion ...opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ public void onFailure(Exception e) {
Instant now = Instant.now();
String iocType = null;
if (tifMetadata.getIocType().equalsIgnoreCase("ip")) {
iocType = IOCType.ipv4_addr.toString();
iocType = IOCType.IPV4_TYPE;
}
satifSourceConfigManagementService.createOrUpdateTifSourceConfig(
new SATIFSourceConfigDto(
Expand Down
11 changes: 6 additions & 5 deletions .../opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.opensearch.rest.RestRequest;
import org.opensearch.search.SearchHit;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.model.STIX2IOC;
import org.opensearch.securityanalytics.model.STIX2IOCDto;
import org.opensearch.securityanalytics.services.STIX2IOCFetchService;
Expand Down Expand Up @@ -200,12 +201,12 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig,
List<STIX2IOC> validStix2IocList = new ArrayList<>();
// If the IOC received is not a type listed for the config, do not add it to the queue
for (STIX2IOC stix2IOC : stix2IOCList) {
if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().name())) {
if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().toString())) {
validStix2IocList.add(stix2IOC);
} else {
log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}",
stix2IOC.getType().name(), saTifSourceConfig.getId(),
stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue()
stix2IOC.getType().toString(), saTifSourceConfig.getId(),
stix2IOC.getId(), stix2IOC.getType().toString(), stix2IOC.getValue()
);
}
}
Expand Down Expand Up @@ -354,7 +355,7 @@ private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListene
Set<String> concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse);

// remove ioc types not specified in list
defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> false == iocTypes.contains(iocToIndexDetails.getIocType().name()));
defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString()));

// get the active indices
defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> activeIndices.add(e.getActiveIndex()));
Expand Down Expand Up @@ -467,7 +468,7 @@ private void downloadAndSaveIocsToRefresh(ActionListener<SATIFSourceConfigDto> l
if (newIocStoreConfig instanceof DefaultIocStoreConfig) {
DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) newIocStoreConfig;
// remove ioc types not specified in list
defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> false == iocTypes.contains(iocToIndexDetails.getIocType().name()));
defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString()));
updatedSourceConfig.setIocStoreConfig(defaultIocStoreConfig);
}
// Update source config as succeeded, change state back to available
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/opensearch/securityanalytics/TestHelpers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2889,7 +2889,7 @@ public static SATIFSourceConfig randomSATIFSourceConfig(
schedule = new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS);
}
if (iocStoreConfig == null) {
iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(IOCType.domain_name, "indexPattern", "writeIndex")));
iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.DOMAIN_NAME_TYPE), "indexPattern", "writeIndex")));
}
if (iocTypes == null) {
iocTypes = List.of("ip");
Expand Down
Loading

0 comments on commit 95d0099

Please sign in to comment.