[Backport 2.x] PR 1142 & PR 1156 (#1158)

* adds new tif source config type - url download (#1142)

* adds new tif source config type - url download

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* set up create default tif configs

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* address review comments

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* add check to block create and delete operation url download type tif source configs

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

---------

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* Adjusted IOCTypes usage. (#1156)

* Removed TODOs.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Refactored how STIX2IOCGenerator creates IOCs of specific types.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Added additional integration tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Changed IOCType usage as it's no longer an enum in SA commons.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Updated jar file.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed unit tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Refactored build.gradle to exclude redundant dependencies from SA commons, instead of the SA commons jar being generated withhout those dependencies.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Updated jar.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Reverted changes to build.gradle.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Updated jar.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed IOCType usage.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed log message.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed tests.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Addressed PR feedback.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

---------

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

* Fixed jar.

Signed-off-by: AWSHurneyt <hurneyt@amazon.com>

---------

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>
Signed-off-by: AWSHurneyt <hurneyt@amazon.com>
Co-authored-by: Surya Sashank Nistala <snistala@amazon.com>

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -157,6 +157,7 @@
 import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestIndexThreatIntelMonitorAction;
 import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction;
 import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestUpdateThreatIntelAlertsStatusAction;
+import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService;
 import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService;
 import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService;
 import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService;
@@ -326,12 +327,13 @@ public Collection<Object> createComponents(Client client,
         IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry);
         ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry);
         SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService);
+        DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService);
         return List.of(
                 detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, threatIntelAlertService,
                 mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService,
                 correlationAlertService, notificationService,
                 tifJobUpdateService, tifJobParameterService, threatIntelLockService, saTifSourceConfigService, saTifSourceConfigManagementService, stix2IOCFetchService,
-                ioCScanService);
+                ioCScanService, defaultTifSourceConfigLoaderService);
     }
 
     @Override

src/main/java/org/opensearch/securityanalytics/model/DetailedSTIX2IOCDto.java

@@ -75,7 +75,6 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
             xcp.nextToken();
 
             switch (fieldName) {
-                // synced up with @hurneyt, parsing the id and version but may need to change ioc id/version logic
                 case STIX2.ID_FIELD:
                     if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
                         id = xcp.text();
@@ -90,7 +89,7 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
                     name = xcp.text();
                     break;
                 case STIX2.TYPE_FIELD:
-                    type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
+                    type = new IOCType(xcp.text().toLowerCase(Locale.ROOT));
                     break;
                 case STIX2.VALUE_FIELD:
                     value = xcp.text();
@@ -177,7 +176,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder.startObject()
                 .field(STIX2IOC.ID_FIELD, ioc.getId())
                 .field(STIX2IOC.NAME_FIELD, ioc.getName())
-                .field(STIX2IOC.TYPE_FIELD, ioc.getType())
+                .field(STIX2IOC.TYPE_FIELD, ioc.getType().toString())
                 .field(STIX2IOC.VALUE_FIELD, ioc.getValue())
                 .field(STIX2IOC.SEVERITY_FIELD, ioc.getSeverity())
                 .timeField(STIX2IOC.CREATED_FIELD, ioc.getCreated())

src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java

@@ -84,7 +84,7 @@ public STIX2IOC(StreamInput sin) throws IOException {
         this(
                 sin.readString(), // id
                 sin.readString(), // name
-                sin.readEnum(IOCType.class), // type
+                new IOCType(sin.readString()), // type
                 sin.readString(), // value
                 sin.readString(), // severity
                 sin.readInstant(), // created
@@ -142,7 +142,7 @@ public static STIX2IOC readFrom(StreamInput sin) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(super.getId());
         out.writeString(super.getName());
-        out.writeEnum(super.getType());
+        out.writeString(super.getType().toString());
         out.writeString(super.getValue());
         out.writeString(super.getSeverity());
         out.writeInstant(super.getCreated());
@@ -160,7 +160,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject()
                 .field(ID_FIELD, super.getId())
                 .field(NAME_FIELD, super.getName())
-                .field(TYPE_FIELD, super.getType())
+                .field(TYPE_FIELD, super.getType().toString())
                 .field(VALUE_FIELD, super.getValue())
                 .field(SEVERITY_FIELD, super.getSeverity());
         XContentUtils.buildInstantAsField(builder, super.getCreated(), CREATED_FIELD);
@@ -205,7 +205,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
                     name = xcp.text();
                     break;
                 case TYPE_FIELD:
-                    type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
+                    type = new IOCType(xcp.text());
                     break;
                 case VALUE_FIELD:
                     value = xcp.text();
@@ -292,8 +292,8 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
     public void validate() throws IllegalArgumentException {
         if (super.getType() == null) {
             throw new IllegalArgumentException(String.format("[%s] is required.", TYPE_FIELD));
-        } else if (!Arrays.asList(IOCType.values()).contains(super.getType())) {
-            logger.debug("Unsupported IOCType: {}", super.getType());
+        } else if (!IOCType.supportedType(super.getType().toString())) {
+            logger.debug("Unsupported IOCType: {}", super.getType().toString());
             throw new IllegalArgumentException(String.format("[%s] is not supported.", TYPE_FIELD));
         }
 

src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java

@@ -102,7 +102,7 @@ public static STIX2IOCDto readFrom(StreamInput sin) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(id);
         out.writeString(name);
-        out.writeEnum(type);
+        out.writeString(type.toString());
         out.writeString(value);
         out.writeString(severity);
         out.writeInstant(created);
@@ -120,7 +120,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder.startObject()
                 .field(STIX2IOC.ID_FIELD, id)
                 .field(STIX2IOC.NAME_FIELD, name)
-                .field(STIX2IOC.TYPE_FIELD, type)
+                .field(STIX2IOC.TYPE_FIELD, type.toString())
                 .field(STIX2IOC.VALUE_FIELD, value)
                 .field(STIX2IOC.SEVERITY_FIELD, severity)
                 .timeField(STIX2IOC.CREATED_FIELD, created)
@@ -161,7 +161,6 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
             xcp.nextToken();
 
             switch (fieldName) {
-                // synced up with @hurneyt, parsing the id and version but may need to change ioc id/version logic
                 case STIX2.ID_FIELD:
                     if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
                         id = xcp.text();
@@ -176,7 +175,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
                     name = xcp.text();
                     break;
                 case STIX2.TYPE_FIELD:
-                    type = IOCType.valueOf(xcp.text().toLowerCase(Locale.ROOT));
+                    type = new IOCType(xcp.text());
                     break;
                 case STIX2.VALUE_FIELD:
                     value = xcp.text();

src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConnectorFactory.java

@@ -6,6 +6,8 @@
 package org.opensearch.securityanalytics.services;
 
 import com.amazonaws.services.s3.AmazonS3;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.opensearch.securityanalytics.commons.connector.Connector;
 import org.opensearch.securityanalytics.commons.connector.S3Connector;
 import org.opensearch.securityanalytics.commons.connector.codec.InputCodec;
@@ -21,6 +23,7 @@
 import java.util.List;
 
 public class STIX2IOCConnectorFactory extends UnaryParameterCachingFactory<FeedConfiguration, Connector<STIX2>> {
+    private static final Logger logger = LogManager.getLogger(STIX2IOCConnectorFactory.class);
     private final InputCodecFactory inputCodecFactory;
     private final S3ClientFactory s3ClientFactory;
 
@@ -31,7 +34,7 @@ public STIX2IOCConnectorFactory(final InputCodecFactory inputCodecFactory, final
 
     protected Connector<STIX2> doCreate(FeedConfiguration feedConfiguration) {
         final FeedLocation feedLocation = FeedLocation.fromFeedConfiguration(feedConfiguration);
-        // TODO hurneyt add debug log for which location gets used
+        logger.debug("FeedLocation: {}", feedLocation);
         switch(feedLocation) {
             case S3: return createS3Connector(feedConfiguration);
             default: throw new IllegalArgumentException("Unsupported feedLocation: " + feedLocation);

src/main/java/org/opensearch/securityanalytics/services/STIX2IOCConsumer.java

@@ -42,11 +42,10 @@ public void accept(final STIX2 ioc) {
                 feedStore.getSaTifSourceConfig().getName()
         );
 
-        // TODO hurneyt refactor once the enum values are updated
         // If the IOC received is not a type listed for the config, do not add it to the queue
-        if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().name())) {
+        if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().toString())) {
             log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}",
-                    stix2IOC.getType().name(), feedStore.getSaTifSourceConfig().getId(),
+                    stix2IOC.getType().toString(), feedStore.getSaTifSourceConfig().getId(),
                     stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue()
             );
             return;

src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java

@@ -57,16 +57,13 @@ public class STIX2IOCFeedStore implements FeedStore {
     public static final String IOC_INDEX_PATTERN = IOC_INDEX_NAME_TEMPLATE + "-" + IOC_TIME_PLACEHOLDER;
 
     private final Logger log = LogManager.getLogger(STIX2IOCFeedStore.class);
+
     Instant startTime = Instant.now();
 
     private Client client;
     private ClusterService clusterService;
     private SATIFSourceConfig saTifSourceConfig;
-
-    // TODO hurneyt FetchIocsActionResponse is just a placeholder response type for now
     private ActionListener<STIX2IOCFetchService.STIX2IOCFetchResponse> baseListener;
-
-    // TODO hurneyt this is using TIF batch size setting. Consider adding IOC-specific setting
     private Integer batchSize;
 
     public STIX2IOCFeedStore(
@@ -97,7 +94,6 @@ public void storeIOCs(Map<IOC, UpdateAction> actionToIOCs) {
         for (Map.Entry<UpdateAction, List<STIX2IOC>> entry : iocsSortedByAction.entrySet()) {
             switch (entry.getKey()) {
                 case DELETE:
-                    // TODO hurneyt consider whether DELETE actions should be handled elsewhere
                     break;
                 case UPSERT:
                     try {
@@ -119,7 +115,7 @@ public void indexIocs(List<STIX2IOC> iocs) throws IOException {
         initFeedIndex(newActiveIndex, ActionListener.wrap(
                 r -> {
                     saTifSourceConfig.getIocTypes().forEach(type -> {
-                        IOCType iocType = IOCType.fromString(type);
+                        IOCType iocType = new IOCType(type);
                         if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
                             List<DefaultIocStoreConfig.IocToIndexDetails> listOfIocToIndexDetails =
                                     ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails();