add tests for ioc to fields for each log type

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

build.gradle

@@ -155,7 +155,7 @@ dependencies {
     implementation group: 'org.apache.commons', name: 'commons-lang3', version: "${versions.commonslang}"
     implementation "org.antlr:antlr4-runtime:4.10.1"
     implementation "com.cronutils:cron-utils:9.1.6"
-    api "org.opensearch:common-utils:${common_utils_version}@jar"
+    api files("/Users/snistala/Documents/opensearch/common-utils/build/libs/common-utils-3.0.0.0-SNAPSHOT.jar")
     api "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
     implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
     implementation "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"

src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java

@@ -10,6 +10,7 @@
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -660,6 +661,13 @@ public void getRuleFieldMappings(String logType, ActionListener<Map<String, Stri
         return;
     }
 
+    public List<LogType.IocFields> getIocFieldsList(String logType) {
+        LogType logTypeByName = builtinLogTypeLoader.getLogTypeByName(logType);
+        if(logTypeByName == null)
+            return Collections.emptyList();
+        return logTypeByName.getIocFieldsList();
+    }
+
     public void getRuleFieldMappingsAllSchemas(String logType, ActionListener<List<LogType.Mapping>> listener) {
 
         if (builtinLogTypeLoader.logTypeExists(logType)) {

src/main/java/org/opensearch/securityanalytics/model/LogType.java

@@ -6,6 +6,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -66,13 +67,16 @@ public LogType(Map<String, Object> logTypeAsMap) {
                     new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF))
             ).collect(Collectors.toList());
         }
-
-        List<Map<String, Object>> iocFieldsList = (List<Map<String, Object>>)logTypeAsMap.get(IOC_FIELDS);
-        if (iocFieldsList.size() > 0) {
-            this.iocFieldsList = new ArrayList<>(mappings.size());
-            this.iocFieldsList = iocFieldsList.stream().map(e ->
-                    new IocFields(e.get(IOC).toString(), (List<String>)e.get(FIELDS))
-            ).collect(Collectors.toList());
+        if(logTypeAsMap.containsKey(IOC_FIELDS)) {
+            List<Map<String, Object>> iocFieldsList = (List<Map<String, Object>>) logTypeAsMap.get(IOC_FIELDS);
+            if (iocFieldsList.size() > 0) {
+                this.iocFieldsList = new ArrayList<>(mappings.size());
+                this.iocFieldsList = iocFieldsList.stream().map(e ->
+                        new IocFields(e.get(IOC).toString(), (List<String>) e.get(FIELDS))
+                ).collect(Collectors.toList());
+            }
+        } else {
+            iocFieldsList = Collections.emptyList();
         }
     }
 

src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java

@@ -1,15 +1,19 @@
 package org.opensearch.securityanalytics.threatIntel;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.client.Client;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.commons.alerting.model.DocLevelQuery;
 import org.opensearch.core.action.ActionListener;
-import org.opensearch.core.rest.RestStatus;
-import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
 import org.opensearch.securityanalytics.model.Detector;
+import org.opensearch.securityanalytics.model.LogType;
 import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
-import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
@@ -19,33 +23,56 @@
 
 public class DetectorThreatIntelService {
 
+    private static final Logger log = LogManager.getLogger(DetectorThreatIntelService.class);
+
     private final ThreatIntelFeedDataService threatIntelFeedDataService;
 
     public DetectorThreatIntelService(ThreatIntelFeedDataService threatIntelFeedDataService) {
         this.threatIntelFeedDataService = threatIntelFeedDataService;
     }
 
+
     /**
      * Convert the feed data IOCs into query string query format to create doc level queries.
      */
-    public DocLevelQuery createDocLevelQueryFromThreatIntelList(
-            List<ThreatIntelFeedData> tifdList, String docLevelQueryId
+    public List<DocLevelQuery> createDocLevelQueriesFromThreatIntelList(
+            List<LogType.IocFields> iocFieldList, List<ThreatIntelFeedData> tifdList, Detector detector
     ) {
+        List<DocLevelQuery> queries = new ArrayList<>();
         Set<String> iocs = tifdList.stream().map(ThreatIntelFeedData::getIocValue).collect(Collectors.toSet());
-        String query = buildQueryStringQueryWithIocList(iocs);
-        return new DocLevelQuery(
-                docLevelQueryId, tifdList.get(0).getFeedId(),
-                Collections.singletonList("*"),
-                query,
-                Collections.singletonList("threat_intel")
-        );
+        //ioc types supported by log type
+        List<String> logTypeIocs = iocFieldList.stream().map(LogType.IocFields::getIoc).collect(Collectors.toList());
+        // filter out ioc types not supported for given log types
+        Map<String, Set<String>> iocTypeToValues = tifdList.stream().filter(t -> logTypeIocs.contains(t.getIocType()))
+                .collect(Collectors.groupingBy(
+                        ThreatIntelFeedData::getIocType,
+                        Collectors.mapping(ThreatIntelFeedData::getIocValue, Collectors.toSet())
+                ));
+
+        for (Map.Entry<String, Set<String>> entry : iocTypeToValues.entrySet()) {
+            String query = buildQueryStringQueryWithIocList(iocs);
+            List<String> fields = iocFieldList.stream().filter(t -> entry.getKey().matches(t.getIoc())).findFirst().get().getFields();
+
+            // create doc
+            for (String field : fields) { //todo increase max clause count from 1024
+                queries.add(new DocLevelQuery(
+                        constructId(detector, entry.getKey()), tifdList.get(0).getFeedId(),
+                        Collections.emptyList(),
+                        "windows-hostname:(120.85.114.146 OR 103.104.106.223 OR 185.191.246.45 OR 120.86.237.94)",
+                        List.of("threat_intel", entry.getKey() /*ioc_type*/)
+                ));
+            }
+        }
+        return queries;
     }
 
     private String buildQueryStringQueryWithIocList(Set<String> iocs) {
         StringBuilder sb = new StringBuilder();
+        sb.append("%s");
+        sb.append(":");
         sb.append("(");
         for (String ioc : iocs) {
-            if (sb.length() > 2) {
+            if (sb.length() > 4) {
                 sb.append(" OR ");
             }
             sb.append(ioc);
@@ -55,42 +82,47 @@ private String buildQueryStringQueryWithIocList(Set<String> iocs) {
         return sb.toString();
     }
 
-    public void createDocLevelQueryFromThreatIntel(Detector detector, ActionListener<DocLevelQuery> listener) {
+    public void createDocLevelQueryFromThreatIntel(List<LogType.IocFields> iocFieldList, Detector detector, ActionListener<List<DocLevelQuery>> listener) {
         try {
-            if (detector.getThreatIntelEnabled() == false) {
-                listener.onResponse(null);
+            if (false == detector.getThreatIntelEnabled() || iocFieldList.isEmpty()) {
+                listener.onResponse(Collections.emptyList());
                 return;
-
             }
+
             CountDownLatch latch = new CountDownLatch(1);
             threatIntelFeedDataService.getThreatIntelFeedData(new ActionListener<>() {
                 @Override
                 public void onResponse(List<ThreatIntelFeedData> threatIntelFeedData) {
                     if (threatIntelFeedData.isEmpty()) {
-                        listener.onResponse(null);
+                        listener.onResponse(Collections.emptyList());
                     } else {
-                        listener.onResponse(createDocLevelQueryFromThreatIntelList(
-                                threatIntelFeedData,
-                                detector.getName() + "_threat_intel" + UUID.randomUUID()
-                        ));
+                        listener.onResponse(
+                                createDocLevelQueriesFromThreatIntelList(iocFieldList, threatIntelFeedData, detector)
+                        );
                     }
                     latch.countDown();
                 }
 
                 @Override
                 public void onFailure(Exception e) {
+                    log.error("Failed to get threat intel feeds for doc level query creation", e);
                     listener.onFailure(e);
                     latch.countDown();
                 }
             });
 
             latch.await(30, TimeUnit.SECONDS);
         } catch (InterruptedException e) {
+            log.error("Failed to create doc level queries from threat intel feeds", e);
             listener.onFailure(e);
         }
 
     }
 
+    private static String constructId(Detector detector, String iocType) {
+        return detector.getName() + "_threat_intel_" + iocType + "_" + UUID.randomUUID();
+    }
+
     public void updateDetectorsWithLatestThreatIntelRules() {
 
     }

src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java

@@ -88,6 +88,7 @@
 import org.opensearch.securityanalytics.model.DetectorInput;
 import org.opensearch.securityanalytics.model.DetectorRule;
 import org.opensearch.securityanalytics.model.DetectorTrigger;
+import org.opensearch.securityanalytics.model.LogType;
 import org.opensearch.securityanalytics.model.Rule;
 import org.opensearch.securityanalytics.model.Value;
 import org.opensearch.securityanalytics.rules.aggregation.AggregationItem;
@@ -323,7 +324,9 @@ private void createMonitorFromQueries(List<Pair<String, Rule>> rulesById, Detect
                         monitorResponses.add(addedFirstMonitorResponse);
                         saveWorkflow(rulesById, detector, monitorResponses, refreshPolicy, listener);
                     },
-                    listener::onFailure
+                    e -> {
+                        listener.onFailure(e);
+                    }
             );
         }
     }
@@ -653,30 +656,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List<Pair<String, Rule>
             DocLevelQuery docLevelQuery = new DocLevelQuery(id, name, Collections.emptyList(), actualQuery, tags);
             docLevelQueries.add(docLevelQuery);
         }
-        try {
-            if (detector.getThreatIntelEnabled()) {
-                CountDownLatch countDownLatch = new CountDownLatch(1);
-                detectorThreatIntelService.createDocLevelQueryFromThreatIntel(detector, new ActionListener<>() {
-                    @Override
-                    public void onResponse(DocLevelQuery dlq) {
-                        if (dlq != null)
-                            docLevelQueries.add(dlq);
-                        countDownLatch.countDown();
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data
-                        log.error("Failed to convert threat intel feed to. Proceeding with detector creation", e);
-                        countDownLatch.countDown();
-                    }
-                });
-                countDownLatch.await();
-            }
-        } catch (Exception e) {
-            // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data
-            log.error("Failed to convert threat intel feed to. Proceeding with detector creation", e);
-        }
+        addThreatIntelBasedDocLevelQueries(detector, docLevelQueries);
         DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries);
         docLevelMonitorInputs.add(docLevelMonitorInput);
 
@@ -707,6 +687,39 @@ public void onFailure(Exception e) {
         return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null);
     }
 
+    private void addThreatIntelBasedDocLevelQueries(Detector detector, List<DocLevelQuery> docLevelQueries) {
+        try {
+
+            if (detector.getThreatIntelEnabled()) {
+                List<LogType.IocFields> iocFieldsList = logTypeService.getIocFieldsList(detector.getDetectorType());
+                if (iocFieldsList == null || iocFieldsList.isEmpty()) {
+
+                } else {
+                    CountDownLatch countDownLatch = new CountDownLatch(1);
+                    detectorThreatIntelService.createDocLevelQueryFromThreatIntel(iocFieldsList, detector, new ActionListener<>() {
+                        @Override
+                        public void onResponse(List<DocLevelQuery> dlqs) {
+                            if (dlqs != null)
+                                docLevelQueries.addAll(dlqs);
+                            countDownLatch.countDown();
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data
+                            log.error("Failed to convert threat intel feed to. Proceeding with detector creation", e);
+                            countDownLatch.countDown();
+                        }
+                    });
+                    countDownLatch.await();
+                }
+            }
+        } catch (Exception e) {
+            // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data
+            log.error("Failed to convert threat intel feed to doc level query. Proceeding with detector creation", e);
+        }
+    }
+
     /**
      * Creates doc level monitor which generates per document alerts for the findings of the bucket level delegate monitors in a workflow.
      * This monitor has match all query applied to generate the alerts per each finding doc.

src/main/resources/OSMapping/test_windows_logtype.json

@@ -2,8 +2,13 @@
   "name": "test_windows",
   "description": "Test Log Type used by tests. It is created as a lightweight log type for integration tests",
   "is_builtin": true,
-  "ioc_fields" : [],
-  "mappings":[
+  "ioc_fields": [
+    {
+      "ioc": "ip",
+      "fields": ["HostName"]
+    }
+  ],
+  "mappings": [
     {
       "raw_field":"EventID",
       "ecs":"event_uid"

src/test/java/org/opensearch/securityanalytics/TestHelpers.java

@@ -1373,6 +1373,46 @@ public static String randomDoc(int severity,  int version, String opCode) {
 
     }
 
+    //Add IPs in HostName field.
+    public static String randomDocWithIpIoc(int severity,  int version, String ioc) {
+        String doc =  "{\n" +
+                "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
+                "\"HostName\":\"%s\",\n" +
+                "\"Keywords\":\"9223372036854775808\",\n" +
+                "\"SeverityValue\":%s,\n" +
+                "\"Severity\":\"INFO\",\n" +
+                "\"EventID\":22,\n" +
+                "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" +
+                "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" +
+                "\"Version\":%s,\n" +
+                "\"TaskValue\":22,\n" +
+                "\"OpcodeValue\":0,\n" +
+                "\"RecordNumber\":9532,\n" +
+                "\"ExecutionProcessID\":1996,\n" +
+                "\"ExecutionThreadID\":2616,\n" +
+                "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
+                "\"Domain\":\"NT AUTHORITY\",\n" +
+                "\"AccountName\":\"SYSTEM\",\n" +
+                "\"UserID\":\"S-1-5-18\",\n" +
+                "\"AccountType\":\"User\",\n" +
+                "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
+                "\"Category\":\"Dns query (rule: DnsQuery)\",\n" +
+                "\"Opcode\":\"%blahblah\",\n" +
+                "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" +
+                "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" +
+                "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" +
+                "\"QueryResults\":\"172.31.46.38;\",\n" +
+                "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" +
+                "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" +
+                "\"SourceModuleName\":\"in\",\n" +
+                "\"SourceModuleType\":\"im_msvistalog\",\n" +
+                "\"CommandLine\": \"eachtest\",\n" +
+                "\"Initiated\": \"true\"\n" +
+                "}";
+        return String.format(Locale.ROOT, ioc, doc, severity, version);
+
+    }
+
     public static String randomDoc() {
         return "{\n" +
                 "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +

src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java

@@ -1088,6 +1088,7 @@ public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatInt
                 "}";
         SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);
 
+
         assertEquals(2, response.getHits().getTotalHits().value);
 
         assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));
@@ -1118,7 +1119,7 @@ public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatInt
         List<String> iocs = getThreatIntelFeedIocs(3);
         int i=1;
         for (String ioc : iocs) {
-            indexDoc(index, i+"", randomDoc(5, 3, ioc));
+            indexDoc(index, i+"", randomDoc(5, 3, i==1? "120.85.114.146" : "120.86.237.94"));
             i++;
         }
         String workflowId = ((List<String>) detectorMap.get("workflow_ids")).get(0);