ioc scan business logic

src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java → src/main/java/org/opensearch/securityanalytics/model/threatintel/IocMatch.java

@@ -1,4 +1,4 @@
-package org.opensearch.securityanalytics.model;
+package org.opensearch.securityanalytics.model.threatintel;
 
 import org.apache.commons.lang3.StringUtils;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -20,7 +20,7 @@
  * IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job
  * It's the inverse of an IoC finding which maps a document to list of IoC's
  */
-public class IoCMatch implements Writeable, ToXContent {
+public class IocMatch implements Writeable, ToXContent {
     //TODO implement IoC_Match interface from security-analytics-commons
     public static final String ID_FIELD = "id";
     public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids";
@@ -42,7 +42,7 @@ public class IoCMatch implements Writeable, ToXContent {
     private final Instant timestamp;
     private final String executionId;
 
-    public IoCMatch(String id, List<String> relatedDocIds, List<String> feedIds, String iocScanJobId,
+    public IocMatch(String id, List<String> relatedDocIds, List<String> feedIds, String iocScanJobId,
                     String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) {
         validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds);
         this.id = id;
@@ -56,7 +56,7 @@ public IoCMatch(String id, List<String> relatedDocIds, List<String> feedIds, Str
         this.executionId = executionId;
     }
 
-    public IoCMatch(StreamInput in) throws IOException {
+    public IocMatch(StreamInput in) throws IOException {
         id = in.readString();
         relatedDocIds = in.readStringList();
         feedIds = in.readStringList();
@@ -133,7 +133,7 @@ public String getExecutionId() {
         return executionId;
     }
 
-    public static IoCMatch parse(XContentParser xcp) throws IOException {
+    public static IocMatch parse(XContentParser xcp) throws IOException {
         String id = null;
         List<String> relatedDocIds = new ArrayList<>();
         List<String> feedIds = new ArrayList<>();
@@ -197,11 +197,11 @@ public static IoCMatch parse(XContentParser xcp) throws IOException {
             }
         }
 
-        return new IoCMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId);
+        return new IocMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId);
     }
 
-    public static IoCMatch readFrom(StreamInput in) throws IOException {
-        return new IoCMatch(in);
+    public static IocMatch readFrom(StreamInput in) throws IOException {
+        return new IocMatch(in);
     }
 
 

src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java

@@ -0,0 +1,23 @@
+package org.opensearch.securityanalytics.threatIntel.iocscan.dto;
+
+import org.opensearch.securityanalytics.threatIntel.iocscan.model.IocScanMonitor;
+
+import java.util.List;
+
+public class IocScanContext<Data> {
+    IocScanMonitor iocScanMonitor;
+    boolean dryRun;
+    List<Data> data;
+
+    public IocScanMonitor getIocScanMonitor() {
+        return iocScanMonitor;
+    }
+
+    public boolean isDryRun() {
+        return dryRun;
+    }
+
+    public List<Data> getData() {
+        return data;
+    }
+}

src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeFieldMappings.java

@@ -0,0 +1,26 @@
+package org.opensearch.securityanalytics.threatIntel.iocscan.dto;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * DTO that contains information about an Ioc type and the list of fields in each index that map to the given
+ */
+public class PerIocTypeFieldMappings {
+
+    private final String iocType;
+    private final Map<String, List<String>> indexToFieldsMap;
+
+    public PerIocTypeFieldMappings(String iocType, Map<String, List<String>> indexToFieldsMap) {
+        this.iocType = iocType;
+        this.indexToFieldsMap = indexToFieldsMap;
+    }
+
+    public String getIocType() {
+        return iocType;
+    }
+
+    public Map<String, List<String>> getIndexToFieldsMap() {
+        return indexToFieldsMap;
+    }
+}

src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/Ioc.java

@@ -0,0 +1,26 @@
+package org.opensearch.securityanalytics.threatIntel.iocscan.model;
+
+public class Ioc {
+
+    private final String feedId;
+    private final String iocValue;
+    private final String iocType;
+
+    public Ioc(String feedId, String iocValue, String iocType) {
+        this.feedId = feedId;
+        this.iocValue = iocValue;
+        this.iocType = iocType;
+    }
+
+    public String getFeedId() {
+        return feedId;
+    }
+
+    public String getIocValue() {
+        return iocValue;
+    }
+
+    public String getIocType() {
+        return iocType;
+    }
+}

src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/IocScanMonitor.java

@@ -0,0 +1,30 @@
+package org.opensearch.securityanalytics.threatIntel.iocscan.model;
+
+import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeFieldMappings;
+
+import java.util.List;
+import java.util.Map;
+
+
+public class IocScanMonitor {
+    String id;
+    String name;
+    List<PerIocTypeFieldMappings> iocTypeToIndexFieldMappings;
+    Map<String, List<String>>  perIoCTypeThreatIntelIndices;
+
+    public String getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public List<PerIocTypeFieldMappings> getIocTypeToIndexFieldMappings() {
+        return iocTypeToIndexFieldMappings;
+    }
+
+    public Map<String, List<String>> getPerIoCTypeThreatIntelIndices() {
+        return perIoCTypeThreatIntelIndices;
+    }
+}

src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java

@@ -0,0 +1,192 @@
+package org.opensearch.securityanalytics.threatIntel.iocscan.service;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.commons.alerting.model.Finding;
+import org.opensearch.securityanalytics.model.threatintel.IocMatch;
+import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext;
+import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeFieldMappings;
+import org.opensearch.securityanalytics.threatIntel.iocscan.model.Ioc;
+import org.opensearch.securityanalytics.threatIntel.iocscan.model.IocScanMonitor;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+
+
+public abstract class IoCScanService<Data> implements IoCScanServiceInterface<Data> {
+    private static final Logger log = LogManager.getLogger(IoCScanService.class);
+
+    @Override
+    public void scanIoCs(IocScanContext<Data> iocScanContext,
+                         BiConsumer<Object, Exception> scanCallback
+    ) {
+        List<Data> data = iocScanContext.getData();
+        IocScanMonitor iocScanMonitor = iocScanContext.getIocScanMonitor();
+
+        long start = System.currentTimeMillis();
+        // log.debug("beginning to scan IoC's")
+        IocLookupDtos iocLookupDtos = extractIocPerTypeSet(data, iocScanMonitor.getIocTypeToIndexFieldMappings());
+        BiConsumer<List<Ioc>, Exception> iocScanResultConsumer = (List<Ioc> maliciousIocs, Exception e) -> {
+            if (e == null) {
+                createIoCMatches(maliciousIocs, iocLookupDtos.iocValueToDocIdMap);
+            } else {
+                //                onIocMatchFailure(e, iocScanMonitor);
+
+            }
+        };
+        matchAgainstThreatIntelAndReturnMaliciousIocs(iocLookupDtos.getIocsPerIocTypeMap(), iocScanMonitor, iocScanResultConsumer);
+    }
+
+    abstract void matchAgainstThreatIntelAndReturnMaliciousIocs(
+            Map<String, Set<String>> iocPerTypeSet,
+            IocScanMonitor iocScanMonitor,
+            BiConsumer<List<Ioc>, Exception> callback);
+
+    /**
+     * For each doc, we extract the list of
+     */
+    private IocLookupDtos extractIocPerTypeSet(List<Data> data, List<PerIocTypeFieldMappings> iocTypeToIndexFieldMappings) {
+        Map<String, Set<String>> iocsPerIocTypeMap = new HashMap<>();
+        Map<String, Set<String>> iocValueToDocIdMap = new HashMap<>();
+        Map<String, Set<String>> docIdToIocsMap = new HashMap<>();
+        for (Data datum : data) {
+            for (PerIocTypeFieldMappings iocTypeToIndexFieldMapping : iocTypeToIndexFieldMappings) {
+                String iocType = iocTypeToIndexFieldMapping.getIocType();
+                String index = getIndexName(datum);
+                List<String> fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index);
+                for (String field : fields) {
+                    List<String> vals = getValuesAsStringList(datum, field);
+                    String docId = getId(datum);
+                    Set<String> iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>());
+                    iocs.addAll(vals);
+                    docIdToIocsMap.put(docId, iocs);
+                    for (String ioc : vals) {
+                        Set<String> docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>());
+                        docIds.add(docId);
+                        iocValueToDocIdMap.put(ioc, docIds);
+                    }
+                    if (false == vals.isEmpty()) {
+                        iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>());
+                        iocs.addAll(vals);
+                        iocsPerIocTypeMap.put(iocType, iocs);
+                    }
+                }
+            }
+        }
+        return new IocLookupDtos(iocsPerIocTypeMap, iocValueToDocIdMap, docIdToIocsMap);
+    }
+
+    public abstract List<String> getValuesAsStringList(Data datum, String field);
+
+    public abstract String getIndexName(Data datum);
+
+    public abstract String getId(Data datum);
+
+    public void createIoCMatches(List<Ioc> iocs, Map<String, Set<String>> iocValueToDocIdMap, IocScanContext iocScanContext) {
+        try {
+            Instant timestamp = Instant.now();
+            IocScanMonitor iocScanMonitor = iocScanContext.getIocScanMonitor();
+            // Map to collect unique IocValue with their respective FeedIds
+            Map<String, Set<String>> iocValueToFeedIds = new HashMap<>();
+
+            for (Ioc ioc : iocs) {
+                String iocValue = ioc.getIocValue();
+                iocValueToFeedIds
+                        .computeIfAbsent(iocValue, k -> new HashSet<>())
+                        .add(ioc.getFeedId());
+            }
+
+            List<IocMatch> iocMatches = new ArrayList<>();
+
+            for (Map.Entry<String, Set<String>> entry : iocValueToFeedIds.entrySet()) {
+                String iocValue = entry.getKey();
+                Set<String> feedIds = entry.getValue();
+
+                List<String> relatedDocIds = new ArrayList<>(iocValueToDocIdMap.getOrDefault(iocValue, new HashSet<>()));
+                List<String> feedIdsList = new ArrayList<>(feedIds);
+
+                IocMatch iocMatch = new IocMatch(
+                        UUID.randomUUID().toString(), // Generating a unique ID
+                        relatedDocIds,
+                        feedIdsList,
+                        iocScanMonitor.getId(),
+                        iocScanMonitor.getName(),
+                        iocValue,
+                        iocs.stream().filter(i -> i.getIocValue().equals(iocValue)).findFirst().orElseThrow().getIocType(),
+                        timestamp,
+                        UUID.randomUUID().toString() // TODO execution ID
+                );
+
+                iocMatches.add(iocMatch);
+            }
+
+
+
+        } catch (Exception e) {
+            log.error("failed to create ioc matches for " + .iocValue + ",moving onto next ioc match. Failure", e);
+        }
+    }
+
+    private static class IocMatchDto {
+        private final String iocValue;
+        private final String iocType;
+        private final List<Ioc> iocs;
+        private final List<String> docIdsContainingIoc;
+
+        public IocMatchDto(String iocValue, String iocType, List<Ioc> iocs, List<String> docIdsContainingIoc) {
+            this.iocValue = iocValue;
+            this.iocType = iocType;
+            this.iocs = iocs;
+            this.docIdsContainingIoc = docIdsContainingIoc;
+        }
+
+        public String getIocValue() {
+            return iocValue;
+        }
+
+        public String getIocType() {
+            return iocType;
+        }
+
+        public List<Ioc> getIocs() {
+            return iocs;
+        }
+
+        public List<String> getDocIdsContainingIoc() {
+            return docIdsContainingIoc;
+        }
+    }
+
+    private static class IocLookupDtos {
+        private final Map<String, Set<String>> iocsPerIocTypeMap;
+        private final Map<String, Set<String>> iocValueToDocIdMap;
+        private final Map<String, Set<String>> docIdToIocsMap;
+
+        public IocLookupDtos(Map<String, Set<String>> iocsPerIocTypeMap, Map<String, Set<String>> iocValueToDocIdMap, Map<String, Set<String>> docIdToIocsMap) {
+            this.iocsPerIocTypeMap = iocsPerIocTypeMap;
+            this.iocValueToDocIdMap = iocValueToDocIdMap;
+            this.docIdToIocsMap = docIdToIocsMap;
+        }
+
+        public Map<String, Set<String>> getIocsPerIocTypeMap() {
+            return iocsPerIocTypeMap;
+        }
+
+        public Map<String, Set<String>> getIocValueToDocIdMap() {
+            return iocValueToDocIdMap;
+        }
+
+        public Map<String, Set<String>> getDocIdToIocsMap() {
+            return docIdToIocsMap;
+        }
+    }
+
+}

src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java

@@ -0,0 +1,20 @@
+package org.opensearch.securityanalytics.threatIntel.iocscan.service;
+
+import org.opensearch.commons.alerting.model.Finding;
+import org.opensearch.securityanalytics.model.threatintel.IocMatch;
+import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext;
+
+import java.util.List;
+import java.util.function.BiConsumer;
+
+public interface IoCScanServiceInterface<Data> {
+
+    void createIoCFindings(List<Finding> findings);
+
+    void createIoCMatch(List<IocMatch> iocMatches);
+
+    void scanIoCs(
+            IocScanContext<Data> iocScanContext,
+            BiConsumer<Object, Exception> scanCallback
+    );
+}

src/test/java/org/opensearch/securityanalytics/TestHelpers.java

@@ -28,7 +28,7 @@
 import org.opensearch.securityanalytics.model.DetectorInput;
 import org.opensearch.securityanalytics.model.DetectorRule;
 import org.opensearch.securityanalytics.model.DetectorTrigger;
-import org.opensearch.securityanalytics.model.IoCMatch;
+import org.opensearch.securityanalytics.model.threatintel.IocMatch;
 import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
@@ -800,7 +800,7 @@ public static String toJsonStringWithUser(Detector detector) throws IOException
         return BytesReference.bytes(builder).utf8ToString();
     }
 
-    public static String toJsonString(IoCMatch iocMatch) throws IOException {
+    public static String toJsonString(IocMatch iocMatch) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
         builder = iocMatch.toXContent(builder, ToXContent.EMPTY_PARAMS);
         return BytesReference.bytes(builder).utf8ToString();