From f1852a6f88d458adcc5b0a15fcf061219baa06fa Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Mon, 13 May 2024 15:49:51 -0700 Subject: [PATCH] ioc match model Signed-off-by: Surya Sashank Nistala --- .../securityanalytics/model/IoCMatch.java | 234 ++++++++++++++++++ .../resources/mappings/ioc_match_mapping.json | 38 +++ .../securityanalytics/TestHelpers.java | 7 + .../model/IoCMatchTests.java | 45 ++++ 4 files changed, 324 insertions(+) create mode 100644 src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java create mode 100644 src/main/resources/mappings/ioc_match_mapping.json create mode 100644 src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java diff --git a/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java b/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java new file mode 100644 index 000000000..04f54699f --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java @@ -0,0 +1,234 @@ +package org.opensearch.securityanalytics.model; + +import org.apache.commons.lang3.StringUtils; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job + * It's the inverse of an IoC finding which maps a document to list of IoC's + */ +public class IoCMatch implements Writeable, ToXContent { + //TODO implement IoC_Match interface from security-analytics-commons + public static final String ID_FIELD = "id"; + public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids"; + public static final String FEED_IDS_FIELD = "feed_ids"; + public static final String IOC_SCAN_JOB_ID_FIELD = "ioc_scan_job_id"; + public static final String IOC_SCAN_JOB_NAME_FIELD = "ioc_scan_job_name"; + public static final String IOC_VALUE_FIELD = "ioc_value"; + public static final String IOC_TYPE_FIELD = "ioc_type"; + public static final String TIMESTAMP_FIELD = "timestamp"; + public static final String EXECUTION_ID_FIELD = "execution_id"; + + private final String id; + private final List relatedDocIds; + private final List feedIds; + private final String iocScanJobId; + private final String iocScanJobName; + private final String iocValue; + private final String iocType; + private final Instant timestamp; + private final String executionId; + + public IoCMatch(String id, List relatedDocIds, List feedIds, String iocScanJobId, + String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) { + validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds); + this.id = id; + this.relatedDocIds = relatedDocIds; + this.feedIds = feedIds; + this.iocScanJobId = iocScanJobId; + this.iocScanJobName = iocScanJobName; + this.iocValue = iocValue; + this.iocType = iocType; + this.timestamp = timestamp; + this.executionId = executionId; + } + + public IoCMatch(StreamInput in) throws IOException { + id = in.readString(); + relatedDocIds = in.readStringList(); + feedIds = in.readStringList(); + iocScanJobId = in.readString(); + iocScanJobName = in.readString(); + iocValue = in.readString(); + iocType = in.readString(); + timestamp = in.readInstant(); + executionId = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeStringCollection(relatedDocIds); + out.writeStringCollection(feedIds); + out.writeString(iocScanJobId); + out.writeString(iocScanJobName); + out.writeString(iocValue); + out.writeString(iocType); + out.writeInstant(timestamp); + out.writeOptionalString(executionId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(ID_FIELD, id) + .field(RELATED_DOC_IDS_FIELD, relatedDocIds) + .field(FEED_IDS_FIELD, feedIds) + .field(IOC_SCAN_JOB_ID_FIELD, iocScanJobId) + .field(IOC_SCAN_JOB_NAME_FIELD, iocScanJobName) + .field(IOC_VALUE_FIELD, iocValue) + .field(IOC_TYPE_FIELD, iocType) + .field(TIMESTAMP_FIELD, timestamp.toEpochMilli()) + .field(EXECUTION_ID_FIELD, executionId) + .endObject(); + return builder; + } + + public String getId() { + return id; + } + + public List getRelatedDocIds() { + return relatedDocIds; + } + + public List getFeedIds() { + return feedIds; + } + + public String getIocScanJobId() { + return iocScanJobId; + } + + public String getIocScanJobName() { + return iocScanJobName; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } + + public Instant getTimestamp() { + return timestamp; + } + + public String getExecutionId() { + return executionId; + } + + public static IoCMatch parse(XContentParser xcp) throws IOException { + String id = null; + List relatedDocIds = new ArrayList<>(); + List feedIds = new ArrayList<>(); + String iocScanJobId = null; + String iocScanName = null; + String iocValue = null; + String iocType = null; + Instant timestamp = null; + String executionId = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case ID_FIELD: + id = xcp.text(); + break; + case RELATED_DOC_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + relatedDocIds.add(xcp.text()); + } + break; + case FEED_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + feedIds.add(xcp.text()); + } + break; + case IOC_SCAN_JOB_ID_FIELD: + iocScanJobId = xcp.textOrNull(); + break; + case IOC_SCAN_JOB_NAME_FIELD: + iocScanName = xcp.textOrNull(); + break; + case IOC_VALUE_FIELD: + iocValue = xcp.textOrNull(); + break; + case IOC_TYPE_FIELD: + iocType = xcp.textOrNull(); + break; + case TIMESTAMP_FIELD: + try { + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + timestamp = null; + } else if (xcp.currentToken().isValue()) { + timestamp = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + timestamp = null; + } + break; + } catch (Exception e) { + throw new IllegalArgumentException("failed to parse timestamp in IoC Match object"); + } + case EXECUTION_ID_FIELD: + executionId = xcp.textOrNull(); + break; + } + } + + return new IoCMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId); + } + + public static IoCMatch readFrom(StreamInput in) throws IOException { + return new IoCMatch(in); + } + + + private static void validateIoCMatch(String id, String iocScanJobId, String iocScanName, String iocValue, Instant timestamp, String executionId, List relatedDocIds) { + if (StringUtils.isBlank(id)) { + throw new IllegalArgumentException("id cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocValue)) { + throw new IllegalArgumentException("ioc_value cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocValue)) { + throw new IllegalArgumentException("ioc_value cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocScanJobId)) { + throw new IllegalArgumentException("ioc_scan_job_id cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocScanName)) { + throw new IllegalArgumentException("ioc_scan_job_name cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(executionId)) { + throw new IllegalArgumentException("execution_id cannot be empty in IoC_Match Object"); + } + if (timestamp == null) { + throw new IllegalArgumentException("timestamp cannot be null in IoC_Match Object"); + } + if(relatedDocIds == null || relatedDocIds.isEmpty()) { + throw new IllegalArgumentException("related_doc_ids cannot be null or empty in IoC_Match Object"); + } + } +} \ No newline at end of file diff --git a/src/main/resources/mappings/ioc_match_mapping.json b/src/main/resources/mappings/ioc_match_mapping.json new file mode 100644 index 000000000..f4573190e --- /dev/null +++ b/src/main/resources/mappings/ioc_match_mapping.json @@ -0,0 +1,38 @@ +{ + "dynamic": "strict", + "_meta" : { + "schema_version": 1 + }, + "properties": { + "schema_version": { + "type": "integer" + }, + "feed_ids" : { + "type": "keyword" + }, + "related_doc_ids": { + "type": "keyword" + }, + "ioc_scan_job_id": { + "type": "keyword" + }, + "ioc_scan_job_name": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "ioc_value" : { + "type": "keyword" + }, + "ioc_type" : { + "type": "keyword" + }, + "timestamp": { + "type": "long" + }, + "execution_id": { + "type": "keyword" + } + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index a1987138d..03dca9281 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -28,6 +28,7 @@ import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.model.IoCMatch; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -799,6 +800,12 @@ public static String toJsonStringWithUser(Detector detector) throws IOException return BytesReference.bytes(builder).utf8ToString(); } + public static String toJsonString(IoCMatch iocMatch) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = iocMatch.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + public static String toJsonString(ThreatIntelFeedData threatIntelFeedData) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); builder = threatIntelFeedData.toXContent(builder, ToXContent.EMPTY_PARAMS); diff --git a/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java b/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java new file mode 100644 index 000000000..454072178 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java @@ -0,0 +1,45 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; + +import static org.opensearch.securityanalytics.TestHelpers.toJsonString; + +public class IoCMatchTests extends OpenSearchTestCase { + + public void testIoCMatchAsAStream() throws IOException { + IoCMatch iocMatch = getRandomIoCMatch(); + String jsonString = toJsonString(iocMatch); + BytesStreamOutput out = new BytesStreamOutput(); + iocMatch.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IoCMatch newIocMatch = new IoCMatch(sin); + assertEquals(iocMatch.getId(), newIocMatch.getId()); + assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); + assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); + assertEquals(iocMatch.getIocValue(), newIocMatch.getIocValue()); + assertEquals(iocMatch.getIocType(), newIocMatch.getIocType()); + assertEquals(iocMatch.getTimestamp(), newIocMatch.getTimestamp()); + assertEquals(iocMatch.getExecutionId(), newIocMatch.getExecutionId()); + assertTrue(iocMatch.getFeedIds().containsAll(newIocMatch.getFeedIds())); + assertTrue(iocMatch.getRelatedDocIds().containsAll(newIocMatch.getRelatedDocIds())); + } + + private static IoCMatch getRandomIoCMatch() { + return new IoCMatch( + randomAlphaOfLength(10), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Instant.now(), + randomAlphaOfLength(10)); + } +}