From ec0721469a7dd03aef26ad11366beb50020fe744 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Fri, 21 Jun 2024 19:07:10 +0000 Subject: [PATCH] add search ioc findings api Signed-off-by: Subhobrata Dey add search ioc findings api Signed-off-by: Subhobrata Dey add search ioc findings api Signed-off-by: Subhobrata Dey --- .../SecurityAnalyticsPlugin.java | 23 +- .../model/{IoCMatch.java => IocFinding.java} | 38 ++-- .../securityanalytics/model/IocWithFeeds.java | 99 +++++++++ .../action/GetIocFindingsAction.java | 17 ++ .../action/GetIocFindingsRequest.java | 91 ++++++++ .../action/GetIocFindingsResponse.java | 62 ++++++ .../iocscan/dao/IocFindingService.java | 209 ++++++++++++++++++ .../resthandler/RestGetIocFindingsAction.java | 102 +++++++++ .../TransportGetIocFindingsAction.java | 148 +++++++++++++ ....alerting.spi.RemoteMonitorRunnerExtension | 6 - ..._mapping.json => ioc_finding_mapping.json} | 15 +- .../SecurityAnalyticsRestTestCase.java | 10 + .../securityanalytics/TestHelpers.java | 7 +- .../model/IoCMatchTests.java | 14 +- .../model/IocFindingTests.java | 80 +++++++ .../dao/IocFindingServiceRestApiIT.java | 72 ++++++ 16 files changed, 940 insertions(+), 53 deletions(-) rename src/main/java/org/opensearch/securityanalytics/model/{IoCMatch.java => IocFinding.java} (86%) create mode 100644 src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java delete mode 100644 src/main/resources/META-INF/services/org.opensearch.alerting.spi.RemoteMonitorRunnerExtension rename src/main/resources/mappings/{ioc_match_mapping.json => ioc_finding_mapping.json} (69%) create mode 100644 src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 6c63d83e2..413808b6f 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -7,8 +7,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionRequest; -import org.opensearch.alerting.spi.RemoteMonitorRunner; -import org.opensearch.alerting.spi.RemoteMonitorRunnerExtension; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; @@ -117,6 +115,7 @@ import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SARefreshTIFSourceConfigAction; @@ -132,6 +131,7 @@ import org.opensearch.securityanalytics.threatIntel.resthandler.RestDeleteTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.model.monitor.SampleRemoteDocLevelMonitorRunner; import org.opensearch.securityanalytics.threatIntel.model.monitor.TransportRemoteDocLevelMonitorFanOutAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestRefreshTIFSourceConfigAction; @@ -180,6 +180,7 @@ import org.opensearch.securityanalytics.transport.TransportSearchRuleAction; import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction; import org.opensearch.securityanalytics.transport.TransportValidateRulesAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.securityanalytics.util.CorrelationRuleIndices; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; @@ -188,7 +189,6 @@ import org.opensearch.securityanalytics.util.RuleTopicIndices; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; -import reactor.util.annotation.NonNull; import java.util.Collection; import java.util.Collections; @@ -199,9 +199,8 @@ import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; -import static org.opensearch.securityanalytics.threatIntel.model.monitor.SampleRemoteDocLevelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; -public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension, RemoteMonitorRunnerExtension { +public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension { private static final Logger log = LogManager.getLogger(SecurityAnalyticsPlugin.class); @@ -345,7 +344,8 @@ public List getRestHandlers(Settings settings, new RestDeleteThreatIntelMonitorAction(), new RestSearchThreatIntelMonitorAction(), new RestRefreshTIFSourceConfigAction(), - new RestListIOCsAction() + new RestListIOCsAction(), + new RestGetIocFindingsAction() ); } @@ -490,7 +490,8 @@ public List> getSettings() { new ActionHandler<>(SASearchTIFSourceConfigsAction.INSTANCE, TransportSearchTIFSourceConfigsAction.class), new ActionHandler<>(SARefreshTIFSourceConfigAction.INSTANCE, TransportRefreshTIFSourceConfigAction.class), new ActionHandler<>(SampleRemoteDocLevelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportRemoteDocLevelMonitorFanOutAction.class), - new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class) + new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class), + new ActionHandler<>(GetIocFindingsAction.INSTANCE, TransportGetIocFindingsAction.class) ); } @@ -509,12 +510,4 @@ public void onFailure(Exception e) { } }); } - - @NonNull - @Override - public Map getMonitorTypesToMonitorRunners() { - return Map.of( - THREAT_INTEL_MONITOR_TYPE, SampleRemoteDocLevelMonitorRunner.getMonitorRunner() - ); - } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java b/src/main/java/org/opensearch/securityanalytics/model/IocFinding.java similarity index 86% rename from src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java rename to src/main/java/org/opensearch/securityanalytics/model/IocFinding.java index 04f54699f..4a66ed3c0 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java +++ b/src/main/java/org/opensearch/securityanalytics/model/IocFinding.java @@ -20,11 +20,11 @@ * IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job * It's the inverse of an IoC finding which maps a document to list of IoC's */ -public class IoCMatch implements Writeable, ToXContent { +public class IocFinding implements Writeable, ToXContent { //TODO implement IoC_Match interface from security-analytics-commons public static final String ID_FIELD = "id"; public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids"; - public static final String FEED_IDS_FIELD = "feed_ids"; + public static final String IOC_WITH_FEED_IDS_FIELD = "ioc_feed_ids"; public static final String IOC_SCAN_JOB_ID_FIELD = "ioc_scan_job_id"; public static final String IOC_SCAN_JOB_NAME_FIELD = "ioc_scan_job_name"; public static final String IOC_VALUE_FIELD = "ioc_value"; @@ -34,7 +34,7 @@ public class IoCMatch implements Writeable, ToXContent { private final String id; private final List relatedDocIds; - private final List feedIds; + private final List iocWithFeeds; private final String iocScanJobId; private final String iocScanJobName; private final String iocValue; @@ -42,12 +42,12 @@ public class IoCMatch implements Writeable, ToXContent { private final Instant timestamp; private final String executionId; - public IoCMatch(String id, List relatedDocIds, List feedIds, String iocScanJobId, - String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) { + public IocFinding(String id, List relatedDocIds, List iocWithFeeds, String iocScanJobId, + String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) { validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds); this.id = id; this.relatedDocIds = relatedDocIds; - this.feedIds = feedIds; + this.iocWithFeeds = iocWithFeeds; this.iocScanJobId = iocScanJobId; this.iocScanJobName = iocScanJobName; this.iocValue = iocValue; @@ -56,10 +56,10 @@ public IoCMatch(String id, List relatedDocIds, List feedIds, Str this.executionId = executionId; } - public IoCMatch(StreamInput in) throws IOException { + public IocFinding(StreamInput in) throws IOException { id = in.readString(); relatedDocIds = in.readStringList(); - feedIds = in.readStringList(); + iocWithFeeds = in.readList(IocWithFeeds::readFrom); iocScanJobId = in.readString(); iocScanJobName = in.readString(); iocValue = in.readString(); @@ -72,7 +72,7 @@ public IoCMatch(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeStringCollection(relatedDocIds); - out.writeStringCollection(feedIds); + out.writeCollection(iocWithFeeds); out.writeString(iocScanJobId); out.writeString(iocScanJobName); out.writeString(iocValue); @@ -86,7 +86,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject() .field(ID_FIELD, id) .field(RELATED_DOC_IDS_FIELD, relatedDocIds) - .field(FEED_IDS_FIELD, feedIds) + .field(IOC_WITH_FEED_IDS_FIELD, iocWithFeeds) .field(IOC_SCAN_JOB_ID_FIELD, iocScanJobId) .field(IOC_SCAN_JOB_NAME_FIELD, iocScanJobName) .field(IOC_VALUE_FIELD, iocValue) @@ -105,8 +105,8 @@ public List getRelatedDocIds() { return relatedDocIds; } - public List getFeedIds() { - return feedIds; + public List getFeedIds() { + return iocWithFeeds; } public String getIocScanJobId() { @@ -133,10 +133,10 @@ public String getExecutionId() { return executionId; } - public static IoCMatch parse(XContentParser xcp) throws IOException { + public static IocFinding parse(XContentParser xcp) throws IOException { String id = null; List relatedDocIds = new ArrayList<>(); - List feedIds = new ArrayList<>(); + List feedIds = new ArrayList<>(); String iocScanJobId = null; String iocScanName = null; String iocValue = null; @@ -159,10 +159,10 @@ public static IoCMatch parse(XContentParser xcp) throws IOException { relatedDocIds.add(xcp.text()); } break; - case FEED_IDS_FIELD: + case IOC_WITH_FEED_IDS_FIELD: ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - feedIds.add(xcp.text()); + feedIds.add(IocWithFeeds.parse(xcp)); } break; case IOC_SCAN_JOB_ID_FIELD: @@ -197,11 +197,11 @@ public static IoCMatch parse(XContentParser xcp) throws IOException { } } - return new IoCMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId); + return new IocFinding(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId); } - public static IoCMatch readFrom(StreamInput in) throws IOException { - return new IoCMatch(in); + public static IocFinding readFrom(StreamInput in) throws IOException { + return new IocFinding(in); } diff --git a/src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java b/src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java new file mode 100644 index 000000000..faee32e76 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java @@ -0,0 +1,99 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +public class IocWithFeeds implements Writeable, ToXContent { + + private static final String FEED_ID_FIELD = "feed_id"; + + private static final String IOC_ID_FIELD = "ioc_id"; + + private static final String INDEX_FIELD = "index"; + + private final String feedId; + + private final String iocId; + + private final String index; + + public IocWithFeeds(String iocId, String feedId, String index) { + this.iocId = iocId; + this.feedId = feedId; + this.index = index; + } + + public IocWithFeeds(StreamInput sin) throws IOException { + this.iocId = sin.readString(); + this.feedId = sin.readString(); + this.index = sin.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(iocId); + out.writeString(feedId); + out.writeString(index); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(IOC_ID_FIELD, iocId) + .field(FEED_ID_FIELD, feedId) + .field(INDEX_FIELD, index) + .endObject(); + return builder; + } + + public String getIocId() { + return iocId; + } + + public String getFeedId() { + return feedId; + } + + public String getIndex() { + return index; + } + + public static IocWithFeeds parse(XContentParser xcp) throws IOException { + String iocId = null; + String feedId = null; + String index = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case IOC_ID_FIELD: + iocId = xcp.text(); + break; + case FEED_ID_FIELD: + feedId = xcp.text(); + break; + case INDEX_FIELD: + index = xcp.text(); + break; + default: + xcp.skipChildren(); + } + } + return new IocWithFeeds(iocId, feedId, index); + } + + public static IocWithFeeds readFrom(StreamInput sin) throws IOException { + return new IocWithFeeds(sin); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java new file mode 100644 index 000000000..f92f3350e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +public class GetIocFindingsAction extends ActionType { + + public static final GetIocFindingsAction INSTANCE = new GetIocFindingsAction(); + public static final String NAME = "cluster:admin/opensearch/securityanalytics/ioc/findings/get"; + + public GetIocFindingsAction() { + super(NAME, GetIocFindingsResponse::new); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java new file mode 100644 index 000000000..1395cff1e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ValidateActions; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +public class GetIocFindingsRequest extends ActionRequest { + + private List findingIds; + + private List iocIds; + + private Instant startTime; + + private Instant endTime; + + private Table table; + + public GetIocFindingsRequest(StreamInput sin) throws IOException { + this( + sin.readOptionalStringList(), + sin.readOptionalStringList(), + sin.readOptionalInstant(), + sin.readOptionalInstant(), + Table.readFrom(sin) + ); + } + + public GetIocFindingsRequest(List findingIds, + List iocIds, + Instant startTime, + Instant endTime, + Table table) { + this.findingIds = findingIds; + this.iocIds = iocIds; + this.startTime = startTime; + this.endTime = endTime; + this.table = table; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (startTime != null && endTime != null && startTime.isAfter(endTime)) { + validationException = ValidateActions.addValidationError(String.format(Locale.getDefault(), + "startTime should be less than endTime"), validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalStringCollection(findingIds); + out.writeOptionalStringCollection(iocIds); + out.writeOptionalInstant(startTime); + out.writeOptionalInstant(endTime); + table.writeTo(out); + } + + public List getFindingIds() { + return findingIds; + } + + public List getIocIds() { + return iocIds; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } + + public Table getTable() { + return table; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java new file mode 100644 index 000000000..4c0dea477 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.model.IocFinding; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class GetIocFindingsResponse extends ActionResponse implements ToXContentObject { + + private static final String TOTAL_IOC_FINDINGS_FIELD = "total_findings"; + + private static final String IOC_FINDINGS_FIELD = "ioc_findings"; + + private Integer totalFindings; + + private List iocFindings; + + public GetIocFindingsResponse(Integer totalFindings, List iocFindings) { + super(); + this.totalFindings = totalFindings; + this.iocFindings = iocFindings; + } + + public GetIocFindingsResponse(StreamInput sin) throws IOException { + this( + sin.readInt(), + Collections.unmodifiableList(sin.readList(IocFinding::new)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(totalFindings); + out.writeCollection(iocFindings); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(TOTAL_IOC_FINDINGS_FIELD, totalFindings) + .field(IOC_FINDINGS_FIELD, iocFindings); + return builder.endObject(); + } + + public Integer getTotalFindings() { + return totalFindings; + } + + public List getIocFindings() { + return iocFindings; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java new file mode 100644 index 000000000..9e91ad207 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java @@ -0,0 +1,209 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.model.IocFinding; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Data layer to perform CRUD operations for threat intel ioc match : store in system index. + */ +public class IocFindingService { + //TODO manage index rollover + public static final String INDEX_NAME = ".opensearch-sap-iocmatch"; + private static final Logger log = LogManager.getLogger(IocFindingService.class); + private final Client client; + private final ClusterService clusterService; + + private final NamedXContentRegistry xContentRegistry; + + public IocFindingService(final Client client, final ClusterService clusterService, final NamedXContentRegistry xContentRegistry) { + this.client = client; + this.clusterService = clusterService; + this.xContentRegistry = xContentRegistry; + } + + public void indexIocMatches(List iocFindings, + final ActionListener actionListener) { + try { + Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + createIndexIfNotExists(ActionListener.wrap( + r -> { + List bulkRequestList = new ArrayList<>(); + BulkRequest bulkRequest = new BulkRequest(INDEX_NAME); + for (int i = 0; i < iocFindings.size(); i++) { + IocFinding iocFinding = iocFindings.get(i); + try { + IndexRequest indexRequest = new IndexRequest(INDEX_NAME) + .source(iocFinding.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .opType(DocWriteRequest.OpType.CREATE); + bulkRequest.add(indexRequest); + if ( + bulkRequest.requests().size() == batchSize + && i != iocFindings.size() - 1 // final bulk request will be added outside for loop with refresh policy none + ) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } catch (IOException e) { + log.error(String.format("Failed to create index request for ioc match %s moving on to next"), e); + } + } + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestList.add(bulkRequest); + GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { + int idx = 0; + for (BulkResponse response : bulkResponses) { + BulkRequest request = bulkRequestList.get(idx); + if (response.hasFailures()) { + log.error("Failed to bulk index {} Ioc Matches. Failure: {}", request.batchSize(), response.buildFailureMessage()); + } + } + actionListener.onResponse(null); + }, actionListener::onFailure), bulkRequestList.size()); + for (BulkRequest req : bulkRequestList) { + try { + client.bulk(req, groupedListener); //todo why stash context here? + } catch (Exception e) { + log.error("Failed to save ioc matches.", e); + } + } + }, e -> { + log.error("Failed to create System Index"); + actionListener.onFailure(e); + })); + + + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + } + + private String getIndexMapping() { + try { + try (InputStream is = IocFindingService.class.getResourceAsStream("/mappings/ioc_finding_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Failed to get the threat intel ioc match index mapping", e); + throw new SecurityAnalyticsException("Failed to get the threat intel ioc match index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + + /** + * Index name: .opensearch-sap-iocmatch + * Mapping: /mappings/ioc_finding_mapping.json + * + * @param listener setup listener + */ + public void createIndexIfNotExists(final ActionListener listener) { + // check if job index exists + try { + if (clusterService.state().metadata().hasIndex(INDEX_NAME) == true) { + listener.onResponse(null); + return; + } + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); + client.admin().indices().create(createIndexRequest, ActionListener.wrap( + r -> { + log.debug("Ioc match index created"); + listener.onResponse(null); + }, e -> { + if (e instanceof ResourceAlreadyExistsException) { + log.debug("index {} already exist", INDEX_NAME); + listener.onResponse(null); + return; + } + log.error("Failed to create security analytics threat intel job index", e); + listener.onFailure(e); + } + )); + } catch (Exception e) { + log.error("Failure in creating ioc_match index", e); + listener.onFailure(e); + } + } + + public void searchIocMatches(SearchSourceBuilder searchSourceBuilder, final ActionListener actionListener) { + createIndexIfNotExists(ActionListener.wrap( + r -> { + SearchRequest searchRequest = new SearchRequest() + .source(searchSourceBuilder) + .indices(INDEX_NAME); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse searchResponse) { + try { + long totalIocFindingsCount = searchResponse.getHits().getTotalHits().value; + List iocFindings = new ArrayList<>(); + + for (SearchHit hit: searchResponse.getHits()) { + XContentParser xcp = XContentType.JSON.xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString()); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + IocFinding iocFinding = IocFinding.parse(xcp); + iocFindings.add(iocFinding); + } + actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocFindings)); + } catch (Exception ex) { + this.onFailure(ex); + } + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + actionListener.onResponse(new GetIocFindingsResponse(0, List.of())); + return; + } + actionListener.onFailure(e); + } + }); + }, e -> { + log.error("Failed to create System Index"); + actionListener.onFailure(e); + })); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java new file mode 100644 index 000000000..36927d35d --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.GetFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; + +import java.io.IOException; +import java.time.DateTimeException; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.opensearch.rest.RestRequest.Method.GET; + +public class RestGetIocFindingsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_ioc_findings_action_sa"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String sortString = request.param("sortString", "timestamp"); + String sortOrder = request.param("sortOrder", "asc"); + String missing = request.param("missing"); + int size = request.paramAsInt("size", 20); + int startIndex = request.paramAsInt("startIndex", 0); + String searchString = request.param("searchString", ""); + + List findingIds = null; + if (request.param("findingIds") != null) { + findingIds = Arrays.asList(request.param("findingIds").split(",")); + } + List iocIds = null; + if (request.param("iocIds") != null) { + iocIds = Arrays.asList(request.param("iocIds").split(",")); + } + Instant startTime = null; + String startTimeParam = request.param("startTime"); + if (startTimeParam != null && !startTimeParam.isEmpty()) { + try { + startTime = Instant.ofEpochMilli(Long.parseLong(startTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + // Handle the parsing error + // For example, log the error or provide a default value + startTime = Instant.now(); // Default value or fallback + } + } + + Instant endTime = null; + String endTimeParam = request.param("endTime"); + if (endTimeParam != null && !endTimeParam.isEmpty()) { + try { + endTime = Instant.ofEpochMilli(Long.parseLong(endTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + // Handle the parsing error + // For example, log the error or provide a default value + endTime = Instant.now(); // Default value or fallback + } + } + + Table table = new Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ); + + GetIocFindingsRequest getIocFindingsRequest = new GetIocFindingsRequest( + findingIds, + iocIds, + startTime, + endTime, + table + ); + return channel -> client.execute( + GetIocFindingsAction.INSTANCE, + getIocFindingsRequest, + new RestToXContentListener<>(channel) + ); + } + + @Override + public List routes() { + return singletonList(new Route(GET, SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings" + "/_search")); + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java new file mode 100644 index 000000000..0e61327f8 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java @@ -0,0 +1,148 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.lucene.search.join.ScoreMode; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.Operator; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.time.Instant; +import java.util.List; + +public class TransportGetIocFindingsAction extends HandledTransportAction implements SecureTransportAction { + + private final IocFindingService iocFindingService; + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private volatile Boolean filterByEnabled; + + @Inject + public TransportGetIocFindingsAction( + TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + Settings settings, + NamedXContentRegistry xContentRegistry, + Client client + ) { + super(GetIocFindingsAction.NAME, transportService, actionFilters, GetIocFindingsRequest::new); + this.settings = settings; + this.clusterService = clusterService; + this.threadPool = client.threadPool(); + this.iocFindingService = new IocFindingService(client, this.clusterService, xContentRegistry); + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + @Override + protected void doExecute(Task task, GetIocFindingsRequest request, ActionListener actionListener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + Table tableProp = request.getTable(); + FieldSortBuilder sortBuilder = SortBuilders + .fieldSort(tableProp.getSortString()) + .order(SortOrder.fromString(tableProp.getSortOrder())); + if (tableProp.getMissing() != null && !tableProp.getMissing().isBlank()) { + sortBuilder.missing(tableProp.getMissing()); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .sort(sortBuilder) + .size(tableProp.getSize()) + .from(tableProp.getStartIndex()) + .fetchSource(new FetchSourceContext(true, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY)) + .seqNoAndPrimaryTerm(true) + .version(true); + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + List findingIds = request.getFindingIds(); + + if (findingIds != null && !findingIds.isEmpty()) { + queryBuilder.filter(QueryBuilders.termsQuery("id", findingIds)); + } + + List iocIds = request.getIocIds(); + if (iocIds != null && !iocIds.isEmpty()) { + queryBuilder.filter(QueryBuilders.termsQuery("ioc_feed_ids.ioc_id", iocIds)); + } + + Instant startTime = request.getStartTime(); + Instant endTime = request.getEndTime(); + if (startTime != null && endTime != null) { + long startTimeMillis = startTime.toEpochMilli(); + long endTimeMillis = endTime.toEpochMilli(); + QueryBuilder timeRangeQuery = QueryBuilders.rangeQuery("timestamp") + .from(startTimeMillis) // Greater than or equal to start time + .to(endTimeMillis); // Less than or equal to end time + queryBuilder.filter(timeRangeQuery); + } + + if (tableProp.getSearchString() != null && !tableProp.getSearchString().isBlank()) { + queryBuilder.should(QueryBuilders + .queryStringQuery(tableProp.getSearchString()) + ).should( + QueryBuilders.nestedQuery( + "queries", + QueryBuilders.boolQuery() + .must( + QueryBuilders + .queryStringQuery(tableProp.getSearchString()) + .defaultOperator(Operator.AND) + .field("queries.tags") + .field("queries.name") + ), + ScoreMode.Avg + ) + ); + } + searchSourceBuilder.query(queryBuilder).trackTotalHits(true); + + this.threadPool.getThreadContext().stashContext(); + iocFindingService.searchIocMatches(searchSourceBuilder, actionListener); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } +} \ No newline at end of file diff --git a/src/main/resources/META-INF/services/org.opensearch.alerting.spi.RemoteMonitorRunnerExtension b/src/main/resources/META-INF/services/org.opensearch.alerting.spi.RemoteMonitorRunnerExtension deleted file mode 100644 index 288e984da..000000000 --- a/src/main/resources/META-INF/services/org.opensearch.alerting.spi.RemoteMonitorRunnerExtension +++ /dev/null @@ -1,6 +0,0 @@ -# -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 -# - -org.opensearch.securityanalytics.SecurityAnalyticsPlugin \ No newline at end of file diff --git a/src/main/resources/mappings/ioc_match_mapping.json b/src/main/resources/mappings/ioc_finding_mapping.json similarity index 69% rename from src/main/resources/mappings/ioc_match_mapping.json rename to src/main/resources/mappings/ioc_finding_mapping.json index f4573190e..d52097e20 100644 --- a/src/main/resources/mappings/ioc_match_mapping.json +++ b/src/main/resources/mappings/ioc_finding_mapping.json @@ -7,8 +7,19 @@ "schema_version": { "type": "integer" }, - "feed_ids" : { - "type": "keyword" + "ioc_feed_ids" : { + "type": "object", + "properties": { + "feed_id": { + "type": "keyword" + }, + "ioc_id": { + "type": "keyword" + }, + "index": { + "type": "keyword" + } + } }, "related_doc_ids": { "type": "keyword" diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 15ec3cd9d..bfa80ece2 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -60,6 +60,7 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.model.IocFinding; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; import org.opensearch.securityanalytics.util.CorrelationIndices; @@ -670,6 +671,10 @@ protected HttpEntity toHttpEntity(ThreatIntelMonitorDto threatIntelMonitorDto) t return new StringEntity(toJsonString(threatIntelMonitorDto), ContentType.APPLICATION_JSON); } + protected HttpEntity toHttpEntity(IocFinding iocFinding) throws IOException { + return new StringEntity(toJsonString(iocFinding), ContentType.APPLICATION_JSON); + } + protected RestStatus restStatus(Response response) { return RestStatus.fromCode(response.getStatusLine().getStatusCode()); } @@ -723,6 +728,11 @@ private String toJsonString(ThreatIntelMonitorDto threatIntelMonitorDto) throws return IndexUtilsKt.string(shuffleXContent(threatIntelMonitorDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); } + private String toJsonString(IocFinding iocFinding) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + private String alertingScheduledJobMappings() { return " \"_meta\" : {\n" + " \"schema_version\": 5\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 77b7c10ab..416991a9c 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -28,12 +28,11 @@ import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.model.IoCMatch; +import org.opensearch.securityanalytics.model.IocFinding; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.RefreshType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.threatIntel.model.Source; @@ -807,9 +806,9 @@ public static String toJsonStringWithUser(Detector detector) throws IOException return BytesReference.bytes(builder).utf8ToString(); } - public static String toJsonString(IoCMatch iocMatch) throws IOException { + public static String toJsonString(IocFinding iocFinding) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); - builder = iocMatch.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder = iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS); return BytesReference.bytes(builder).utf8ToString(); } diff --git a/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java b/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java index 4b56c7eb5..b2c383aa1 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java @@ -16,12 +16,12 @@ public class IoCMatchTests extends OpenSearchTestCase { public void testIoCMatchAsAStream() throws IOException { - IoCMatch iocMatch = getRandomIoCMatch(); + IocFinding iocMatch = getRandomIoCMatch(); String jsonString = toJsonString(iocMatch); BytesStreamOutput out = new BytesStreamOutput(); iocMatch.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); - IoCMatch newIocMatch = new IoCMatch(sin); + IocFinding newIocMatch = new IocFinding(sin); assertEquals(iocMatch.getId(), newIocMatch.getId()); assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); @@ -38,11 +38,11 @@ public void testIoCMatchParse() throws IOException { "\"relatedDocId2\"], \"feed_ids\": [\"feedId1\", \"feedId2\"], \"ioc_scan_job_id\":" + " \"scanJob123\", \"ioc_scan_job_name\": \"Example Scan Job\", \"ioc_value\": \"exampleIocValue\", " + "\"ioc_type\": \"exampleIocType\", \"timestamp\": 1620912896000, \"execution_id\": \"execution123\" }"; - IoCMatch iocMatch = IoCMatch.parse((getParser(iocMatchString))); + IocFinding iocMatch = IocFinding.parse((getParser(iocMatchString))); BytesStreamOutput out = new BytesStreamOutput(); iocMatch.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); - IoCMatch newIocMatch = new IoCMatch(sin); + IocFinding newIocMatch = new IocFinding(sin); assertEquals(iocMatch.getId(), newIocMatch.getId()); assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); @@ -61,11 +61,11 @@ public XContentParser getParser(String xc) throws IOException { } - private static IoCMatch getRandomIoCMatch() { - return new IoCMatch( + private static IocFinding getRandomIoCMatch() { + return new IocFinding( randomAlphaOfLength(10), List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), - List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), + List.of(new IocWithFeeds(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10))), randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java new file mode 100644 index 000000000..5b3bb5492 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java @@ -0,0 +1,80 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.securityanalytics.model.IocFinding; +import org.opensearch.securityanalytics.model.IocWithFeeds; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; + +import static org.opensearch.securityanalytics.TestHelpers.toJsonString; + +public class IocFindingTests extends OpenSearchTestCase { + + public void testIoCMatchAsAStream() throws IOException { + IocFinding iocFinding = getRandomIoCMatch(); + String jsonString = toJsonString(iocFinding); + BytesStreamOutput out = new BytesStreamOutput(); + iocFinding.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getIocScanJobId(), newIocFinding.getIocScanJobId()); + assertEquals(iocFinding.getIocScanJobName(), newIocFinding.getIocScanJobName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); + } + + public void testIoCMatchParse() throws IOException { + String iocMatchString = "{ \"id\": \"exampleId123\", \"related_doc_ids\": [\"relatedDocId1\", " + + "\"relatedDocId2\"], \"feed_ids\": [\"feedId1\", \"feedId2\"], \"ioc_scan_job_id\":" + + " \"scanJob123\", \"ioc_scan_job_name\": \"Example Scan Job\", \"ioc_value\": \"exampleIocValue\", " + + "\"ioc_type\": \"exampleIocType\", \"timestamp\": 1620912896000, \"execution_id\": \"execution123\" }"; + IocFinding iocFinding = IocFinding.parse((getParser(iocMatchString))); + BytesStreamOutput out = new BytesStreamOutput(); + iocFinding.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getIocScanJobId(), newIocFinding.getIocScanJobId()); + assertEquals(iocFinding.getIocScanJobName(), newIocFinding.getIocScanJobName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); + } + + public XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } + + private static IocFinding getRandomIoCMatch() { + return new IocFinding( + randomAlphaOfLength(10), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), + List.of(new IocWithFeeds(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10))), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Instant.now(), + randomAlphaOfLength(10)); + } + + +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java new file mode 100644 index 000000000..2e5cc062e --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java @@ -0,0 +1,72 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.model.IocFinding; +import org.opensearch.securityanalytics.model.IocWithFeeds; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class IocFindingServiceRestApiIT extends SecurityAnalyticsRestTestCase { + + @SuppressWarnings("unchecked") + public void testGetIocFindings() throws IOException { + List iocFindings = generateIocMatches(10); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.INDEX_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + + Response response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + Map responseAsMap = responseAsMap(response); + Assert.assertEquals(5, ((List>) responseAsMap.get("ioc_findings")).size()); + } + + @SuppressWarnings("unchecked") + public void testGetIocFindingsWithIocIdFilter() throws IOException { + Response response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + List iocFindings = generateIocMatches(10); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.INDEX_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + String iocId = iocFindings.stream().map(iocFinding -> iocFinding.getFeedIds().get(0).getIocId()).findFirst().get(); + + response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?iocIds=" + iocId, + Map.of(), null); + Map responseAsMap = responseAsMap(response); + Assert.assertEquals(1, ((List>) responseAsMap.get("ioc_findings")).size()); + } + + private List generateIocMatches(int i) { + List iocFindings = new ArrayList<>(); + String monitorId = randomAlphaOfLength(10); + String monitorName = randomAlphaOfLength(10); + for (int i1 = 0; i1 < i; i1++) { + iocFindings.add(new IocFinding( + randomAlphaOfLength(10), + randomList(1, 10, () -> randomAlphaOfLength(10)),//docids + randomList(1, 10, () -> new IocWithFeeds(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10))), //feedids + monitorId, + monitorName, + randomAlphaOfLength(10), + "IP", + Instant.now(), + randomAlphaOfLength(10) + )); + } + return iocFindings; + } +} \ No newline at end of file