Skip to content

Commit

Permalink
add search ioc findings api
Browse files Browse the repository at this point in the history
Signed-off-by: Subhobrata Dey <sbcd90@gmail.com>

add search ioc findings api

Signed-off-by: Subhobrata Dey <sbcd90@gmail.com>

add search ioc findings api

Signed-off-by: Subhobrata Dey <sbcd90@gmail.com>
  • Loading branch information
sbcd90 committed Jun 21, 2024
1 parent 1927009 commit ec07214
Show file tree
Hide file tree
Showing 16 changed files with 940 additions and 53 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.action.ActionRequest;
import org.opensearch.alerting.spi.RemoteMonitorRunner;
import org.opensearch.alerting.spi.RemoteMonitorRunnerExtension;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
Expand Down Expand Up @@ -117,6 +115,7 @@
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction;
import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.action.SARefreshTIFSourceConfigAction;
Expand All @@ -132,6 +131,7 @@
import org.opensearch.securityanalytics.threatIntel.resthandler.RestDeleteTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.model.monitor.SampleRemoteDocLevelMonitorRunner;
import org.opensearch.securityanalytics.threatIntel.model.monitor.TransportRemoteDocLevelMonitorFanOutAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetIocFindingsAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestRefreshTIFSourceConfigAction;
Expand Down Expand Up @@ -180,6 +180,7 @@
import org.opensearch.securityanalytics.transport.TransportSearchRuleAction;
import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction;
import org.opensearch.securityanalytics.transport.TransportValidateRulesAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction;
import org.opensearch.securityanalytics.util.CorrelationIndices;
import org.opensearch.securityanalytics.util.CorrelationRuleIndices;
import org.opensearch.securityanalytics.util.CustomLogTypeIndices;
Expand All @@ -188,7 +189,6 @@
import org.opensearch.securityanalytics.util.RuleTopicIndices;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.watcher.ResourceWatcherService;
import reactor.util.annotation.NonNull;

import java.util.Collection;
import java.util.Collections;
Expand All @@ -199,9 +199,8 @@

import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD;
import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX;
import static org.opensearch.securityanalytics.threatIntel.model.monitor.SampleRemoteDocLevelMonitorRunner.THREAT_INTEL_MONITOR_TYPE;

public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension, RemoteMonitorRunnerExtension {
public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension {

private static final Logger log = LogManager.getLogger(SecurityAnalyticsPlugin.class);

Expand Down Expand Up @@ -345,7 +344,8 @@ public List<RestHandler> getRestHandlers(Settings settings,
new RestDeleteThreatIntelMonitorAction(),
new RestSearchThreatIntelMonitorAction(),
new RestRefreshTIFSourceConfigAction(),
new RestListIOCsAction()
new RestListIOCsAction(),
new RestGetIocFindingsAction()
);
}

Expand Down Expand Up @@ -490,7 +490,8 @@ public List<Setting<?>> getSettings() {
new ActionHandler<>(SASearchTIFSourceConfigsAction.INSTANCE, TransportSearchTIFSourceConfigsAction.class),
new ActionHandler<>(SARefreshTIFSourceConfigAction.INSTANCE, TransportRefreshTIFSourceConfigAction.class),
new ActionHandler<>(SampleRemoteDocLevelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportRemoteDocLevelMonitorFanOutAction.class),
new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class)
new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class),
new ActionHandler<>(GetIocFindingsAction.INSTANCE, TransportGetIocFindingsAction.class)
);
}

Expand All @@ -509,12 +510,4 @@ public void onFailure(Exception e) {
}
});
}

@NonNull
@Override
public Map<String, RemoteMonitorRunner> getMonitorTypesToMonitorRunners() {
return Map.of(
THREAT_INTEL_MONITOR_TYPE, SampleRemoteDocLevelMonitorRunner.getMonitorRunner()
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@
* IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job
* It's the inverse of an IoC finding which maps a document to list of IoC's
*/
public class IoCMatch implements Writeable, ToXContent {
public class IocFinding implements Writeable, ToXContent {
//TODO implement IoC_Match interface from security-analytics-commons
public static final String ID_FIELD = "id";
public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids";
public static final String FEED_IDS_FIELD = "feed_ids";
public static final String IOC_WITH_FEED_IDS_FIELD = "ioc_feed_ids";
public static final String IOC_SCAN_JOB_ID_FIELD = "ioc_scan_job_id";
public static final String IOC_SCAN_JOB_NAME_FIELD = "ioc_scan_job_name";
public static final String IOC_VALUE_FIELD = "ioc_value";
Expand All @@ -34,20 +34,20 @@ public class IoCMatch implements Writeable, ToXContent {

private final String id;
private final List<String> relatedDocIds;
private final List<String> feedIds;
private final List<IocWithFeeds> iocWithFeeds;
private final String iocScanJobId;
private final String iocScanJobName;
private final String iocValue;
private final String iocType;
private final Instant timestamp;
private final String executionId;

public IoCMatch(String id, List<String> relatedDocIds, List<String> feedIds, String iocScanJobId,
String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) {
public IocFinding(String id, List<String> relatedDocIds, List<IocWithFeeds> iocWithFeeds, String iocScanJobId,
String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) {
validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds);
this.id = id;
this.relatedDocIds = relatedDocIds;
this.feedIds = feedIds;
this.iocWithFeeds = iocWithFeeds;
this.iocScanJobId = iocScanJobId;
this.iocScanJobName = iocScanJobName;
this.iocValue = iocValue;
Expand All @@ -56,10 +56,10 @@ public IoCMatch(String id, List<String> relatedDocIds, List<String> feedIds, Str
this.executionId = executionId;
}

public IoCMatch(StreamInput in) throws IOException {
public IocFinding(StreamInput in) throws IOException {
id = in.readString();
relatedDocIds = in.readStringList();
feedIds = in.readStringList();
iocWithFeeds = in.readList(IocWithFeeds::readFrom);
iocScanJobId = in.readString();
iocScanJobName = in.readString();
iocValue = in.readString();
Expand All @@ -72,7 +72,7 @@ public IoCMatch(StreamInput in) throws IOException {
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
out.writeStringCollection(relatedDocIds);
out.writeStringCollection(feedIds);
out.writeCollection(iocWithFeeds);
out.writeString(iocScanJobId);
out.writeString(iocScanJobName);
out.writeString(iocValue);
Expand All @@ -86,7 +86,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject()
.field(ID_FIELD, id)
.field(RELATED_DOC_IDS_FIELD, relatedDocIds)
.field(FEED_IDS_FIELD, feedIds)
.field(IOC_WITH_FEED_IDS_FIELD, iocWithFeeds)
.field(IOC_SCAN_JOB_ID_FIELD, iocScanJobId)
.field(IOC_SCAN_JOB_NAME_FIELD, iocScanJobName)
.field(IOC_VALUE_FIELD, iocValue)
Expand All @@ -105,8 +105,8 @@ public List<String> getRelatedDocIds() {
return relatedDocIds;
}

public List<String> getFeedIds() {
return feedIds;
public List<IocWithFeeds> getFeedIds() {
return iocWithFeeds;
}

public String getIocScanJobId() {
Expand All @@ -133,10 +133,10 @@ public String getExecutionId() {
return executionId;
}

public static IoCMatch parse(XContentParser xcp) throws IOException {
public static IocFinding parse(XContentParser xcp) throws IOException {
String id = null;
List<String> relatedDocIds = new ArrayList<>();
List<String> feedIds = new ArrayList<>();
List<IocWithFeeds> feedIds = new ArrayList<>();
String iocScanJobId = null;
String iocScanName = null;
String iocValue = null;
Expand All @@ -159,10 +159,10 @@ public static IoCMatch parse(XContentParser xcp) throws IOException {
relatedDocIds.add(xcp.text());
}
break;
case FEED_IDS_FIELD:
case IOC_WITH_FEED_IDS_FIELD:
ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp);
while (xcp.nextToken() != XContentParser.Token.END_ARRAY) {
feedIds.add(xcp.text());
feedIds.add(IocWithFeeds.parse(xcp));
}
break;
case IOC_SCAN_JOB_ID_FIELD:
Expand Down Expand Up @@ -197,11 +197,11 @@ public static IoCMatch parse(XContentParser xcp) throws IOException {
}
}

return new IoCMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId);
return new IocFinding(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId);
}

public static IoCMatch readFrom(StreamInput in) throws IOException {
return new IoCMatch(in);
public static IocFinding readFrom(StreamInput in) throws IOException {
return new IocFinding(in);
}


Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package org.opensearch.securityanalytics.model;

import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;

import java.io.IOException;

import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken;

public class IocWithFeeds implements Writeable, ToXContent {

private static final String FEED_ID_FIELD = "feed_id";

private static final String IOC_ID_FIELD = "ioc_id";

private static final String INDEX_FIELD = "index";

private final String feedId;

private final String iocId;

private final String index;

public IocWithFeeds(String iocId, String feedId, String index) {
this.iocId = iocId;
this.feedId = feedId;
this.index = index;
}

public IocWithFeeds(StreamInput sin) throws IOException {
this.iocId = sin.readString();
this.feedId = sin.readString();
this.index = sin.readString();
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(iocId);
out.writeString(feedId);
out.writeString(index);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject()
.field(IOC_ID_FIELD, iocId)
.field(FEED_ID_FIELD, feedId)
.field(INDEX_FIELD, index)
.endObject();
return builder;
}

public String getIocId() {
return iocId;
}

public String getFeedId() {
return feedId;
}

public String getIndex() {
return index;
}

public static IocWithFeeds parse(XContentParser xcp) throws IOException {
String iocId = null;
String feedId = null;
String index = null;

ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp);
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
String fieldName = xcp.currentName();
xcp.nextToken();

switch (fieldName) {
case IOC_ID_FIELD:
iocId = xcp.text();
break;
case FEED_ID_FIELD:
feedId = xcp.text();
break;
case INDEX_FIELD:
index = xcp.text();
break;
default:
xcp.skipChildren();
}
}
return new IocWithFeeds(iocId, feedId, index);
}

public static IocWithFeeds readFrom(StreamInput sin) throws IOException {
return new IocWithFeeds(sin);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/
package org.opensearch.securityanalytics.threatIntel.action;

import org.opensearch.action.ActionType;

public class GetIocFindingsAction extends ActionType<GetIocFindingsResponse> {

public static final GetIocFindingsAction INSTANCE = new GetIocFindingsAction();
public static final String NAME = "cluster:admin/opensearch/securityanalytics/ioc/findings/get";

public GetIocFindingsAction() {
super(NAME, GetIocFindingsResponse::new);
}
}
Loading

0 comments on commit ec07214

Please sign in to comment.