Skip to content

Commit

Permalink
Merge branch 'feature/threat_intel' into ioc_findings
Browse files Browse the repository at this point in the history
Signed-off-by: Subhobrata Dey <sbcd90@gmail.com>
  • Loading branch information
sbcd90 committed Jun 25, 2024
2 parents b236fbe + 23a6b6d commit 534fd45
Show file tree
Hide file tree
Showing 39 changed files with 1,757 additions and 290 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction;
import org.opensearch.securityanalytics.action.SearchDetectorAction;
import org.opensearch.securityanalytics.action.SearchRuleAction;
import org.opensearch.securityanalytics.action.TestS3ConnectionAction;
import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction;
import org.opensearch.securityanalytics.action.ValidateRulesAction;
import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService;
Expand Down Expand Up @@ -111,6 +112,7 @@
import org.opensearch.securityanalytics.resthandler.RestSearchCustomLogTypeAction;
import org.opensearch.securityanalytics.resthandler.RestSearchDetectorAction;
import org.opensearch.securityanalytics.resthandler.RestSearchRuleAction;
import org.opensearch.securityanalytics.resthandler.RestTestS3ConnectionAction;
import org.opensearch.securityanalytics.resthandler.RestUpdateIndexMappingsAction;
import org.opensearch.securityanalytics.resthandler.RestValidateRulesAction;
import org.opensearch.securityanalytics.services.STIX2IOCFetchService;
Expand Down Expand Up @@ -180,6 +182,7 @@
import org.opensearch.securityanalytics.transport.TransportSearchCustomLogTypeAction;
import org.opensearch.securityanalytics.transport.TransportSearchDetectorAction;
import org.opensearch.securityanalytics.transport.TransportSearchRuleAction;
import org.opensearch.securityanalytics.transport.TransportTestS3ConnectionAction;
import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction;
import org.opensearch.securityanalytics.transport.TransportValidateRulesAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction;
Expand Down Expand Up @@ -221,7 +224,10 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map
public static final String THREAT_INTEL_BASE_URI = PLUGINS_BASE_URI + "/threat_intel";
public static final String THREAT_INTEL_SOURCE_URI = PLUGINS_BASE_URI + "/threat_intel/source";
public static final String THREAT_INTEL_MONITOR_URI = PLUGINS_BASE_URI + "/threat_intel/monitor";
public static final String LIST_IOCS_URI = PLUGINS_BASE_URI + "/iocs/list";
public static final String IOCS_URI = PLUGINS_BASE_URI + "/iocs";
public static final String LIST_IOCS_URI = IOCS_URI + "/list";
public static final String TEST_CONNECTION_BASE_URI = PLUGINS_BASE_URI + "/connections/%s/test";
public static final String TEST_S3_CONNECTION_URI = String.format(TEST_CONNECTION_BASE_URI, "s3");

public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype";
public static final String JOB_INDEX_NAME = ".opensearch-sap--job";
Expand Down Expand Up @@ -291,7 +297,7 @@ public Collection<Object> createComponents(Client client,
TIFLockService threatIntelLockService = new TIFLockService(clusterService, client);
saTifSourceConfigService = new SATIFSourceConfigService(client, clusterService, threadPool, xContentRegistry, threatIntelLockService);
STIX2IOCFetchService stix2IOCFetchService = new STIX2IOCFetchService(client, clusterService);
SATIFSourceConfigManagementService saTifSourceConfigManagementService = new SATIFSourceConfigManagementService(saTifSourceConfigService, threatIntelLockService, stix2IOCFetchService, xContentRegistry);
SATIFSourceConfigManagementService saTifSourceConfigManagementService = new SATIFSourceConfigManagementService(saTifSourceConfigService, threatIntelLockService, stix2IOCFetchService, xContentRegistry, clusterService);
SecurityAnalyticsRunner.getJobRunnerInstance();
TIFSourceConfigRunner.getJobRunnerInstance().initialize(clusterService, threatIntelLockService, threadPool, saTifSourceConfigManagementService, saTifSourceConfigService);
TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService);
Expand Down Expand Up @@ -349,7 +355,8 @@ public List<RestHandler> getRestHandlers(Settings settings,
new RestSearchThreatIntelMonitorAction(),
new RestRefreshTIFSourceConfigAction(),
new RestListIOCsAction(),
new RestGetIocFindingsAction()
new RestGetIocFindingsAction(),
new RestTestS3ConnectionAction()
);
}

Expand All @@ -370,6 +377,7 @@ public ScheduledJobRunner getJobRunner() {

@Override
public ScheduledJobParser getJobParser() {
// TODO: @jowg fix the job parser to parse previous tif job
return (xcp, id, jobDocVersion) -> {
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp);
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
Expand Down Expand Up @@ -457,7 +465,9 @@ public List<Setting<?>> getSettings() {
SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE,
SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL,
SecurityAnalyticsSettings.BATCH_SIZE,
SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT
SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT,
SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD,
SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_ALIAS
);
}

Expand Down Expand Up @@ -500,7 +510,8 @@ public List<Setting<?>> getSettings() {
new ActionHandler<>(SARefreshTIFSourceConfigAction.INSTANCE, TransportRefreshTIFSourceConfigAction.class),
new ActionHandler<>(SampleRemoteDocLevelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportRemoteDocLevelMonitorFanOutAction.class),
new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class),
new ActionHandler<>(GetIocFindingsAction.INSTANCE, TransportGetIocFindingsAction.class)
new ActionHandler<>(GetIocFindingsAction.INSTANCE, TransportGetIocFindingsAction.class),
new ActionHandler<>(TestS3ConnectionAction.INSTANCE, TransportTestS3ConnectionAction.class)
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@
import org.opensearch.securityanalytics.commons.model.IOCType;

import java.io.IOException;
import java.util.List;
import java.util.Locale;
import java.util.stream.Collectors;

public class ListIOCsActionRequest extends ActionRequest {
public static String START_INDEX_FIELD = "start";
Expand All @@ -22,6 +24,7 @@ public class ListIOCsActionRequest extends ActionRequest {
public static String SORT_STRING_FIELD = "sort_string";
public static String SEARCH_FIELD = "search";
public static String TYPE_FIELD = "type";

public static String ALL_TYPES_FILTER = "ALL";

private int startIndex;
Expand All @@ -30,18 +33,20 @@ public class ListIOCsActionRequest extends ActionRequest {
private String sortString;

private String search;
private String type;
private String feedId;
private List<String> types;
private List<String> feedIds;

public ListIOCsActionRequest(int startIndex, int size, String sortOrder, String sortString, String search, String type, String feedId) {
public ListIOCsActionRequest(int startIndex, int size, String sortOrder, String sortString, String search, List<String> types, List<String> feedIds) {
super();
this.startIndex = startIndex;
this.size = size;
this.sortOrder = SortOrder.valueOf(sortOrder.toLowerCase(Locale.ROOT));
this.sortString = sortString;
this.search = search;
this.type = type.toLowerCase(Locale.ROOT);
this.feedId = feedId;
this.types = types == null
? null
: types.stream().map(t -> t.toLowerCase(Locale.ROOT)).collect(Collectors.toList());
this.feedIds = feedIds;
}

public ListIOCsActionRequest(StreamInput sin) throws IOException {
Expand All @@ -51,8 +56,8 @@ public ListIOCsActionRequest(StreamInput sin) throws IOException {
sin.readString(), // sortOrder
sin.readString(), // sortString
sin.readOptionalString(), // search
sin.readOptionalString(), // type
sin.readOptionalString() //feedId
sin.readOptionalStringList(), // type
sin.readOptionalStringList() //feedId
);
}

Expand All @@ -62,8 +67,8 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeEnum(sortOrder);
out.writeString(sortString);
out.writeOptionalString(search);
out.writeOptionalString(type);
out.writeOptionalString(feedId);
out.writeOptionalStringCollection(types);
out.writeOptionalStringCollection(feedIds);
}

@Override
Expand All @@ -75,12 +80,17 @@ public ActionRequestValidationException validate() {
} else if (size < 0 || size > 10000) {
validationException = ValidateActions
.addValidationError(String.format("[%s] param must be between 0 and 10,000.", SIZE_FIELD), validationException);
} else if (!ALL_TYPES_FILTER.equalsIgnoreCase(type)) {
try {
IOCType.valueOf(type);
} catch (Exception e) {
validationException = ValidateActions
.addValidationError(String.format("Unrecognized [%s] param.", TYPE_FIELD), validationException);
} else {
for (String type : types) {
if (!ALL_TYPES_FILTER.equalsIgnoreCase(type)) {
try {
IOCType.valueOf(type);
} catch (IllegalArgumentException e) {
validationException = ValidateActions
.addValidationError(String.format("Unrecognized [%s] param.", TYPE_FIELD), validationException);
break;
}
}
}
}
return validationException;
Expand All @@ -106,12 +116,12 @@ public String getSearch() {
return search;
}

public String getType() {
return type;
public List<String> getTypes() {
return types;
}

public String getFeedId() {
return feedId;
public List<String> getFeedIds() {
return feedIds;
}

public enum SortOrder {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.securityanalytics.model.DetailedSTIX2IOCDto;
import org.opensearch.securityanalytics.model.STIX2IOCDto;

import java.io.IOException;
Expand All @@ -23,16 +24,16 @@ public class ListIOCsActionResponse extends ActionResponse implements ToXContent
public static ListIOCsActionResponse EMPTY_RESPONSE = new ListIOCsActionResponse(0, Collections.emptyList());

private long totalHits;
private List<STIX2IOCDto> hits;
private List<DetailedSTIX2IOCDto> hits;

public ListIOCsActionResponse(long totalHits, List<STIX2IOCDto> hits) {
public ListIOCsActionResponse(long totalHits, List<DetailedSTIX2IOCDto> hits) {
super();
this.totalHits = totalHits;
this.hits = hits;
}

public ListIOCsActionResponse(StreamInput sin) throws IOException {
this(sin.readInt(), sin.readList(STIX2IOCDto::new));
this(sin.readInt(), sin.readList(DetailedSTIX2IOCDto::new));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.securityanalytics.action;

import org.opensearch.action.ActionType;

public class TestS3ConnectionAction extends ActionType<TestS3ConnectionResponse> {
public static final String NAME = "cluster:admin/opensearch/securityanalytics/connections/test/s3";
public static final TestS3ConnectionAction INSTANCE = new TestS3ConnectionAction();

public TestS3ConnectionAction() {
super(NAME, TestS3ConnectionResponse::new);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.securityanalytics.action;

import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionRequestValidationException;
import org.opensearch.action.ValidateActions;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig;
import org.opensearch.securityanalytics.threatIntel.model.S3Source;

import java.io.IOException;

public class TestS3ConnectionRequest extends ActionRequest implements ToXContentObject {
private final S3Source s3Source;

public TestS3ConnectionRequest(S3Source s3Source) {
super();
this.s3Source = s3Source;
}

public TestS3ConnectionRequest(String bucketName, String objectKey, String region, String roleArn) {
this(new S3Source(bucketName, objectKey, region, roleArn));
}

public TestS3ConnectionRequest(StreamInput sin) throws IOException {
this(new S3Source(sin));
}

public void writeTo(StreamOutput out) throws IOException {
s3Source.writeTo(out);
}

@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if (s3Source.getBucketName() == null || s3Source.getBucketName().isEmpty()) {
validationException = ValidateActions.addValidationError("Must provide bucket name.", validationException);
}
if (s3Source.getObjectKey() == null || s3Source.getObjectKey().isEmpty()) {
validationException = ValidateActions.addValidationError("Must provide object key.", validationException);
}
if (s3Source.getObjectKey() == null || s3Source.getObjectKey().isEmpty()) {
validationException = ValidateActions.addValidationError("Must provide region.", validationException);
}
if (s3Source.getRoleArn() == null || s3Source.getRoleArn().isEmpty()) {
validationException = ValidateActions.addValidationError("Must provide role ARN.", validationException);
}
return validationException;
}

public static TestS3ConnectionRequest parse(XContentParser xcp) throws IOException {
return new TestS3ConnectionRequest(S3Source.parse(xcp));
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return s3Source.toXContent(builder, params);
}

public S3ConnectorConfig constructS3ConnectorConfig() {
return new S3ConnectorConfig(
s3Source.getBucketName(),
s3Source.getObjectKey(),
s3Source.getRegion(),
s3Source.getRoleArn()
);
}

public S3Source getS3Source() {
return s3Source;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.securityanalytics.action;

import org.opensearch.core.action.ActionResponse;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;

import java.io.IOException;

public class TestS3ConnectionResponse extends ActionResponse implements ToXContentObject {
public static final String STATUS_FIELD = "status";
public static final String ERROR_FIELD = "error";

private RestStatus status;
private String error;

public TestS3ConnectionResponse(RestStatus status, String error) {
super();
this.status = status;
this.error = error;
}

public TestS3ConnectionResponse(StreamInput sin) throws IOException {
this(sin.readEnum(RestStatus.class), sin.readOptionalString());
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeEnum(status);
out.writeOptionalString(error);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.startObject()
.field(STATUS_FIELD, status)
.field(ERROR_FIELD, error)
.endObject();
}

public RestStatus getStatus() {
return status;
}

public String getError() {
return error;
}
}
Loading

0 comments on commit 534fd45

Please sign in to comment.