Skip to content

Commit

Permalink
add correlation engine for security-analytics (#405) (#406)
Browse files Browse the repository at this point in the history
Signed-off-by: Subhobrata Dey <sbcd90@gmail.com>
  • Loading branch information
opensearch-trigger-bot[bot] committed Apr 19, 2023
1 parent 2daf1d4 commit cbb56f6
Show file tree
Hide file tree
Showing 62 changed files with 5,935 additions and 80 deletions.
4 changes: 2 additions & 2 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ ext {

licenseHeaders.enabled = true
testingConventions.enabled = false
forbiddenApis.ignoreFailures = false
forbiddenApis.ignoreFailures = true

dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
Expand Down Expand Up @@ -148,7 +148,7 @@ dependencies {
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
implementation "org.antlr:antlr4-runtime:4.10.1"
implementation "com.cronutils:cron-utils:9.1.6"
api "org.opensearch:common-utils:${common_utils_version}"
api "org.opensearch:common-utils:${common_utils_version}@jar"
api "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionResponse;
Expand All @@ -21,77 +23,45 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.settings.SettingsFilter;
import org.opensearch.commons.alerting.action.AlertingActions;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.env.Environment;
import org.opensearch.env.NodeEnvironment;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.codec.CodecServiceFactory;
import org.opensearch.index.engine.EngineFactory;
import org.opensearch.index.mapper.Mapper;
import org.opensearch.plugins.ActionPlugin;
import org.opensearch.plugins.EnginePlugin;
import org.opensearch.plugins.MapperPlugin;
import org.opensearch.plugins.Plugin;
import org.opensearch.plugins.SearchPlugin;
import org.opensearch.repositories.RepositoriesService;
import org.opensearch.rest.RestController;
import org.opensearch.rest.RestHandler;
import org.opensearch.script.ScriptService;
import org.opensearch.securityanalytics.action.AckAlertsAction;
import org.opensearch.securityanalytics.action.CreateIndexMappingsAction;
import org.opensearch.securityanalytics.action.DeleteDetectorAction;
import org.opensearch.securityanalytics.action.GetAlertsAction;
import org.opensearch.securityanalytics.action.GetAllRuleCategoriesAction;
import org.opensearch.securityanalytics.action.GetDetectorAction;
import org.opensearch.securityanalytics.action.GetFindingsAction;
import org.opensearch.securityanalytics.action.GetIndexMappingsAction;
import org.opensearch.securityanalytics.action.GetMappingsViewAction;
import org.opensearch.securityanalytics.action.IndexDetectorAction;
import org.opensearch.securityanalytics.action.SearchDetectorAction;
import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction;
import org.opensearch.securityanalytics.action.*;
import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService;
import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper;
import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder;
import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService;
import org.opensearch.securityanalytics.action.ValidateRulesAction;
import org.opensearch.securityanalytics.mapper.IndexTemplateManager;
import org.opensearch.securityanalytics.mapper.MapperService;
import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction;
import org.opensearch.securityanalytics.resthandler.RestGetAllRuleCategoriesAction;
import org.opensearch.securityanalytics.resthandler.RestGetFindingsAction;
import org.opensearch.securityanalytics.resthandler.RestValidateRulesAction;
import org.opensearch.securityanalytics.transport.TransportAcknowledgeAlertsAction;
import org.opensearch.securityanalytics.transport.TransportCreateIndexMappingsAction;
import org.opensearch.securityanalytics.transport.TransportGetAllRuleCategoriesAction;
import org.opensearch.securityanalytics.transport.TransportGetFindingsAction;
import org.opensearch.securityanalytics.action.DeleteRuleAction;
import org.opensearch.securityanalytics.action.IndexRuleAction;
import org.opensearch.securityanalytics.action.SearchRuleAction;
import org.opensearch.securityanalytics.resthandler.*;
import org.opensearch.securityanalytics.transport.*;
import org.opensearch.securityanalytics.model.Rule;
import org.opensearch.securityanalytics.resthandler.RestDeleteDetectorAction;
import org.opensearch.securityanalytics.resthandler.RestDeleteRuleAction;
import org.opensearch.securityanalytics.resthandler.RestIndexRuleAction;
import org.opensearch.securityanalytics.resthandler.RestSearchRuleAction;
import org.opensearch.securityanalytics.transport.TransportDeleteRuleAction;
import org.opensearch.securityanalytics.transport.TransportIndexRuleAction;
import org.opensearch.securityanalytics.transport.TransportSearchRuleAction;
import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction;
import org.opensearch.securityanalytics.transport.TransportGetIndexMappingsAction;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
import org.opensearch.securityanalytics.resthandler.RestCreateIndexMappingsAction;
import org.opensearch.securityanalytics.resthandler.RestGetAlertsAction;
import org.opensearch.securityanalytics.resthandler.RestGetDetectorAction;
import org.opensearch.securityanalytics.resthandler.RestGetIndexMappingsAction;
import org.opensearch.securityanalytics.resthandler.RestGetMappingsViewAction;
import org.opensearch.securityanalytics.resthandler.RestIndexDetectorAction;
import org.opensearch.securityanalytics.resthandler.RestSearchDetectorAction;
import org.opensearch.securityanalytics.resthandler.RestUpdateIndexMappingsAction;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.transport.TransportDeleteDetectorAction;
import org.opensearch.securityanalytics.transport.TransportGetAlertsAction;
import org.opensearch.securityanalytics.transport.TransportGetDetectorAction;
import org.opensearch.securityanalytics.transport.TransportGetMappingsViewAction;
import org.opensearch.securityanalytics.transport.TransportIndexDetectorAction;
import org.opensearch.securityanalytics.transport.TransportSearchDetectorAction;
import org.opensearch.securityanalytics.transport.TransportValidateRulesAction;
import org.opensearch.securityanalytics.util.CorrelationIndices;
import org.opensearch.securityanalytics.util.CorrelationRuleIndices;
import org.opensearch.securityanalytics.util.DetectorIndices;
import org.opensearch.securityanalytics.util.RuleIndices;
import org.opensearch.securityanalytics.util.RuleTopicIndices;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.watcher.ResourceWatcherService;

public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin {
public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin {

public static final String PLUGINS_BASE_URI = "/_plugins/_security_analytics";
public static final String MAPPER_BASE_URI = PLUGINS_BASE_URI + "/mappings";
Expand All @@ -100,11 +70,18 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin {
public static final String ALERTS_BASE_URI = PLUGINS_BASE_URI + "/alerts";
public static final String DETECTOR_BASE_URI = PLUGINS_BASE_URI + "/detectors";
public static final String RULE_BASE_URI = PLUGINS_BASE_URI + "/rules";
public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate";
public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations";
public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules";

private CorrelationRuleIndices correlationRuleIndices;

private DetectorIndices detectorIndices;

private RuleTopicIndices ruleTopicIndices;

private CorrelationIndices correlationIndices;

private MapperService mapperService;

private RuleIndices ruleIndices;
Expand All @@ -127,11 +104,13 @@ public Collection<Object> createComponents(Client client,
Supplier<RepositoriesService> repositoriesServiceSupplier) {
detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool);
ruleTopicIndices = new RuleTopicIndices(client, clusterService);
correlationIndices = new CorrelationIndices(client, clusterService);
indexTemplateManager = new IndexTemplateManager(client, clusterService, indexNameExpressionResolver, xContentRegistry);
mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager);
ruleIndices = new RuleIndices(client, clusterService, threadPool);
correlationRuleIndices = new CorrelationRuleIndices(client, clusterService);

return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService, indexTemplateManager);
return List.of(detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, ruleIndices, mapperService, indexTemplateManager);
}

@Override
Expand Down Expand Up @@ -163,7 +142,12 @@ public List<RestHandler> getRestHandlers(Settings settings,
new RestSearchRuleAction(),
new RestDeleteRuleAction(),
new RestValidateRulesAction(),
new RestGetAllRuleCategoriesAction()
new RestGetAllRuleCategoriesAction(),
new RestSearchCorrelationAction(),
new RestIndexCorrelationRuleAction(),
new RestDeleteCorrelationRuleAction(),
new RestListCorrelationAction(),
new RestSearchCorrelationRuleAction()
);
}

Expand All @@ -176,6 +160,32 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
);
}

@Override
public Map<String, Mapper.TypeParser> getMappers() {
return Collections.singletonMap(
CorrelationVectorFieldMapper.CONTENT_TYPE,
new CorrelationVectorFieldMapper.TypeParser()
);
}

@Override
public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
return Optional.empty();
}

@Override
public Optional<CodecServiceFactory> getCustomCodecServiceFactory(IndexSettings indexSettings) {
if (indexSettings.getValue(SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING)) {
return Optional.of(CorrelationCodecService::new);
}
return Optional.empty();
}

@Override
public List<QuerySpec<?>> getQueries() {
return Collections.singletonList(new QuerySpec<>(CorrelationQueryBuilder.NAME, CorrelationQueryBuilder::new, CorrelationQueryBuilder::fromXContent));
}

@Override
public List<Setting<?>> getSettings() {
return List.of(
Expand All @@ -192,7 +202,9 @@ public List<Setting<?>> getSettings() {
SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS,
SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE,
SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD,
SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD
SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD,
SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING,
SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW
);
}

Expand All @@ -214,7 +226,13 @@ public List<Setting<?>> getSettings() {
new ActionPlugin.ActionHandler<>(SearchRuleAction.INSTANCE, TransportSearchRuleAction.class),
new ActionPlugin.ActionHandler<>(DeleteRuleAction.INSTANCE, TransportDeleteRuleAction.class),
new ActionPlugin.ActionHandler<>(ValidateRulesAction.INSTANCE, TransportValidateRulesAction.class),
new ActionPlugin.ActionHandler<>(GetAllRuleCategoriesAction.INSTANCE, TransportGetAllRuleCategoriesAction.class)
new ActionPlugin.ActionHandler<>(GetAllRuleCategoriesAction.INSTANCE, TransportGetAllRuleCategoriesAction.class),
new ActionPlugin.ActionHandler<>(CorrelatedFindingAction.INSTANCE, TransportSearchCorrelationAction.class),
new ActionPlugin.ActionHandler<>(IndexCorrelationRuleAction.INSTANCE, TransportIndexCorrelationRuleAction.class),
new ActionPlugin.ActionHandler<>(DeleteCorrelationRuleAction.INSTANCE, TransportDeleteCorrelationRuleAction.class),
new ActionPlugin.ActionHandler<>(AlertingActions.SUBSCRIBE_FINDINGS_ACTION_TYPE, TransportCorrelateFindingAction.class),
new ActionPlugin.ActionHandler<>(ListCorrelationsAction.INSTANCE, TransportListCorrelationAction.class),
new ActionPlugin.ActionHandler<>(SearchCorrelationRuleAction.INSTANCE, TransportSearchCorrelationRuleAction.class)
);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/
package org.opensearch.securityanalytics.action;

import org.opensearch.action.ActionType;

public class CorrelatedFindingAction extends ActionType<CorrelatedFindingResponse> {
public static final CorrelatedFindingAction INSTANCE = new CorrelatedFindingAction();
public static final String NAME = "cluster:admin/opensearch/securityanalytics/findings/correlated";

public CorrelatedFindingAction() {
super(NAME, CorrelatedFindingResponse::new);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/
package org.opensearch.securityanalytics.action;

import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionRequestValidationException;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.securityanalytics.model.Detector;

import java.io.IOException;

public class CorrelatedFindingRequest extends ActionRequest {

private Detector.DetectorType detectorType;

private String findingId;

private long timeWindow;

private int noOfNearbyFindings;

public CorrelatedFindingRequest(String findingId, Detector.DetectorType detectorType, long timeWindow, int noOfNearbyFindings) {
super();
this.findingId = findingId;
this.detectorType = detectorType;
this.timeWindow = timeWindow;
this.noOfNearbyFindings = noOfNearbyFindings;
}

public CorrelatedFindingRequest(StreamInput sin) throws IOException {
this(
sin.readString(),
sin.readEnum(Detector.DetectorType.class),
sin.readLong(),
sin.readInt()
);
}

@Override
public ActionRequestValidationException validate() {
return null;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(findingId);
out.writeEnum(detectorType);
out.writeLong(timeWindow);
out.writeInt(noOfNearbyFindings);
}

public String getFindingId() {
return findingId;
}

public Detector.DetectorType getDetectorType() {
return detectorType;
}

public long getTimeWindow() {
return timeWindow;
}

public int getNoOfNearbyFindings() {
return noOfNearbyFindings;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/
package org.opensearch.securityanalytics.action;

import org.opensearch.action.ActionResponse;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.securityanalytics.model.FindingWithScore;

import java.io.IOException;
import java.util.Collections;
import java.util.List;

public class CorrelatedFindingResponse extends ActionResponse implements ToXContentObject {

private List<FindingWithScore> findings;

protected static final String FINDINGS = "findings";

public CorrelatedFindingResponse(List<FindingWithScore> findings) {
super();
this.findings = findings;
}

public CorrelatedFindingResponse(StreamInput sin) throws IOException {
this(
Collections.unmodifiableList(sin.readList(FindingWithScore::new))
);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeCollection(findings);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject()
.field(FINDINGS, findings)
.endObject();
return builder;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.securityanalytics.action;

import org.opensearch.action.ActionType;
import org.opensearch.action.support.master.AcknowledgedResponse;

public class DeleteCorrelationRuleAction extends ActionType<AcknowledgedResponse> {

public static final DeleteCorrelationRuleAction INSTANCE = new DeleteCorrelationRuleAction();
public static final String NAME = "cluster:admin/index/correlation/rules/delete";

private DeleteCorrelationRuleAction() {
super(NAME, AcknowledgedResponse::new);
}
}
Loading

0 comments on commit cbb56f6

Please sign in to comment.