[Backport 2.7] add correlation engine for security-analytics

build.gradle

@@ -53,7 +53,7 @@ ext {
 
 licenseHeaders.enabled = true
 testingConventions.enabled = false
-forbiddenApis.ignoreFailures = false
+forbiddenApis.ignoreFailures = true
 
 dependencyLicenses.enabled = false
 thirdPartyAudit.enabled = false
@@ -148,7 +148,7 @@ dependencies {
     implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
     implementation "org.antlr:antlr4-runtime:4.10.1"
     implementation "com.cronutils:cron-utils:9.1.6"
-    api "org.opensearch:common-utils:${common_utils_version}"
+    api "org.opensearch:common-utils:${common_utils_version}@jar"
     api "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
     implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
 

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -7,6 +7,8 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.function.Supplier;
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionResponse;
@@ -21,77 +23,45 @@
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsFilter;
+import org.opensearch.commons.alerting.action.AlertingActions;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.codec.CodecServiceFactory;
+import org.opensearch.index.engine.EngineFactory;
+import org.opensearch.index.mapper.Mapper;
 import org.opensearch.plugins.ActionPlugin;
+import org.opensearch.plugins.EnginePlugin;
+import org.opensearch.plugins.MapperPlugin;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.plugins.SearchPlugin;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.rest.RestController;
 import org.opensearch.rest.RestHandler;
 import org.opensearch.script.ScriptService;
-import org.opensearch.securityanalytics.action.AckAlertsAction;
-import org.opensearch.securityanalytics.action.CreateIndexMappingsAction;
-import org.opensearch.securityanalytics.action.DeleteDetectorAction;
-import org.opensearch.securityanalytics.action.GetAlertsAction;
-import org.opensearch.securityanalytics.action.GetAllRuleCategoriesAction;
-import org.opensearch.securityanalytics.action.GetDetectorAction;
-import org.opensearch.securityanalytics.action.GetFindingsAction;
-import org.opensearch.securityanalytics.action.GetIndexMappingsAction;
-import org.opensearch.securityanalytics.action.GetMappingsViewAction;
-import org.opensearch.securityanalytics.action.IndexDetectorAction;
-import org.opensearch.securityanalytics.action.SearchDetectorAction;
-import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction;
+import org.opensearch.securityanalytics.action.*;
+import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService;
+import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper;
+import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder;
 import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService;
-import org.opensearch.securityanalytics.action.ValidateRulesAction;
 import org.opensearch.securityanalytics.mapper.IndexTemplateManager;
 import org.opensearch.securityanalytics.mapper.MapperService;
-import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction;
-import org.opensearch.securityanalytics.resthandler.RestGetAllRuleCategoriesAction;
-import org.opensearch.securityanalytics.resthandler.RestGetFindingsAction;
-import org.opensearch.securityanalytics.resthandler.RestValidateRulesAction;
-import org.opensearch.securityanalytics.transport.TransportAcknowledgeAlertsAction;
-import org.opensearch.securityanalytics.transport.TransportCreateIndexMappingsAction;
-import org.opensearch.securityanalytics.transport.TransportGetAllRuleCategoriesAction;
-import org.opensearch.securityanalytics.transport.TransportGetFindingsAction;
-import org.opensearch.securityanalytics.action.DeleteRuleAction;
-import org.opensearch.securityanalytics.action.IndexRuleAction;
-import org.opensearch.securityanalytics.action.SearchRuleAction;
+import org.opensearch.securityanalytics.resthandler.*;
+import org.opensearch.securityanalytics.transport.*;
 import org.opensearch.securityanalytics.model.Rule;
-import org.opensearch.securityanalytics.resthandler.RestDeleteDetectorAction;
-import org.opensearch.securityanalytics.resthandler.RestDeleteRuleAction;
-import org.opensearch.securityanalytics.resthandler.RestIndexRuleAction;
-import org.opensearch.securityanalytics.resthandler.RestSearchRuleAction;
-import org.opensearch.securityanalytics.transport.TransportDeleteRuleAction;
-import org.opensearch.securityanalytics.transport.TransportIndexRuleAction;
-import org.opensearch.securityanalytics.transport.TransportSearchRuleAction;
-import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction;
-import org.opensearch.securityanalytics.transport.TransportGetIndexMappingsAction;
 import org.opensearch.securityanalytics.model.Detector;
 import org.opensearch.securityanalytics.model.DetectorInput;
-import org.opensearch.securityanalytics.resthandler.RestCreateIndexMappingsAction;
-import org.opensearch.securityanalytics.resthandler.RestGetAlertsAction;
-import org.opensearch.securityanalytics.resthandler.RestGetDetectorAction;
-import org.opensearch.securityanalytics.resthandler.RestGetIndexMappingsAction;
-import org.opensearch.securityanalytics.resthandler.RestGetMappingsViewAction;
-import org.opensearch.securityanalytics.resthandler.RestIndexDetectorAction;
-import org.opensearch.securityanalytics.resthandler.RestSearchDetectorAction;
-import org.opensearch.securityanalytics.resthandler.RestUpdateIndexMappingsAction;
 import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
-import org.opensearch.securityanalytics.transport.TransportDeleteDetectorAction;
-import org.opensearch.securityanalytics.transport.TransportGetAlertsAction;
-import org.opensearch.securityanalytics.transport.TransportGetDetectorAction;
-import org.opensearch.securityanalytics.transport.TransportGetMappingsViewAction;
-import org.opensearch.securityanalytics.transport.TransportIndexDetectorAction;
-import org.opensearch.securityanalytics.transport.TransportSearchDetectorAction;
-import org.opensearch.securityanalytics.transport.TransportValidateRulesAction;
+import org.opensearch.securityanalytics.util.CorrelationIndices;
+import org.opensearch.securityanalytics.util.CorrelationRuleIndices;
 import org.opensearch.securityanalytics.util.DetectorIndices;
 import org.opensearch.securityanalytics.util.RuleIndices;
 import org.opensearch.securityanalytics.util.RuleTopicIndices;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.watcher.ResourceWatcherService;
 
-public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin {
+public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin {
 
     public static final String PLUGINS_BASE_URI = "/_plugins/_security_analytics";
     public static final String MAPPER_BASE_URI = PLUGINS_BASE_URI + "/mappings";
@@ -100,11 +70,18 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin {
     public static final String ALERTS_BASE_URI = PLUGINS_BASE_URI + "/alerts";
     public static final String DETECTOR_BASE_URI = PLUGINS_BASE_URI + "/detectors";
     public static final String RULE_BASE_URI = PLUGINS_BASE_URI + "/rules";
+    public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate";
+    public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations";
+    public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules";
+
+    private CorrelationRuleIndices correlationRuleIndices;
 
     private DetectorIndices detectorIndices;
 
     private RuleTopicIndices ruleTopicIndices;
 
+    private CorrelationIndices correlationIndices;
+
     private MapperService mapperService;
 
     private RuleIndices ruleIndices;
@@ -127,11 +104,13 @@ public Collection<Object> createComponents(Client client,
                                                Supplier<RepositoriesService> repositoriesServiceSupplier) {
         detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool);
         ruleTopicIndices = new RuleTopicIndices(client, clusterService);
+        correlationIndices = new CorrelationIndices(client, clusterService);
         indexTemplateManager = new IndexTemplateManager(client, clusterService, indexNameExpressionResolver, xContentRegistry);
         mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager);
         ruleIndices = new RuleIndices(client, clusterService, threadPool);
+        correlationRuleIndices = new CorrelationRuleIndices(client, clusterService);
 
-        return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService, indexTemplateManager);
+        return List.of(detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, ruleIndices, mapperService, indexTemplateManager);
     }
 
     @Override
@@ -163,7 +142,12 @@ public List<RestHandler> getRestHandlers(Settings settings,
                 new RestSearchRuleAction(),
                 new RestDeleteRuleAction(),
                 new RestValidateRulesAction(),
-                new RestGetAllRuleCategoriesAction()
+                new RestGetAllRuleCategoriesAction(),
+                new RestSearchCorrelationAction(),
+                new RestIndexCorrelationRuleAction(),
+                new RestDeleteCorrelationRuleAction(),
+                new RestListCorrelationAction(),
+                new RestSearchCorrelationRuleAction()
         );
     }
 
@@ -176,6 +160,32 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
         );
     }
 
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        return Collections.singletonMap(
+                CorrelationVectorFieldMapper.CONTENT_TYPE,
+                new CorrelationVectorFieldMapper.TypeParser()
+        );
+    }
+
+    @Override
+    public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
+        return Optional.empty();
+    }
+
+    @Override
+    public Optional<CodecServiceFactory> getCustomCodecServiceFactory(IndexSettings indexSettings) {
+        if (indexSettings.getValue(SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING)) {
+            return Optional.of(CorrelationCodecService::new);
+        }
+        return Optional.empty();
+    }
+
+    @Override
+    public List<QuerySpec<?>> getQueries() {
+        return Collections.singletonList(new QuerySpec<>(CorrelationQueryBuilder.NAME, CorrelationQueryBuilder::new, CorrelationQueryBuilder::fromXContent));
+    }
+
     @Override
     public List<Setting<?>> getSettings() {
         return List.of(
@@ -192,7 +202,9 @@ public List<Setting<?>> getSettings() {
                 SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS,
                 SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE,
                 SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD,
-                SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD
+                SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD,
+                SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING,
+                SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW
         );
     }
 
@@ -214,7 +226,13 @@ public List<Setting<?>> getSettings() {
                 new ActionPlugin.ActionHandler<>(SearchRuleAction.INSTANCE, TransportSearchRuleAction.class),
                 new ActionPlugin.ActionHandler<>(DeleteRuleAction.INSTANCE, TransportDeleteRuleAction.class),
                 new ActionPlugin.ActionHandler<>(ValidateRulesAction.INSTANCE, TransportValidateRulesAction.class),
-                new ActionPlugin.ActionHandler<>(GetAllRuleCategoriesAction.INSTANCE, TransportGetAllRuleCategoriesAction.class)
+                new ActionPlugin.ActionHandler<>(GetAllRuleCategoriesAction.INSTANCE, TransportGetAllRuleCategoriesAction.class),
+                new ActionPlugin.ActionHandler<>(CorrelatedFindingAction.INSTANCE, TransportSearchCorrelationAction.class),
+                new ActionPlugin.ActionHandler<>(IndexCorrelationRuleAction.INSTANCE, TransportIndexCorrelationRuleAction.class),
+                new ActionPlugin.ActionHandler<>(DeleteCorrelationRuleAction.INSTANCE, TransportDeleteCorrelationRuleAction.class),
+                new ActionPlugin.ActionHandler<>(AlertingActions.SUBSCRIBE_FINDINGS_ACTION_TYPE, TransportCorrelateFindingAction.class),
+                new ActionPlugin.ActionHandler<>(ListCorrelationsAction.INSTANCE, TransportListCorrelationAction.class),
+                new ActionPlugin.ActionHandler<>(SearchCorrelationRuleAction.INSTANCE, TransportSearchCorrelationRuleAction.class)
         );
     }
 }

src/main/java/org/opensearch/securityanalytics/action/CorrelatedFindingAction.java

@@ -0,0 +1,16 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.opensearch.securityanalytics.action;
+
+import org.opensearch.action.ActionType;
+
+public class CorrelatedFindingAction extends ActionType<CorrelatedFindingResponse> {
+    public static final CorrelatedFindingAction INSTANCE = new CorrelatedFindingAction();
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/findings/correlated";
+
+    public CorrelatedFindingAction() {
+        super(NAME, CorrelatedFindingResponse::new);
+    }
+}

src/main/java/org/opensearch/securityanalytics/action/CorrelatedFindingRequest.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.opensearch.securityanalytics.action;
+
+import org.opensearch.action.ActionRequest;
+import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.common.io.stream.StreamInput;
+import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.securityanalytics.model.Detector;
+
+import java.io.IOException;
+
+public class CorrelatedFindingRequest extends ActionRequest {
+
+    private Detector.DetectorType detectorType;
+
+    private String findingId;
+
+    private long timeWindow;
+
+    private int noOfNearbyFindings;
+
+    public CorrelatedFindingRequest(String findingId, Detector.DetectorType detectorType, long timeWindow, int noOfNearbyFindings) {
+        super();
+        this.findingId = findingId;
+        this.detectorType = detectorType;
+        this.timeWindow = timeWindow;
+        this.noOfNearbyFindings = noOfNearbyFindings;
+    }
+
+    public CorrelatedFindingRequest(StreamInput sin) throws IOException {
+        this(
+                sin.readString(),
+                sin.readEnum(Detector.DetectorType.class),
+                sin.readLong(),
+                sin.readInt()
+        );
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(findingId);
+        out.writeEnum(detectorType);
+        out.writeLong(timeWindow);
+        out.writeInt(noOfNearbyFindings);
+    }
+
+    public String getFindingId() {
+        return findingId;
+    }
+
+    public Detector.DetectorType getDetectorType() {
+        return detectorType;
+    }
+
+    public long getTimeWindow() {
+        return timeWindow;
+    }
+
+    public int getNoOfNearbyFindings() {
+        return noOfNearbyFindings;
+    }
+}

src/main/java/org/opensearch/securityanalytics/action/CorrelatedFindingResponse.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.opensearch.securityanalytics.action;
+
+import org.opensearch.action.ActionResponse;
+import org.opensearch.common.io.stream.StreamInput;
+import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.securityanalytics.model.FindingWithScore;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+public class CorrelatedFindingResponse extends ActionResponse implements ToXContentObject {
+
+    private List<FindingWithScore> findings;
+
+    protected static final String FINDINGS = "findings";
+
+    public CorrelatedFindingResponse(List<FindingWithScore> findings) {
+        super();
+        this.findings = findings;
+    }
+
+    public CorrelatedFindingResponse(StreamInput sin) throws IOException {
+        this(
+                Collections.unmodifiableList(sin.readList(FindingWithScore::new))
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeCollection(findings);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject()
+                .field(FINDINGS, findings)
+                .endObject();
+        return builder;
+    }
+}

src/main/java/org/opensearch/securityanalytics/action/DeleteCorrelationRuleAction.java

@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.securityanalytics.action;
+
+import org.opensearch.action.ActionType;
+import org.opensearch.action.support.master.AcknowledgedResponse;
+
+public class DeleteCorrelationRuleAction extends ActionType<AcknowledgedResponse> {
+
+    public static final DeleteCorrelationRuleAction INSTANCE = new DeleteCorrelationRuleAction();
+    public static final String NAME = "cluster:admin/index/correlation/rules/delete";
+
+    private DeleteCorrelationRuleAction() {
+        super(NAME, AcknowledgedResponse::new);
+    }
+}