From c7ea1ea0918be9c80fbc1cc688460f3d0a10ccf6 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 18:05:26 -0700 Subject: [PATCH] entire custom logtype implementation (#500) (#504) Signed-off-by: Subhobrata Dey --- .../SecurityAnalyticsPlugin.java | 22 +- .../action/CorrelatedFindingRequest.java | 10 +- .../action/DeleteCustomLogTypeAction.java | 21 + .../action/DeleteCustomLogTypeRequest.java | 54 ++ .../action/DeleteCustomLogTypeResponse.java | 59 ++ .../action/IndexCustomLogTypeAction.java | 21 + .../action/IndexCustomLogTypeRequest.java | 90 +++ .../action/IndexCustomLogTypeResponse.java | 78 ++ .../action/SearchCustomLogTypeAction.java | 22 + .../action/SearchCustomLogTypeRequest.java | 45 ++ .../correlation/VectorEmbeddingsEngine.java | 38 +- .../logtype/BuiltinLogTypeLoader.java | 36 + .../logtype/LogTypeService.java | 168 ++++- .../model/CustomLogType.java | 191 +++++ .../securityanalytics/model/Detector.java | 41 -- .../RestDeleteCustomLogTypeAction.java | 66 ++ .../RestIndexCustomLogTypeAction.java | 95 +++ .../RestSearchCorrelationAction.java | 2 +- .../RestSearchCustomLogTypeAction.java | 106 +++ .../rules/backend/OSQueryBackend.java | 2 +- .../TransportCorrelateFindingAction.java | 110 ++- .../TransportDeleteCustomLogTypeAction.java | 297 ++++++++ .../TransportGetAllRuleCategoriesAction.java | 4 +- .../TransportIndexCustomLogTypeAction.java | 477 ++++++++++++ .../TransportIndexDetectorAction.java | 97 ++- .../transport/TransportIndexRuleAction.java | 80 +- .../TransportSearchCorrelationAction.java | 4 +- .../TransportSearchCustomLogTypeAction.java | 91 +++ .../util/CustomLogTypeIndices.java | 68 ++ .../securityanalytics/util/IndexUtils.java | 6 + .../securityanalytics/util/RuleIndices.java | 66 +- .../util/RuleTopicIndices.java | 56 +- .../util/SecurityAnalyticsException.java | 18 +- .../OSMapping/ad_ldap/fieldmappings.yml | 25 - src/main/resources/OSMapping/logtypes.json | 178 +++++ .../mappings/log_type_config_mapping.json | 35 + .../SecurityAnalyticsRestTestCase.java | 10 + .../securityanalytics/TestHelpers.java | 68 +- .../action/IndexDetectorResponseTests.java | 11 +- .../alerts/AlertingServiceTests.java | 16 +- .../securityanalytics/alerts/AlertsIT.java | 4 +- .../CorrelationEngineRestApiIT.java | 8 +- .../securityanalytics/findings/FindingIT.java | 2 +- .../findings/FindingServiceTests.java | 16 +- .../findings/SecureFindingRestApiIT.java | 2 +- .../mapper/MapperRestApiIT.java | 24 +- .../resthandler/CustomLogTypeRestApiIT.java | 691 ++++++++++++++++++ .../resthandler/DetectorRestApiIT.java | 23 + .../resthandler/OCSFDetectorRestApiIT.java | 16 +- .../resthandler/RuleRestApiIT.java | 4 +- 50 files changed, 3371 insertions(+), 303 deletions(-) create mode 100644 src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeRequest.java create mode 100644 src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeResponse.java create mode 100644 src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeRequest.java create mode 100644 src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeResponse.java create mode 100644 src/main/java/org/opensearch/securityanalytics/action/SearchCustomLogTypeAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/action/SearchCustomLogTypeRequest.java create mode 100644 src/main/java/org/opensearch/securityanalytics/model/CustomLogType.java create mode 100644 src/main/java/org/opensearch/securityanalytics/resthandler/RestDeleteCustomLogTypeAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexCustomLogTypeAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/resthandler/RestSearchCustomLogTypeAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/transport/TransportSearchCustomLogTypeAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java delete mode 100644 src/main/resources/OSMapping/ad_ldap/fieldmappings.yml create mode 100644 src/main/resources/OSMapping/logtypes.json create mode 100644 src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 9b57436eb..26b029cde 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -58,6 +58,7 @@ import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.mapper.IndexTemplateManager; import org.opensearch.securityanalytics.mapper.MapperService; +import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.resthandler.*; import org.opensearch.securityanalytics.transport.*; import org.opensearch.securityanalytics.model.Rule; @@ -66,6 +67,7 @@ import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.securityanalytics.util.CorrelationRuleIndices; +import org.opensearch.securityanalytics.util.CustomLogTypeIndices; import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.securityanalytics.util.RuleIndices; import org.opensearch.securityanalytics.util.RuleTopicIndices; @@ -87,6 +89,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations"; public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; + public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; + private CorrelationRuleIndices correlationRuleIndices; private DetectorIndices detectorIndices; @@ -95,6 +99,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private CorrelationIndices correlationIndices; + private CustomLogTypeIndices customLogTypeIndices; + private MapperService mapperService; private RuleIndices ruleIndices; @@ -126,6 +132,7 @@ public Collection createComponents(Client client, detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool); ruleTopicIndices = new RuleTopicIndices(client, clusterService, logTypeService); correlationIndices = new CorrelationIndices(client, clusterService); + customLogTypeIndices = new CustomLogTypeIndices(client.admin(), clusterService); indexTemplateManager = new IndexTemplateManager(client, clusterService, indexNameExpressionResolver, xContentRegistry); mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager, logTypeService); ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool); @@ -133,7 +140,7 @@ public Collection createComponents(Client client, this.client = client; return List.of( - detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, ruleIndices, + detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, mapperService, indexTemplateManager, builtinLogTypeLoader ); } @@ -172,7 +179,10 @@ public List getRestHandlers(Settings settings, new RestIndexCorrelationRuleAction(), new RestDeleteCorrelationRuleAction(), new RestListCorrelationAction(), - new RestSearchCorrelationRuleAction() + new RestSearchCorrelationRuleAction(), + new RestIndexCustomLogTypeAction(), + new RestSearchCustomLogTypeAction(), + new RestDeleteCustomLogTypeAction() ); } @@ -181,7 +191,8 @@ public List getNamedXContent() { return List.of( Detector.XCONTENT_REGISTRY, DetectorInput.XCONTENT_REGISTRY, - Rule.XCONTENT_REGISTRY + Rule.XCONTENT_REGISTRY, + CustomLogType.XCONTENT_REGISTRY ); } @@ -258,7 +269,10 @@ public List> getSettings() { new ActionPlugin.ActionHandler<>(DeleteCorrelationRuleAction.INSTANCE, TransportDeleteCorrelationRuleAction.class), new ActionPlugin.ActionHandler<>(AlertingActions.SUBSCRIBE_FINDINGS_ACTION_TYPE, TransportCorrelateFindingAction.class), new ActionPlugin.ActionHandler<>(ListCorrelationsAction.INSTANCE, TransportListCorrelationAction.class), - new ActionPlugin.ActionHandler<>(SearchCorrelationRuleAction.INSTANCE, TransportSearchCorrelationRuleAction.class) + new ActionPlugin.ActionHandler<>(SearchCorrelationRuleAction.INSTANCE, TransportSearchCorrelationRuleAction.class), + new ActionHandler<>(IndexCustomLogTypeAction.INSTANCE, TransportIndexCustomLogTypeAction.class), + new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class), + new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class) ); } diff --git a/src/main/java/org/opensearch/securityanalytics/action/CorrelatedFindingRequest.java b/src/main/java/org/opensearch/securityanalytics/action/CorrelatedFindingRequest.java index bbc22a9ac..6eaf9a0ca 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/CorrelatedFindingRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/action/CorrelatedFindingRequest.java @@ -14,7 +14,7 @@ public class CorrelatedFindingRequest extends ActionRequest { - private Detector.DetectorType detectorType; + private String detectorType; private String findingId; @@ -22,7 +22,7 @@ public class CorrelatedFindingRequest extends ActionRequest { private int noOfNearbyFindings; - public CorrelatedFindingRequest(String findingId, Detector.DetectorType detectorType, long timeWindow, int noOfNearbyFindings) { + public CorrelatedFindingRequest(String findingId, String detectorType, long timeWindow, int noOfNearbyFindings) { super(); this.findingId = findingId; this.detectorType = detectorType; @@ -33,7 +33,7 @@ public CorrelatedFindingRequest(String findingId, Detector.DetectorType detector public CorrelatedFindingRequest(StreamInput sin) throws IOException { this( sin.readString(), - sin.readEnum(Detector.DetectorType.class), + sin.readString(), sin.readLong(), sin.readInt() ); @@ -47,7 +47,7 @@ public ActionRequestValidationException validate() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(findingId); - out.writeEnum(detectorType); + out.writeString(detectorType); out.writeLong(timeWindow); out.writeInt(noOfNearbyFindings); } @@ -56,7 +56,7 @@ public String getFindingId() { return findingId; } - public Detector.DetectorType getDetectorType() { + public String getDetectorType() { return detectorType; } diff --git a/src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeAction.java new file mode 100644 index 000000000..f6b34a8c0 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeAction.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionType; + +public class DeleteCustomLogTypeAction extends ActionType { + + public static final DeleteCustomLogTypeAction INSTANCE = new DeleteCustomLogTypeAction(); + public static final String NAME = "cluster:admin/opensearch/securityanalytics/logtype/delete"; + + public DeleteCustomLogTypeAction() { + super(NAME, DeleteCustomLogTypeResponse::new); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeRequest.java b/src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeRequest.java new file mode 100644 index 000000000..1bce40fe2 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeRequest.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +public class DeleteCustomLogTypeRequest extends ActionRequest { + + private String logTypeId; + + private WriteRequest.RefreshPolicy refreshPolicy; + + public DeleteCustomLogTypeRequest(String logTypeId, WriteRequest.RefreshPolicy refreshPolicy) { + super(); + this.logTypeId = logTypeId; + this.refreshPolicy = refreshPolicy; + } + + public DeleteCustomLogTypeRequest(StreamInput sin) throws IOException { + this(sin.readString(), + WriteRequest.RefreshPolicy.readFrom(sin)); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(logTypeId); + refreshPolicy.writeTo(out); + } + + public String getLogTypeId() { + return logTypeId; + } + + public WriteRequest.RefreshPolicy getRefreshPolicy() { + return refreshPolicy; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeResponse.java b/src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeResponse.java new file mode 100644 index 000000000..6347a42a1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/DeleteCustomLogTypeResponse.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionResponse; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.rest.RestStatus; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; + +public class DeleteCustomLogTypeResponse extends ActionResponse implements ToXContentObject { + + private String id; + + private Long version; + + private RestStatus status; + + public DeleteCustomLogTypeResponse(String id, Long version, RestStatus status) { + super(); + this.id = id; + this.version = version; + this.status = status; + } + + public DeleteCustomLogTypeResponse(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readLong(), + sin.readEnum(RestStatus.class) + ); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(_ID, id) + .field(_VERSION, version); + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeAction.java new file mode 100644 index 000000000..5960941c8 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeAction.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionType; + +public class IndexCustomLogTypeAction extends ActionType { + + public static final IndexCustomLogTypeAction INSTANCE = new IndexCustomLogTypeAction(); + public static final String NAME = "cluster:admin/opensearch/securityanalytics/logtype/write"; + + public IndexCustomLogTypeAction() { + super(NAME, IndexCustomLogTypeResponse::new); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeRequest.java b/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeRequest.java new file mode 100644 index 000000000..b0e368151 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeRequest.java @@ -0,0 +1,90 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.model.CustomLogType; + +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class IndexCustomLogTypeRequest extends ActionRequest { + + private String logTypeId; + + private WriteRequest.RefreshPolicy refreshPolicy; + + private RestRequest.Method method; + + private CustomLogType customLogType; + + private static final Pattern IS_VALID_CUSTOM_LOG_NAME = Pattern.compile("[a-zA-Z0-9 _,-.]{5,50}"); + + public IndexCustomLogTypeRequest( + String logTypeId, + WriteRequest.RefreshPolicy refreshPolicy, + RestRequest.Method method, + CustomLogType customLogType + ) { + super(); + this.logTypeId = logTypeId; + this.refreshPolicy = refreshPolicy; + this.method = method; + this.customLogType = customLogType; + } + + public IndexCustomLogTypeRequest(StreamInput sin) throws IOException { + this( + sin.readString(), + WriteRequest.RefreshPolicy.readFrom(sin), + sin.readEnum(RestRequest.Method.class), + CustomLogType.readFrom(sin) + ); + } + + @Override + public ActionRequestValidationException validate() { + Matcher matcher = IS_VALID_CUSTOM_LOG_NAME.matcher(customLogType.getName()); + boolean find = matcher.matches(); + if (!find) { + throw new ActionRequestValidationException(); + } + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(logTypeId); + refreshPolicy.writeTo(out); + out.writeEnum(method); + customLogType.writeTo(out); + } + + public String getLogTypeId() { + return logTypeId; + } + + public WriteRequest.RefreshPolicy getRefreshPolicy() { + return refreshPolicy; + } + + public RestRequest.Method getMethod() { + return method; + } + + public CustomLogType getCustomLogType() { + return customLogType; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeResponse.java b/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeResponse.java new file mode 100644 index 000000000..3a95d99b9 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeResponse.java @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionResponse; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.rest.RestStatus; +import org.opensearch.securityanalytics.model.CustomLogType; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; + +public class IndexCustomLogTypeResponse extends ActionResponse implements ToXContentObject { + + public static final String CUSTOM_LOG_TYPES_FIELD = "logType"; + + private String id; + + private Long version; + + private RestStatus status; + + private CustomLogType customLogType; + + public IndexCustomLogTypeResponse( + String id, + Long version, + RestStatus status, + CustomLogType customLogType + ) { + super(); + this.id = id; + this.version = version; + this.status = status; + this.customLogType = customLogType; + } + + public IndexCustomLogTypeResponse(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readLong(), + sin.readEnum(RestStatus.class), + CustomLogType.readFrom(sin) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeEnum(status); + customLogType.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(_ID, id) + .field(_VERSION, version) + .field(CUSTOM_LOG_TYPES_FIELD, customLogType) + .endObject(); + } + + public String getId() { + return id; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/SearchCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/action/SearchCustomLogTypeAction.java new file mode 100644 index 000000000..6ada49c63 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/SearchCustomLogTypeAction.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionType; +import org.opensearch.action.search.SearchResponse; + +public class SearchCustomLogTypeAction extends ActionType { + + public static final SearchCustomLogTypeAction INSTANCE = new SearchCustomLogTypeAction(); + public static final String NAME = "cluster:admin/opensearch/securityanalytics/logtype/search"; + + public SearchCustomLogTypeAction() { + super(NAME, SearchResponse::new); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/SearchCustomLogTypeRequest.java b/src/main/java/org/opensearch/securityanalytics/action/SearchCustomLogTypeRequest.java new file mode 100644 index 000000000..f315a09ff --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/action/SearchCustomLogTypeRequest.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +public class SearchCustomLogTypeRequest extends ActionRequest { + + private SearchRequest searchRequest; + + public SearchCustomLogTypeRequest(SearchRequest request) { + super(); + this.searchRequest = request; + } + + public SearchCustomLogTypeRequest(StreamInput sin) throws IOException { + searchRequest = new SearchRequest(sin); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + searchRequest.writeTo(out); + } + + public SearchRequest searchRequest() { + return searchRequest; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java b/src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java index 1e91835f6..f0f433d5e 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java @@ -29,6 +29,7 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder; +import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.transport.TransportCorrelateFindingAction; import org.opensearch.securityanalytics.util.CorrelationIndices; @@ -58,7 +59,10 @@ public VectorEmbeddingsEngine(Client client, TimeValue indexTimeout, long corrTi this.correlateFindingAction = correlateFindingAction; } - public void insertCorrelatedFindings(String detectorType, Finding finding, String logType, List correlatedFindings, float timestampFeature, List correlationRules) { + public void insertCorrelatedFindings(String detectorType, Finding finding, String logType, List correlatedFindings, float timestampFeature, List correlationRules, Map logTypes) { + Map tags = logTypes.get(detectorType).getTags(); + String correlationId = tags.get("correlation_id").toString(); + long findingTimestamp = finding.getTimestamp().toEpochMilli(); MatchQueryBuilder queryBuilder = QueryBuilders.matchQuery( "root", true @@ -133,7 +137,8 @@ public void onResponse(MultiSearchResponse items) { for (int i = 0; i < 100; ++i) { corrVector[i] = ((float) counter) - 50.0f; } - corrVector[Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)).getDim()] = (float) counter; + + corrVector[Integer.parseInt(correlationId)] = (float) counter; corrVector[100] = timestampFeature; XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); @@ -141,7 +146,7 @@ public void onResponse(MultiSearchResponse items) { builder.field("counter", counter); builder.field("finding1", finding.getId()); builder.field("finding2", ""); - builder.field("logType", Integer.valueOf(Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)).getDim()).toString()); + builder.field("logType", correlationId); builder.field("timestamp", findingTimestamp); builder.field("corr_vector", corrVector); builder.field("recordType", "finding"); @@ -158,8 +163,8 @@ public void onResponse(MultiSearchResponse items) { for (int i = 0; i < 100; ++i) { corrVector[i] = ((float) counter) - 50.0f; } - corrVector[Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)).getDim()] = (2.0f * ((float) counter) - 50.0f) / 2.0f; - corrVector[Detector.DetectorType.valueOf(logType.toUpperCase(Locale.ROOT)).getDim()] = (2.0f * ((float) neighborCounter) - 50.0f) / 2.0f; + corrVector[Integer.parseInt(correlationId)] = (2.0f * ((float) counter) - 50.0f) / 2.0f; + corrVector[Integer.parseInt(correlationId)] = (2.0f * ((float) neighborCounter) - 50.0f) / 2.0f; corrVector[100] = timestampFeature; XContentBuilder corrBuilder = XContentFactory.jsonBuilder().startObject(); @@ -202,7 +207,7 @@ public void onFailure(Exception e) { } }); } else { - insertOrphanFindings(detectorType, finding, timestampFeature); + insertOrphanFindings(detectorType, finding, timestampFeature, logTypes); } } @@ -220,7 +225,10 @@ public void onFailure(Exception e) { }); } - public void insertOrphanFindings(String detectorType, Finding finding, float timestampFeature) { + public void insertOrphanFindings(String detectorType, Finding finding, float timestampFeature, Map logTypes) { + Map tags = logTypes.get(detectorType).getTags(); + String correlationId = tags.get("correlation_id").toString(); + long findingTimestamp = finding.getTimestamp().toEpochMilli(); MatchQueryBuilder queryBuilder = QueryBuilders.matchQuery( "root", true @@ -268,7 +276,7 @@ public void onResponse(IndexResponse response) { if (response.status().equals(RestStatus.OK)) { try { float[] corrVector = new float[101]; - corrVector[Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)).getDim()] = 50.0f; + corrVector[Integer.parseInt(correlationId)] = 50.0f; corrVector[100] = timestampFeature; XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); @@ -276,7 +284,7 @@ public void onResponse(IndexResponse response) { builder.field("counter", 50L); builder.field("finding1", finding.getId()); builder.field("finding2", ""); - builder.field("logType", Integer.valueOf(Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)).getDim()).toString()); + builder.field("logType", correlationId); builder.field("timestamp", findingTimestamp); builder.field("corr_vector", corrVector); builder.field("recordType", "finding"); @@ -339,7 +347,7 @@ public void onResponse(IndexResponse response) { correlateFindingAction.onOperation(); try { float[] corrVector = new float[101]; - corrVector[Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)).getDim()] = 50.0f; + corrVector[Integer.parseInt(logTypes.get(detectorType).getTags().get("correlation_id").toString())] = 50.0f; corrVector[100] = timestampFeature; XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); @@ -347,7 +355,7 @@ public void onResponse(IndexResponse response) { builder.field("counter", 50L); builder.field("finding1", finding.getId()); builder.field("finding2", ""); - builder.field("logType", Integer.valueOf(Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)).getDim()).toString()); + builder.field("logType", Integer.valueOf(logTypes.get(detectorType).getTags().get("correlation_id").toString()).toString()); builder.field("timestamp", findingTimestamp); builder.field("corr_vector", corrVector); builder.field("recordType", "finding"); @@ -430,7 +438,7 @@ public void onResponse(SearchResponse response) { for (int i = 0; i < 100; ++i) { corrVector[i] = ((float) counter) - 50.0f; } - corrVector[Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)).getDim()] = (float) counter; + corrVector[Integer.parseInt(logTypes.get(detectorType).getTags().get("correlation_id").toString())] = (float) counter; corrVector[100] = timestampFeature; XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); @@ -438,7 +446,7 @@ public void onResponse(SearchResponse response) { builder.field("counter", counter); builder.field("finding1", finding.getId()); builder.field("finding2", ""); - builder.field("logType", Integer.valueOf(Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)).getDim()).toString()); + builder.field("logType", Integer.valueOf(logTypes.get(detectorType).getTags().get("correlation_id").toString()).toString()); builder.field("timestamp", findingTimestamp); builder.field("corr_vector", corrVector); builder.field("recordType", "finding"); @@ -495,7 +503,7 @@ public void onResponse(IndexResponse response) { for (int i = 0; i < 100; ++i) { corrVector[i] = (float) counter; } - corrVector[Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)).getDim()] = counter + 50.0f; + corrVector[Integer.parseInt(logTypes.get(detectorType).getTags().get("correlation_id").toString())] = counter + 50.0f; corrVector[100] = timestampFeature; XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); @@ -503,7 +511,7 @@ public void onResponse(IndexResponse response) { builder.field("counter", counter + 50L); builder.field("finding1", finding.getId()); builder.field("finding2", ""); - builder.field("logType", Integer.valueOf(Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)).getDim()).toString()); + builder.field("logType", Integer.valueOf(logTypes.get(detectorType).getTags().get("correlation_id").toString()).toString()); builder.field("timestamp", findingTimestamp); builder.field("corr_vector", corrVector); builder.field("recordType", "finding"); diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java index 604744486..14dad9e4c 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java @@ -23,6 +23,7 @@ import org.opensearch.common.settings.SettingsException; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.util.FileUtils; @@ -103,6 +104,41 @@ private List loadBuiltinLogTypes() throws URISyntaxException, IOExcepti return logTypes; } + @SuppressWarnings("unchecked") + protected List loadBuiltinLogTypesMetadata() throws URISyntaxException, IOException { + List customLogTypes = new ArrayList<>(); + + final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH), + "Built-in log type metadata file not found").toURI().toString(); + Path dirPath = null; + if (url.contains("!")) { + final String[] paths = url.split("!"); + dirPath = FileUtils.getFs().getPath(paths[1]); + } else { + dirPath = Path.of(url); + } + + Stream folder = Files.list(dirPath); + Path logTypePath = folder.filter(e -> e.toString().endsWith("logtypes.json")).collect(Collectors.toList()).get(0); + try ( + InputStream is = BuiltinLogTypeLoader.class.getResourceAsStream(logTypePath.toString()) + ) { + String logTypeFilePayload = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); + + if (logTypeFilePayload != null) { + Map logTypeFileAsMap = + XContentHelper.convertToMap(JsonXContent.jsonXContent, logTypeFilePayload, false); + + for (Map.Entry logType: logTypeFileAsMap.entrySet()) { + customLogTypes.add(new CustomLogType((Map) logType.getValue())); + } + } + } catch (Exception e) { + throw new SettingsException("Failed to load builtin log types", e); + } + return customLogTypes; + } + @Override protected void doStart() { ensureLogTypesLoaded(); diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java index 8fb978144..0c49abc9d 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -7,6 +7,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.AbstractMap; import java.util.ArrayList; @@ -22,6 +23,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.ExceptionsHelper; +import org.opensearch.OpenSearchStatusException; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; @@ -30,6 +32,7 @@ import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; @@ -44,11 +47,14 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.FieldMappingDoc; import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -128,6 +134,92 @@ public void getAllLogTypes(ActionListener> listener) { }, listener::onFailure)); } + public void getAllLogTypesMetadata(ActionListener> listener) { + ensureConfigIndexIsInitialized(ActionListener.wrap(e -> { + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.existsQuery("source")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(10000); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(LogTypeService.LOG_TYPE_INDEX); + searchRequest.source(searchSourceBuilder); + searchRequest.preference("_primary"); + client.search( + searchRequest, + ActionListener.delegateFailure( + listener, + (delegatedListener, searchResponse) -> { + List logTypes = new ArrayList<>(); + SearchHit[] hits = searchResponse.getHits().getHits(); + + for (SearchHit hit: hits) { + Map source = hit.getSourceAsMap(); + logTypes.add(source.get("name").toString()); + } + delegatedListener.onResponse(logTypes); + } + ) + ); + }, listener::onFailure)); + } + + public void doesLogTypeExist(String logType, ActionListener listener) { + ensureConfigIndexIsInitialized(ActionListener.wrap(e -> { + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.matchQuery("name", logType)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(10000); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(LogTypeService.LOG_TYPE_INDEX); + searchRequest.source(searchSourceBuilder); + searchRequest.preference("_primary"); + client.search( + searchRequest, + ActionListener.delegateFailure( + listener, + (delegatedListener, searchResponse) -> { + SearchHit[] hits = searchResponse.getHits().getHits(); + delegatedListener.onResponse(hits.length > 0); + } + ) + ); + }, listener::onFailure)); + } + + public void searchLogTypes(SearchRequest request, ActionListener listener) { + ensureConfigIndexIsInitialized(ActionListener.wrap(e -> { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.existsQuery("source")); + if (request.source().query() != null) { + queryBuilder.must(request.source().query()); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(10000); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(LogTypeService.LOG_TYPE_INDEX); + searchRequest.source(searchSourceBuilder); + searchRequest.preference("_primary"); + client.search( + searchRequest, + ActionListener.delegateFailure( + listener, + (delegatedListener, searchResponse) -> { + delegatedListener.onResponse(searchResponse); + } + ) + ); + }, listener::onFailure)); + } + private void doIndexFieldMappings(List fieldMappingDocs, ActionListener listener) { if (fieldMappingDocs.isEmpty()) { listener.onResponse(null); @@ -170,6 +262,69 @@ private void doIndexFieldMappings(List fieldMappingDocs, Action }, listener::onFailure)); } + private void doIndexLogTypeMetadata(ActionListener listener) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.existsQuery("source")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(false); + searchSourceBuilder.size(0); + searchSourceBuilder.trackTotalHits(true); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(LogTypeService.LOG_TYPE_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + listener.onFailure(new OpenSearchStatusException("Unknown error", RestStatus.INTERNAL_SERVER_ERROR)); + } + if (response.getHits().getTotalHits().value > 0) { + listener.onResponse(null); + } else { + try { + List customLogTypes = builtinLogTypeLoader.loadBuiltinLogTypesMetadata(); + BulkRequest bulkRequest = new BulkRequest(); + + for (CustomLogType customLogType: customLogTypes) { + IndexRequest indexRequest = new IndexRequest(LOG_TYPE_INDEX).id(customLogType.getName()); + indexRequest.source(customLogType.toXContent(XContentFactory.jsonBuilder(), null)); + indexRequest.opType(DocWriteRequest.OpType.INDEX); + bulkRequest.add(indexRequest); + } + + if (bulkRequest.numberOfActions() > 0) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + logger.info("Indexing [" + bulkRequest.numberOfActions() + "] customLogTypes"); + client.bulk( + bulkRequest, + ActionListener.delegateFailure(listener, (l, r) -> { + if (r.hasFailures()) { + logger.error("Custom LogType Bulk Index had failures:\n ", r.buildFailureMessage()); + listener.onFailure(new IllegalStateException(r.buildFailureMessage())); + } else { + logger.info("Loaded [" + r.getItems().length + "] customLogType docs successfully!"); + listener.onResponse(null); + } + }) + ); + } else { + listener.onResponse(null); + } + } catch (URISyntaxException | IOException e) { + listener.onFailure(e); + } + } + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + private String generateFieldMappingDocId(FieldMappingDoc fieldMappingDoc) { String generatedId = fieldMappingDoc.getRawField() + "|"; if (fieldMappingDoc.getSchemaFields().containsKey(defaultSchemaField)) { @@ -215,7 +370,8 @@ private List mergeFieldMappings(List existingF public void getAllFieldMappings(ActionListener> listener) { SearchRequest searchRequest = new SearchRequest(LOG_TYPE_INDEX); - searchRequest.source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(10000)); + searchRequest.source(new SearchSourceBuilder().query(QueryBuilders.boolQuery() + .mustNot(QueryBuilders.existsQuery("source"))).size(10000)); client.search( searchRequest, ActionListener.delegateFailure( @@ -293,7 +449,7 @@ public void onResponse(CreateIndexResponse result) { listener, (delegatedListener, unused) -> { isConfigIndexInitialized = true; - delegatedListener.onResponse(null); + doIndexLogTypeMetadata(listener); }) ); } @@ -306,7 +462,7 @@ public void onFailure(Exception e) { listener, (delegatedListener, unused) -> { isConfigIndexInitialized = true; - delegatedListener.onResponse(null); + doIndexLogTypeMetadata(listener); }) ); } else { @@ -328,20 +484,20 @@ public void onFailure(Exception e) { listener, (delegatedListener, unused) -> { isConfigIndexInitialized = true; - delegatedListener.onResponse(null); + doIndexLogTypeMetadata(listener); }) ); })); } else { if (isConfigIndexInitialized) { - listener.onResponse(null); + doIndexLogTypeMetadata(listener); return; } loadBuiltinLogTypes(ActionListener.delegateFailure( listener, (delegatedListener, unused) -> { isConfigIndexInitialized = true; - delegatedListener.onResponse(null); + doIndexLogTypeMetadata(listener); }) ); } diff --git a/src/main/java/org/opensearch/securityanalytics/model/CustomLogType.java b/src/main/java/org/opensearch/securityanalytics/model/CustomLogType.java new file mode 100644 index 000000000..37a4217cf --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/CustomLogType.java @@ -0,0 +1,191 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; +import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Map; + +import static org.opensearch.securityanalytics.action.IndexCustomLogTypeResponse.CUSTOM_LOG_TYPES_FIELD; +import static org.opensearch.securityanalytics.model.Detector.NO_ID; +import static org.opensearch.securityanalytics.model.Detector.NO_VERSION; + +public class CustomLogType implements Writeable, ToXContentObject { + + private static final Logger log = LogManager.getLogger(CustomLogType.class); + + public static final String CUSTOM_LOG_TYPE_ID_FIELD = "custom_logtype_id"; + + private static final String NAME_FIELD = "name"; + + private static final String DESCRIPTION_FIELD = "description"; + private static final String SOURCE_FIELD = "source"; + + private static final String TAGS_FIELD = "tags"; + + private String id; + + private Long version; + + private String name; + + private String description; + + private String source; + + private Map tags; + + public static final NamedXContentRegistry.Entry XCONTENT_REGISTRY = new NamedXContentRegistry.Entry( + CustomLogType.class, + new ParseField(CUSTOM_LOG_TYPES_FIELD), + xcp -> parse(xcp, null, null) + ); + + public CustomLogType(String id, + Long version, + String name, + String description, + String source, + Map tags) { + this.id = id != null ? id : NO_ID; + this.version = version != null ? version : NO_VERSION; + this.name = name; + this.description = description; + this.source = source; + this.tags = tags; + } + + public CustomLogType(StreamInput sin) throws IOException { + this( + sin.readString(), + sin.readLong(), + sin.readString(), + sin.readString(), + sin.readString(), + sin.readMap() + ); + } + + @SuppressWarnings("unchecked") + public CustomLogType(Map input) { + this( + null, + null, + input.get(NAME_FIELD).toString(), + input.get(DESCRIPTION_FIELD).toString(), + input.get(SOURCE_FIELD).toString(), + (Map) input.get(TAGS_FIELD) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeString(name); + out.writeString(description); + out.writeString(source); + out.writeMap(tags); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(NAME_FIELD, name) + .field(DESCRIPTION_FIELD, description) + .field(SOURCE_FIELD, source) + .field(TAGS_FIELD, tags) + .endObject(); + } + + public static CustomLogType parse(XContentParser xcp, String id, Long version) throws IOException { + if (id == null) { + id = NO_ID; + } + if (version == null) { + version = NO_VERSION; + } + + String name = null; + String description = null; + String source = null; + Map tags = null; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case NAME_FIELD: + name = xcp.text(); + break; + case DESCRIPTION_FIELD: + description = xcp.text(); + break; + case SOURCE_FIELD: + source = xcp.text(); + break; + case TAGS_FIELD: + tags = xcp.map(); + break; + default: + xcp.skipChildren(); + } + } + return new CustomLogType(id, version, name, description, source, tags); + } + + public static CustomLogType readFrom(StreamInput sin) throws IOException { + return new CustomLogType(sin); + } + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setVersion(Long version) { + this.version = version; + } + + public Long getVersion() { + return version; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getSource() { + return source; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public Map getTags() { + return tags; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/model/Detector.java b/src/main/java/org/opensearch/securityanalytics/model/Detector.java index fa9f0fa81..0769e74f8 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/Detector.java +++ b/src/main/java/org/opensearch/securityanalytics/model/Detector.java @@ -211,47 +211,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return createXContentBuilder(builder, params, true); } - public enum DetectorType { - OTHERS_APPLICATION("others_application", 0), - OTHERS_APT("others_apt", 1), - OTHERS_CLOUD("others_cloud", 2), - OTHERS_COMPLIANCE("others_compliance", 4), - LINUX("linux", 5), - OTHERS_MACOS("others_macos", 6), - NETWORK("network", 7), - OTHERS_PROXY("others_proxy", 8), - OTHERS_WEB("others_web", 9), - WINDOWS("windows", 10), - AD_LDAP("ad_ldap", 11), - APACHE_ACCESS("apache_access", 12), - CLOUDTRAIL("cloudtrail", 14), - DNS("dns", 15), - GITHUB("github", 16), - M365("m365", 17), - GWORKSPACE("gworkspace", 18), - OKTA("okta", 19), - AZURE("azure", 20), - S3("s3", 21), - TEST_WINDOWS("test_windows", 22), - VPCFLOW("vpcflow", 23); - - private String type; - private int dim; - - DetectorType(String type, int dim) { - this.type = type; - this.dim = dim; - } - - public String getDetectorType() { - return type; - } - - public int getDim() { - return dim; - } - } - private XContentBuilder createXContentBuilder(XContentBuilder builder, ToXContent.Params params, Boolean secure) throws IOException { builder.startObject(); if (params.paramAsBoolean("with_type", false)) { diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestDeleteCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestDeleteCustomLogTypeAction.java new file mode 100644 index 000000000..692bd6e2c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestDeleteCustomLogTypeAction.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestStatus; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.DeleteCustomLogTypeAction; +import org.opensearch.securityanalytics.action.DeleteCustomLogTypeRequest; +import org.opensearch.securityanalytics.model.CustomLogType; +import org.opensearch.securityanalytics.util.RestHandlerUtils; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +public class RestDeleteCustomLogTypeAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestDeleteCustomLogTypeAction.class); + + @Override + public String getName() { + return "delete_custom_log_type_action"; + } + + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.DELETE, String.format(Locale.getDefault(), + "%s/{%s}", + SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, + CustomLogType.CUSTOM_LOG_TYPE_ID_FIELD)) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI)); + + WriteRequest.RefreshPolicy refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE; + if (request.hasParam(RestHandlerUtils.REFRESH)) { + refreshPolicy = WriteRequest.RefreshPolicy.parse(request.param(RestHandlerUtils.REFRESH)); + } + + String id = request.param(CustomLogType.CUSTOM_LOG_TYPE_ID_FIELD); + if (id == null) { + throw new OpenSearchStatusException("Log Type id is null", RestStatus.BAD_REQUEST); + } + + DeleteCustomLogTypeRequest deleteCustomLogTypeRequest = new DeleteCustomLogTypeRequest(id, refreshPolicy); + return channel -> client.execute(DeleteCustomLogTypeAction.INSTANCE, deleteCustomLogTypeRequest, new RestToXContentListener<>(channel)); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexCustomLogTypeAction.java new file mode 100644 index 000000000..d927fdf55 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestIndexCustomLogTypeAction.java @@ -0,0 +1,95 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.RestStatus; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.IndexCustomLogTypeAction; +import org.opensearch.securityanalytics.action.IndexCustomLogTypeRequest; +import org.opensearch.securityanalytics.action.IndexCustomLogTypeResponse; +import org.opensearch.securityanalytics.model.CustomLogType; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.util.RestHandlerUtils; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +public class RestIndexCustomLogTypeAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestIndexCustomLogTypeAction.class); + + @Override + public String getName() { + return "index_custom_log_type_action"; + } + + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI), + new Route(RestRequest.Method.PUT, String.format(Locale.getDefault(), + "%s/{%s}", + SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, + CustomLogType.CUSTOM_LOG_TYPE_ID_FIELD)) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI)); + + WriteRequest.RefreshPolicy refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE; + if (request.hasParam(RestHandlerUtils.REFRESH)) { + refreshPolicy = WriteRequest.RefreshPolicy.parse(request.param(RestHandlerUtils.REFRESH)); + } + + String id = request.param(CustomLogType.CUSTOM_LOG_TYPE_ID_FIELD, Detector.NO_ID); + + XContentParser xcp = request.contentParser(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + + CustomLogType logType = CustomLogType.parse(xcp, id, null); + IndexCustomLogTypeRequest customLogTypeRequest = new IndexCustomLogTypeRequest(id, refreshPolicy, request.method(), logType); + return channel -> client.execute(IndexCustomLogTypeAction.INSTANCE, customLogTypeRequest, indexCustomLogTypeResponse(channel, request.method())); + } + + private RestResponseListener indexCustomLogTypeResponse(RestChannel channel, RestRequest.Method restMethod) { + return new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(IndexCustomLogTypeResponse response) throws Exception { + RestStatus returnStatus = RestStatus.CREATED; + if (restMethod == RestRequest.Method.PUT) { + returnStatus = RestStatus.OK; + } + + BytesRestResponse restResponse = new BytesRestResponse(returnStatus, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + + if (restMethod == RestRequest.Method.POST) { + String location = String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, response.getId()); + restResponse.addHeader("Location", location); + } + + return restResponse; + } + }; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestSearchCorrelationAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestSearchCorrelationAction.java index c90b74753..e66e3ba7b 100644 --- a/src/main/java/org/opensearch/securityanalytics/resthandler/RestSearchCorrelationAction.java +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestSearchCorrelationAction.java @@ -60,7 +60,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli long timeWindow = request.paramAsLong("time_window", 300000L); int noOfNearbyFindings = request.paramAsInt("nearby_findings", 10); - CorrelatedFindingRequest correlatedFindingRequest = new CorrelatedFindingRequest(findingId, Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)), timeWindow, noOfNearbyFindings); + CorrelatedFindingRequest correlatedFindingRequest = new CorrelatedFindingRequest(findingId, detectorType, timeWindow, noOfNearbyFindings); return channel -> { client.execute(CorrelatedFindingAction.INSTANCE, correlatedFindingRequest, new RestCorrelatedFindingResponseListener(channel, request)); diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestSearchCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestSearchCustomLogTypeAction.java new file mode 100644 index 000000000..83f4ffa83 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestSearchCustomLogTypeAction.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction; +import org.opensearch.securityanalytics.action.SearchCustomLogTypeRequest; +import org.opensearch.securityanalytics.logtype.LogTypeService; +import org.opensearch.securityanalytics.model.CustomLogType; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.opensearch.rest.RestStatus.OK; + +public class RestSearchCustomLogTypeAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestSearchCustomLogTypeAction.class); + + @Override + public String getName() { + return "search_custom_log_type_action"; + } + + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + "_search"), + new Route(RestRequest.Method.GET, SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + "_search") + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + "_search")); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.parseXContent(request.contentOrSourceParamParser()); + searchSourceBuilder.fetchSource(FetchSourceContext.parseFromRestRequest(request)); + searchSourceBuilder.seqNoAndPrimaryTerm(true); + searchSourceBuilder.version(true); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(searchSourceBuilder); + searchRequest.indices(LogTypeService.LOG_TYPE_INDEX); + + SearchCustomLogTypeRequest searchCustomLogTypeRequest = new SearchCustomLogTypeRequest(searchRequest); + return channel -> { + client.execute(SearchCustomLogTypeAction.INSTANCE, searchCustomLogTypeRequest, new RestSearchCustomLogTypeResponseListener(channel, request)); + }; + } + + static class RestSearchCustomLogTypeResponseListener extends RestResponseListener { + private final RestRequest request; + + RestSearchCustomLogTypeResponseListener(RestChannel channel, RestRequest request) { + super(channel); + this.request = request; + } + + @Override + public RestResponse buildResponse(final SearchResponse response) throws Exception { + for (SearchHit hit : response.getHits()) { + Map sourceMap = hit.getSourceAsMap(); + + CustomLogType logType = new CustomLogType(sourceMap); + logType.setId(hit.getId()); + logType.setVersion(hit.getVersion()); + + XContentBuilder xcb = logType.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS); + hit.sourceRef(BytesReference.bytes(xcb)); + } + return new BytesRestResponse(OK, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + } + + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 958a2c132..8335c51f6 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -438,7 +438,7 @@ private Object convertValueCidr(SigmaCIDRExpression ip) { } private String getMappedField(String field) { - if (this.enableFieldMappings && this.fieldMappings.containsKey(field)) { + if (this.enableFieldMappings && this.fieldMappings.containsKey(field) && this.fieldMappings.get(field) != null) { return this.fieldMappings.get(field); } return field; diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java index f84b433db..a8d4d8003 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java @@ -47,6 +47,8 @@ import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.correlation.JoinEngine; import org.opensearch.securityanalytics.correlation.VectorEmbeddingsEngine; +import org.opensearch.securityanalytics.logtype.LogTypeService; +import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.util.CorrelationIndices; @@ -60,6 +62,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -118,7 +121,6 @@ public TransportCorrelateFindingAction(TransportService transportService, @Override protected void doExecute(Task task, ActionRequest request, ActionListener actionListener) { try { - log.info("hit here1"); PublishFindingsRequest transformedRequest = transformRequest(request); if (!this.correlationIndices.correlationIndexExists()) { @@ -158,7 +160,6 @@ public void onFailure(Exception e) { log.error(ex); } } else { - log.info("hit here2"); AsyncCorrelateFindingAction correlateFindingAction = new AsyncCorrelateFindingAction(task, transformedRequest, actionListener); correlateFindingAction.start(); } @@ -186,11 +187,9 @@ public class AsyncCorrelateFindingAction { this.joinEngine = new JoinEngine(client, request, xContentRegistry, corrTimeWindow, this); this.vectorEmbeddingsEngine = new VectorEmbeddingsEngine(client, indexTimeout, corrTimeWindow, this); - log.info("hit here5"); } void start() { - log.info("hit here4"); TransportCorrelateFindingAction.this.threadPool.getThreadContext().stashContext(); String monitorId = request.getMonitorId(); Finding finding = request.getFinding(); @@ -252,7 +251,6 @@ public void onFailure(Exception e) { public void initCorrelationIndex(String detectorType, Map> correlatedFindings, List correlationRules) { try { - log.info("hit here6"); if (!IndexUtils.correlationIndexUpdated) { IndexUtils.updateIndexMapping( CorrelationIndices.CORRELATION_INDEX, @@ -283,7 +281,6 @@ public void onFailure(Exception e) { } public void getTimestampFeature(String detectorType, Map> correlatedFindings, Finding orphanFinding, List correlationRules) { - log.info("hit here7"); long findingTimestamp = this.request.getFinding().getTimestamp().toEpochMilli(); BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() .mustNot(QueryBuilders.termQuery("scoreTimestamp", 0L)); @@ -318,17 +315,49 @@ public void onResponse(SearchResponse response) { client.index(scoreIndexRequest, new ActionListener<>() { @Override public void onResponse(IndexResponse response) { - if (correlatedFindings != null) { - if (correlatedFindings.isEmpty()) { - vectorEmbeddingsEngine.insertOrphanFindings(detectorType, request.getFinding(), Long.valueOf(CorrelationIndices.FIXED_HISTORICAL_INTERVAL / 1000L).floatValue()); + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.existsQuery("source")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(10000); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(LogTypeService.LOG_TYPE_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR)); + } + + SearchHit[] hits = response.getHits().getHits(); + Map logTypes = new HashMap<>(); + for (SearchHit hit : hits) { + Map sourceMap = hit.getSourceAsMap(); + logTypes.put(sourceMap.get("name").toString(), + new CustomLogType(sourceMap)); + } + + if (correlatedFindings != null) { + if (correlatedFindings.isEmpty()) { + vectorEmbeddingsEngine.insertOrphanFindings(detectorType, request.getFinding(), Long.valueOf(CorrelationIndices.FIXED_HISTORICAL_INTERVAL / 1000L).floatValue(), logTypes); + } + for (Map.Entry> correlatedFinding : correlatedFindings.entrySet()) { + vectorEmbeddingsEngine.insertCorrelatedFindings(detectorType, request.getFinding(), correlatedFinding.getKey(), correlatedFinding.getValue(), + Long.valueOf(CorrelationIndices.FIXED_HISTORICAL_INTERVAL / 1000L).floatValue(), correlationRules, logTypes); + } + } else { + vectorEmbeddingsEngine.insertOrphanFindings(detectorType, orphanFinding, Long.valueOf(CorrelationIndices.FIXED_HISTORICAL_INTERVAL / 1000L).floatValue(), logTypes); + } } - for (Map.Entry> correlatedFinding : correlatedFindings.entrySet()) { - vectorEmbeddingsEngine.insertCorrelatedFindings(detectorType, request.getFinding(), correlatedFinding.getKey(), correlatedFinding.getValue(), - Long.valueOf(CorrelationIndices.FIXED_HISTORICAL_INTERVAL / 1000L).floatValue(), correlationRules); + + @Override + public void onFailure(Exception e) { + onFailures(e); } - } else { - vectorEmbeddingsEngine.insertOrphanFindings(detectorType, orphanFinding, Long.valueOf(CorrelationIndices.FIXED_HISTORICAL_INTERVAL / 1000L).floatValue()); - } + }); } @Override @@ -341,17 +370,50 @@ public void onFailure(Exception e) { } } else { float timestampFeature = Long.valueOf((findingTimestamp - scoreTimestamp) / 1000L).floatValue(); - if (correlatedFindings != null) { - if (correlatedFindings.isEmpty()) { - vectorEmbeddingsEngine.insertOrphanFindings(detectorType, request.getFinding(), timestampFeature); + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.existsQuery("source")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(10000); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(LogTypeService.LOG_TYPE_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR)); + } + + SearchHit[] hits = response.getHits().getHits(); + Map logTypes = new HashMap<>(); + for (SearchHit hit : hits) { + Map sourceMap = hit.getSourceAsMap(); + logTypes.put(sourceMap.get("name").toString(), + new CustomLogType(sourceMap)); + } + + if (correlatedFindings != null) { + if (correlatedFindings.isEmpty()) { + vectorEmbeddingsEngine.insertOrphanFindings(detectorType, request.getFinding(), timestampFeature, logTypes); + } + for (Map.Entry> correlatedFinding : correlatedFindings.entrySet()) { + vectorEmbeddingsEngine.insertCorrelatedFindings(detectorType, request.getFinding(), correlatedFinding.getKey(), correlatedFinding.getValue(), + timestampFeature, correlationRules, logTypes); + } + } else { + vectorEmbeddingsEngine.insertOrphanFindings(detectorType, orphanFinding, timestampFeature, logTypes); + } } - for (Map.Entry> correlatedFinding : correlatedFindings.entrySet()) { - vectorEmbeddingsEngine.insertCorrelatedFindings(detectorType, request.getFinding(), correlatedFinding.getKey(), correlatedFinding.getValue(), - timestampFeature, correlationRules); + + @Override + public void onFailure(Exception e) { + onFailures(e); } - } else { - vectorEmbeddingsEngine.insertOrphanFindings(detectorType, orphanFinding, timestampFeature); - } + }); } } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java new file mode 100644 index 000000000..f375e84ba --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java @@ -0,0 +1,297 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.transport; + +import org.apache.lucene.search.join.ScoreMode; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.ActionRunnable; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.commons.authuser.User; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.rest.RestStatus; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.action.DeleteCustomLogTypeAction; +import org.opensearch.securityanalytics.action.DeleteCustomLogTypeRequest; +import org.opensearch.securityanalytics.action.DeleteCustomLogTypeResponse; +import org.opensearch.securityanalytics.logtype.LogTypeService; +import org.opensearch.securityanalytics.model.CustomLogType; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.Rule; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.util.CustomLogTypeIndices; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.opensearch.securityanalytics.model.Detector.NO_VERSION; + +public class TransportDeleteCustomLogTypeAction extends HandledTransportAction implements SecureTransportAction { + + private final Client client; + + private final ClusterService clusterService; + + private final ThreadPool threadPool; + + private final Settings settings; + + private final CustomLogTypeIndices customLogTypeIndices; + + private volatile Boolean filterByEnabled; + + private volatile TimeValue indexTimeout; + + @Inject + public TransportDeleteCustomLogTypeAction(TransportService transportService, + Client client, + ActionFilters actionFilters, + ClusterService clusterService, + CustomLogTypeIndices customLogTypeIndices, + Settings settings, + ThreadPool threadPool) { + super(DeleteCustomLogTypeAction.NAME, transportService, actionFilters, DeleteCustomLogTypeRequest::new); + this.client = client; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.customLogTypeIndices = customLogTypeIndices; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings); + + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.INDEX_TIMEOUT, this::setIndexTimeout); + } + + @Override + protected void doExecute(Task task, DeleteCustomLogTypeRequest request, ActionListener listener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); + return; + } + this.threadPool.getThreadContext().stashContext(); + AsyncDeleteCustomLogTypeAction deleteCustomLogTypeAction = new AsyncDeleteCustomLogTypeAction(task, request, listener); + deleteCustomLogTypeAction.start(); + } + + class AsyncDeleteCustomLogTypeAction { + + private final DeleteCustomLogTypeRequest request; + + private final ActionListener listener; + + private final AtomicReference response; + private final AtomicBoolean counter = new AtomicBoolean(); + + private Task task; + + AsyncDeleteCustomLogTypeAction( + Task task, + DeleteCustomLogTypeRequest request, + ActionListener listener + ) { + this.task = task; + this.request = request; + this.listener = listener; + this.response = new AtomicReference<>(); + } + + void start() { + if (!customLogTypeIndices.customLogTypeIndexExists()) { + onFailures(new OpenSearchStatusException( + String.format(Locale.getDefault(), + "Log Type with id %s is not found", + request.getLogTypeId()), + RestStatus.NOT_FOUND)); + return; + } + String logTypeId = request.getLogTypeId(); + GetRequest getRequest = new GetRequest(LogTypeService.LOG_TYPE_INDEX, logTypeId); + client.get(getRequest, new ActionListener<>() { + @Override + public void onResponse(GetResponse response) { + if (!response.isExists()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s is not found", logTypeId), RestStatus.NOT_FOUND)); + return; + } + + Map sourceMap = response.getSourceAsMap(); + CustomLogType logType = new CustomLogType(sourceMap); + logType.setId(response.getId()); + logType.setVersion(response.getVersion()); + + onGetResponse(logType); + } + + @Override + public void onFailure(Exception e) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s is not found", logTypeId), RestStatus.NOT_FOUND)); + } + }); + } + + private void onGetResponse(CustomLogType logType) { + if (logType.getSource().equals("Sigma")) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because source is sigma", logType.getId()), RestStatus.BAD_REQUEST)); + } + + searchDetectors(logType.getName(), new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } + + if (response.getHits().getTotalHits().value > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active detectors exist", logType.getId()), RestStatus.BAD_REQUEST)); + return; + } + + searchRules(logType.getName(), new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } + + if (response.getHits().getTotalHits().value > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active rules exist", logType.getId()), RestStatus.BAD_REQUEST)); + return; + } + + DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId()) + .setRefreshPolicy(request.getRefreshPolicy()) + .timeout(indexTimeout); + + client.delete(deleteRequest, new ActionListener<>() { + @Override + public void onResponse(DeleteResponse response) { + if (response.status() != RestStatus.OK) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); + } + onOperation(response); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + private void searchDetectors(String logTypeName, ActionListener listener) { + QueryBuilder queryBuilder = + QueryBuilders.nestedQuery("detector", + QueryBuilders.boolQuery().must( + QueryBuilders.matchQuery("detector.detector_type", logTypeName) + ), ScoreMode.Avg); + + SearchRequest searchRequest = new SearchRequest(Detector.DETECTORS_INDEX) + .source(new SearchSourceBuilder() + .seqNoAndPrimaryTerm(true) + .version(true) + .query(queryBuilder) + .size(0)); + + client.search(searchRequest, listener); + } + + private void searchRules(String logTypeName, ActionListener listener) { + QueryBuilder queryBuilder = + QueryBuilders.nestedQuery("rule", + QueryBuilders.boolQuery().must( + QueryBuilders.matchQuery("rule.category", logTypeName) + ), ScoreMode.Avg); + + SearchRequest searchRequest = new SearchRequest(Rule.CUSTOM_RULES_INDEX) + .source(new SearchSourceBuilder() + .seqNoAndPrimaryTerm(true) + .version(true) + .query(queryBuilder) + .size(0)); + + client.search(searchRequest, listener); + } + + private void onOperation(DeleteResponse response) { + this.response.set(response); + if (counter.compareAndSet(false, true)) { + finishHim(response.getId(), null); + } + } + + private void onFailures(Exception t) { + log.error(String.format(Locale.ROOT, "Failed to delete detector")); + if (counter.compareAndSet(false, true)) { + finishHim(null, t); + } + } + + private void finishHim(String logTypeId, Exception t) { + threadPool.executor(ThreadPool.Names.GENERIC).execute(ActionRunnable.supply(listener, () -> { + if (t != null) { + log.error(String.format(Locale.ROOT, "Failed to delete log type %s",logTypeId), t); + if (t instanceof OpenSearchStatusException) { + throw t; + } + throw SecurityAnalyticsException.wrap(t); + } else { + return new DeleteCustomLogTypeResponse(logTypeId, NO_VERSION, RestStatus.NO_CONTENT); + } + })); + } + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } + + private void setIndexTimeout(TimeValue indexTimeout) { + this.indexTimeout = indexTimeout; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAllRuleCategoriesAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAllRuleCategoriesAction.java index 0f26d95ad..c8eedaa33 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAllRuleCategoriesAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAllRuleCategoriesAction.java @@ -20,7 +20,7 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; -public class TransportGetAllRuleCategoriesAction extends HandledTransportAction { +public class TransportGetAllRuleCategoriesAction extends HandledTransportAction implements SecureTransportAction { private final ThreadPool threadPool; private final LogTypeService logTypeService; @@ -41,7 +41,7 @@ public TransportGetAllRuleCategoriesAction( @Override protected void doExecute(Task task, GetAllRuleCategoriesRequest request, ActionListener actionListener) { this.threadPool.getThreadContext().stashContext(); - logTypeService.getAllLogTypes(ActionListener.wrap(logTypes -> { + logTypeService.getAllLogTypesMetadata(ActionListener.wrap(logTypes -> { actionListener.onResponse( new GetAllRuleCategoriesResponse( logTypes.stream().map(logType -> new RuleCategory(logType, logType)).collect(Collectors.toList()) diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java new file mode 100644 index 000000000..70c576963 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java @@ -0,0 +1,477 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.transport; + +import org.apache.lucene.search.join.ScoreMode; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.ActionRunnable; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestStatus; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.metrics.Max; +import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.action.IndexCustomLogTypeAction; +import org.opensearch.securityanalytics.action.IndexCustomLogTypeRequest; +import org.opensearch.securityanalytics.action.IndexCustomLogTypeResponse; +import org.opensearch.securityanalytics.logtype.LogTypeService; +import org.opensearch.securityanalytics.model.CustomLogType; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.Rule; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.util.CustomLogTypeIndices; +import org.opensearch.securityanalytics.util.IndexUtils; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +public class TransportIndexCustomLogTypeAction extends HandledTransportAction implements SecureTransportAction { + + private final Client client; + + private final ClusterService clusterService; + + private final ThreadPool threadPool; + + private final Settings settings; + + private final CustomLogTypeIndices customLogTypeIndices; + + private final LogTypeService logTypeService; + + private volatile Boolean filterByEnabled; + + private volatile TimeValue indexTimeout; + + @Inject + public TransportIndexCustomLogTypeAction(TransportService transportService, + Client client, + ActionFilters actionFilters, + ClusterService clusterService, + CustomLogTypeIndices customLogTypeIndices, + LogTypeService logTypeService, + Settings settings, + ThreadPool threadPool) { + super(IndexCustomLogTypeAction.NAME, transportService, actionFilters, IndexCustomLogTypeRequest::new); + this.client = client; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.customLogTypeIndices = customLogTypeIndices; + this.logTypeService = logTypeService; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings); + + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.INDEX_TIMEOUT, this::setIndexTimeout); + } + + @Override + protected void doExecute(Task task, IndexCustomLogTypeRequest request, ActionListener listener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); + return; + } + AsyncIndexCustomLogTypeAction asyncAction = new AsyncIndexCustomLogTypeAction(task, request, listener); + asyncAction.start(); + } + + public void onCreateMappingsResponse(CreateIndexResponse response) { + if (response.isAcknowledged()) { + log.info(String.format(Locale.getDefault(), "Created %s with mappings.", LogTypeService.LOG_TYPE_INDEX)); + IndexUtils.customLogTypeIndexUpdated(); + } else { + log.error(String.format(Locale.getDefault(), "Create %s mappings call not acknowledged.", LogTypeService.LOG_TYPE_INDEX)); + throw new OpenSearchStatusException(String.format(Locale.getDefault(), "Create %s mappings call not acknowledged", LogTypeService.LOG_TYPE_INDEX), RestStatus.INTERNAL_SERVER_ERROR); + } + } + + public void onUpdateMappingsResponse(AcknowledgedResponse response) { + if (response.isAcknowledged()) { + log.info(String.format(Locale.getDefault(), "Updated %s with mappings.", LogTypeService.LOG_TYPE_INDEX)); + IndexUtils.customLogTypeIndexUpdated(); + } else { + log.error(String.format(Locale.getDefault(), "Update %s mappings call not acknowledged.", LogTypeService.LOG_TYPE_INDEX)); + throw new OpenSearchStatusException(String.format(Locale.getDefault(), "Update %s mappings call not acknowledged.", LogTypeService.LOG_TYPE_INDEX), RestStatus.INTERNAL_SERVER_ERROR); + } + } + + class AsyncIndexCustomLogTypeAction { + private final IndexCustomLogTypeRequest request; + + private final ActionListener listener; + + private final AtomicReference response; + + private final AtomicBoolean counter = new AtomicBoolean(); + + private final Task task; + + AsyncIndexCustomLogTypeAction(Task task, IndexCustomLogTypeRequest request, ActionListener listener) { + this.task = task; + this.request = request; + this.listener = listener; + + this.response = new AtomicReference<>(); + } + + void start() { + TransportIndexCustomLogTypeAction.this.threadPool.getThreadContext().stashContext(); + try { + if (!customLogTypeIndices.customLogTypeIndexExists()) { + customLogTypeIndices.initCustomLogTypeIndex(new ActionListener<>() { + @Override + public void onResponse(CreateIndexResponse response) { + try { + onCreateMappingsResponse(response); + prepareCustomLogTypeIndexing(); + } catch (IOException ex) { + onFailures(ex); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else if (!IndexUtils.customLogTypeIndexUpdated) { + IndexUtils.updateIndexMapping(LogTypeService.LOG_TYPE_INDEX, + CustomLogTypeIndices.customLogTypeMappings(), + clusterService.state(), + client.admin().indices(), + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse response) { + try { + onUpdateMappingsResponse(response); + prepareCustomLogTypeIndexing(); + } catch (IOException ex) { + onFailures(ex); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + prepareCustomLogTypeIndexing(); + } + } catch (IOException ex) { + onFailures(ex); + } + } + + private void prepareCustomLogTypeIndexing() throws IOException { + String logTypeId = request.getLogTypeId(); + String source = request.getCustomLogType().getSource(); + if (source.equals("Sigma")) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated because source is sigma", logTypeId), RestStatus.BAD_REQUEST)); + } + + if (request.getMethod() == RestRequest.Method.PUT) { + searchLogTypes(logTypeId, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } + + if (response.getHits().getTotalHits().value != 1) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } + + try { + Map sourceMap = response.getHits().getHits()[0].getSourceAsMap(); + CustomLogType existingLogType = new CustomLogType(sourceMap); + existingLogType.setId(request.getCustomLogType().getId()); + existingLogType.setVersion(request.getCustomLogType().getVersion()); + + if (existingLogType.getSource().equals("Sigma")) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated because source is sigma", logTypeId), RestStatus.BAD_REQUEST)); + } + if (!existingLogType.getName().equals(request.getCustomLogType().getName())) { + searchDetectors(existingLogType.getName(), new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } + + if (response.getHits().getTotalHits().value > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Name of Log Type with id %s cannot be updated because active detectors exist", logTypeId), RestStatus.BAD_REQUEST)); + return; + } + + searchRules(existingLogType.getName(), new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } + + if (response.getHits().getTotalHits().value > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Name of Log Type with id %s cannot be updated because active rules exist", logTypeId), RestStatus.BAD_REQUEST)); + return; + } + + try { + request.getCustomLogType().setTags(existingLogType.getTags()); + IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) + .setRefreshPolicy(request.getRefreshPolicy()) + .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .id(request.getLogTypeId()) + .timeout(indexTimeout); + + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status() != RestStatus.OK) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + } + onOperation(response, request.getCustomLogType()); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } catch (IOException e) { + onFailures(e); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + request.getCustomLogType().setTags(existingLogType.getTags()); + IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) + .setRefreshPolicy(request.getRefreshPolicy()) + .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .id(request.getLogTypeId()) + .timeout(indexTimeout); + + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status() != RestStatus.OK) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + } + + request.getCustomLogType().setId(response.getId()); + onOperation(response, request.getCustomLogType()); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + } catch (IOException e) { + onFailures(e); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + logTypeService.ensureConfigIndexIsInitialized(new ActionListener() { + @Override + public void onResponse(Void unused) { + MaxAggregationBuilder queryBuilder = AggregationBuilders.max("agg").field("tags.correlation_id"); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.aggregation(queryBuilder); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(LogTypeService.LOG_TYPE_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } + + try { + Max agg = response.getAggregations().get("agg"); + int value = Double.valueOf(agg.getValue()).intValue(); + request.getCustomLogType().setTags(Map.of("correlation_id", value+1)); + IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) + .setRefreshPolicy(request.getRefreshPolicy()) + .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .timeout(indexTimeout); + + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status() != RestStatus.CREATED) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + } + request.getCustomLogType().setId(response.getId()); + onOperation(response, request.getCustomLogType()); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } catch (IOException ex) { + onFailures(ex); + } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } + } + + private void searchLogTypes(String logTypeId, ActionListener listener) { + QueryBuilder queryBuilder = QueryBuilders.matchQuery("_id", logTypeId); + SearchRequest searchRequest = new SearchRequest(LogTypeService.LOG_TYPE_INDEX) + .source(new SearchSourceBuilder() + .seqNoAndPrimaryTerm(true) + .version(true) + .query(queryBuilder) + .size(1)); + client.search(searchRequest, listener); + } + + private void searchDetectors(String logTypeName, ActionListener listener) { + QueryBuilder queryBuilder = + QueryBuilders.nestedQuery("detector", + QueryBuilders.boolQuery().must( + QueryBuilders.matchQuery("detector.detector_type", logTypeName) + ), ScoreMode.Avg); + + SearchRequest searchRequest = new SearchRequest(Detector.DETECTORS_INDEX) + .source(new SearchSourceBuilder() + .seqNoAndPrimaryTerm(true) + .version(true) + .query(queryBuilder) + .size(0)); + + client.search(searchRequest, listener); + } + + private void searchRules(String logTypeName, ActionListener listener) { + QueryBuilder queryBuilder = + QueryBuilders.nestedQuery("rule", + QueryBuilders.boolQuery().must( + QueryBuilders.matchQuery("rule.category", logTypeName) + ), ScoreMode.Avg); + + SearchRequest searchRequest = new SearchRequest(Rule.CUSTOM_RULES_INDEX) + .source(new SearchSourceBuilder() + .seqNoAndPrimaryTerm(true) + .version(true) + .query(queryBuilder) + .size(0)); + + client.search(searchRequest, listener); + } + + private void onOperation(IndexResponse response, CustomLogType logType) { + this.response.set(response); + if (counter.compareAndSet(false, true)) { + finishHim(logType); + } + } + + private void onFailures(Exception... t) { + if (counter.compareAndSet(false, true)) { + finishHim(null, t); + } + } + + private void finishHim(CustomLogType logType, Exception... t) { + threadPool.executor(ThreadPool.Names.GENERIC).execute(ActionRunnable.supply(listener, () -> { + if (t != null && t.length > 0) { + if (t.length > 1) { + throw SecurityAnalyticsException.wrap(Arrays.asList(t)); + } else { + throw SecurityAnalyticsException.wrap(t[0]); + } + } else { + return new IndexCustomLogTypeResponse(logType.getId(), logType.getVersion(), RestStatus.CREATED, logType); + } + })); + } + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } + + private void setIndexTimeout(TimeValue indexTimeout) { + this.indexTimeout = indexTimeout; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 50e78653c..d8d4c99bf 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -774,53 +774,69 @@ class AsyncIndexDetectorsAction { } void start() { - try { - TransportIndexDetectorAction.this.threadPool.getThreadContext().stashContext(); + TransportIndexDetectorAction.this.threadPool.getThreadContext().stashContext(); - if (!detectorIndices.detectorIndexExists()) { - detectorIndices.initDetectorIndex(new ActionListener<>() { - @Override - public void onResponse(CreateIndexResponse response) { - try { - onCreateMappingsResponse(response); - prepareDetectorIndexing(); - } catch (IOException e) { - onFailures(e); - } - } + logTypeService.doesLogTypeExist(request.getDetector().getDetectorType().toLowerCase(Locale.ROOT), new ActionListener<>() { + @Override + public void onResponse(Boolean exist) { + if (exist) { + try { + if (!detectorIndices.detectorIndexExists()) { + detectorIndices.initDetectorIndex(new ActionListener<>() { + @Override + public void onResponse(CreateIndexResponse response) { + try { + onCreateMappingsResponse(response); + prepareDetectorIndexing(); + } catch (IOException e) { + onFailures(e); + } + } - @Override - public void onFailure(Exception e) { - onFailures(e); - } - }); - } else if (!IndexUtils.detectorIndexUpdated) { - IndexUtils.updateIndexMapping( - Detector.DETECTORS_INDEX, - DetectorIndices.detectorMappings(), clusterService.state(), client.admin().indices(), - new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse response) { - onUpdateMappingsResponse(response); - try { - prepareDetectorIndexing(); - } catch (IOException e) { + @Override + public void onFailure(Exception e) { onFailures(e); } - } + }); + } else if (!IndexUtils.detectorIndexUpdated) { + IndexUtils.updateIndexMapping( + Detector.DETECTORS_INDEX, + DetectorIndices.detectorMappings(), clusterService.state(), client.admin().indices(), + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse response) { + onUpdateMappingsResponse(response); + try { + prepareDetectorIndexing(); + } catch (IOException e) { + onFailures(e); + } + } - @Override - public void onFailure(Exception e) { - onFailures(e); - } + @Override + public void onFailure(Exception e) { + onFailures(e); + } + } + ); + } else { + prepareDetectorIndexing(); } - ); - } else { - prepareDetectorIndexing(); + } catch (IOException e) { + onFailures(e); + } + } else { + onFailures(new OpenSearchStatusException(String.format("Detector cannot be created as logtype %s does not exist", + request.getDetector().getDetectorType().toLowerCase(Locale.ROOT)), RestStatus.BAD_REQUEST)); + } } - } catch (IOException e) { - onFailures(e); - } + + @Override + public void onFailure(Exception e) { + + } + }); + } void prepareDetectorIndexing() throws IOException { @@ -1262,6 +1278,7 @@ private void onFailures(Exception t) { private void finishHim(Detector detector, Exception t) { threadPool.executor(ThreadPool.Names.GENERIC).execute(ActionRunnable.supply(listener, () -> { if (t != null) { + log.error("exception:", t); if (t instanceof OpenSearchStatusException) { throw t; } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java index 180616636..da0afefc0 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java @@ -138,43 +138,57 @@ class AsyncIndexRulesAction { void start() { TransportIndexRuleAction.this.threadPool.getThreadContext().stashContext(); - try { - if (!ruleIndices.ruleIndexExists(false)) { - ruleIndices.initRuleIndex(new ActionListener<>() { - @Override - public void onResponse(CreateIndexResponse response) { - ruleIndices.onCreateMappingsResponse(response, false); - prepareRuleIndexing(); - } + logTypeService.doesLogTypeExist(request.getLogType().toLowerCase(Locale.ROOT), new ActionListener<>() { + @Override + public void onResponse(Boolean exist) { + if (exist) { + try { + if (!ruleIndices.ruleIndexExists(false)) { + ruleIndices.initRuleIndex(new ActionListener<>() { + @Override + public void onResponse(CreateIndexResponse response) { + ruleIndices.onCreateMappingsResponse(response, false); + prepareRuleIndexing(); + } - @Override - public void onFailure(Exception e) { - onFailures(e); + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }, false); + } else if (!IndexUtils.customRuleIndexUpdated) { + IndexUtils.updateIndexMapping( + Rule.CUSTOM_RULES_INDEX, + RuleIndices.ruleMappings(), clusterService.state(), client.admin().indices(), + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse response) { + ruleIndices.onUpdateMappingsResponse(response, false); + prepareRuleIndexing(); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + } + ); + } else { + prepareRuleIndexing(); + } + } catch (IOException ex) { + onFailures(ex); } - }, false); - } else if (!IndexUtils.customRuleIndexUpdated) { - IndexUtils.updateIndexMapping( - Rule.CUSTOM_RULES_INDEX, - RuleIndices.ruleMappings(), clusterService.state(), client.admin().indices(), - new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse response) { - ruleIndices.onUpdateMappingsResponse(response, false); - prepareRuleIndexing(); - } + } else { + onFailures(new OpenSearchStatusException(String.format("Invalid rule category %s", request.getLogType().toLowerCase(Locale.ROOT)), RestStatus.BAD_REQUEST)); + } + } - @Override - public void onFailure(Exception e) { - onFailures(e); - } - } - ); - } else { - prepareRuleIndexing(); + @Override + public void onFailure(Exception e) { + onFailures(e); } - } catch (IOException ex) { - onFailures(ex); - } + }); } void prepareRuleIndexing() { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchCorrelationAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchCorrelationAction.java index dde82e31f..9152f0042 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchCorrelationAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchCorrelationAction.java @@ -102,7 +102,7 @@ class AsyncSearchCorrelationAction { @SuppressWarnings("unchecked") void start() { String findingId = request.getFindingId(); - Detector.DetectorType detectorType = request.getDetectorType(); + String detectorType = request.getDetectorType(); long timeWindow = request.getTimeWindow(); int noOfNearbyFindings = request.getNoOfNearbyFindings(); @@ -115,7 +115,7 @@ void start() { searchSourceBuilder.fetchField("timestamp"); searchSourceBuilder.size(1); SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices(DetectorMonitorConfig.getAllFindingsIndicesPattern(detectorType.getDetectorType())); + searchRequest.indices(DetectorMonitorConfig.getAllFindingsIndicesPattern(detectorType)); searchRequest.source(searchSourceBuilder); client.search(searchRequest, new ActionListener<>() { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchCustomLogTypeAction.java new file mode 100644 index 000000000..340cfd9e0 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchCustomLogTypeAction.java @@ -0,0 +1,91 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.securityanalytics.transport; + +import org.opensearch.action.ActionListener; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; +import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction; +import org.opensearch.securityanalytics.action.SearchCustomLogTypeRequest; +import org.opensearch.securityanalytics.logtype.LogTypeService; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class TransportSearchCustomLogTypeAction extends HandledTransportAction implements SecureTransportAction { + + private final Client client; + + private final Settings settings; + + private volatile Boolean filterByEnabled; + + private final ClusterService clusterService; + + private final ThreadPool threadPool; + + private final LogTypeService logTypeService; + + @Inject + public TransportSearchCustomLogTypeAction( + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + ThreadPool threadPool, + Settings settings, + Client client, + LogTypeService logTypeService + ) { + super(SearchCustomLogTypeAction.NAME, transportService, actionFilters, SearchCustomLogTypeRequest::new); + this.client = client; + this.settings = settings; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.logTypeService = logTypeService; + + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + @Override + protected void doExecute(Task task, SearchCustomLogTypeRequest request, ActionListener listener) { + User user = readUserFromThreadContext(this.threadPool); + + if (doFilterForUser(user, this.filterByEnabled)) { + // security is enabled and filterby is enabled + log.info("Filtering result by: {}", user.getBackendRoles()); + addFilter(user, request.searchRequest().source(), "detector.user.backend_roles.keyword"); + } + + this.threadPool.getThreadContext().stashContext(); + logTypeService.searchLogTypes(request.searchRequest(), new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + listener.onResponse(response); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java new file mode 100644 index 000000000..7b9e6e066 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.util; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.ActionListener; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.client.AdminClient; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.health.ClusterIndexHealth; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.securityanalytics.logtype.LogTypeService; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.Objects; + +public class CustomLogTypeIndices { + + private static final Logger log = LogManager.getLogger(CustomLogTypeIndices.class); + + + private final AdminClient client; + + private final ClusterService clusterService; + + public CustomLogTypeIndices(AdminClient client, ClusterService clusterService) { + this.client = client; + this.clusterService = clusterService; + } + + public static String customLogTypeMappings() throws IOException { + return new String(Objects.requireNonNull(CustomLogTypeIndices.class.getClassLoader().getResourceAsStream("mappings/log_type_config_mapping.json")).readAllBytes(), Charset.defaultCharset()); + } + + public void initCustomLogTypeIndex(ActionListener actionListener) throws IOException { + if (!customLogTypeIndexExists()) { + CreateIndexRequest indexRequest = new CreateIndexRequest(LogTypeService.LOG_TYPE_INDEX) + .mapping(customLogTypeMappings()) + .settings(Settings.builder().put("index.hidden", true).build()); + client.indices().create(indexRequest, actionListener); + } + } + + public boolean customLogTypeIndexExists() { + ClusterState clusterState = clusterService.state(); + return clusterState.getRoutingTable().hasIndex(LogTypeService.LOG_TYPE_INDEX); + } + + public ClusterIndexHealth customLogTypeIndexHealth() { + ClusterIndexHealth indexHealth = null; + + if (customLogTypeIndexExists()) { + IndexRoutingTable indexRoutingTable = clusterService.state().routingTable().index(LogTypeService.LOG_TYPE_INDEX); + IndexMetadata indexMetadata = clusterService.state().metadata().index(LogTypeService.LOG_TYPE_INDEX); + + indexHealth = new ClusterIndexHealth(indexMetadata, indexRoutingTable); + } + return indexHealth; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java index 0280c93c4..aaff505bc 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java @@ -37,6 +37,8 @@ public class IndexUtils { public static Boolean correlationIndexUpdated = false; public static Boolean correlationRuleIndexUpdated = false; + public static Boolean customLogTypeIndexUpdated = false; + public static void detectorIndexUpdated() { detectorIndexUpdated = true; } @@ -55,6 +57,10 @@ public static void correlationRuleIndexUpdated() { correlationRuleIndexUpdated = true; } + public static void customLogTypeIndexUpdated() { + customLogTypeIndexUpdated = true; + } + public static Integer getSchemaVersion(String mapping) throws IOException { XContentParser xcp = XContentType.JSON.xContent().createParser( NamedXContentRegistry.EMPTY, diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index ea0e86806..5a0da3a2e 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -13,6 +13,7 @@ import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.opensearch.action.bulk.BulkItemResponse; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.index.IndexRequest; @@ -32,11 +33,14 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.reindex.BulkByScrollResponse; import org.opensearch.index.reindex.DeleteByQueryAction; import org.opensearch.index.reindex.DeleteByQueryRequestBuilder; import org.opensearch.rest.RestStatus; +import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.mapper.MapperUtils; @@ -106,6 +110,10 @@ public void loadRules(List rules, WriteRequest.RefreshPolicy refreshPolicy String ruleIndex = getRuleIndex(isPrepackaged); BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(refreshPolicy).timeout(indexTimeout); + if (rules.isEmpty()) { + actionListener.onResponse(new BulkResponse(new BulkItemResponse[]{}, 1)); + return; + } for (Rule rule: rules) { IndexRequest indexRequest = new IndexRequest(ruleIndex) .id(rule.getId()) @@ -245,13 +253,9 @@ private void loadQueries(Path path, WriteRequest.RefreshPolicy refreshPolicy, Ti for (Path folderPath: folderPaths) { List rules = getRules(List.of(folderPath)); String ruleCategory = getRuleCategory(folderPath); - - if (Arrays.stream(Detector.DetectorType.values()) - .anyMatch(detectorType -> detectorType.getDetectorType().equals(ruleCategory))) { - logIndexToRules.put(ruleCategory, rules); - } + logIndexToRules.put(ruleCategory, rules); } - ingestQueries(logIndexToRules, refreshPolicy, indexTimeout, listener); + checkLogTypes(logIndexToRules, refreshPolicy, indexTimeout, listener); } private String getRuleCategory(Path folderPath) { @@ -292,4 +296,54 @@ private List getQueries(QueryBackend backend, String category, List> logIndexToRules, WriteRequest.RefreshPolicy refreshPolicy, TimeValue indexTimeout, ActionListener listener) { + logTypeService.ensureConfigIndexIsInitialized(new ActionListener<>() { + @Override + public void onResponse(Void unused) { + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() + .must(QueryBuilders.existsQuery("source")); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(queryBuilder); + searchSourceBuilder.fetchSource(true); + searchSourceBuilder.size(10000); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(LogTypeService.LOG_TYPE_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + listener.onFailure(new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR)); + } + try { + SearchHit[] hits = response.getHits().getHits(); + Map> filteredLogIndexToRules = new HashMap<>(); + for (SearchHit hit : hits) { + String name = hit.getSourceAsMap().get("name").toString(); + + if (logIndexToRules.containsKey(name)) { + filteredLogIndexToRules.put(name, logIndexToRules.get(name)); + } + } + ingestQueries(filteredLogIndexToRules, refreshPolicy, indexTimeout, listener); + } catch (SigmaError | IOException e) { + onFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java index 69639d4e2..13c4478ac 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java @@ -44,39 +44,35 @@ public static String ruleTopicIndexSettings() throws IOException { } public void initRuleTopicIndexTemplate(ActionListener actionListener) throws IOException { - if (!ruleTopicIndexTemplateExists()) { - getAllRuleIndices(ActionListener.wrap(allRuleIndices -> { - // Compose list of all patterns to cover all query indices - List indexPatterns = new ArrayList<>(); - for(String ruleIndex : allRuleIndices) { - indexPatterns.add(ruleIndex + "*"); - } + getAllRuleIndices(ActionListener.wrap(allRuleIndices -> { + // Compose list of all patterns to cover all query indices + List indexPatterns = new ArrayList<>(); + for(String ruleIndex : allRuleIndices) { + indexPatterns.add(ruleIndex + "*"); + } - ComposableIndexTemplate template = new ComposableIndexTemplate( - indexPatterns, - new Template( - Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(), - null, - null - ), - null, - 500L, - null, - null - ); + ComposableIndexTemplate template = new ComposableIndexTemplate( + indexPatterns, + new Template( + Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(), + null, + null + ), + null, + 500L, + null, + null + ); - client.execute( - PutComposableIndexTemplateAction.INSTANCE, - new PutComposableIndexTemplateAction.Request(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE) - .indexTemplate(template) - .create(true), - actionListener - ); + client.execute( + PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE) + .indexTemplate(template) + .create(false), + actionListener + ); - }, actionListener::onFailure)); - } else { - actionListener.onResponse(new AcknowledgedResponse(true)); - } + }, actionListener::onFailure)); } public boolean ruleTopicIndexTemplateExists() { diff --git a/src/main/java/org/opensearch/securityanalytics/util/SecurityAnalyticsException.java b/src/main/java/org/opensearch/securityanalytics/util/SecurityAnalyticsException.java index b9acb215f..b54ed9072 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/SecurityAnalyticsException.java +++ b/src/main/java/org/opensearch/securityanalytics/util/SecurityAnalyticsException.java @@ -39,16 +39,20 @@ public RestStatus status() { } public static OpenSearchException wrap(Exception ex) { - log.error("Security Analytics error:", ex); + if (ex instanceof OpenSearchException) { + return wrap((OpenSearchException) ex); + } else { + log.error("Security Analytics error:", ex); - String friendlyMsg = "Unknown error"; - RestStatus status = RestStatus.INTERNAL_SERVER_ERROR; + String friendlyMsg = "Unknown error"; + RestStatus status = RestStatus.INTERNAL_SERVER_ERROR; - if (!Strings.isNullOrEmpty(ex.getMessage())) { - friendlyMsg = ex.getMessage(); - } + if (!Strings.isNullOrEmpty(ex.getMessage())) { + friendlyMsg = ex.getMessage(); + } - return new SecurityAnalyticsException(friendlyMsg, status, new Exception(String.format(Locale.getDefault(), "%s: %s", ex.getClass().getName(), ex.getMessage()))); + return new SecurityAnalyticsException(friendlyMsg, status, new Exception(String.format(Locale.getDefault(), "%s: %s", ex.getClass().getName(), ex.getMessage()))); + } } public static OpenSearchException wrap(OpenSearchException ex) { diff --git a/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml b/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml deleted file mode 100644 index 5ec90ee14..000000000 --- a/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml +++ /dev/null @@ -1,25 +0,0 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under windows log group to their corresponding ECS Fields. -fieldmappings: - TargetUserName: azure-signinlogs-properties-user_id - creationTime: timestamp - Category: azure-activitylogs-category - OperationName: azure-platformlogs-operation_name - ModifiedProperties_NewValue: modified_properties-new_value - ResourceProviderValue: azure-resource-provider - conditionalAccessStatus: azure-signinlogs-properties-conditional_access_status - SearchFilter: SearchFilter - Operation: azure-platformlogs-operation_name - ResultType: azure-platformlogs-result_type - DeviceDetail_isCompliant: azure-signinlogs-properties-device_detail-is_compliant - ResourceDisplayName: resource_display_name - AuthenticationRequirement: azure-signinlogs-properties-authentication_requirement - TargetResources: target_resources - Workload: workload - DeviceDetail.deviceId: azure-signinlogs-properties-device_detail-device_id - OperationNameValue: azure-platformlogs-operation_name - ResourceId: azure-signinlogs-properties-resource_id - ResultDescription: azure-signinlogs-result_description - EventID: EventID - NetworkLocationDetails: azure-signinlogs-properties-network_location_details - CategoryValue: azure-activitylogs-category - ActivityDisplayName: azure-auditlogs-properties-activity_display_name diff --git a/src/main/resources/OSMapping/logtypes.json b/src/main/resources/OSMapping/logtypes.json new file mode 100644 index 000000000..4185696e5 --- /dev/null +++ b/src/main/resources/OSMapping/logtypes.json @@ -0,0 +1,178 @@ +{ + "others_application": { + "name": "others_application", + "description": "Application logs", + "source": "Sigma", + "tags": { + "correlation_id": 0 + } + }, + "others_apt": { + "name": "others_apt", + "description": "Apt logs", + "source": "Sigma", + "tags": { + "correlation_id": 1 + } + }, + "others_cloud": { + "name": "others_cloud", + "description": "Cloud logs", + "source": "Sigma", + "tags": { + "correlation_id": 2 + } + }, + "others_compliance": { + "name": "others_compliance", + "description": "Compliance logs", + "source": "Sigma", + "tags": { + "correlation_id": 4 + } + }, + "linux": { + "name": "linux", + "description": "Sys logs", + "source": "Sigma", + "tags": { + "correlation_id": 5 + } + }, + "others_macos": { + "name": "others_macos", + "description": "MacOS logs", + "source": "Sigma", + "tags": { + "correlation_id": 6 + } + }, + "network": { + "name": "network", + "description": "Network logs", + "source": "Sigma", + "tags": { + "correlation_id": 7 + } + }, + "others_proxy": { + "name": "others_proxy", + "description": "Proxy logs", + "source": "Sigma", + "tags": { + "correlation_id": 8 + } + }, + "others_web": { + "name": "others_web", + "description": "Web logs", + "source": "Sigma", + "tags": { + "correlation_id": 9 + } + }, + "windows": { + "name": "windows", + "description": "Windows logs", + "source": "Sigma", + "tags": { + "correlation_id": 10 + } + }, + "ad_ldap": { + "name": "ad_ldap", + "description": "Ad/ldap logs", + "source": "Sigma", + "tags": { + "correlation_id": 11 + } + }, + "apache_access": { + "name": "apache_access", + "description": "Apt logs", + "source": "Sigma", + "tags": { + "correlation_id": 12 + } + }, + "cloudtrail": { + "name": "cloudtrail", + "description": "Cloudtrail Raw or OCSF based logs", + "source": "Sigma", + "tags": { + "correlation_id": 14 + } + }, + "dns": { + "name": "dns", + "description": "DNS Raw or Route53 OCSF based logs", + "source": "Sigma", + "tags": { + "correlation_id": 15 + } + }, + "github": { + "name": "github", + "description": "Github logs", + "source": "Sigma", + "tags": { + "correlation_id": 16 + } + }, + "m365": { + "name": "m365", + "description": "M365 logs", + "source": "Sigma", + "tags": { + "correlation_id": 17 + } + }, + "gworkspace": { + "name": "gworkspace", + "description": "GWorkspace logs", + "source": "Sigma", + "tags": { + "correlation_id": 18 + } + }, + "okta": { + "name": "okta", + "description": "Okta logs", + "source": "Sigma", + "tags": { + "correlation_id": 19 + } + }, + "azure": { + "name": "azure", + "description": "Azure logs", + "source": "Sigma", + "tags": { + "correlation_id": 20 + } + }, + "s3": { + "name": "s3", + "description": "S3 logs", + "source": "Sigma", + "tags": { + "correlation_id": 21 + } + }, + "test_windows": { + "name": "test_windows", + "description": "Test Windows Log Type for integ tests. Please do not use.", + "source": "Sigma", + "tags": { + "correlation_id": 22 + } + }, + "vpcflow": { + "name": "vpcflow", + "description": "VPC Flow Raw or OCSF based logs", + "source": "Sigma", + "tags": { + "correlation_id": 23 + } + } +} \ No newline at end of file diff --git a/src/main/resources/mappings/log_type_config_mapping.json b/src/main/resources/mappings/log_type_config_mapping.json index c150cc466..cc5910d19 100644 --- a/src/main/resources/mappings/log_type_config_mapping.json +++ b/src/main/resources/mappings/log_type_config_mapping.json @@ -30,6 +30,41 @@ }, "log_types": { "type": "keyword" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "source": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tags": { + "dynamic": true, + "properties": { + "correlation_id": { + "type": "integer" + } + } } } } diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 0246ccf3e..8dbff6bb4 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -58,6 +58,7 @@ import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder; import org.opensearch.securityanalytics.mapper.MappingsTraverser; import org.opensearch.securityanalytics.model.CorrelationRule; +import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -562,6 +563,10 @@ protected HttpEntity toHttpEntity(CreateIndexMappingsRequest request) throws IOE return new StringEntity(toJsonString(request), ContentType.APPLICATION_JSON); } + protected HttpEntity toHttpEntity(CustomLogType logType) throws IOException { + return new StringEntity(toJsonString(logType), ContentType.APPLICATION_JSON); + } + protected HttpEntity toHttpEntity(UpdateIndexMappingsRequest request) throws IOException { return new StringEntity(toJsonString(request), ContentType.APPLICATION_JSON); } @@ -583,6 +588,11 @@ private String toJsonString(Detector detector) throws IOException { return IndexUtilsKt.string(shuffleXContent(detector.toXContent(builder, ToXContent.EMPTY_PARAMS))); } + private String toJsonString(CustomLogType logType) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(logType.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + private String toJsonString(Rule rule) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); return IndexUtilsKt.string(shuffleXContent(rule.toXContent(builder, ToXContent.EMPTY_PARAMS))); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index df731b7ee..20605d8f8 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -23,6 +23,7 @@ import org.opensearch.script.ScriptType; import org.opensearch.securityanalytics.model.CorrelationQuery; import org.opensearch.securityanalytics.model.CorrelationRule; +import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; @@ -55,12 +56,21 @@ public static Detector randomDetector(List rules) { return randomDetector(null, null, null, List.of(input), List.of(), null, null, null, null); } + public static Detector randomDetector(List rules, String detectorType) { + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), + rules.stream().map(DetectorRule::new).collect(Collectors.toList())); + return randomDetector(null, detectorType, null, List.of(input), List.of(), null, null, null, null); + } + public static Detector randomDetectorWithInputs(List inputs) { return randomDetector(null, null, null, inputs, List.of(), null, null, null, null); } - public static Detector randomDetectorWithInputs(List inputs, Detector.DetectorType detectorType) { + public static Detector randomDetectorWithInputs(List inputs, String detectorType) { return randomDetector(null, detectorType, null, inputs, List.of(), null, null, null, null); } + + + public static Detector randomDetectorWithTriggers(List triggers) { return randomDetector(null, null, null, List.of(), triggers, null, null, null, null); } @@ -78,16 +88,16 @@ public static Detector randomDetectorWithInputsAndTriggers(List i return randomDetector(null, null, null, inputs, triggers, null, null, null, null); } - public static Detector randomDetectorWithTriggers(List rules, List triggers, Detector.DetectorType detectorType, DetectorInput input) { + public static Detector randomDetectorWithTriggers(List rules, List triggers, String detectorType, DetectorInput input) { return randomDetector(null, detectorType, null, List.of(input), triggers, null, null, null, null); } - public static Detector randomDetectorWithInputsAndTriggersAndType(List inputs, List triggers, Detector.DetectorType detectorType) { + public static Detector randomDetectorWithInputsAndTriggersAndType(List inputs, List triggers, String detectorType) { return randomDetector(null, detectorType, null, inputs, triggers, null, null, null, null); } public static Detector randomDetector(String name, - Detector.DetectorType detectorType, + String detectorType, User user, List inputs, List triggers, @@ -99,7 +109,7 @@ public static Detector randomDetector(String name, name = OpenSearchRestTestCase.randomAlphaOfLength(10); } if (detectorType == null) { - detectorType = Detector.DetectorType.valueOf(randomDetectorType().toUpperCase(Locale.ROOT)); + detectorType = randomDetectorType(); } if (user == null) { user = randomUser(); @@ -133,19 +143,32 @@ public static Detector randomDetector(String name, DetectorTrigger trigger = new DetectorTrigger(null, "windows-trigger", "1", List.of(randomDetectorType()), List.of("QuarksPwDump Clearing Access History"), List.of("high"), List.of("T0008"), List.of()); triggers.add(trigger); } - return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType.getDetectorType(), user, inputs, triggers, Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap()); + return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType, user, inputs, triggers, Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap()); + } + + public static CustomLogType randomCustomLogType(String name, String description, String source) { + if (name == null) { + name = "custom-log-type"; + } + if (description == null) { + description = "custom-log-type-desc"; + } + if (source == null) { + source = "Sigma"; + } + return new CustomLogType(null, null, name, description, source, null); } public static Detector randomDetectorWithNoUser() { String name = OpenSearchRestTestCase.randomAlphaOfLength(10); - Detector.DetectorType detectorType = Detector.DetectorType.valueOf(randomDetectorType().toUpperCase(Locale.ROOT)); + String detectorType = randomDetectorType(); List inputs = Collections.emptyList(); Schedule schedule = new IntervalSchedule(5, ChronoUnit.MINUTES, null); Boolean enabled = OpenSearchTestCase.randomBoolean(); Instant enabledTime = enabled ? Instant.now().truncatedTo(ChronoUnit.MILLIS) : null; Instant lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); - return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType.getDetectorType(), null, inputs, Collections.emptyList(),Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap()); + return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType, null, inputs, Collections.emptyList(),Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap()); } public static CorrelationRule randomCorrelationRule(String name) { @@ -186,6 +209,35 @@ public static String randomRule() { "level: high"; } + public static String randomRuleWithAlias() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " event_uid: 22\n" + + " condition: selection\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String countAggregationTestRule() { return " title: Test\n" + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java index b60fd6738..30df1ed5a 100644 --- a/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java @@ -29,8 +29,7 @@ public void testIndexDetectorPostResponse() throws IOException { CronSchedule cronSchedule = new CronSchedule(cronExpression, ZoneId.of("Asia/Kolkata"), testInstance); - Detector.DetectorType detectorType = Detector.DetectorType.LINUX; - String detectorTypeString = detectorType.getDetectorType(); + String detectorType = "linux"; Detector detector = new Detector( "123", 0L, @@ -39,17 +38,17 @@ public void testIndexDetectorPostResponse() throws IOException { cronSchedule, Instant.now(), Instant.now(), - detectorType.getDetectorType(), + detectorType, randomUser(), List.of(), List.of(), List.of("1", "2", "3"), - DetectorMonitorConfig.getRuleIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getRuleIndex("others_application"), null, - DetectorMonitorConfig.getAlertsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getAlertsIndex("others_application"), null, null, - DetectorMonitorConfig.getFindingsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getFindingsIndex("others_application"), Collections.emptyMap() ); IndexDetectorResponse response = new IndexDetectorResponse("1234", 1L, RestStatus.OK, detector); diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index f3d234974..dacccb685 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -53,17 +53,17 @@ public void testGetAlerts_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION.getDetectorType(), + "others_application", null, List.of(), List.of(), List.of("monitor_id1", "monitor_id2"), - DetectorMonitorConfig.getRuleIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getRuleIndex("others_application"), null, - DetectorMonitorConfig.getAlertsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getAlertsIndex("others_application"), null, null, - DetectorMonitorConfig.getFindingsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getFindingsIndex("others_application"), Collections.emptyMap() ); GetDetectorResponse getDetectorResponse = new GetDetectorResponse("detector_id123", 1L, RestStatus.OK, detector); @@ -229,17 +229,17 @@ public void testGetFindings_getFindingsByMonitorIdFailures() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION.getDetectorType(), + "others_application", null, List.of(), List.of(), List.of("monitor_id1", "monitor_id2"), - DetectorMonitorConfig.getRuleIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getRuleIndex("others_application"), null, - DetectorMonitorConfig.getAlertsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getAlertsIndex("others_application"), null, null, - DetectorMonitorConfig.getFindingsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getFindingsIndex("others_application"), Collections.emptyMap() ); GetDetectorResponse getDetectorResponse = new GetDetectorResponse("detector_id123", 1L, RestStatus.OK, detector); diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index c50abfa82..eb6ebd9c8 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -516,7 +516,7 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx Detector detector2 = randomDetectorWithTriggers( getPrePackagedRules("network"), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of())), - Detector.DetectorType.NETWORK, + "network", inputNetflow ); @@ -580,7 +580,7 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx Assert.assertEquals(1, getAlertsBody.get("total_alerts")); // Call GetAlerts API for NETWORK detector params = new HashMap<>(); - params.put("detectorType", Detector.DetectorType.NETWORK.getDetectorType()); + params.put("detectorType", "network"); getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); getAlertsBody = asMap(getAlertsResponse); Assert.assertEquals(1, getAlertsBody.get("total_alerts")); diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index e5111804a..1a001311d 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -170,7 +170,7 @@ private String createWindowsToAppLogsToS3LogsRule(LogIndices indices) throws IOE private String createVpcFlowDetector(String indexName) throws IOException { Detector vpcFlowDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("vpc flow detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("network").stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of())), Detector.DetectorType.NETWORK); + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of())), "network"); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(vpcFlowDetector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -230,7 +230,7 @@ private String createAdLdapDetector(String indexName) throws IOException { Detector adLdapDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("ad_ldap logs detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("ad_ldap").stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("ad_ldap"), List.of(), List.of(), List.of(), List.of())), Detector.DetectorType.AD_LDAP); + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("ad_ldap"), List.of(), List.of(), List.of(), List.of())), "ad_ldap"); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(adLdapDetector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -295,7 +295,7 @@ private String createTestWindowsDetector(String indexName) throws IOException { private String createAppLogsDetector(String indexName) throws IOException { Detector appLogsDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("app logs detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("others_application").stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("others_application"), List.of(), List.of(), List.of(), List.of())), Detector.DetectorType.OTHERS_APPLICATION); + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("others_application"), List.of(), List.of(), List.of(), List.of())), "others_application"); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(appLogsDetector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -347,7 +347,7 @@ private String createS3Detector(String indexName) throws IOException { Detector s3AccessLogsDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("s3 access logs detector for security analytics", List.of(indexName), List.of(), getPrePackagedRules("s3").stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("s3"), List.of(), List.of(), List.of(), List.of())), Detector.DetectorType.S3); + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("s3"), List.of(), List.of(), List.of(), List.of())), "s3"); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(s3AccessLogsDetector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index daac0bb87..9a1875fed 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -209,7 +209,7 @@ public void testGetFindings_byDetectorType_success() throws IOException { Detector detector2 = randomDetectorWithTriggers( getPrePackagedRules("network"), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of())), - Detector.DetectorType.NETWORK, + "network", inputNetflow ); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java index 1b23270c5..13231e732 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java @@ -53,17 +53,17 @@ public void testGetFindings_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION.getDetectorType(), + "others_application", null, List.of(), List.of(), List.of("monitor_id1", "monitor_id2"), - DetectorMonitorConfig.getRuleIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getRuleIndex("others_application"), null, - DetectorMonitorConfig.getAlertsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getAlertsIndex("others_application"), null, null, - DetectorMonitorConfig.getFindingsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getFindingsIndex("others_application"), Collections.emptyMap() ); GetDetectorResponse getDetectorResponse = new GetDetectorResponse("detector_id123", 1L, RestStatus.OK, detector); @@ -173,17 +173,17 @@ public void testGetFindings_getFindingsByMonitorIdFailure() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION.getDetectorType(), + "others_application", null, List.of(), List.of(), List.of("monitor_id1", "monitor_id2"), - DetectorMonitorConfig.getRuleIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getRuleIndex("others_application"), null, - DetectorMonitorConfig.getAlertsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getAlertsIndex("others_application"), null, null, - DetectorMonitorConfig.getFindingsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + DetectorMonitorConfig.getFindingsIndex("others_application"), Collections.emptyMap() ); GetDetectorResponse getDetectorResponse = new GetDetectorResponse("detector_id123", 1L, RestStatus.OK, detector); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java index 8bb395dec..26fce0efc 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java @@ -230,7 +230,7 @@ public void testGetFindings_byDetectorType_success() throws IOException { Detector detector2 = randomDetectorWithTriggers( getPrePackagedRules("network"), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of())), - Detector.DetectorType.NETWORK, + "network", inputNetflow ); diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index bcc3e56d9..fc22c258f 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -1514,7 +1514,7 @@ public void testAzureMappings() throws IOException { indexDoc(indexName, "1", sampleDoc); - createMappingsAPI(indexName, Detector.DetectorType.AZURE.getDetectorType()); + createMappingsAPI(indexName, "azure"); //Expect only "timestamp" alias to be applied Map mappings = getIndexMappingsSAFlat(indexName); @@ -1522,8 +1522,8 @@ public void testAzureMappings() throws IOException { // Verify that all rules are working DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), - getPrePackagedRules(Detector.DetectorType.AZURE.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList())); - Detector detector = randomDetectorWithInputs(List.of(input), Detector.DetectorType.AZURE); + getPrePackagedRules("azure").stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input), "azure"); createDetector(detector); String request = "{\n" + @@ -1545,7 +1545,7 @@ public void testADLDAPMappings() throws IOException { indexDoc(indexName, "1", sampleDoc); - createMappingsAPI(indexName, Detector.DetectorType.AD_LDAP.getDetectorType()); + createMappingsAPI(indexName, "ad_ldap"); //Expect only "timestamp" alias to be applied Map mappings = getIndexMappingsSAFlat(indexName); @@ -1553,8 +1553,8 @@ public void testADLDAPMappings() throws IOException { // Verify that all rules are working DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), - getPrePackagedRules(Detector.DetectorType.AD_LDAP.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList())); - Detector detector = randomDetectorWithInputs(List.of(input), Detector.DetectorType.AD_LDAP); + getPrePackagedRules("ad_ldap").stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input), "ad_ldap"); createDetector(detector); String request = "{\n" + @@ -1576,7 +1576,7 @@ public void testCloudtrailMappings() throws IOException { indexDoc(indexName, "1", sampleDoc); - createMappingsAPI(indexName, Detector.DetectorType.CLOUDTRAIL.getDetectorType()); + createMappingsAPI(indexName, "cloudtrail"); //Expect only "timestamp" alias to be applied Map mappings = getIndexMappingsSAFlat(indexName); @@ -1584,8 +1584,8 @@ public void testCloudtrailMappings() throws IOException { // Verify that all rules are working DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), - getPrePackagedRules(Detector.DetectorType.CLOUDTRAIL.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList())); - Detector detector = randomDetectorWithInputs(List.of(input), Detector.DetectorType.CLOUDTRAIL); + getPrePackagedRules("cloudtrail").stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input), "cloudtrail"); createDetector(detector); String request = "{\n" + @@ -1607,7 +1607,7 @@ public void testS3Mappings() throws IOException { indexDoc(indexName, "1", sampleDoc); - createMappingsAPI(indexName, Detector.DetectorType.S3.getDetectorType()); + createMappingsAPI(indexName, "s3"); //Expect only "timestamp" alias to be applied Map mappings = getIndexMappingsSAFlat(indexName); @@ -1615,8 +1615,8 @@ public void testS3Mappings() throws IOException { // Verify that all rules are working DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), - getPrePackagedRules(Detector.DetectorType.S3.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList())); - Detector detector = randomDetectorWithInputs(List.of(input), Detector.DetectorType.S3); + getPrePackagedRules("s3").stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input), "s3"); createDetector(detector); String request = "{\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java new file mode 100644 index 000000000..1bc37485d --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java @@ -0,0 +1,691 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.resthandler; + +import org.apache.http.HttpStatus; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; +import org.junit.Assert; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.rest.RestStatus; +import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.TestHelpers; +import org.opensearch.securityanalytics.model.CustomLogType; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorInput; +import org.opensearch.securityanalytics.model.DetectorRule; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.opensearch.securityanalytics.TestHelpers.*; + +public class CustomLogTypeRestApiIT extends SecurityAnalyticsRestTestCase { + + @SuppressWarnings("unchecked") + public void testCreateACustomLogType() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + createdId = responseBody.get("_id").toString(); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", customLogType.getName(), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + } + + @SuppressWarnings("unchecked") + public void testCreateACustomLogTypeWithMappings() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{\"properties\":{\"event_uid\":{\"type\":\"alias\",\"path\":\"EventID\"}}}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRuleWithAlias(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + createdId = responseBody.get("_id").toString(); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", customLogType.getName(), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + } + + @SuppressWarnings("unchecked") + public void testEditACustomLogTypeDescription() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + int correlationId = Integer.parseInt(((Map)(((Map) responseBody.get("logType")).get("tags"))).get("correlation_id").toString()); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + customLogType = TestHelpers.randomCustomLogType(null, "updated desc", "Custom"); + Response updatedResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.OK, restStatus(updatedResponse)); + + responseBody = asMap(updatedResponse); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + Assert.assertEquals(correlationId, Integer.parseInt(((Map)(((Map) responseBody.get("logType")).get("tags"))).get("correlation_id").toString())); + } + + @SuppressWarnings("unchecked") + public void testEditACustomLogTypeNameFailsAsDetectorExist() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + expectThrows(ResponseException.class, () -> { + CustomLogType updatedCustomLogType = TestHelpers.randomCustomLogType("updated name", null, "Custom"); + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), toHttpEntity(updatedCustomLogType)); + }); + } + + @SuppressWarnings("unchecked") + public void testEditACustomLogTypeNameFailsAsCustomRuleExist() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + createdId = responseBody.get("_id").toString(); + + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + createdId, Collections.emptyMap(), null); + Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); + + expectThrows(ResponseException.class, () -> { + CustomLogType updatedCustomLogType = TestHelpers.randomCustomLogType("updated name", null, "Custom"); + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), toHttpEntity(updatedCustomLogType)); + }); + } + + @SuppressWarnings("unchecked") + public void testEditACustomLogTypeName() throws IOException, InterruptedException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String ruleId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(ruleId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + createdId, Collections.emptyMap(), null); + Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); + + deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.RULE_BASE_URI + "/" + ruleId, Collections.emptyMap(), null); + Assert.assertEquals("Delete rule failed", RestStatus.OK, restStatus(deleteResponse)); + Thread.sleep(5000); + + CustomLogType updatedCustomLogType = TestHelpers.randomCustomLogType("updated name", null, "Custom"); + Response updatedResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), toHttpEntity(updatedCustomLogType)); + responseBody = asMap(updatedResponse); + Assert.assertEquals(updatedCustomLogType.getName(), ((Map) responseBody.get("logType")).get("name")); + } + + @SuppressWarnings("unchecked") + public void testSearchLogTypes() throws IOException, InterruptedException { + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + Thread.sleep(5000); + + String request = "{\n" + + " \"query\": {\n" + + " \"match_all\": {\n" + + " }\n" + + " }\n" + + "}"; + + Response searchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI),Collections.emptyMap(), + new StringEntity(request), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(searchResponse)); + + Map responseBody = asMap(searchResponse); + Assert.assertEquals(23, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + + request = "{\n" + + " \"query\": {\n" + + " \"match\": {\n" + + " \"name\": {\"query\": \"" + customLogType.getName() + "\"}" + + " }\n" + + " }\n" + + "}"; + + searchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI),Collections.emptyMap(), + new StringEntity(request), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(searchResponse)); + + responseBody = asMap(searchResponse); + Assert.assertEquals(1, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + } + + @SuppressWarnings("unchecked") + public void testDeleteCustomLogTypeFailsAsDetectorExist() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + expectThrows(ResponseException.class, () -> { + makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), new StringEntity("")); + }); + } + + @SuppressWarnings("unchecked") + public void testDeleteCustomLogTypeFailsAsRulesExist() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + createdId = responseBody.get("_id").toString(); + + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + createdId, Collections.emptyMap(), null); + Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); + + expectThrows(ResponseException.class, () -> { + makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), new StringEntity("")); + }); + } + + @SuppressWarnings("unchecked") + public void testDeleteCustomLogType() throws IOException, InterruptedException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String ruleId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(ruleId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + createdId, Collections.emptyMap(), null); + Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); + + deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.RULE_BASE_URI + "/" + ruleId, Collections.emptyMap(), null); + Assert.assertEquals("Delete rule failed", RestStatus.OK, restStatus(deleteResponse)); + Thread.sleep(5000); + makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), new StringEntity("")); + } + + @SuppressWarnings("unchecked") + public void testCreateMultipleLogTypes() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + createdId = responseBody.get("_id").toString(); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", customLogType.getName(), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + customLogType = TestHelpers.randomCustomLogType("custom-log-type-again", null, "Custom"); + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + // Execute CreateMappingsAction to add alias mapping for index + createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + createdId = responseBody.get("_id").toString(); + + input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + createdId = responseBody.get("_id").toString(); + + detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", customLogType.getName(), detectorTypeInResponse); + + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(Detector.DETECTORS_INDEX, request); + hit = hits.get(0); + + String againMonitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + + indexDoc(index, "2", randomDoc()); + + executeResponse = executeAlertingMonitor(againMonitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 23215edf4..2e27d7835 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -347,6 +347,29 @@ public void testCreateDetectorWithoutRules() throws IOException { Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); } + public void testCreateDetectorWithInvalidCategory() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + Detector detector = randomDetector(Collections.emptyList(), "unknown"); + + expectThrows(ResponseException.class, () -> { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + }); + } + public void testGettingADetector() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java index 676f3f4ca..62e4287c3 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java @@ -47,7 +47,7 @@ public void testCloudTrailAPIActivityOCSFDetector() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("ocsf logs based cloudtrail detector for security analytics", List.of(index), List.of(), - getPrePackagedRules(Detector.DetectorType.CLOUDTRAIL.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.CLOUDTRAIL); + getPrePackagedRules("cloudtrail").stream().map(DetectorRule::new).collect(Collectors.toList()))), "cloudtrail"); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -105,7 +105,7 @@ public void testCloudTrailAPIActivityRawDetector() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("raw logs based cloudtrail detector for security analytics", List.of(index), List.of(), - getPrePackagedRules(Detector.DetectorType.CLOUDTRAIL.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.CLOUDTRAIL); + getPrePackagedRules("cloudtrail").stream().map(DetectorRule::new).collect(Collectors.toList()))), "cloudtrail"); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -173,7 +173,7 @@ public void testRoute53OCSFDetector() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("raw logs based route53 detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), - getPrePackagedRules(Detector.DetectorType.DNS.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.DNS); + getPrePackagedRules("dns").stream().map(DetectorRule::new).collect(Collectors.toList()))), "dns"); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -241,7 +241,7 @@ public void testRoute53RawDetector() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("raw logs based route53 detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), - getPrePackagedRules(Detector.DetectorType.DNS.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.DNS); + getPrePackagedRules("dns").stream().map(DetectorRule::new).collect(Collectors.toList()))), "dns"); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -309,7 +309,7 @@ public void testVpcflowOcsfDetector() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("raw logs based vpcflow detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), - getPrePackagedRules(Detector.DetectorType.DNS.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.DNS); + getPrePackagedRules("vpcflow").stream().map(DetectorRule::new).collect(Collectors.toList()))), "vpcflow"); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -326,7 +326,7 @@ public void testVpcflowOcsfDetector() throws IOException { Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", "dns", detectorTypeInResponse); + Assert.assertEquals("Detector type incorrect", "vpcflow", detectorTypeInResponse); String request = "{\n" + " \"query\" : {\n" + @@ -377,7 +377,7 @@ public void testVpcflowRawDetector() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("raw logs based vpcflow detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), - getPrePackagedRules(Detector.DetectorType.DNS.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.DNS); + getPrePackagedRules("vpcflow").stream().map(DetectorRule::new).collect(Collectors.toList()))), "vpcflow"); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -394,7 +394,7 @@ public void testVpcflowRawDetector() throws IOException { Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", "dns", detectorTypeInResponse); + Assert.assertEquals("Detector type incorrect", "vpcflow", detectorTypeInResponse); String request = "{\n" + " \"query\" : {\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index a57897d85..cfca3800a 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -104,14 +104,13 @@ public void testCreatingARule() throws IOException { Assert.assertEquals(0, hits.size()); } - @Ignore public void testCreatingARule_custom_category() throws IOException { String rule = randomRule(); try { makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "unknown_category"), new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); -// fail("expected exception due to invalid category"); + fail("expected exception due to invalid category"); } catch (ResponseException e) { assertEquals(HttpStatus.SC_BAD_REQUEST, e.getResponse().getStatusLine().getStatusCode()); Assert.assertTrue( @@ -357,7 +356,6 @@ public void testUpdatingUnusedRule() throws IOException { Assert.assertEquals("Update rule failed", RestStatus.OK, restStatus(updateResponse)); } - @Ignore public void testUpdatingARule_custom_category() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping());