Security Analytics 2.7 Backports (#938)

* Version bump to 2.7.1 Signed-off-by: Megha Goyal <goyamegh@amazon.com> * fix detector writeTo() method missing fields (#695) * fix detector writeTo() method missing fields Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * fix test Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> --------- Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * fix null query filter conversion from sigma to query string query (#722) * fix null query filter conversion from sigma to query string query Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * fix rule to query conversion tests for null filter Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * enhance test to verify non null doc doesnt match null query Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> --------- Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * * #709 Return empty response for empty mappings and no applied aliases Signed-off-by: Megha Goyal <goyamegh@amazon.com> * Adding integ tests for empty mappings/aliases use-cases Signed-off-by: Megha Goyal <goyamegh@amazon.com> * Fix unit tests for MappingsTraverser Signed-off-by: Megha Goyal <goyamegh@amazon.com> --------- Signed-off-by: Megha Goyal <goyamegh@amazon.com> * fixing build failures for previous cherry-pick PR for SAP2.7 (to be squashed) Signed-off-by: Dennis Toepker <toepkerd@amazon.com> * Pass rule field names in doc level queries during monitor/creation. Remove blocking actionGet() calls (#873) * pass query field names in doc level queries during monitor creation/updation Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * remove actionGet() and change get index mapping call to event driven flow Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * fix chained findings monitor Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * add finding mappings Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * remove test messages from logs Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * revert build.gradle change Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> --------- Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * fixing an import Signed-off-by: Dennis Toepker <toepkerd@amazon.com> --------- Signed-off-by: Megha Goyal <goyamegh@amazon.com> Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> Signed-off-by: Dennis Toepker <toepkerd@amazon.com> Co-authored-by: Megha Goyal <goyamegh@amazon.com> Co-authored-by: Surya Sashank Nistala <snistala@amazon.com> Co-authored-by: Megha Goyal <56077967+goyamegh@users.noreply.github.com> Co-authored-by: Dennis Toepker <toepkerd@amazon.com>
opensearch-project · Mar 19, 2024 · 0f8369c · 0f8369c
1 parent 5dac2c1
commit 0f8369c
Show file tree

Hide file tree

Showing 10 changed files with 788 additions and 405 deletions.
diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
@@ -72,9 +72,11 @@ public void createMappingAction(String indexName, String ruleTopic, String alias
         // since you can't update documents in non-write indices
         String index = indexName;
         boolean shouldUpsertIndexTemplate = IndexUtils.isConcreteIndex(indexName, this.clusterService.state()) == false;
-        if (IndexUtils.isDataStream(indexName, this.clusterService.state())) {
+        if (IndexUtils.isDataStream(indexName, this.clusterService.state()) || IndexUtils.isAlias(indexName, this.clusterService.state())) {
+            log.debug("{} is an alias or datastream. Fetching write index for create mapping action.", indexName);
             String writeIndex = IndexUtils.getWriteIndex(indexName, this.clusterService.state());
             if (writeIndex != null) {
+                log.debug("Write index for {} is {}", indexName, writeIndex);
                 index = writeIndex;
             }
         }
@@ -86,6 +88,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
                 applyAliasMappings(getMappingsResponse.getMappings(), ruleTopic, aliasMappings, partial, new ActionListener<>() {
                     @Override
                     public void onResponse(Collection<CreateMappingResult> createMappingResponse) {
+                        log.debug("Completed create mappings for {}", indexName);
                         // We will return ack==false if one of the requests returned that
                         // else return ack==true
                         Optional<AcknowledgedResponse> notAckd = createMappingResponse.stream()
@@ -104,6 +107,7 @@ public void onResponse(Collection<CreateMappingResult> createMappingResponse) {
 
                     @Override
                     public void onFailure(Exception e) {
+                        log.debug("Failed to create mappings for {}", indexName );
                         actionListener.onFailure(e);
                     }
                 });

diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java
@@ -5,19 +5,16 @@
 
 package org.opensearch.securityanalytics.mapper;
 
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
 import org.apache.commons.lang3.tuple.Pair;
 import org.opensearch.cluster.metadata.MappingMetadata;
-import org.opensearch.common.collect.ImmutableOpenMap;
-
+import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
 import java.util.List;
 import java.util.Set;
-import java.util.stream.Collectors;
-import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 
 public class MapperUtils {
 
@@ -249,7 +246,6 @@ public void onError(String error) {
             }
         });
         mappingsTraverser.traverse();
-
         return presentPathsMappings;
     }
 }
diff --git a/src/main/java/org/opensearch/securityanalytics/model/Detector.java b/src/main/java/org/opensearch/securityanalytics/model/Detector.java
@@ -158,12 +158,12 @@ public Detector(StreamInput sin) throws IOException {
                 sin.readList(DetectorInput::readFrom),
                 sin.readList(DetectorTrigger::readFrom),
                 sin.readStringList(),
-                sin.readString(),
-                sin.readString(),
-                sin.readString(),
-                sin.readString(),
-                sin.readString(),
-                sin.readString(),
+                sin.readOptionalString(),
+                sin.readOptionalString(),
+                sin.readOptionalString(),
+                sin.readOptionalString(),
+                sin.readOptionalString(),
+                sin.readOptionalString(),
                 sin.readMap(StreamInput::readString, StreamInput::readString)
             );
     }
@@ -196,8 +196,12 @@ public void writeTo(StreamOutput out) throws IOException {
             it.writeTo(out);
         }
         out.writeStringCollection(monitorIds);
-        out.writeString(ruleIndex);
-
+        out.writeOptionalString(ruleIndex);
+        out.writeOptionalString(alertsIndex);
+        out.writeOptionalString(alertsHistoryIndex);
+        out.writeOptionalString(alertsHistoryIndexPattern);
+        out.writeOptionalString(findingsIndex);
+        out.writeOptionalString(findingsIndexPattern);
         out.writeMap(ruleIdMonitorIdMap, StreamOutput::writeString, StreamOutput::writeString);
     }
 

diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java
@@ -131,7 +131,7 @@ public OSQueryBackend(String ruleCategory, boolean collectErrors, boolean enable
         this.reEscapeChar = "\\";
         this.reExpression = "%s: /%s/";
         this.cidrExpression = "%s: \"%s\"";
-        this.fieldNullExpression = "%s: null";
+        this.fieldNullExpression = "%s: (NOT [* TO *])";
         this.unboundValueStrExpression = "\"%s\"";
         this.unboundValueNumExpression = "\"%s\"";
         this.unboundWildcardExpression = "%s";
@@ -331,9 +331,12 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre
 
     @Override
     public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError {
+        String field = getFinalValueField();
+        ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer"));
         SigmaString value = (SigmaString) condition.getValue();
         boolean containsWildcard = value.containsWildcard();
-        return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue()));
+        return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression),
+                this.convertValueStr((SigmaString) condition.getValue()));
     }
 
     @Override