Updating of bucket level monitors supported

Added integration tests for checking update of bucket level monitors

Signed-off-by: Stevan Buzejic <stevan.buzejic@htecgroup.com>

src/main/java/org/opensearch/securityanalytics/model/Detector.java

@@ -4,6 +4,8 @@
  */
 package org.opensearch.securityanalytics.model;
 
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.common.ParseField;
@@ -49,6 +51,8 @@ public class Detector implements Writeable, ToXContentObject {
     public static final String LAST_UPDATE_TIME_FIELD = "last_update_time";
     public static final String ENABLED_TIME_FIELD = "enabled_time";
     public static final String ALERTING_MONITOR_ID = "monitor_id";
+
+    public static final String BUCKET_MONITOR_ID_RULE_ID = "bucket_monitor_id_rule_id";
     private static final String RULE_TOPIC_INDEX = "rule_topic_index";
 
     private static final String ALERTS_INDEX = "alert_index";
@@ -59,6 +63,9 @@ public class Detector implements Writeable, ToXContentObject {
 
     public static final String DETECTORS_INDEX = ".opensearch-detectors-config";
 
+    // Used as a key in rule-monitor map for the purpose of easy detection of the doc level monitor
+    public static final String DOC_LEVEL_MONITOR = "-1";
+
     public static final NamedXContentRegistry.Entry XCONTENT_REGISTRY = new NamedXContentRegistry.Entry(
             Detector.class,
             new ParseField(DETECTOR_TYPE),
@@ -90,6 +97,8 @@ public class Detector implements Writeable, ToXContentObject {
 
     private List<String> monitorIds;
 
+    private Map<String, String> ruleIdMonitorId;
+
     private String ruleIndex;
 
     private String alertsIndex;
@@ -108,7 +117,7 @@ public Detector(String id, Long version, String name, Boolean enabled, Schedule
                     Instant lastUpdateTime, Instant enabledTime, DetectorType detectorType,
                     User user, List<DetectorInput> inputs, List<DetectorTrigger> triggers, List<String> monitorIds,
                     String ruleIndex, String alertsIndex, String alertsHistoryIndex, String alertsHistoryIndexPattern,
-                    String findingsIndex, String findingsIndexPattern) {
+                    String findingsIndex, String findingsIndexPattern, Map<String, String> rulePerMonitor) {
         this.type = DETECTOR_TYPE;
 
         this.id = id != null ? id : NO_ID;
@@ -129,6 +138,7 @@ public Detector(String id, Long version, String name, Boolean enabled, Schedule
         this.alertsHistoryIndexPattern = alertsHistoryIndexPattern;
         this.findingsIndex = findingsIndex;
         this.findingsIndexPattern = findingsIndexPattern;
+        this.ruleIdMonitorId = rulePerMonitor;
 
         if (enabled) {
             Objects.requireNonNull(enabledTime);
@@ -154,7 +164,9 @@ public Detector(StreamInput sin) throws IOException {
                 sin.readString(),
                 sin.readString(),
                 sin.readString(),
-                sin.readString());
+                sin.readString(),
+                sin.readMap(StreamInput::readString, StreamInput::readString)
+            );
     }
 
     @Override
@@ -186,6 +198,8 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeStringCollection(monitorIds);
         out.writeString(ruleIndex);
+
+        out.writeMap(ruleIdMonitorId, StreamOutput::writeString, StreamOutput::writeString);
     }
 
     public XContentBuilder toXContentWithUser(XContentBuilder builder, Params params) throws IOException {
@@ -268,6 +282,7 @@ private XContentBuilder createXContentBuilder(XContentBuilder builder, ToXConten
         }
 
         builder.field(ALERTING_MONITOR_ID, monitorIds);
+        builder.field(BUCKET_MONITOR_ID_RULE_ID, ruleIdMonitorId);
         builder.field(RULE_TOPIC_INDEX, ruleIndex);
         builder.field(ALERTS_INDEX, alertsIndex);
         builder.field(ALERTS_HISTORY_INDEX, alertsHistoryIndex);
@@ -312,6 +327,8 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws
         List<DetectorInput> inputs = new ArrayList<>();
         List<DetectorTrigger> triggers = new ArrayList<>();
         List<String> monitorIds = new ArrayList<>();
+        Map<String, String> rulePerMonitor = new HashMap<>();
+
         String ruleIndex = null;
         String alertsIndex = null;
         String alertsHistoryIndex = null;
@@ -390,6 +407,9 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws
                         monitorIds.add(monitorId);
                     }
                     break;
+                case BUCKET_MONITOR_ID_RULE_ID:
+                    rulePerMonitor= xcp.mapStrings();
+                    break;
                 case RULE_TOPIC_INDEX:
                     ruleIndex = xcp.text();
                     break;
@@ -437,7 +457,8 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws
                 alertsHistoryIndex,
                 alertsHistoryIndexPattern,
                 findingsIndex,
-                findingsIndexPattern);
+                findingsIndexPattern,
+                rulePerMonitor);
     }
 
     public static Detector readFrom(StreamInput sin) throws IOException {
@@ -516,6 +537,8 @@ public List<String> getMonitorIds() {
         return monitorIds;
     }
 
+    public Map<String, String> getRuleIdMonitorId() {return ruleIdMonitorId; }
+
     public void setId(String id) {
         this.id = id;
     }
@@ -563,6 +586,13 @@ public void setInputs(List<DetectorInput> inputs) {
     public void setMonitorIds(List<String> monitorIds) {
         this.monitorIds = monitorIds;
     }
+    public void setRuleIdMonitorId(Map<String, String> ruleIdMonitorId) {
+        this.ruleIdMonitorId = ruleIdMonitorId;
+    }
+
+    public String getDocLevelMonitorId() {
+        return ruleIdMonitorId.get(DOC_LEVEL_MONITOR);
+    }
 
     @Override
     public boolean equals(Object o) {

src/main/java/org/opensearch/securityanalytics/model/Rule.java

@@ -452,7 +452,6 @@ public boolean isAggregationRule() {
         return aggregationQueries != null && !aggregationQueries.isEmpty();
     }
 
-    // TODO - temp method; Replace once you have some more inputs from Shubo and Surya
     public List<AggregationItem> getAggregationItemsFromRule () throws SigmaError {
         SigmaRule sigmaRule = SigmaRule.fromYaml(rule, true);
         List<AggregationItem> aggregationItems = new ArrayList<>();

src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java

@@ -1,76 +1,29 @@
 package org.opensearch.securityanalytics.rules.backend;
 
 import org.opensearch.search.aggregations.AggregationBuilder;
-import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.histogram.VariableWidthHistogramAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.range.DateRangeAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.range.GeoDistanceAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.range.IpRangeAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.terms.RareTermsAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.terms.SignificantTermsAggregationBuilder;
 import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder;
-import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder;
-import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder;
-import org.opensearch.search.aggregations.metrics.GeoCentroidAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.MedianAbsoluteDeviationAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.MinAggregationBuilder;
-import org.opensearch.search.aggregations.metrics.PercentileRanksAggregationBuilder;
-import org.opensearch.search.aggregations.metrics.PercentilesAggregationBuilder;
-import org.opensearch.search.aggregations.metrics.StatsAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.SumAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.ValueCountAggregationBuilder;
+
 public final class AggregationBuilders {
 
     /**
      * Finds the builder aggregation based on the forwarded function
      *
-     * @param aggregationFunction - aggregation function
-     * @param name - name of the aggregation
-     * @return
+     * @param aggregationFunction Aggregation function
+     * @param name                Name of the aggregation
+     * @return Aggregation builder
      */
-    public static AggregationBuilder getAggregationBuilderByFunction(String aggregationFunction, String name){
+    public static AggregationBuilder getAggregationBuilderByFunction(String aggregationFunction, String name) {
         AggregationBuilder aggregationBuilder;
-        switch (aggregationFunction){
-            case AutoDateHistogramAggregationBuilder.NAME:
-                aggregationBuilder = new AutoDateHistogramAggregationBuilder(name).field(name);
-                break;
+        switch (aggregationFunction.toLowerCase()) {
             case AvgAggregationBuilder.NAME:
                 aggregationBuilder = new AvgAggregationBuilder(name).field(name);
                 break;
-            case CardinalityAggregationBuilder.NAME:
-                aggregationBuilder = new CardinalityAggregationBuilder(name).field(name);
-                break;
-            case DateHistogramAggregationBuilder.NAME:
-                aggregationBuilder = new DateHistogramAggregationBuilder(name).field(name);
-                break;
-            case DateRangeAggregationBuilder.NAME:
-                aggregationBuilder = new DateRangeAggregationBuilder(name).field(name);
-                break;
-            case DiversifiedAggregationBuilder.NAME:
-                aggregationBuilder = new DiversifiedAggregationBuilder(name).field(name);
-                break;
-            case ExtendedStatsAggregationBuilder.NAME:
-                aggregationBuilder = new ExtendedStatsAggregationBuilder(name).field(name);
-                break;
-            case GeoCentroidAggregationBuilder.NAME:
-                aggregationBuilder = new GeoCentroidAggregationBuilder(name).field(name);
-                break;
-                // TODO ?
-            case GeoDistanceAggregationBuilder.NAME:
-                aggregationBuilder = new GeoDistanceAggregationBuilder(name, null).field(name);
-                break;
-            case HistogramAggregationBuilder.NAME:
-                aggregationBuilder = new HistogramAggregationBuilder(name).field(name);
-                break;
-            case IpRangeAggregationBuilder.NAME:
-                aggregationBuilder = new IpRangeAggregationBuilder(name).field(name);
-                break;
             case MaxAggregationBuilder.NAME:
                 aggregationBuilder = new MaxAggregationBuilder(name).field(name);
                 break;
@@ -80,38 +33,17 @@ public static AggregationBuilder getAggregationBuilderByFunction(String aggregat
             case MinAggregationBuilder.NAME:
                 aggregationBuilder = new MinAggregationBuilder(name).field(name);
                 break;
-                // TODO - do we need this?
-            case PercentileRanksAggregationBuilder.NAME:
-                aggregationBuilder = new PercentileRanksAggregationBuilder(name, null).field(name);
-                break;
-            case PercentilesAggregationBuilder.NAME:
-                aggregationBuilder = new PercentilesAggregationBuilder(name).field(name);
-                break;
-            case RangeAggregationBuilder.NAME:
-                aggregationBuilder = new RangeAggregationBuilder(name).field(name);
-                break;
-            case RareTermsAggregationBuilder.NAME:
-                aggregationBuilder = new RareTermsAggregationBuilder(name).field(name);
-                break;
-            case SignificantTermsAggregationBuilder.NAME:
-                aggregationBuilder = new SignificantTermsAggregationBuilder(name).field(name);
-                break;
-            case StatsAggregationBuilder.NAME:
-                aggregationBuilder = new StatsAggregationBuilder(name).field(name);
-                break;
             case SumAggregationBuilder.NAME:
                 aggregationBuilder = new SumAggregationBuilder(name).field(name);
                 break;
             case TermsAggregationBuilder.NAME:
                 aggregationBuilder = new TermsAggregationBuilder(name).field(name);
                 break;
-            case ValueCountAggregationBuilder.NAME:
+            case "count":
                 aggregationBuilder = new ValueCountAggregationBuilder(name).field(name);
                 break;
-            case VariableWidthHistogramAggregationBuilder.NAME:
-                aggregationBuilder = new VariableWidthHistogramAggregationBuilder(name).field(name);
-                break;
-            default: return null;
+            default:
+                return null;
         }
         return aggregationBuilder;
     }

src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java

@@ -389,8 +389,11 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) {
             fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", aggregation.getGroupByField(), aggregation.getAggField(), aggregation.getAggFunction(), aggregation.getAggField());
             fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, aggregation.getAggField(), aggregation.getAggField(), "result_agg", aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold());
 
+            // Add subaggregation
             AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), aggregation.getAggField());
-            aggBuilder.field(aggregation.getGroupByField()).subAggregation(subAgg);
+            if (subAgg != null) {
+                aggBuilder.field(aggregation.getGroupByField()).subAggregation(subAgg);
+            }
 
             Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold()));
             condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(aggregation.getAggField(), aggregation.getAggField()), script, "result_agg", null);