Watcher Threshold Alert add Filter and Search

[Kuermel]

With Threshold alert, the interface should provide options for filtering and searching.

[elasticmachine]

Pinging @elastic/es-ui

[FrankyBoy]

So much +1. Just had to manually write JSON for an "advanced watcher" just because you can't even add the simplest of filters in the threshold alert.

[cpmoore]

I would really love to see this feature implemented.
Most of the watches i need to create are simple, alert me when a message from server x matches y type alerts. It’s really inconvenient to have to write the full json just for that.

[elasticmachine]

Pinging @elastic/kibana-alerting-services (Team:Alerting Services)

[shahargl]

@FrankyBoy @cpmoore

Can you please attach a JSON example for a watch with a filter?
Just need it for my alerts but couldn't find a way to do it.

[FrankyBoy]

Sorry switched companies since then and not working with Kibana any more.

[cpmoore]

@shahargl
Sure, here's an advanced example. In this case I'm actually formatting the alerts and sending it to prometheus alertmanager to handle the notifications
The filters are defined in the filters aggregation, then i can set the min and max thresholds in the overrides in the metadata field
Each alert gets grouped multiple fields for aggreagtion and includes the latest 5 hits

{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "chain": {
      "inputs": [
        {
          "query": {
            "search": {
              "request": {
                "search_type": "query_then_fetch",
                "indices": [
                  "filebeat-*"
                ],
                "rest_total_hits_as_int": true,
                "body": {
                  "size": 0,
                  "track_total_hits": false,
                  "query": {
                    "range": {
                      "@timestamp": {
                        "gte": "now/m-5m",
                        "lte": "now/m"
                      }
                    }
                  },
                  "aggs": {
                    "filters": {
                      "filters": {
                        "filters": {
                          "Alert One":{
                            "query_string": {
                              "query": "message: ERROR"
                            }
                          },
                          "Alert Two": {
                            "query_string": {
                              "query": "message: WARN"
                            }
                          }
                        }
                      },
                      "aggs": {
                        "environments": {
                          "terms": {
                            "field": "service.environment",
                            "size": 1000
                          },
                          "aggs": {
                            "groups": {
                              "terms": {
                                "field": "service.group",
                                "size": 1000
                              },
                              "aggs": {
                                "hosts": {
                                  "terms": {
                                    "field": "host.name",
                                    "size": 1000
                                  },
                                  "aggs": {
                                    "no_service": {
                                      "filter": {
                                        "bool": {
                                          "must_not": {
                                            "exists": {
                                              "field": "service.name"
                                            }
                                          }
                                        }
                                      },
                                      "aggs": {
                                        "hits": {
                                          "top_hits": {
                                            "sort": {
                                              "@timestamp": "desc"
                                            },
                                            "_source": {
                                              "includes": [
                                                "message"
                                              ]
                                            },
                                            "size": 5
                                          }
                                        }
                                      }
                                    },
                                    "has_service": {
                                      "filter": {
                                        "exists": {
                                          "field": "service.name"
                                        }
                                      },
                                      "aggs": {
                                        "services": {
                                          "terms": {
                                            "field": "service.name",
                                            "size": 1000
                                          },
                                          "aggs": {
                                            "hits": {
                                              "top_hits": {
                                                "sort": {
                                                  "@timestamp": "desc"
                                                },
                                                "_source": {
                                                  "includes": [
                                                    "message"
                                                  ]
                                                },
                                                "size": 5
                                              }
                                            }
                                          }
                                        }
                                      }
                                    }
                                  }
                                }
                              }
                            }
                          }
                        }
                      }
                    }
                  }
                }
              }
            }
          }
        },
        {
          "alerts": {
            "transform": {
              "script": {
                "source": """
                  def addToList(def startsAt,def endsAt,def generatorURL,def list,def filter,def config,def env,def group,def host,def service){
                    try{
                     def count=service.doc_count;
                     def hasMax=config.containsKey('max');
                     def hasMin=config.containsKey('min');
                     if(hasMax && hasMin){
                        if(count<config['min'] || count>config['max']){
                          return 0;
                        } 
                      }else if(hasMax){
                        if(count>config['max']){
                          return 0;
                        }
                      }else if(hasMin){
                        if(count<config['min']){
                          return 0;
                        }
                     }
                     def annotations=['summary':''];
                     try{
                       
                        def i=0;
                        for(def hit:service.hits.hits.hits){
                          annotations['hit'+i]=hit._source.message.toString();
                          i++;
                        }
                       
                     }catch(Exception e2){
                       annotations['error']=e2.getMessage();
                     }
                     
                     def labels=[
                                'alertname':filter,
                                'host': host.key,
                                'group':group.key,
                                'environment':env.key
                            ];
                      def s=count.toString();
                      if(service.containsKey('key')){
                        labels['app']=service.key;
                        annotations['description']='There have been '+s+' hits for '+filter+' for service '+service.key+' on the '+env.key+' '+group.key+' server '+host.key;
                        annotations['summary']=env.key.toUpperCase()+' - '+group.key+' - '+service.key+' - '+host.key+' - '+filter+' - '+s+' hits';
                      }else{
                        annotations['description']='There have been '+s+' hits for '+filter+' on the '+env.key+' '+group.key+' server '+host.key;
                        annotations['summary']=env.key.toUpperCase()+' - '+group.key+' - '+host.key+' - '+filter+' - '+s+' hits';
                      }
                      if(config.containsKey('labels')){
                        for(def label:config['labels'].keySet()){
                           labels[label]=config.labels[label]
                        }
                      }
                      list.add([
                              'startsAt':startsAt,
                              'endsAt':endsAt,
                              'generatorURL':generatorURL,
                              'annotations':annotations,
                              'labels':labels
                            ]);
                     return 0;
                    }catch(Exception e){
                      return 1;
                    }
                  }
                  
                  
                  
                  def list=[];  
                  def endsAt = ctx.trigger.triggered_time.plusSeconds(75);
                  def generatorURL='https://kibana.elk.omf.cloud/app/kibana#/management/elasticsearch/watcher/watches/watch/'+ctx.watch_id+'/history-item/'+ctx.id;
                  def startsAt=ctx.trigger.triggered_time;
                  for(def filter:ctx.payload.query.aggregations.filters.buckets.keySet()){
                      def config=['min':1];
                      if(ctx.metadata.overrides.containsKey(filter)){
                         config=ctx.metadata.overrides[filter]   
                      }
                      def value=ctx.payload.query.aggregations.filters.buckets[filter];
                      for(def env:value.environments.buckets){
                       for(def group:env.groups.buckets){
                        for(def host:group.hosts.buckets){
                            for(def service:host.has_service.services.buckets){
                              addToList(startsAt,endsAt,generatorURL,list,filter,config,env,group,host,service);
                            }
                            addToList(startsAt,endsAt,generatorURL,list,filter,config,env,group,host,host.no_service);
                        }
                       }
                      }    
                  }
                  return ['list':list,'size':list.size()];
                  
                  """,
                "lang": "painless"
              }
            }
          }
        }
      ]
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.alerts.size": {
        "gt": 0
      }
    }
  },
  "actions": {
    "send_to_alertmanager": {
      "throttle_period_in_millis": 0,
      "webhook": {
        "scheme": "https",
        "host": "REDACTED",
        "port": 443,
        "method": "post",
        "path": "/alertmanager/api/v1/alerts",
        "params": {},
        "headers": {
          "Content-Type": "application/json"
        },
        "auth": {
          "basic": {
            "username": "REDACTED",
            "password": "REDACTED"
          }
        },
        "body": "{{#toJson}}ctx.payload.alerts.list{{/toJson}}"
      }
    }
  },
  "metadata": {
    "overrides": {
      "Alert Two": {
        "min": 15
      }
    }
  }
}

[shahargl]

Thanks mate @cpmoore