Merge pull request Yelp#75 from nicholasgibson2/add_prometheus_metric…

…s_jertel_fork

added optional Prometheus metrics endpoint

README-old.md

@@ -115,6 +115,8 @@ Eg: ``--rule this_rule.yaml``
 
 ``--config`` allows you to specify the location of the configuration. By default, it is will look for config.yaml in the current directory.
 
+``--prometheus_port`` exposes ElastAlert Prometheus metrics on the specified port. Prometheus metrics disabled by default.
+
 ## Third Party Tools And Extras
 ### Kibana plugin
 ![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/showcase.gif)

elastalert/elastalert.py

@@ -25,6 +25,7 @@
 from elasticsearch.exceptions import ElasticsearchException
 from elasticsearch.exceptions import NotFoundError
 from elasticsearch.exceptions import TransportError
+from .prometheus_wrapper import PrometheusWrapper
 
 from . import kibana
 from .alerts import DebugAlerter
@@ -111,6 +112,7 @@ def parse_args(self, args):
             dest='es_debug_trace',
             help='Enable logging from Elasticsearch queries as curl command. Queries will be logged to file. Note that '
                  'this will incorrectly display localhost:9200 as the host/port')
+        parser.add_argument('--prometheus_port', type=int, dest='prometheus_port', help='Enables Prometheus metrics on specified port.')
         self.args = parser.parse_args(args)
 
     def __init__(self, args):
@@ -171,6 +173,7 @@ def __init__(self, args):
         self.scheduler = BackgroundScheduler()
         self.string_multi_field_name = self.conf.get('string_multi_field_name', False)
         self.add_metadata_alert = self.conf.get('add_metadata_alert', False)
+        self.prometheus_port = self.args.prometheus_port
         self.show_disabled_rules = self.conf.get('show_disabled_rules', True)
 
         self.writeback_es = elasticsearch_client(self.conf)
@@ -2076,6 +2079,11 @@ def main(args=None):
     if not args:
         args = sys.argv[1:]
     client = ElastAlerter(args)
+
+    if client.prometheus_port and not client.debug:
+        p = PrometheusWrapper(client)
+        p.start()
+
     if not client.args.silence:
         client.start()
 

elastalert/prometheus_wrapper.py

@@ -0,0 +1,55 @@
+import prometheus_client
+
+
+class PrometheusWrapper:
+    """ Exposes ElastAlert metrics on a Prometheus metrics endpoint.
+        Wraps ElastAlerter run_rule and writeback to collect metrics. """
+
+    def __init__(self, client):
+        self.prometheus_port = client.prometheus_port
+        self.run_rule = client.run_rule
+        self.writeback = client.writeback
+
+        client.run_rule = self.metrics_run_rule
+        client.writeback = self.metrics_writeback
+
+        # initialize prometheus metrics to be exposed
+        self.prom_scrapes = prometheus_client.Counter('elastalert_scrapes', 'Number of scrapes for rule', ['rule_name'])
+        self.prom_hits = prometheus_client.Counter('elastalert_hits', 'Number of hits for rule', ['rule_name'])
+        self.prom_matches = prometheus_client.Counter('elastalert_matches', 'Number of matches for rule', ['rule_name'])
+        self.prom_time_taken = prometheus_client.Counter('elastalert_time_taken', 'Time taken to evaluate rule', ['rule_name'])
+        self.prom_alerts_sent = prometheus_client.Counter('elastalert_alerts_sent', 'Number of alerts sent for rule', ['rule_name'])
+        self.prom_alerts_not_sent = prometheus_client.Counter('elastalert_alerts_not_sent', 'Number of alerts not sent', ['rule_name'])
+        self.prom_errors = prometheus_client.Counter('elastalert_errors', 'Number of errors for rule')
+        self.prom_alerts_silenced = prometheus_client.Counter('elastalert_alerts_silenced', 'Number of silenced alerts', ['rule_name'])
+
+    def start(self):
+        prometheus_client.start_http_server(self.prometheus_port)
+
+    def metrics_run_rule(self, rule, endtime, starttime=None):
+        """ Increment counter every time rule is run """
+        try:
+            self.prom_scrapes.labels(rule['name']).inc()
+        finally:
+            return self.run_rule(rule, endtime, starttime)
+
+    def metrics_writeback(self, doc_type, body):
+        """ Update various prometheus metrics accoording to the doc_type """
+
+        res = self.writeback(doc_type, body)
+        try:
+            if doc_type == 'elastalert_status':
+                self.prom_hits.labels(body['rule_name']).inc(int(body['hits']))
+                self.prom_matches.labels(body['rule_name']).inc(int(body['matches']))
+                self.prom_time_taken.labels(body['rule_name']).inc(float(body['time_taken']))
+            elif doc_type == 'elastalert':
+                if body['alert_sent']:
+                    self.prom_alerts_sent.labels(body['rule_name']).inc()
+                else:
+                    self.prom_alerts_not_sent.labels(body['rule_name']).inc()
+            elif doc_type == 'elastalert_error':
+                self.prom_errors.inc()
+            elif doc_type == 'silence':
+                self.prom_alerts_silenced.labels(body['rule_name']).inc()
+        finally:
+            return res

requirements.txt

@@ -13,6 +13,7 @@ jira>=2.0.0
 jsonschema>=3.0.2
 mock>=2.0.0
 prison>=0.1.2
+prometheus_client>=0.10.1
 py-zabbix>=1.1.3
 PyStaticConfiguration>=0.10.3
 python-dateutil>=2.6.0,<2.7.0

setup.py

@@ -41,6 +41,7 @@
         'jsonschema>=3.0.2',
         'mock>=2.0.0',
         'prison>=0.1.2',
+        'prometheus_client>=0.10.1',
         'py-zabbix>=1.1.3',
         'PyStaticConfiguration>=0.10.3',
         'python-dateutil>=2.6.0,<2.7.0',