Merge pull request #1293 from eht16/logging_config

Read logging configuration from config file

config.yaml.example

@@ -62,3 +62,53 @@ writeback_index: elastalert_status
 # sending the alert until this time period has elapsed
 alert_time_limit:
   days: 2
+
+# Custom logging configuration
+# If you want to setup your own logging configuration to log into
+# files as well or to Logstash and/or modify log levels, use
+# the configuration below and adjust to your needs.
+# Note: if you run ElastAlert with --verbose/--debug, the log level of
+# the "elastalert" logger is changed to INFO, if not already INFO/DEBUG.
+#logging:
+#  version: 1
+#  incremental: false
+#  disable_existing_loggers: false
+#  formatters:
+#    logline:
+#      format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s'
+#
+#    handlers:
+#      console:
+#        class: logging.StreamHandler
+#        formatter: logline
+#        level: DEBUG
+#        stream: ext://sys.stderr
+#
+#      file:
+#        class : logging.FileHandler
+#        formatter: logline
+#        level: DEBUG
+#        filename: elastalert.log
+#
+#    loggers:
+#      elastalert:
+#        level: WARN
+#        handlers: []
+#        propagate: true
+#
+#      elasticsearch:
+#        level: WARN
+#        handlers: []
+#        propagate: true
+#
+#      elasticsearch.trace:
+#        level: WARN
+#        handlers: []
+#        propagate: true
+#
+#      '':  # root logger
+#        level: WARN
+#          handlers:
+#            - console
+#            - file
+#        propagate: false

docs/source/elastalert.rst

@@ -194,6 +194,20 @@ The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticse
 
 ``skip_invalid``: If ``True``, skip invalid files instead of exiting.
 
+=======
+Logging
+-------
+
+By default, ElastAlert uses a simple basic logging configuration to print log messages to standard error.
+You can change the log level to ``INFO`` messages by using the ``--verbose`` or ``--debug`` command line options.
+
+If you need a more sophisticated logging configuration, you can provide a full logging configuration
+in the config file. This way you can also configure logging to a file, to Logstash and
+adjust the logging format.
+
+For details, see the end of ``config.yaml.example`` where you can find an example logging
+configuration.
+
 
 .. _runningelastalert:
 

elastalert/config.py

@@ -3,6 +3,7 @@
 import datetime
 import hashlib
 import logging
+import logging.config
 import os
 import sys
 
@@ -19,6 +20,7 @@
 from util import dt_to_ts_with_format
 from util import dt_to_unix
 from util import dt_to_unixms
+from util import elastalert_logger
 from util import EAException
 from util import ts_to_dt
 from util import ts_to_dt_with_format
@@ -452,6 +454,9 @@ def load_rules(args):
     conf = yaml_loader(filename)
     use_rule = args.rule
 
+    # init logging from config and set log levels according to command line options
+    configure_logging(args, conf)
+
     for env_var, conf_var in env_settings.items():
         val = env(env_var, None)
         if val is not None:
@@ -509,6 +514,38 @@ def load_rules(args):
     return conf
 
 
+def configure_logging(args, conf):
+    # configure logging from config file if provided
+    if 'logging' in conf:
+        # load new logging config
+        logging.config.dictConfig(conf['logging'])
+
+    if args.verbose and args.debug:
+        elastalert_logger.info(
+            "Note: --debug and --verbose flags are set. --debug takes precedent."
+        )
+
+    # re-enable INFO log level on elastalert_logger in verbose/debug mode
+    # (but don't touch it if it is already set to INFO or below by config)
+    if args.verbose or args.debug:
+        if elastalert_logger.level > logging.INFO or elastalert_logger.level == logging.NOTSET:
+            elastalert_logger.setLevel(logging.INFO)
+
+    if args.debug:
+        elastalert_logger.info(
+            """Note: In debug mode, alerts will be logged to console but NOT actually sent.
+            To send them but remain verbose, use --verbose instead."""
+        )
+
+    if not args.es_debug and 'logging' not in conf:
+        logging.getLogger('elasticsearch').setLevel(logging.WARNING)
+
+    if args.es_debug_trace:
+        tracer = logging.getLogger('elasticsearch.trace')
+        tracer.setLevel(logging.INFO)
+        tracer.addHandler(logging.FileHandler(args.es_debug_trace))
+
+
 def get_rule_hashes(conf, use_rule=None):
     rule_files = get_file_paths(conf, use_rule)
     rule_mod_times = {}

elastalert/elastalert.py

@@ -105,28 +105,6 @@ def __init__(self, args):
         self.debug = self.args.debug
         self.verbose = self.args.verbose
 
-        if self.verbose and self.debug:
-            elastalert_logger.info(
-                "Note: --debug and --verbose flags are set. --debug takes precedent."
-            )
-
-        if self.verbose or self.debug:
-            elastalert_logger.setLevel(logging.INFO)
-
-        if self.debug:
-            elastalert_logger.info(
-                """Note: In debug mode, alerts will be logged to console but NOT actually sent.
-                To send them but remain verbose, use --verbose instead."""
-            )
-
-        if not self.args.es_debug:
-            logging.getLogger('elasticsearch').setLevel(logging.WARNING)
-
-        if self.args.es_debug_trace:
-            tracer = logging.getLogger('elasticsearch.trace')
-            tracer.setLevel(logging.INFO)
-            tracer.addHandler(logging.FileHandler(self.args.es_debug_trace))
-
         self.conf = load_rules(self.args)
         self.max_query_size = self.conf['max_query_size']
         self.scroll_keepalive = self.conf['scroll_keepalive']

tests/config_test.py

@@ -45,6 +45,7 @@
 test_args.config = 'test_config'
 test_args.rule = None
 test_args.debug = False
+test_args.es_debug_trace = None
 
 
 def test_import_rules():