Add ECS log format (#1529)

curator/cli_singletons/utils.py

@@ -155,7 +155,7 @@ def config_override(ctx, config_dict):
         elif k == 'host':
             if 'host' in ctx.params and ctx.params['host'] is not None:
                 config_dict['client']['hosts'] = ctx.params[k]
-        elif k in ['loglevel', 'logfile', 'logformat']:
+        elif k in ['loglevel', 'logfile', 'logformat', 'ecs']:
             if k in ctx.params and ctx.params[k] is not None:
                 config_dict['logging'][k] = ctx.params[k]
         else:

curator/defaults/client_defaults.py

@@ -38,6 +38,6 @@ def config_logging():
                 ),
         Optional('logfile', default=None): Any(None, *string_types),
         Optional('logformat', default='default'):
-            Any(None, All(Any(*string_types), Any('default', 'json', 'logstash'))),
+            Any(None, All(Any(*string_types), Any('default', 'json', 'logstash', 'ecs'))),
         Optional('blacklist', default=['elasticsearch', 'urllib3']): Any(None, list),
     }

curator/logtools.py

@@ -27,6 +27,15 @@ def format(self, record):
             result[self.WANTED_ATTRS[attribute]] = getattr(record, attribute)
         return json.dumps(result, sort_keys=True)
 
+class ECSFormatter(LogstashFormatter):
+    """Elastic Common Schema formatting (ECS)"""
+    # Overload LogstashFormatter attribute
+    WANTED_ATTRS = {'levelname': 'log.level',
+                    'funcName': 'log.origin.function',
+                    'lineno': 'log.origin.file.line',
+                    'message': 'message',
+                    'name': 'log.logger'}
+
 class Whitelist(logging.Filter):
     """How to whitelist logs"""
     def __init__(self, *whitelist):
@@ -63,5 +72,7 @@ def __init__(self, cfg):
 
         if cfg['logformat'] == 'json' or cfg['logformat'] == 'logstash':
             self.handler.setFormatter(LogstashFormatter())
+        elif cfg['logformat'] == 'ecs':
+            self.handler.setFormatter(ECSFormatter())
         else:
             self.handler.setFormatter(logging.Formatter(self.format_string))

curator/singletons.py

@@ -56,7 +56,7 @@
 @click.option('--dry-run', is_flag=True, help='Do not perform any changes.')
 @click.option('--loglevel', help='Log level')
 @click.option('--logfile', help='log file')
-@click.option('--logformat', help='Log output format [default|logstash|json].')
+@click.option('--logformat', help='Log output format [default|logstash|json|ecs].')
 @click.version_option(version=__version__)
 @click.pass_context
 def cli(

docs/asciidoc/command-line.asciidoc

@@ -95,7 +95,7 @@ Options:
   --dry-run           Do not perform any changes.
   --loglevel TEXT     Log level
   --logfile TEXT      log file
-  --logformat TEXT    Log output format [default|logstash|json].
+  --logformat TEXT    Log output format [default|logstash|json|ecs].
   --version           Show the version and exit.
   --help              Show this message and exit.
 

docs/asciidoc/configuration.asciidoc

@@ -587,7 +587,7 @@ console.
 [[logformat]]
 === logformat
 
-This should `default`, `json`, `logstash`, or left empty.
+This should `default`, `json`, `logstash`, `ecs` or left empty.
 
 [source,sh]
 -----------
@@ -609,6 +609,13 @@ The `json` or `logstash` formats look like:
 "loglevel": "INFO", "message": "Action #1: ACTIONNAME", "name": "curator.cli"}
 -----------
 
+The `ecs` format looks like:
+[source,sh]
+-----------
+{"@timestamp": "2020-02-22T11:55:00.022Z", "log.origin.function": "cli", "log.origin.file.line": 178,
+"log.level": "INFO", "message": "Action #1: ACTIONNAME", "log.logger": "curator.cli"}
+-----------
+
 The default value is `default`.
 
 [[blacklist]]