Allow for custom log formats to be defined in stacks.yaml (#705)

Allow for custom log formats to be defined in stacker.yaml #705 
Docs and config model adjustment.

	modified:   .gitignore
	modified:   docs/config.rst
	modified:   stacker/commands/stacker/__init__.py
	modified:   stacker/commands/stacker/base.py
	modified:   stacker/config/__init__.py
	modified:   stacker/logger/__init__.py
	new file:   stacker/tests/fixtures/not-basic.env
	new file:   stacker/tests/fixtures/vpc-custom-log-format-info.yaml
	modified:   stacker/tests/test_stacker.py

.gitignore

@@ -68,3 +68,5 @@ dev.yaml
 dev.env
 
 tests/fixtures/blueprints/*-result
+
+FakeKey.pem

docs/config.rst

@@ -469,6 +469,28 @@ Here's an example of a target that will execute all "database" stacks::
       required_by:
         - databases
 
+Custom Log Formats
+------------------
+
+By default, stacker uses the following `log_formats`::
+
+  log_formats:
+    info: "[%(asctime)s] %(message)s"
+    color: "[%(asctime)s] \033[%(color)sm%(message)s\033[39m"
+    debug: "[%(asctime)s] %(levelname)s %(threadName)s %(name)s:%(lineno)d(%(funcName)s): %(message)s"
+
+You may optionally provide custom `log_formats`.
+
+You may use any of the standard Python
+[logging module format attributes](https://docs.python.org/2.7/library/logging.html#logrecord-attributes)
+when building your `log_formats`.
+
+In this example, we add the environment name to each log line::
+
+  log_formats:
+    info: "[%(asctime)s] ${environment} %(message)s"
+    color: "[%(asctime)s] ${environment} \033[%(color)sm%(message)s\033[39m"
+
 Variables
 ==========
 

stacker/commands/stacker/__init__.py

@@ -24,35 +24,37 @@ class Stacker(BaseCommand):
     subcommands = (Build, Destroy, Info, Diff, Graph)
 
     def configure(self, options, **kwargs):
-        super(Stacker, self).configure(options, **kwargs)
-        if options.interactive:
-            logger.info("Using interactive AWS provider mode.")
-        else:
-            logger.info("Using default AWS provider mode")
 
         session_cache.default_profile = options.profile
 
-        config = load_config(
+        self.config = load_config(
             options.config.read(),
             environment=options.environment,
-            validate=True)
+            validate=True,
+        )
 
         options.provider_builder = default.ProviderBuilder(
             region=options.region,
             interactive=options.interactive,
             replacements_only=options.replacements_only,
             recreate_failed=options.recreate_failed,
-            service_role=config.service_role,
+            service_role=self.config.service_role,
         )
 
         options.context = Context(
             environment=options.environment,
-            config=config,
+            config=self.config,
             # Allow subcommands to provide any specific kwargs to the Context
             # that it wants.
             **options.get_context_kwargs(options)
         )
 
+        super(Stacker, self).configure(options, **kwargs)
+        if options.interactive:
+            logger.info("Using interactive AWS provider mode.")
+        else:
+            logger.info("Using default AWS provider mode")
+
     def add_arguments(self, parser):
         parser.add_argument("--version", action="version",
                             version="%%(prog)s %s" % (__version__,))

stacker/commands/stacker/base.py

@@ -107,12 +107,6 @@ def add_subcommands(self, parser):
                 subparser.set_defaults(
                     get_context_kwargs=subcommand.get_context_kwargs)
 
-    @property
-    def logger(self):
-        if not hasattr(self, "_logger"):
-            self._logger = logging.getLogger(self.name)
-        return self._logger
-
     def parse_args(self, *vargs):
         parser = argparse.ArgumentParser(description=self.description)
         self.add_subcommands(parser)
@@ -126,7 +120,7 @@ def run(self, options, **kwargs):
 
     def configure(self, options, **kwargs):
         if self.setup_logging:
-            self.setup_logging(options.verbose)
+            self.setup_logging(options.verbose, self.config.log_formats)
 
     def get_context_kwargs(self, options, **kwargs):
         """Return a dictionary of kwargs that will be used with the Context.

stacker/config/__init__.py

@@ -433,6 +433,8 @@ class Config(Model):
     stacks = ListType(
         ModelType(Stack), default=[])
 
+    log_formats = DictType(StringType, serialize_when_none=False)
+
     def _remove_excess_keys(self, data):
         excess_keys = set(data.keys())
         excess_keys -= self._schema.valid_input_keys

stacker/logger/__init__.py

@@ -21,15 +21,29 @@ def format(self, record):
         return msg
 
 
-def setup_logging(verbosity):
+def setup_logging(verbosity, formats=None):
+    """
+    Configure a proper logger based on verbosity and optional log formats.
+
+    Args:
+        verbosity (int): 0, 1, 2
+        formats (dict): Optional, looks for `info`, `color`, and `debug` keys
+                        which may override the associated default log formats.
+    """
+    if formats is None:
+        formats = {}
+
     log_level = logging.INFO
-    log_format = INFO_FORMAT
+
+    log_format = formats.get("info", INFO_FORMAT)
+
     if sys.stdout.isatty():
-        log_format = COLOR_FORMAT
+        log_format = formats.get("color", COLOR_FORMAT)
 
     if verbosity > 0:
         log_level = logging.DEBUG
-        log_format = DEBUG_FORMAT
+        log_format = formats.get("debug", DEBUG_FORMAT)
+
     if verbosity < 2:
         logging.getLogger("botocore").setLevel(logging.CRITICAL)
 

stacker/tests/fixtures/not-basic.env

@@ -0,0 +1,2 @@
+namespace: test.stacker
+environment: test

stacker/tests/fixtures/vpc-custom-log-format-info.yaml

@@ -0,0 +1,18 @@
+log_formats:
+  info: "[%(asctime)s] ${environment} custom log format - %(message)s"
+
+stacks:
+  - name: vpc
+    class_path: stacker.tests.fixtures.mock_blueprints.VPC
+    variables:
+      InstanceType: m3.medium
+      SshKeyName: default
+      ImageName: NAT
+      # Only build 2 AZs, can be overridden with -p on the command line
+      # Note: If you want more than 4 AZs you should add more subnets below
+      #       Also you need at least 2 AZs in order to use the DB because
+      #       of the fact that the DB blueprint uses MultiAZ
+      AZCount: 2
+      # Enough subnets for 4 AZs
+      PublicSubnets: 10.128.0.0/24,10.128.1.0/24,10.128.2.0/24,10.128.3.0/24
+      PrivateSubnets: 10.128.8.0/22,10.128.12.0/22,10.128.16.0/22,10.128.20.0/22

stacker/tests/test_stacker.py

@@ -92,6 +92,27 @@ def test_stacker_build_fail_when_parameters_in_stack_def(self):
         with self.assertRaises(InvalidConfig):
             stacker.configure(args)
 
+    def test_stacker_build_custom_info_log_format(self):
+        stacker = Stacker()
+        args = stacker.parse_args(
+            [
+                "build", "-r", "us-west-2",
+                "stacker/tests/fixtures/not-basic.env",
+                "stacker/tests/fixtures/vpc-custom-log-format-info.yaml"
+            ]
+        )
+        stacker.configure(args)
+        self.assertEqual(
+            stacker.config.log_formats["info"],
+            '[%(asctime)s] test custom log format - %(message)s'
+        )
+        self.assertIsNone(
+            stacker.config.log_formats.get("color")
+        )
+        self.assertIsNone(
+            stacker.config.log_formats.get("debug")
+        )
+
 
 if __name__ == '__main__':
     unittest.main()