Merge pull request #144 from jverhoeven/logdriver

Introduced logdriver (logconfig) support, including tests.

docs/index.md

@@ -212,6 +212,8 @@ be placed on (by name). Additionally, it may define:
       container;
     - `swap`, the swap limit of the container (in bytes, or with one
       of the `k`, `m` or `g` suffixes, also valid in uppercase);
+  - `log_driver`, one of the supported log drivers, e.g. syslog or json-file.
+  - `log_opt`, a set of key value pairs that provide additional logging parameters. E.g. the syslog-address to redirect syslog output to another address.
   - `command`, to specify or override the command executed by the
     container;
   - `net`, to specify the container's network mode (one of `bridge` --

maestro/entities.py

@@ -4,6 +4,7 @@
 # Docker container orchestration utility.
 
 import bgtunnel
+import collections
 import datetime
 import time
 import os
@@ -19,6 +20,10 @@
     # Fall back to <= 0.3.1 location
     from docker.client import APIError
 
+from docker.utils import create_host_config
+from docker.utils.types import LogConfigTypesEnum
+
+
 import multiprocessing.pool
 import re
 import six
@@ -390,6 +395,10 @@ def env_list_expand(elt):
         self.mem_limit = self._parse_bytes(limits.get('memory'))
         self.memswap_limit = self._parse_bytes(limits.get('swap'))
 
+        # Get logging config.
+        self.log_config = self._parse_log_config(
+            config.get('log_driver'), config.get('log_opt'))
+
         # Additional LXC configuration options. See the LXC documentation for a
         # reference of the available settings. Those are only supported if the
         # remote Docker daemon uses the lxc execution driver.
@@ -398,6 +407,32 @@ def env_list_expand(elt):
         # Work directory for the container
         self.workdir = config.get('workdir')
 
+        # Reformat port structure
+        ports = collections.defaultdict(list) if self.ports else None
+        if ports is not None:
+            for port in self.ports.values():
+                ports[port['exposed']].append(
+                    (port['external'][0], port['external'][1].split('/')[0]))
+
+        # host_config now contains all settings previously passed in container
+        # start().
+        self.host_config = create_host_config(
+            log_config=self.log_config, 
+            mem_limit=self.mem_limit, 
+            memswap_limit=self.memswap_limit,
+            binds=self.volumes,
+            port_bindings=ports,
+            lxc_conf=self.lxc_conf,
+            privileged=self.privileged,
+            cap_add=self.cap_add,
+            cap_drop=self.cap_drop,
+            extra_hosts=self.extra_hosts,
+            network_mode=self.network_mode,
+            restart_policy=self.restart_policy,
+            dns=self.dns,
+            links=self.links,
+            volumes_from=list(self.volumes_from))
+
         # Seed the service name, container name and host address as part of the
         # container's environment.
         self.env.update({
@@ -636,6 +671,35 @@ def _parse_spec(src, spec):
         for src, spec in volumes.items():
             _parse_spec(src, spec)
         return result
+
+
+    def _parse_log_config(self, log_driver, log_opt):
+        """ Parse the log config found in the container's configuration.
+        
+        Args:
+            log_driver (enum): Should be a valid value as defined by 
+                docker/docker-py, e.g. json-file, syslog, none. 
+            log_opt (dict): Should be a valid dictionary with additional log driver
+                settings. Values are not interpreted.
+        Returns: A dictionary that can be passed to to docker-py via the
+            host_config.LogConfig variable.
+        """
+        if log_driver:
+            if log_driver not in LogConfigTypesEnum._values:
+                raise exceptions.InvalidLogConfigurationException(
+                    "log_driver must be one of ({0})".format(
+                    ', '.join(LogConfigTypesEnum._values)
+                ))
+            if log_opt and not type(log_opt) == dict:
+                raise exceptions.InvalidLogConfigurationException(
+                    "log_opt must be a dictionary")
+            if log_opt:
+                return {"Type": log_driver, "Config": log_opt}
+            else:
+                return {"Type": log_driver}
+
+        return None
+
 
     def _parse_go_time(self, s):
         """Parse a time string found in the container status into a Python

maestro/exceptions.py

@@ -66,3 +66,8 @@ class InvalidVolumeConfigurationException(MaestroException):
 
 class InvalidAuditorConfigurationException(MaestroException):
     """Invalid configuration of one of the specified auditors."""
+
+
+class InvalidLogConfigurationException(MaestroException):
+    """Error thrown when a log_driver or log_opt is in an invalid format."""
+    pass

maestro/plays/tasks.py

@@ -208,9 +208,8 @@ def _create_and_start_container(self):
                 name=self.container.name,
                 environment=self.container.env,
                 volumes=list(self.container.get_volumes()),
-                mem_limit=self.container.mem_limit,
-                memswap_limit=self.container.memswap_limit,
                 cpu_shares=self.container.cpu_shares,
+                host_config=self.container.host_config,
                 ports=ports,
                 detach=True,
                 working_dir=self.container.workdir,
@@ -232,19 +231,7 @@ def _create_and_start_container(self):
         self.o.pending('starting container {}...'
                        .format(self.container.id[:7]))
         self.container.ship.backend.start(
-            self.container.id,
-            binds=self.container.volumes,
-            port_bindings=ports,
-            lxc_conf=self.container.lxc_conf,
-            privileged=self.container.privileged,
-            cap_add=self.container.cap_add,
-            cap_drop=self.container.cap_drop,
-            extra_hosts=self.container.extra_hosts,
-            network_mode=self.container.network_mode,
-            restart_policy=self.container.restart_policy,
-            dns=self.container.dns,
-            links=self.container.links,
-            volumes_from=list(self.container.volumes_from))
+            self.container.id)
 
         # Waiting one second and checking container state again to make sure
         # initialization didn't fail.

tests/unittests.py

@@ -123,6 +123,36 @@ def test_swap_limit_string_with_suffix(self):
         container = self._cntr(config={'limits': {'swap': '42k'}})
         self.assertEqual(container.memswap_limit, 42*1024)
 
+    def test_log_config_default(self):
+        self.assertTrue("LogConfig" not in self._cntr().host_config)
+
+    def test_log_config_syslog(self):
+        container = self._cntr(config={'log_driver': 'syslog'})
+        self.assertTrue("LogConfig" in container.host_config)
+        self.assertEqual(container.host_config['LogConfig'],
+                         {'Type': 'syslog', 'Config': {}})
+
+    def test_log_config_syslog_with_opts(self):
+        container = self._cntr(config={'log_driver': 'syslog', 'log_opt': {
+            'syslog-address': 'tcp://127.0.0.1:514'
+        }})
+        self.assertTrue("LogConfig" in container.host_config)
+        self.assertEqual(container.host_config['LogConfig'], {
+            'Type': 'syslog', 'Config': {
+                'syslog-address': 'tcp://127.0.0.1:514'
+            }})
+
+    def test_log_config_wrong_driver_type(self):
+        self.assertRaises(
+            exceptions.InvalidLogConfigurationException,
+            lambda: self._cntr(config={'log_driver': 'notvalid'}))
+
+    def test_log_config_wrong_opt_type(self):
+        self.assertRaises(
+            exceptions.InvalidLogConfigurationException,
+            lambda: self._cntr(
+                config={'log_driver': 'syslog', "log_opt": 'shouldbeadict'}))
+
     def test_restart_policy_default(self):
         self.assertEqual(self._cntr().restart_policy,
                          {'Name': 'no', 'MaximumRetryCount': 0})