Using config_template

Thanks to @cloudnull great patch at
ansible/ansible#12555
we now have the ability to add more configuration options instead of
having to push a PR to add a new option to the template. So you can
dynamically add and remove flags.

To use it, edit `ceph_conf_overrides` in `group_vars/all` like so:

```
ceph_conf_overrides
 global:
     foo: 12345
     bar: 6789
```

Signed-off-by: Sébastien Han <seb@redhat.com>

.gitignore

@@ -11,3 +11,4 @@ group_vars/rgws
 group_vars/restapis
 *.DS_Store
 site.yml
+*.pyc

ansible.cfg

@@ -1,2 +1,3 @@
 [defaults]
 ansible_managed = Please do not change this file directly since it is managed by Ansible and will be overwritten
+action_plugins = plugins/actions

group_vars/all.sample

@@ -230,6 +230,24 @@ dummy:
 #common_single_host_mode: true
 
 
+###################
+# CONFIG OVERRIDE #
+###################
+
+# Ceph configuration file override.
+# This allows you to specify more configuration options
+# using an INI style format.
+# The following sections are supported: [global], [mon], [osd], [mds], [rgw]
+#
+# Example:
+# ceph_conf_overrides:
+#   global:
+#     foo: 1234
+#     bar: 5678
+#
+#ceph_conf_overrides: {}
+
+
 #############
 # OS TUNING #
 #############

library/config_template

@@ -0,0 +1,66 @@
+# this is a virtual module that is entirely implemented server side
+
+DOCUMENTATION = """
+---
+module: config_template
+version_added: 1.9.2
+short_description: Renders template files providing a create/update override interface
+description:
+  - The module contains the template functionality with the ability to override items
+    in config, in transit, though the use of an simple dictionary without having to
+    write out various temp files on target machines. The module renders all of the
+    potential jinja a user could provide in both the template file and in the override
+    dictionary which is ideal for deployers whom may have lots of different configs
+    using a similar code base.
+  - The module is an extension of the **copy** module and all of attributes that can be
+    set there are available to be set here.
+options:
+  src:
+    description:
+      - Path of a Jinja2 formatted template on the local server. This can be a relative
+        or absolute path.
+    required: true
+    default: null
+  dest:
+    description:
+      - Location to render the template to on the remote machine.
+    required: true
+    default: null
+  config_overrides:
+    description:
+      - A dictionary used to update or override items within a configuration template.
+        The dictionary data structure may be nested. If the target config file is an ini
+        file the nested keys in the ``config_overrides`` will be used as section
+        headers.
+  config_type:
+    description:
+      - A string value describing the target config type.
+    choices:
+      - ini
+      - json
+      - yaml
+author: Kevin Carter
+"""
+
+EXAMPLES = """
+  - name: run config template ini
+    config_template:
+      src: templates/test.ini.j2
+      dest: /tmp/test.ini
+      config_overrides: {}
+      config_type: ini
+
+  - name: run config template json
+    config_template:
+      src: templates/test.json.j2
+      dest: /tmp/test.json
+      config_overrides: {}
+      config_type: json
+
+  - name: run config template yaml
+    config_template:
+      src: templates/test.yaml.j2
+      dest: /tmp/test.yaml
+      config_overrides: {}
+      config_type: yaml
+"""

plugins/actions/config_template.py

@@ -0,0 +1,240 @@
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import ConfigParser
+import io
+import json
+import os
+import yaml
+
+from ansible import errors
+from ansible.runner.return_data import ReturnData
+from ansible import utils
+from ansible.utils import template
+
+
+CONFIG_TYPES = {
+    'ini': 'return_config_overrides_ini',
+    'json': 'return_config_overrides_json',
+    'yaml': 'return_config_overrides_yaml'
+}
+
+
+class ActionModule(object):
+    TRANSFERS_FILES = True
+
+    def __init__(self, runner):
+        self.runner = runner
+
+    def grab_options(self, complex_args, module_args):
+        """Grab passed options from Ansible complex and module args.
+
+        :param complex_args: ``dict``
+        :param module_args: ``dict``
+        :returns: ``dict``
+        """
+        options = dict()
+        if complex_args:
+            options.update(complex_args)
+
+        options.update(utils.parse_kv(module_args))
+        return options
+
+    @staticmethod
+    def return_config_overrides_ini(config_overrides, resultant):
+        """Returns string value from a modified config file.
+
+        :param config_overrides: ``dict``
+        :param resultant: ``str`` || ``unicode``
+        :returns: ``str``
+        """
+        config = ConfigParser.RawConfigParser(allow_no_value=True)
+        config_object = io.BytesIO(resultant.encode('utf-8'))
+        config.readfp(config_object)
+        for section, items in config_overrides.items():
+            # If the items value is not a dictionary it is assumed that the
+            #  value is a default item for this config type.
+            if not isinstance(items, dict):
+                config.set('DEFAULT', section, str(items))
+            else:
+                # Attempt to add a section to the config file passing if
+                #  an error is raised that is related to the section
+                #  already existing.
+                try:
+                    config.add_section(section)
+                except (ConfigParser.DuplicateSectionError, ValueError):
+                    pass
+                for key, value in items.items():
+                    config.set(section, key, str(value))
+        else:
+            config_object.close()
+
+        resultant_bytesio = io.BytesIO()
+        try:
+            config.write(resultant_bytesio)
+            return resultant_bytesio.getvalue()
+        finally:
+            resultant_bytesio.close()
+
+    def return_config_overrides_json(self, config_overrides, resultant):
+        """Returns config json
+
+        Its important to note that file ordering will not be preserved as the
+        information within the json file will be sorted by keys.
+
+        :param config_overrides: ``dict``
+        :param resultant: ``str`` || ``unicode``
+        :returns: ``str``
+        """
+        original_resultant = json.loads(resultant)
+        merged_resultant = self._merge_dict(
+            base_items=original_resultant,
+            new_items=config_overrides
+        )
+        return json.dumps(
+            merged_resultant,
+            indent=4,
+            sort_keys=True
+        )
+
+    def return_config_overrides_yaml(self, config_overrides, resultant):
+        """Return config yaml.
+
+        :param config_overrides: ``dict``
+        :param resultant: ``str`` || ``unicode``
+        :returns: ``str``
+        """
+        original_resultant = yaml.safe_load(resultant)
+        merged_resultant = self._merge_dict(
+            base_items=original_resultant,
+            new_items=config_overrides
+        )
+        return yaml.safe_dump(
+            merged_resultant,
+            default_flow_style=False,
+            width=1000,
+        )
+
+    def _merge_dict(self, base_items, new_items):
+        """Recursively merge new_items into base_items.
+
+        :param base_items: ``dict``
+        :param new_items: ``dict``
+        :returns: ``dict``
+        """
+        for key, value in new_items.iteritems():
+            if isinstance(value, dict):
+                base_items[key] = self._merge_dict(
+                    base_items.get(key, {}),
+                    value
+                )
+            elif isinstance(value, list):
+                if key in base_items and isinstance(base_items[key], list):
+                    base_items[key].extend(value)
+                else:
+                    base_items[key] = value
+            else:
+                base_items[key] = new_items[key]
+        return base_items
+
+    def run(self, conn, tmp, module_name, module_args, inject,
+            complex_args=None, **kwargs):
+        """Run the method"""
+        if not self.runner.is_playbook:
+            raise errors.AnsibleError(
+                'FAILED: `config_templates` are only available in playbooks'
+            )
+
+        options = self.grab_options(complex_args, module_args)
+        try:
+            source = options['src']
+            dest = options['dest']
+
+            config_overrides = options.get('config_overrides', dict())
+            config_type = options['config_type']
+            assert config_type.lower() in ['ini', 'json', 'yaml']
+        except KeyError as exp:
+            result = dict(failed=True, msg=exp)
+            return ReturnData(conn=conn, comm_ok=False, result=result)
+
+        source_template = template.template(
+            self.runner.basedir,
+            source,
+            inject
+        )
+
+        if '_original_file' in inject:
+            source_file = utils.path_dwim_relative(
+                inject['_original_file'],
+                'templates',
+                source_template,
+                self.runner.basedir
+            )
+        else:
+            source_file = utils.path_dwim(self.runner.basedir, source_template)
+
+        # Open the template file and return the data as a string. This is
+        #  being done here so that the file can be a vault encrypted file.
+        resultant = template.template_from_file(
+            self.runner.basedir,
+            source_file,
+            inject,
+            vault_password=self.runner.vault_pass
+        )
+
+        if config_overrides:
+            type_merger = getattr(self, CONFIG_TYPES.get(config_type))
+            resultant = type_merger(
+                config_overrides=config_overrides,
+                resultant=resultant
+            )
+
+        # Retemplate the resultant object as it may have new data within it
+        #  as provided by an override variable.
+        template.template_from_string(
+            basedir=self.runner.basedir,
+            data=resultant,
+            vars=inject,
+            fail_on_undefined=True
+        )
+
+        # Access to protected method is unavoidable in Ansible 1.x.
+        new_module_args = dict(
+            src=self.runner._transfer_str(conn, tmp, 'source', resultant),
+            dest=dest,
+            original_basename=os.path.basename(source),
+            follow=True,
+        )
+
+        module_args_tmp = utils.merge_module_args(
+            module_args,
+            new_module_args
+        )
+
+        # Remove data types that are not available to the copy module
+        complex_args.pop('config_overrides')
+        complex_args.pop('config_type')
+
+        # Return the copy module status. Access to protected method is
+        #  unavoidable in Ansible 1.x.
+        return self.runner._execute_module(
+            conn,
+            tmp,
+            'copy',
+            module_args_tmp,
+            inject=inject,
+            complex_args=complex_args
+        )
+

roles/ceph-common/defaults/main.yml

@@ -239,6 +239,24 @@ restapi_log_level: warning # available level are: critical, error, warning, info
 #common_single_host_mode: true
 
 
+###################
+# CONFIG OVERRIDE #
+###################
+
+# Ceph configuration file override.
+# This allows you to specify more configuration options
+# using an INI style format.
+# The following sections are supported: [global], [mon], [osd], [mds], [rgw]
+#
+# Example:
+# ceph_conf_overrides:
+#   global:
+#     foo: 1234
+#     bar: 5678
+#
+ceph_conf_overrides: {}
+
+
 #############
 # OS TUNING #
 #############

roles/ceph-common/tasks/main.yml

@@ -79,12 +79,14 @@
       mode: 0644
 
 - name: generate ceph configuration file
-  template:
+  config_template:
     src: ceph.conf.j2
     dest: /etc/ceph/ceph.conf
-    owner: root
-    group: root
-    mode: 0644
+    owner: "root"
+    group: "root"
+    mode: "0644"
+    config_overrides: "{{ ceph_conf_overrides }}"
+    config_type: ini
   notify:
     - restart ceph mons
     - restart ceph mons on ubuntu