devstack_local_conf.py

# Copyright (C) 2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re


class DependencyGraph(object):
    # This is based on the JobGraph from Zuul.

    def __init__(self):
        self._names = set()
        self._dependencies = {}  # dependent_name -> set(parent_names)

    def add(self, name, dependencies):
        # Append the dependency information
        self._dependencies.setdefault(name, set())
        try:
            for dependency in dependencies:
                # Make sure a circular dependency is never created
                ancestors = self._getParentNamesRecursively(
                    dependency, soft=True)
                ancestors.add(dependency)
                if name in ancestors:
                    raise Exception("Dependency cycle detected in {}".
                                    format(name))
                self._dependencies[name].add(dependency)
        except Exception:
            del self._dependencies[name]
            raise

    def getDependenciesRecursively(self, parent):
        dependencies = []

        current_dependencies = self._dependencies[parent]
        for current in current_dependencies:
            if current not in dependencies:
                dependencies.append(current)
            for dep in self.getDependenciesRecursively(current):
                if dep not in dependencies:
                    dependencies.append(dep)
        return dependencies

    def _getParentNamesRecursively(self, dependent, soft=False):
        all_parent_items = set()
        items_to_iterate = set([dependent])
        while len(items_to_iterate) > 0:
            current_item = items_to_iterate.pop()
            current_parent_items = self._dependencies.get(current_item)
            if current_parent_items is None:
                if soft:
                    current_parent_items = set()
                else:
                    raise Exception("Dependent item {} not found: ".format(
                                    dependent))
            new_parent_items = current_parent_items - all_parent_items
            items_to_iterate |= new_parent_items
            all_parent_items |= new_parent_items
        return all_parent_items


class VarGraph(DependencyGraph):
    def __init__(self, vars):
        super(VarGraph, self).__init__()
        self.vars = {}
        self._varnames = set()
        for k, v in vars.items():
            self._varnames.add(k)
        for k, v in vars.items():
            self._addVar(k, str(v))

    bash_var_re = re.compile(r'\$\{?(\w+)')
    def getDependencies(self, value):
        return self.bash_var_re.findall(value)

    def _addVar(self, key, value):
        if key in self.vars:
            raise Exception("Variable {} already added".format(key))
        self.vars[key] = value
        # Append the dependency information
        dependencies = set()
        for dependency in self.getDependencies(value):
            if dependency == key:
                # A variable is allowed to reference itself; no
                # dependency link needed in that case.
                continue
            if dependency not in self._varnames:
                # It's not necessary to create a link for an
                # external variable.
                continue
            dependencies.add(dependency)
        try:
            self.add(key, dependencies)
        except Exception:
            del self.vars[key]
            raise

    def getVars(self):
        ret = []
        keys = sorted(self.vars.keys())
        seen = set()
        for key in keys:
            dependencies = self.getDependenciesRecursively(key)
            for var in dependencies + [key]:
                if var not in seen:
                    ret.append((var, self.vars[var]))
                    seen.add(var)
        return ret


class PluginGraph(DependencyGraph):
    def __init__(self, base_dir, plugins):
        super(PluginGraph, self).__init__()
        # The dependency trees expressed by all the plugins we found
        # (which may be more than those the job is using).
        self._plugin_dependencies = {}
        self.loadPluginNames(base_dir)

        self.plugins = {}
        self._pluginnames = set()
        for k, v in plugins.items():
            self._pluginnames.add(k)
        for k, v in plugins.items():
            self._addPlugin(k, str(v))

    def loadPluginNames(self, base_dir):
        if base_dir is None:
            return
        git_roots = []
        for root, dirs, files in os.walk(base_dir):
            if '.git' not in dirs:
                continue
            # Don't go deeper than git roots
            dirs[:] = []
            git_roots.append(root)
        for root in git_roots:
            devstack = os.path.join(root, 'devstack')
            if not (os.path.exists(devstack) and os.path.isdir(devstack)):
                continue
            settings = os.path.join(devstack, 'settings')
            if not (os.path.exists(settings) and os.path.isfile(settings)):
                continue
            self.loadDevstackPluginInfo(settings)

    define_re = re.compile(r'^define_plugin\s+(\S+).*')
    require_re = re.compile(r'^plugin_requires\s+(\S+)\s+(\S+).*')
    def loadDevstackPluginInfo(self, fn):
        name = None
        reqs = set()
        with open(fn) as f:
            for line in f:
                m = self.define_re.match(line)
                if m:
                    name = m.group(1)
                m = self.require_re.match(line)
                if m:
                    if name == m.group(1):
                        reqs.add(m.group(2))
        if name and reqs:
            self._plugin_dependencies[name] = reqs

    def getDependencies(self, value):
        return self._plugin_dependencies.get(value, [])

    def _addPlugin(self, key, value):
        if key in self.plugins:
            raise Exception("Plugin {} already added".format(key))
        self.plugins[key] = value
        # Append the dependency information
        dependencies = set()
        for dependency in self.getDependencies(key):
            if dependency == key:
                continue
            dependencies.add(dependency)
        try:
            self.add(key, dependencies)
        except Exception:
            del self.plugins[key]
            raise

    def getPlugins(self):
        ret = []
        keys = sorted(self.plugins.keys())
        seen = set()
        for key in keys:
            dependencies = self.getDependenciesRecursively(key)
            for plugin in dependencies + [key]:
                if plugin not in seen:
                    ret.append((plugin, self.plugins[plugin]))
                    seen.add(plugin)
        return ret


class LocalConf(object):

    def __init__(self, localrc, localconf, base_services, services, plugins,
                 base_dir, projects, project, tempest_plugins):
        self.localrc = []
        self.warnings = []
        self.meta_sections = {}
        self.plugin_deps = {}
        self.base_dir = base_dir
        self.projects = projects
        self.project = project
        self.tempest_plugins = tempest_plugins
        if services or base_services:
            self.handle_services(base_services, services or {})
        self.handle_localrc(localrc)
        # Plugins must be the last items in localrc, otherwise
        # the configuration lines which follows them in the file are
        # not applied to the plugins (for example, the value of DEST.)
        if plugins:
            self.handle_plugins(plugins)
        if localconf:
            self.handle_localconf(localconf)

    def handle_plugins(self, plugins):
        pg = PluginGraph(self.base_dir, plugins)
        for k, v in pg.getPlugins():
            if v:
                self.localrc.append('enable_plugin {} {}'.format(k, v))

    def handle_services(self, base_services, services):
        enable_base_services = services.pop('base', True)
        if enable_base_services and base_services:
            self.localrc.append('ENABLED_SERVICES={}'.format(
                ",".join(base_services)))
        else:
            self.localrc.append('disable_all_services')
        for k, v in services.items():
            if v is False:
                self.localrc.append('disable_service {}'.format(k))
            elif v is True:
                self.localrc.append('enable_service {}'.format(k))

    def handle_localrc(self, localrc):
        lfg = False
        tp = False
        if localrc:
            vg = VarGraph(localrc)
            for k, v in vg.getVars():
                self.localrc.append('{}={}'.format(k, v))
                if k == 'LIBS_FROM_GIT':
                    lfg = True
                elif k == 'TEMPEST_PLUGINS':
                    tp = True

        if not lfg and (self.projects or self.project):
            required_projects = []
            if self.projects:
                for project_name, project_info in self.projects.items():
                    if project_info.get('required'):
                        required_projects.append(project_info['short_name'])
            if self.project:
                if self.project['short_name'] not in required_projects:
                    required_projects.append(self.project['short_name'])
            if required_projects:
                self.localrc.append('LIBS_FROM_GIT={}'.format(
                    ','.join(required_projects)))

        if self.tempest_plugins:
            if not tp:
                tp_dirs = []
                for tempest_plugin in self.tempest_plugins:
                    tp_dirs.append(os.path.join(self.base_dir, tempest_plugin))
                self.localrc.append('TEMPEST_PLUGINS="{}"'.format(
                        ' '.join(tp_dirs)))
            else:
                self.warnings.append('TEMPEST_PLUGINS already defined ({}),'
                                     'requested value {} ignored'.format(
                                         tp, self.tempest_plugins))


    def handle_localconf(self, localconf):
        for phase, phase_data in localconf.items():
            for fn, fn_data in phase_data.items():
                ms_name = '[[{}|{}]]'.format(phase, fn)
                ms_data = []
                for section, section_data in fn_data.items():
                    ms_data.append('[{}]'.format(section))
                    for k, v in section_data.items():
                        ms_data.append('{} = {}'.format(k, v))
                    ms_data.append('')
                self.meta_sections[ms_name] = ms_data

    def write(self, path):
        with open(path, 'w') as f:
            f.write('[[local|localrc]]\n')
            f.write('\n'.join(self.localrc))
            f.write('\n\n')
            for section, lines in self.meta_sections.items():
                f.write('{}\n'.format(section))
                f.write('\n'.join(lines))


def main():
    module = AnsibleModule(
        argument_spec=dict(
            plugins=dict(type='dict'),
            base_services=dict(type='list'),
            services=dict(type='dict'),
            localrc=dict(type='dict'),
            local_conf=dict(type='dict'),
            base_dir=dict(type='path'),
            path=dict(type='str'),
            projects=dict(type='dict'),
            project=dict(type='dict'),
            tempest_plugins=dict(type='list'),
        )
    )

    p = module.params
    lc = LocalConf(p.get('localrc'),
                   p.get('local_conf'),
                   p.get('base_services'),
                   p.get('services'),
                   p.get('plugins'),
                   p.get('base_dir'),
                   p.get('projects'),
                   p.get('project'),
                   p.get('tempest_plugins'))
    lc.write(p['path'])

    module.exit_json(warnings=lc.warnings)


try:
    from ansible.module_utils.basic import *  # noqa
    from ansible.module_utils.basic import AnsibleModule
except ImportError:
    pass

if __name__ == '__main__':
    main()