From 2fc5ceb95fd2fd30ebaefb1b73d3f831cd82ab93 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 14 Sep 2016 15:40:42 -0500 Subject: [PATCH] Fix proprietary regression in py_pkgs.py The change backports the newton version of the lookup plugin into mitaka to resolve a regression where the proprietary package listings are being indexed by the lookup plugin. This is a combined backport as a clean backport from "7af6721b4520ee96209e482bd66247afcfa7b4e5" is not possible. Closes-Bug: 1622275 Depends-On: I6b0ae96ccf0f774be1c8a19374fbca295bb40a3b Change-Id: I79f408dcee81ef59432e943bfc5e81644e3003b9 Signed-off-by: Kevin Carter --- lookup/py_pkgs.py | 179 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 138 insertions(+), 41 deletions(-) diff --git a/lookup/py_pkgs.py b/lookup/py_pkgs.py index 471fa62..68e192b 100644 --- a/lookup/py_pkgs.py +++ b/lookup/py_pkgs.py @@ -36,6 +36,9 @@ GIT_PACKAGE_DEFAULT_PARTS = dict() +# Role based package indexes +ROLE_DISTRO_BREAKOUT_PACKAGES = dict() +ROLE_BREAKOUT_REQUIREMENTS = dict() ROLE_PACKAGES = dict() ROLE_REQUIREMENTS = dict() @@ -58,13 +61,20 @@ 'pip_packages' ] +BUILT_IN_DISTRO_PACKAGE_VARS = [ + 'distro_packages', + 'apt_packages', + 'yum_packages' +] + PACKAGE_MAPPING = { 'packages': set(), 'remote_packages': set(), 'remote_package_parts': list(), 'role_packages': dict(), - 'role_project_groups': dict() + 'role_project_groups': dict(), + 'distro_packages': set() } @@ -294,6 +304,7 @@ def __init__(self, local_path): def _py_pkg_extend(self, packages, py_package=None): if py_package is None: py_package = self.pip['py_package'] + for pkg in packages: pkg_name = _pip_requirement_split(pkg)[0] for py_pkg in py_package: @@ -301,7 +312,8 @@ def _py_pkg_extend(self, packages, py_package=None): if pkg_name == py_pkg_name: py_package.remove(py_pkg) else: - py_package.extend([i.lower() for i in packages]) + norm_pkgs = [i.lower() for i in packages if not i.startswith('{{')] + py_package.extend(norm_pkgs) return py_package @staticmethod @@ -457,30 +469,51 @@ def _process_git(self, loaded_yaml, git_item, yaml_file_name): git_data=git_data ) - def _package_build_index(self, packages, role_name, var_name, - pkg_index=ROLE_PACKAGES, project_group='all'): - self._py_pkg_extend(packages) + def _package_build_index(self, packages, role_name, var_name, pkg_index, + project_group='all', var_file_name=None, + pip_packages=True): + if pip_packages: + self._py_pkg_extend(packages) + if role_name: if role_name in pkg_index: role_pkgs = pkg_index[role_name] else: role_pkgs = pkg_index[role_name] = dict() + role_pkgs['project_group'] = project_group - pkgs = role_pkgs.get(var_name, list()) - if 'optional' not in var_name: + if var_file_name: + _name = os.path.splitext(os.path.basename(var_file_name))[0] + file_name_index = pkg_index[role_name][_name] = dict() + pkgs = file_name_index.get(var_name, list()) pkgs = self._py_pkg_extend(packages, pkgs) - pkg_index[role_name][var_name] = pkgs + file_name_index[var_name] = sorted(set(pkgs)) + else: + pkgs = role_pkgs.get(var_name, list()) + pkgs.extend(packages) + if 'pip' in var_name: + pkgs = [i.lower() for i in pkgs if not i.startswith('{{')] + else: + pkgs = [i for i in pkgs if not i.startswith('{{')] + if pkgs: + pkg_index[role_name][var_name] = sorted(set(pkgs)) else: for k, v in pkg_index.items(): for item_name in v.keys(): if var_name == item_name: - pkg_index[k][item_name] = self._py_pkg_extend(packages, pkg_index[k][item_name]) + pkg_index[k][item_name] = self._py_pkg_extend( + packages, + pkg_index[k][item_name] + ) def _process_files(self): - """Process files.""" + """Process all of the requirement files.""" + self._process_files_defaults() + self._process_files_requirements() - role_name = None + def _process_files_defaults(self): + """Process files.""" for file_name in self._filter_files(self.file_names, ('yaml', 'yml')): with open(file_name, 'r') as f: # If there is an exception loading the file continue @@ -494,9 +527,11 @@ def _process_files(self): if not loaded_config or not isinstance(loaded_config, dict): continue - if 'roles' in file_name: - _role_name = file_name.split('roles%s' % os.sep)[-1] - role_name = _role_name.split(os.sep)[0] + if 'roles' in file_name: + _role_name = file_name.split('roles%s' % os.sep)[-1] + role_name = _role_name.split(os.sep)[0] + else: + role_name = None for key, value in loaded_config.items(): if key.endswith('role_project_group'): @@ -506,39 +541,95 @@ def _process_files(self): project_group = 'all' for key, values in loaded_config.items(): + key = key.lower() if key.endswith('git_repo'): self._process_git( loaded_yaml=loaded_config, git_item=key, yaml_file_name=file_name ) + # Process pip packages + self._process_packages( + pkg_constant=BUILT_IN_PIP_PACKAGE_VARS, + pkg_breakout_index=ROLE_BREAKOUT_REQUIREMENTS, + pkg_role_index=ROLE_PACKAGES, + pkg_var_name=key, + packages=values, + role_name=role_name, + project_group=project_group + ) + # Process distro packages + self._process_packages( + pkg_constant=BUILT_IN_DISTRO_PACKAGE_VARS, + pkg_breakout_index=ROLE_DISTRO_BREAKOUT_PACKAGES, + pkg_role_index=dict(), # this is not used here + pkg_var_name=key, + packages=values, + role_name=role_name, + project_group=project_group, + role_index=False, + var_file_name=file_name, + pip_packages=False + ) - if [i for i in BUILT_IN_PIP_PACKAGE_VARS - if i in key - if 'proprietary' not in key]: - self._package_build_index( - packages=values, - role_name=role_name, - var_name=key, - project_group=project_group - ) + def _process_packages(self, pkg_constant, pkg_breakout_index, + pkg_role_index, pkg_var_name, packages, role_name, + project_group, role_index=True, var_file_name=None, + pip_packages=True): + """Process variables to build the package data structures. + + :param pkg_constant: CONSTANT used to validate package names + :type pkg_constant: ``list`` + :param pkg_breakout_index: CONSTANT used to store indexed packages + :type pkg_breakout_index: ``dict`` + :param pkg_role_index: CONSTANT used to store role indexed packages + :type pkg_role_index: ``dict`` + :param pkg_var_name: package variable name + :type pkg_var_name: ``str`` + :param packages: list of packages to index + :type packages: ``list`` + :param role_name: Name of the role where the packages came from + :type role_name: ``str`` + :param project_group: Name of the group being indexed + :type project_group: ``str`` + :param role_index: Enable or disable the use of the role index + :type role_index: ``bool`` + :param var_file_name: Variable file name used to index packages + :type var_file_name: ``str`` + :param pip_packages: Enable or disable pip index types + :type pip_packages: ``bool`` + """ + if [i for i in pkg_constant if i in pkg_var_name]: + if 'proprietary' in pkg_var_name: + return - for key, values in loaded_config.items(): - if 'proprietary' in key: - proprietary_pkgs = [ - i for i in values - if GIT_PACKAGE_DEFAULT_PARTS.get(i) - ] - if proprietary_pkgs: - self._package_build_index( - packages=proprietary_pkgs, - role_name=role_name, - var_name=key, - project_group=project_group - ) - else: - role_name = None + self._package_build_index( + packages=packages, + role_name=role_name, + var_name=pkg_var_name, + pkg_index=pkg_breakout_index, + project_group=project_group, + var_file_name=var_file_name, + pip_packages=pip_packages + ) + if not role_index: + return + elif 'optional' in pkg_var_name: + return + else: + self._package_build_index( + packages=packages, + role_name=role_name, + var_name=pkg_var_name, + pkg_index=pkg_role_index, + project_group=project_group, + var_file_name=var_file_name, + pip_packages=pip_packages + ) + + def _process_files_requirements(self): + """Process requirements files.""" return_list = self._filter_files(self.file_names, 'txt') for file_name in return_list: base_name = os.path.basename(file_name) @@ -550,10 +641,12 @@ def _process_files(self): for file_name in return_list: if file_name.endswith('other-requirements.txt'): continue - if 'roles' in file_name: + elif file_name.endswith('bindep.txt'): + continue + elif 'roles' in file_name: _role_name = file_name.split('roles%s' % os.sep)[-1] role_name = _role_name.split(os.sep)[0] - if not role_name: + else: role_name = 'default' with open(file_name, 'r') as f: packages = [ @@ -646,6 +739,8 @@ def run_v2(self, terms, variables=None, **kwargs): if isinstance(value, (list, set)): return_data[key] = sorted(value) return_data['role_requirement_files'] = ROLE_REQUIREMENTS + return_data['role_requirements'] = ROLE_BREAKOUT_REQUIREMENTS + return_data['role_distro_packages'] = ROLE_DISTRO_BREAKOUT_PACKAGES return [return_data] def run_v1(self, terms, inject=None, **kwargs): @@ -695,10 +790,12 @@ def run_v1(self, terms, inject=None, **kwargs): if isinstance(value, (list, set)): return_data[key] = sorted(value) return_data['role_requirement_files'] = ROLE_REQUIREMENTS + return_data['role_requirements'] = ROLE_BREAKOUT_REQUIREMENTS + return_data['role_distro_packages'] = ROLE_DISTRO_BREAKOUT_PACKAGES return [return_data] # Used for testing and debuging usage: `python plugins/lookups/py_pkgs.py ../` if __name__ == '__main__': import sys import json - print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4)) + print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4, sort_keys=True))