Skip to content

Commit

Permalink
Fix proprietary regression in py_pkgs.py
Browse files Browse the repository at this point in the history 
The change backports the newton version of the lookup plugin into mitaka
to resolve a regression where the proprietary package listings are being
indexed by the lookup plugin.

This is a combined backport as a clean backport from
"7af6721b4520ee96209e482bd66247afcfa7b4e5" is not possible.

Closes-Bug: 1622275
Depends-On: I6b0ae96ccf0f774be1c8a19374fbca295bb40a3b
Change-Id: I79f408dcee81ef59432e943bfc5e81644e3003b9
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
  • Loading branch information
@cloudnull
cloudnull authored and Jesse Pretorius (odyssey4me) committed Sep 14, 2016
1 parent 620c64e commit 2fc5ceb
Showing 1 changed file with 138 additions and 41 deletions.
179 changes: 138 additions & 41 deletions lookup/py_pkgs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@
GIT_PACKAGE_DEFAULT_PARTS = dict()


# Role based package indexes
ROLE_DISTRO_BREAKOUT_PACKAGES = dict()
ROLE_BREAKOUT_REQUIREMENTS = dict()
ROLE_PACKAGES = dict()
ROLE_REQUIREMENTS = dict()

Expand All @@ -58,13 +61,20 @@
'pip_packages'
]

BUILT_IN_DISTRO_PACKAGE_VARS = [
'distro_packages',
'apt_packages',
'yum_packages'
]


PACKAGE_MAPPING = {
'packages': set(),
'remote_packages': set(),
'remote_package_parts': list(),
'role_packages': dict(),
'role_project_groups': dict()
'role_project_groups': dict(),
'distro_packages': set()
}


Expand Down Expand Up @@ -294,14 +304,16 @@ def __init__(self, local_path):
def _py_pkg_extend(self, packages, py_package=None):
if py_package is None:
py_package = self.pip['py_package']

for pkg in packages:
pkg_name = _pip_requirement_split(pkg)[0]
for py_pkg in py_package:
py_pkg_name = _pip_requirement_split(py_pkg)[0]
if pkg_name == py_pkg_name:
py_package.remove(py_pkg)
else:
py_package.extend([i.lower() for i in packages])
norm_pkgs = [i.lower() for i in packages if not i.startswith('{{')]
py_package.extend(norm_pkgs)
return py_package

@staticmethod
Expand Down Expand Up @@ -457,30 +469,51 @@ def _process_git(self, loaded_yaml, git_item, yaml_file_name):
git_data=git_data
)

def _package_build_index(self, packages, role_name, var_name,
pkg_index=ROLE_PACKAGES, project_group='all'):
self._py_pkg_extend(packages)
def _package_build_index(self, packages, role_name, var_name, pkg_index,
project_group='all', var_file_name=None,
pip_packages=True):
if pip_packages:
self._py_pkg_extend(packages)

if role_name:
if role_name in pkg_index:
role_pkgs = pkg_index[role_name]
else:
role_pkgs = pkg_index[role_name] = dict()

role_pkgs['project_group'] = project_group

pkgs = role_pkgs.get(var_name, list())
if 'optional' not in var_name:
if var_file_name:
_name = os.path.splitext(os.path.basename(var_file_name))[0]
file_name_index = pkg_index[role_name][_name] = dict()
pkgs = file_name_index.get(var_name, list())
pkgs = self._py_pkg_extend(packages, pkgs)
pkg_index[role_name][var_name] = pkgs
file_name_index[var_name] = sorted(set(pkgs))
else:
pkgs = role_pkgs.get(var_name, list())
pkgs.extend(packages)
if 'pip' in var_name:
pkgs = [i.lower() for i in pkgs if not i.startswith('{{')]
else:
pkgs = [i for i in pkgs if not i.startswith('{{')]
if pkgs:
pkg_index[role_name][var_name] = sorted(set(pkgs))
else:
for k, v in pkg_index.items():
for item_name in v.keys():
if var_name == item_name:
pkg_index[k][item_name] = self._py_pkg_extend(packages, pkg_index[k][item_name])
pkg_index[k][item_name] = self._py_pkg_extend(
packages,
pkg_index[k][item_name]
)

def _process_files(self):
"""Process files."""
"""Process all of the requirement files."""
self._process_files_defaults()
self._process_files_requirements()

role_name = None
def _process_files_defaults(self):
"""Process files."""
for file_name in self._filter_files(self.file_names, ('yaml', 'yml')):
with open(file_name, 'r') as f:
# If there is an exception loading the file continue
Expand All @@ -494,9 +527,11 @@ def _process_files(self):
if not loaded_config or not isinstance(loaded_config, dict):
continue

if 'roles' in file_name:
_role_name = file_name.split('roles%s' % os.sep)[-1]
role_name = _role_name.split(os.sep)[0]
if 'roles' in file_name:
_role_name = file_name.split('roles%s' % os.sep)[-1]
role_name = _role_name.split(os.sep)[0]
else:
role_name = None

for key, value in loaded_config.items():
if key.endswith('role_project_group'):
Expand All @@ -506,39 +541,95 @@ def _process_files(self):
project_group = 'all'

for key, values in loaded_config.items():
key = key.lower()
if key.endswith('git_repo'):
self._process_git(
loaded_yaml=loaded_config,
git_item=key,
yaml_file_name=file_name
)
# Process pip packages
self._process_packages(
pkg_constant=BUILT_IN_PIP_PACKAGE_VARS,
pkg_breakout_index=ROLE_BREAKOUT_REQUIREMENTS,
pkg_role_index=ROLE_PACKAGES,
pkg_var_name=key,
packages=values,
role_name=role_name,
project_group=project_group
)
# Process distro packages
self._process_packages(
pkg_constant=BUILT_IN_DISTRO_PACKAGE_VARS,
pkg_breakout_index=ROLE_DISTRO_BREAKOUT_PACKAGES,
pkg_role_index=dict(), # this is not used here
pkg_var_name=key,
packages=values,
role_name=role_name,
project_group=project_group,
role_index=False,
var_file_name=file_name,
pip_packages=False
)

if [i for i in BUILT_IN_PIP_PACKAGE_VARS
if i in key
if 'proprietary' not in key]:
self._package_build_index(
packages=values,
role_name=role_name,
var_name=key,
project_group=project_group
)
def _process_packages(self, pkg_constant, pkg_breakout_index,
pkg_role_index, pkg_var_name, packages, role_name,
project_group, role_index=True, var_file_name=None,
pip_packages=True):
"""Process variables to build the package data structures.
:param pkg_constant: CONSTANT used to validate package names
:type pkg_constant: ``list``
:param pkg_breakout_index: CONSTANT used to store indexed packages
:type pkg_breakout_index: ``dict``
:param pkg_role_index: CONSTANT used to store role indexed packages
:type pkg_role_index: ``dict``
:param pkg_var_name: package variable name
:type pkg_var_name: ``str``
:param packages: list of packages to index
:type packages: ``list``
:param role_name: Name of the role where the packages came from
:type role_name: ``str``
:param project_group: Name of the group being indexed
:type project_group: ``str``
:param role_index: Enable or disable the use of the role index
:type role_index: ``bool``
:param var_file_name: Variable file name used to index packages
:type var_file_name: ``str``
:param pip_packages: Enable or disable pip index types
:type pip_packages: ``bool``
"""
if [i for i in pkg_constant if i in pkg_var_name]:
if 'proprietary' in pkg_var_name:
return

for key, values in loaded_config.items():
if 'proprietary' in key:
proprietary_pkgs = [
i for i in values
if GIT_PACKAGE_DEFAULT_PARTS.get(i)
]
if proprietary_pkgs:
self._package_build_index(
packages=proprietary_pkgs,
role_name=role_name,
var_name=key,
project_group=project_group
)
else:
role_name = None
self._package_build_index(
packages=packages,
role_name=role_name,
var_name=pkg_var_name,
pkg_index=pkg_breakout_index,
project_group=project_group,
var_file_name=var_file_name,
pip_packages=pip_packages
)

if not role_index:
return
elif 'optional' in pkg_var_name:
return
else:
self._package_build_index(
packages=packages,
role_name=role_name,
var_name=pkg_var_name,
pkg_index=pkg_role_index,
project_group=project_group,
var_file_name=var_file_name,
pip_packages=pip_packages
)

def _process_files_requirements(self):
"""Process requirements files."""
return_list = self._filter_files(self.file_names, 'txt')
for file_name in return_list:
base_name = os.path.basename(file_name)
Expand All @@ -550,10 +641,12 @@ def _process_files(self):
for file_name in return_list:
if file_name.endswith('other-requirements.txt'):
continue
if 'roles' in file_name:
elif file_name.endswith('bindep.txt'):
continue
elif 'roles' in file_name:
_role_name = file_name.split('roles%s' % os.sep)[-1]
role_name = _role_name.split(os.sep)[0]
if not role_name:
else:
role_name = 'default'
with open(file_name, 'r') as f:
packages = [
Expand Down Expand Up @@ -646,6 +739,8 @@ def run_v2(self, terms, variables=None, **kwargs):
if isinstance(value, (list, set)):
return_data[key] = sorted(value)
return_data['role_requirement_files'] = ROLE_REQUIREMENTS
return_data['role_requirements'] = ROLE_BREAKOUT_REQUIREMENTS
return_data['role_distro_packages'] = ROLE_DISTRO_BREAKOUT_PACKAGES
return [return_data]

def run_v1(self, terms, inject=None, **kwargs):
Expand Down Expand Up @@ -695,10 +790,12 @@ def run_v1(self, terms, inject=None, **kwargs):
if isinstance(value, (list, set)):
return_data[key] = sorted(value)
return_data['role_requirement_files'] = ROLE_REQUIREMENTS
return_data['role_requirements'] = ROLE_BREAKOUT_REQUIREMENTS
return_data['role_distro_packages'] = ROLE_DISTRO_BREAKOUT_PACKAGES
return [return_data]

# Used for testing and debuging usage: `python plugins/lookups/py_pkgs.py ../`
if __name__ == '__main__':
import sys
import json
print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4))
print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4, sort_keys=True))

0 comments on commit 2fc5ceb

Please sign in to comment.