From 884ca6fc9ee1cbd6619364f61050df7836f30d0f Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Tue, 27 Sep 2016 17:07:36 -0400 Subject: [PATCH 1/5] Remove backported modules now included in our Salt These modules were backported from Salt 2015.8 or Salt 2016.3, but they should no longer be necessary now that we are on Salt 2016.3.3. --- _modules/mac_service.py | 601 -------------------------- _states/pip_state.py | 917 ---------------------------------------- 2 files changed, 1518 deletions(-) delete mode 100644 _modules/mac_service.py delete mode 100644 _states/pip_state.py diff --git a/_modules/mac_service.py b/_modules/mac_service.py deleted file mode 100644 index 01aa11e44..000000000 --- a/_modules/mac_service.py +++ /dev/null @@ -1,601 +0,0 @@ -# This module is a backported copy of the salt/modules/mac_service.py module -# from Salt 2016.3.0 (at git revision 3e5218daea73f3f24b82a3078764ccb82c2a1ec9) -# without any other changes applied. -# -# The original copyright and licensing notice for this module is reproduced -# below in the double-# comment block: -# -## Salt - Remote execution system -## -## Copyright 2014-2015 SaltStack Team -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. - -# -*- coding: utf-8 -*- -''' -The service module for Mac OS X -.. versionadded:: 2016.3.0 -''' -from __future__ import absolute_import - -# Import python libs -import os -import re -import plistlib -from distutils.version import LooseVersion - -# Import salt libs -import salt.utils -import salt.utils.decorators as decorators -from salt.exceptions import CommandExecutionError - -# Import 3rd party libs -import salt.ext.six as six - -# Define the module's virtual name -__virtualname__ = 'service' - -__func_alias__ = { - 'list_': 'list', -} - - -def __virtual__(): - ''' - Only for Mac OS X with launchctl - ''' - if not salt.utils.is_darwin(): - return (False, 'Failed to load the mac_service module:\n' - 'Only available on Mac OS X systems.') - - if not salt.utils.which('launchctl'): - return (False, 'Failed to load the mac_service module:\n' - 'Required binary not found: "launchctl"') - - if not salt.utils.which('plutil'): - return (False, 'Failed to load the mac_service module:\n' - 'Required binary not found: "plutil"') - - if LooseVersion(__grains__['osrelease']) < LooseVersion('10.11'): - return (False, 'Failed to load the mac_service module:\n' - 'Requires OS X 10.11 or newer') - - return __virtualname__ - - -def _launchd_paths(): - ''' - Paths where launchd services can be found - ''' - return [ - '/Library/LaunchAgents', - '/Library/LaunchDaemons', - '/System/Library/LaunchAgents', - '/System/Library/LaunchDaemons', - ] - - -@decorators.memoize -def _available_services(): - ''' - Return a dictionary of all available services on the system - ''' - available_services = dict() - for launch_dir in _launchd_paths(): - for root, dirs, files in os.walk(launch_dir): - for file_name in files: - - # Must be a plist file - if not file_name.endswith('.plist'): - continue - - # Follow symbolic links of files in _launchd_paths - file_path = os.path.join(root, file_name) - true_path = os.path.realpath(file_path) - - # ignore broken symlinks - if not os.path.exists(true_path): - continue - - try: - # This assumes most of the plist files - # will be already in XML format - with salt.utils.fopen(file_path): - plist = plistlib.readPlist(true_path) - - except Exception: - # If plistlib is unable to read the file we'll need to use - # the system provided plutil program to do the conversion - cmd = '/usr/bin/plutil -convert xml1 -o - -- "{0}"'.format( - true_path) - plist_xml = __salt__['cmd.run'](cmd, output_loglevel='quiet') - if six.PY2: - plist = plistlib.readPlistFromString(plist_xml) - else: - plist = plistlib.readPlistFromBytes( - salt.utils.to_bytes(plist_xml)) - - try: - available_services[plist.Label.lower()] = { - 'file_name': file_name, - 'file_path': true_path, - 'plist': plist} - except AttributeError: - # Handle malformed plist files - available_services[os.path.basename(file_name).lower()] = { - 'file_name': file_name, - 'file_path': true_path, - 'plist': plist} - - return available_services - - -def _get_service(name): - ''' - Get information about a service. If the service is not found, raise an - error - - :param str name: Service label, file name, or full path - - :return: The service information for the service, otherwise an Error - :rtype: dict - ''' - services = _available_services() - name = name.lower() - - if name in services: - # Match on label - return services[name] - - for service in six.itervalues(services): - if service['file_path'].lower() == name: - # Match on full path - return service - basename, ext = os.path.splitext(service['file_name']) - if basename.lower() == name: - # Match on basename - return service - - # Could not find service - raise CommandExecutionError('Service not found: {0}'.format(name)) - - -def show(name): - ''' - Show properties of a launchctl service - - :param str name: Service label, file name, or full path - - :return: The service information if the service is found - :rtype: dict - - CLI Example: - - .. code-block:: bash - - salt '*' service.show org.cups.cupsd # service label - salt '*' service.show org.cups.cupsd.plist # file name - salt '*' service.show /System/Library/LaunchDaemons/org.cups.cupsd.plist # full path - ''' - return _get_service(name) - - -def launchctl(sub_cmd, *args, **kwargs): - ''' - Run a launchctl command and raise an error if it fails - - :param str sub_cmd: Sub command supplied to launchctl - - :param tuple args: Tuple containing additional arguments to pass to - launchctl - - :param dict kwargs: Dictionary containing arguments to pass to - ``cmd.run_all`` - - :param bool return_stdout: A keyword argument. If true return the stdout - of the launchctl command - - :return: ``True`` if successful, raise ``CommandExecutionError`` if not, or - the stdout of the launchctl command if requested - :rtype: bool, str - - CLI Example: - - .. code-block:: bash - - salt '*' service.launchctl debug org.cups.cupsd - ''' - # Get return type - return_stdout = kwargs.pop('return_stdout', False) - - # Construct command - cmd = ['launchctl', sub_cmd] - cmd.extend(args) - - # Run command - kwargs['python_shell'] = False - ret = __salt__['cmd.run_all'](cmd, **kwargs) - - # Raise an error or return successful result - if ret['retcode']: - out = 'Failed to {0} service:\n'.format(sub_cmd) - out += 'stdout: {0}\n'.format(ret['stdout']) - out += 'stderr: {0}\n'.format(ret['stderr']) - out += 'retcode: {0}\n'.format(ret['retcode']) - raise CommandExecutionError(out) - else: - return ret['stdout'] if return_stdout else True - - -def list_(name=None, runas=None): - ''' - Run launchctl list and return the output - - :param str name: The name of the service to list - - :param str runas: User to run launchctl commands - - :return: If a name is passed returns information about the named service, - otherwise returns a list of all services and pids - :rtype: str - - CLI Example: - - .. code-block:: bash - - salt '*' service.list - salt '*' service.list org.cups.cupsd - ''' - if name: - # Get service information and label - service = _get_service(name) - label = service['plist']['Label'] - - # Collect information on service: will raise an error if it fails - return launchctl('list', - label, - return_stdout=True, - output_loglevel='trace', - runas=runas) - - # Collect information on all services: will raise an error if it fails - return launchctl('list', - return_stdout=True, - output_loglevel='trace', - runas=runas) - - -def enable(name, runas=None): - ''' - Enable a launchd service. Raises an error if the service fails to be enabled - - :param str name: Service label, file name, or full path - - :param str runas: User to run launchctl commands - - :return: ``True`` if successful or if the service is already enabled - :rtype: bool - - CLI Example: - - .. code-block:: bash - - salt '*' service.enable org.cups.cupsd - ''' - # Get service information and label - service = _get_service(name) - label = service['plist']['Label'] - - # Enable the service: will raise an error if it fails - return launchctl('enable', 'system/{0}'.format(label), runas=runas) - - -def disable(name, runas=None): - ''' - Disable a launchd service. Raises an error if the service fails to be - disabled - - :param str name: Service label, file name, or full path - - :param str runas: User to run launchctl commands - - :return: ``True`` if successful or if the service is already disabled - :rtype: bool - - CLI Example: - - .. code-block:: bash - - salt '*' service.disable org.cups.cupsd - ''' - # Get service information and label - service = _get_service(name) - label = service['plist']['Label'] - - # disable the service: will raise an error if it fails - return launchctl('disable', 'system/{0}'.format(label), runas=runas) - - -def start(name, runas=None): - ''' - Start a launchd service. Raises an error if the service fails to start - - .. note:: - To start a service in Mac OS X the service must be enabled first. Use - ``service.enable`` to enable the service. - - :param str name: Service label, file name, or full path - - :param str runas: User to run launchctl commands - - :return: ``True`` if successful or if the service is already running - :rtype: bool - - CLI Example: - - .. code-block:: bash - - salt '*' service.start org.cups.cupsd - ''' - # Get service information and file path - service = _get_service(name) - path = service['file_path'] - - # Load the service: will raise an error if it fails - return launchctl('load', path, runas=runas) - - -def stop(name, runas=None): - ''' - Stop a launchd service. Raises an error if the service fails to stop - - .. note:: - Though ``service.stop`` will unload a service in Mac OS X, the service - will start on next boot unless it is disabled. Use ``service.disable`` - to disable the service - - :param str name: Service label, file name, or full path - - :param str runas: User to run launchctl commands - - :return: ``True`` if successful or if the service is already stopped - :rtype: bool - - CLI Example: - - .. code-block:: bash - - salt '*' service.stop org.cups.cupsd - ''' - # Get service information and file path - service = _get_service(name) - path = service['file_path'] - - # Disable the Launch Daemon: will raise an error if it fails - return launchctl('unload', path, runas=runas) - - -def restart(name, runas=None): - ''' - Unloads and reloads a launchd service. Raises an error if the service - fails to reload - - :param str name: Service label, file name, or full path - - :param str runas: User to run launchctl commands - - :return: ``True`` if successful - :rtype: bool - - CLI Example: - - .. code-block:: bash - - salt '*' service.restart org.cups.cupsd - ''' - # Restart the service: will raise an error if it fails - if enabled(name): - stop(name, runas=runas) - start(name, runas=runas) - - return True - - -def status(name, sig=None, runas=None): - ''' - Return the status for a service. - - :param str name: Used to find the service from launchctl. Can be any part - of the service name or a regex expression. - - :param str sig: Find the service with status.pid instead. Note that - ``name`` must still be provided. - - :param str runas: User to run launchctl commands - - :return: The PID for the service if it is running, otherwise an empty string - :rtype: str - - CLI Example: - - .. code-block:: bash - - salt '*' service.status cups - ''' - # Find service with ps - if sig: - return __salt__['status.pid'](sig) - - output = list_(runas=runas) - - # Used a string here instead of a list because that's what the linux version - # of this module does - pids = '' - for line in output.splitlines(): - if 'PID' in line: - continue - if re.search(name, line): - if line.split()[0].isdigit(): - if pids: - pids += '\n' - pids += line.split()[0] - - return pids - - -def available(name): - ''' - Check that the given service is available. - - :param str name: The name of the service - - :return: True if the service is available, otherwise False - :rtype: bool - - CLI Example: - - .. code-block:: bash - - salt '*' service.available com.openssh.sshd - ''' - try: - _get_service(name) - return True - except CommandExecutionError: - return False - - -def missing(name): - ''' - The inverse of service.available - Check that the given service is not available. - - :param str name: The name of the service - - :return: True if the service is not available, otherwise False - :rtype: bool - - CLI Example: - - .. code-block:: bash - - salt '*' service.missing com.openssh.sshd - ''' - return not available(name) - - -def enabled(name, runas=None): - ''' - Check if the specified service is enabled - - :param str name: The name of the service to look up - - :param str runas: User to run launchctl commands - - :return: True if the specified service enabled, otherwise False - :rtype: bool - - CLI Example: - - .. code-block:: bash - - salt '*' service.enabled org.cups.cupsd - ''' - # Try to list the service. If it can't be listed, it's not enabled - try: - list_(name=name, runas=runas) - return True - except CommandExecutionError: - return False - - -def disabled(name, runas=None): - ''' - Check if the specified service is not enabled. This is the opposite of - ``service.enabled`` - - :param str name: The name to look up - - :param str runas: User to run launchctl commands - - :return: True if the specified service is NOT enabled, otherwise False - :rtype: bool - - CLI Example: - - .. code-block:: bash - - salt '*' service.disabled org.cups.cupsd - ''' - # A service is disabled if it is not enabled - return not enabled(name, runas=runas) - - -def get_all(runas=None): - ''' - Return a list of services that are enabled or available. Can be used to - find the name of a service. - - :param str runas: User to run launchctl commands - - :return: A list of all the services available or enabled - :rtype: list - - CLI Example: - - .. code-block:: bash - - salt '*' service.get_all - ''' - # Get list of enabled services - enabled = get_enabled(runas=runas) - - # Get list of all services - available = list(_available_services().keys()) - - # Return composite list - return sorted(set(enabled + available)) - - -def get_enabled(runas=None): - ''' - Return a list of all services that are enabled. Can be used to find the - name of a service. - - :param str runas: User to run launchctl commands - - :return: A list of all the services enabled on the system - :rtype: list - - CLI Example: - - .. code-block:: bash - - salt '*' service.get_enabled - salt '*' service.get_enabled running=True - ''' - # Collect list of enabled services - stdout = list_(runas=runas) - service_lines = [line for line in stdout.splitlines()] - - # Construct list of enabled services - enabled = [] - for line in service_lines: - # Skip header line - if line.startswith('PID'): - continue - - pid, status, label = line.split('\t') - enabled.append(label) - - return sorted(set(enabled)) diff --git a/_states/pip_state.py b/_states/pip_state.py deleted file mode 100644 index f2443f064..000000000 --- a/_states/pip_state.py +++ /dev/null @@ -1,917 +0,0 @@ -# This module is a copy of the salt/states/pip_state.py module from Salt -# version 2015.5.8 (git revision a26c10a811fef3c7deca0559f431713f69a83289), -# with the patches from the following two PRs backported: -# - https://github.com/saltstack/salt/pull/33180 -# - https://github.com/saltstack/salt/pull/33383 -# These patches allow the module to work with old and new pips after an -# change in the internals of pip in pip version 8.1.2. -# -# The original copyright and licensing notice for this module is reproduced -# below in the double-# comment block: -# -## Salt - Remote execution system -## -## Copyright 2014-2015 SaltStack Team -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. - -# -*- coding: utf-8 -*- -''' -Installation of Python Packages Using pip -========================================= - -These states manage system installed python packages. Note that pip must be -installed for these states to be available, so pip states should include a -requisite to a pkg.installed state for the package which provides pip -(``python-pip`` in most cases). Example: - -.. code-block:: yaml - - python-pip: - pkg.installed - - virtualenvwrapper: - pip.installed: - - require: - - pkg: python-pip -''' - -# Import python libs -from __future__ import absolute_import -import re -import logging - -# Import salt libs -import salt.utils -from salt.version import SaltStackVersion as _SaltStackVersion -from salt.exceptions import CommandExecutionError, CommandNotFoundError - -# Import 3rd-party libs -import salt.ext.six as six -try: - import pip - HAS_PIP = True -except ImportError: - HAS_PIP = False - -if HAS_PIP is True: - try: - import pip.req - except ImportError: - HAS_PIP = False - # Remove references to the loaded pip module above so reloading works - import sys - del pip - if 'pip' in sys.modules: - del sys.modules['pip'] - -logger = logging.getLogger(__name__) - -# Define the module's virtual name -__virtualname__ = 'pip' - - -def __virtual__(): - ''' - Only load if the pip module is available in __salt__ - ''' - if 'pip.list' in __salt__: - return __virtualname__ - return False - - -def _find_key(prefix, pip_list): - ''' - Does a case-insensitive match in the pip_list for the desired package. - ''' - try: - match = next( - iter(x for x in pip_list if x.lower() == prefix.lower()) - ) - except StopIteration: - return None - else: - return match - - -def _fulfills_version_spec(version, version_spec): - ''' - Check version number against version specification info and return a - boolean value based on whether or not the version number meets the - specified version. - ''' - for oper, spec in version_spec: - if oper is None: - continue - if not salt.utils.compare_versions(ver1=version, oper=oper, ver2=spec): - return False - return True - - -def _check_pkg_version_format(pkg): - ''' - Takes a package name and version specification (if any) and checks it using - the pip library. - ''' - - ret = {'result': False, 'comment': None, - 'prefix': None, 'version_spec': None} - - if not HAS_PIP: - ret['comment'] = ( - 'An importable pip module is required but could not be found on ' - 'your system. This usually means that the system''s pip package ' - 'is not installed properly.' - ) - - return ret - - from_vcs = False - try: - # Get the requirement object from the pip library - try: - # With pip < 1.2, the __version__ attribute does not exist and - # vcs+URL urls are not properly parsed. - # The next line is meant to trigger an AttributeError and - # handle lower pip versions - logger.debug( - 'Installed pip version: {0}'.format(pip.__version__) - ) - install_req = pip.req.InstallRequirement.from_line(pkg) - except AttributeError: - logger.debug('Installed pip version is lower than 1.2') - supported_vcs = ('git', 'svn', 'hg', 'bzr') - if pkg.startswith(supported_vcs): - for vcs in supported_vcs: - if pkg.startswith(vcs): - from_vcs = True - install_req = pip.req.InstallRequirement.from_line( - pkg.split('{0}+'.format(vcs))[-1] - ) - break - else: - install_req = pip.req.InstallRequirement.from_line(pkg) - except ValueError as exc: - ret['result'] = False - if not from_vcs and '=' in pkg and '==' not in pkg: - ret['comment'] = ( - 'Invalid version specification in package {0}. \'=\' is ' - 'not supported, use \'==\' instead.'.format(pkg) - ) - return ret - ret['comment'] = ( - 'pip raised an exception while parsing {0!r}: {1}'.format( - pkg, exc - ) - ) - return ret - - if install_req.req is None: - # This is most likely an url and there's no way to know what will - # be installed before actually installing it. - ret['result'] = True - ret['prefix'] = '' - ret['version_spec'] = [] - else: - ret['result'] = True - try: - ret['prefix'] = install_req.req.project_name - ret['version_spec'] = install_req.req.specs - except Exception: - ret['prefix'] = re.sub('[^A-Za-z0-9.]+', '-', install_req.name) - if hasattr(install_req, "specifier"): - specifier = install_req.specifier - else: - specifier = install_req.req.specifier - ret['version_spec'] = [(spec.operator, spec.version) for spec in specifier] - - return ret - - -def _check_if_installed(prefix, state_pkg_name, version_spec, - ignore_installed, force_reinstall, - upgrade, user, cwd, bin_env): - - # result: None means the command failed to run - # result: True means the package is installed - # result: False means the package is not installed - ret = {'result': False, 'comment': None} - - # Check if the requested packated is already installed. - try: - pip_list = __salt__['pip.list'](prefix, bin_env=bin_env, - user=user, cwd=cwd) - prefix_realname = _find_key(prefix, pip_list) - except (CommandNotFoundError, CommandExecutionError) as err: - ret['result'] = None - ret['comment'] = 'Error installing {0!r}: {1}'.format(state_pkg_name, - err) - return ret - - # If the package was already installed, check - # the ignore_installed and force_reinstall flags - if ignore_installed is False and prefix_realname is not None: - if force_reinstall is False and not upgrade: - # Check desired version (if any) against currently-installed - if ( - any(version_spec) and - _fulfills_version_spec(pip_list[prefix_realname], - version_spec) - ) or (not any(version_spec)): - ret['result'] = True - ret['comment'] = ('Python package {0} was already ' - 'installed'.format(state_pkg_name)) - return ret - - return ret - - -def installed(name, - pkgs=None, - pip_bin=None, - requirements=None, - env=None, - bin_env=None, - use_wheel=False, - no_use_wheel=False, - log=None, - proxy=None, - timeout=None, - repo=None, - editable=None, - find_links=None, - index_url=None, - extra_index_url=None, - no_index=False, - mirrors=None, - build=None, - target=None, - download=None, - download_cache=None, - source=None, - upgrade=False, - force_reinstall=False, - ignore_installed=False, - exists_action=None, - no_deps=False, - no_install=False, - no_download=False, - install_options=None, - global_options=None, - user=None, - no_chown=False, - cwd=None, - activate=False, - pre_releases=False, - cert=None, - allow_all_external=False, - allow_external=None, - allow_unverified=None, - process_dependency_links=False, - env_vars=None, - use_vt=False): - ''' - Make sure the package is installed - - name - The name of the python package to install. You can also specify version - numbers here using the standard operators ``==, >=, <=``. If - ``requirements`` is given, this parameter will be ignored. - - Example: - - .. code-block:: yaml - - django: - pip.installed: - - name: django >= 1.6, <= 1.7 - - require: - - pkg: python-pip - - This will install the latest Django version greater than 1.6 but less - than 1.7. - - requirements - Path to a pip requirements file. If the path begins with salt:// - the file will be transferred from the master file server. - - user - The user under which to run pip - - use_wheel : False - Prefer wheel archives (requires pip>=1.4) - - no_use_wheel : False - Force to not use wheel archives (requires pip>=1.4) - - log - Log file where a complete (maximum verbosity) record will be kept - - proxy - Specify a proxy in the form - user:passwd@proxy.server:port. Note that the - user:password@ is optional and required only if you - are behind an authenticated proxy. If you provide - user@proxy.server:port then you will be prompted for a - password. - - timeout - Set the socket timeout (default 15 seconds) - - editable - install something editable (i.e. - git+https://github.com/worldcompany/djangoembed.git#egg=djangoembed) - - find_links - URL to look for packages at - - index_url - Base URL of Python Package Index - - extra_index_url - Extra URLs of package indexes to use in addition to ``index_url`` - - no_index - Ignore package index - - mirrors - Specific mirror URL(s) to query (automatically adds --use-mirrors) - - build - Unpack packages into ``build`` dir - - target - Install packages into ``target`` dir - - download - Download packages into ``download`` instead of installing them - - download_cache - Cache downloaded packages in ``download_cache`` dir - - source - Check out ``editable`` packages into ``source`` dir - - upgrade - Upgrade all packages to the newest available version - - force_reinstall - When upgrading, reinstall all packages even if they are already - up-to-date. - - ignore_installed - Ignore the installed packages (reinstalling instead) - - exists_action - Default action when a path already exists: (s)witch, (i)gnore, (w)ipe, - (b)ackup - - no_deps - Ignore package dependencies - - no_install - Download and unpack all packages, but don't actually install them - - no_chown - When user is given, do not attempt to copy and chown - a requirements file - - cwd - Current working directory to run pip from - - activate - Activates the virtual environment, if given via bin_env, - before running install. - - .. deprecated:: 2014.7.2 - If `bin_env` is given, pip will already be sourced from that - virualenv, making `activate` effectively a noop. - - pre_releases - Include pre-releases in the available versions - - cert - Provide a path to an alternate CA bundle - - allow_all_external - Allow the installation of all externally hosted files - - allow_external - Allow the installation of externally hosted files (comma separated list) - - allow_unverified - Allow the installation of insecure and unverifiable files (comma separated list) - - process_dependency_links - Enable the processing of dependency links - - bin_env : None - Absolute path to a virtual environment directory or absolute path to - a pip executable. The example below assumes a virtual environment - has been created at ``/foo/.virtualenvs/bar``. - - env_vars - Add or modify environment variables. Useful for tweaking build steps, - such as specifying INCLUDE or LIBRARY paths in Makefiles, build scripts or - compiler calls. This must be in the form of a dictionary or a mapping. - - Example: - - .. code-block:: yaml - - django: - pip.installed: - - name: django_app - - env_vars: - CUSTOM_PATH: /opt/django_app - VERBOSE: True - - use_vt - Use VT terminal emulation (see ouptut while installing) - - Example: - - .. code-block:: yaml - - django: - pip.installed: - - name: django >= 1.6, <= 1.7 - - bin_env: /foo/.virtualenvs/bar - - require: - - pkg: python-pip - - Or - - Example: - - .. code-block:: yaml - - django: - pip.installed: - - name: django >= 1.6, <= 1.7 - - bin_env: /foo/.virtualenvs/bar/bin/pip - - require: - - pkg: python-pip - - .. admonition:: Attention - - The following arguments are deprecated, do not use. - - pip_bin : None - Deprecated, use ``bin_env`` - - env : None - Deprecated, use ``bin_env`` - - .. versionchanged:: 0.17.0 - ``use_wheel`` option added. - - install_options - - Extra arguments to be supplied to the setup.py install command. - If you are using an option with a directory path, be sure to use - absolute path. - - Example: - - .. code-block:: yaml - - django: - pip.installed: - - name: django - - install_options: - - --prefix=/blah - - require: - - pkg: python-pip - - global_options - Extra global options to be supplied to the setup.py call before the - install command. - - .. versionadded:: 2014.1.3 - - .. admonition:: Attention - - As of Salt 0.17.0 the pip state **needs** an importable pip module. - This usually means having the system's pip package installed or running - Salt from an active `virtualenv`_. - - The reason for this requirement is because ``pip`` already does a - pretty good job parsing its own requirements. It makes no sense for - Salt to do ``pip`` requirements parsing and validation before passing - them to the ``pip`` library. It's functionality duplication and it's - more error prone. - - .. _`virtualenv`: http://www.virtualenv.org/en/latest/ - ''' - - if pip_bin and not bin_env: - bin_env = pip_bin - elif env and not bin_env: - bin_env = env - - # If pkgs is present, ignore name - if pkgs: - if not isinstance(pkgs, list): - return {'name': name, - 'result': False, - 'changes': {}, - 'comment': 'pkgs argument must be formatted as a list'} - else: - pkgs = [name] - - # Assumption: If `pkg` is not an `string`, it's a `collections.OrderedDict` - # prepro = lambda pkg: pkg if type(pkg) == str else \ - # ' '.join((pkg.items()[0][0], pkg.items()[0][1].replace(',', ';'))) - # pkgs = ','.join([prepro(pkg) for pkg in pkgs]) - prepro = lambda pkg: pkg if isinstance(pkg, str) else \ - ' '.join((six.iteritems(pkg)[0][0], six.iteritems(pkg)[0][1])) - pkgs = [prepro(pkg) for pkg in pkgs] - - ret = {'name': ';'.join(pkgs), 'result': None, - 'comment': '', 'changes': {}} - - # Check that the pip binary supports the 'use_wheel' option - if use_wheel: - min_version = '1.4' - cur_version = __salt__['pip.version'](bin_env) - if not salt.utils.compare_versions(ver1=cur_version, oper='>=', - ver2=min_version): - ret['result'] = False - ret['comment'] = ('The \'use_wheel\' option is only supported in ' - 'pip {0} and newer. The version of pip detected ' - 'was {1}.').format(min_version, cur_version) - return ret - - # Check that the pip binary supports the 'no_use_wheel' option - if no_use_wheel: - min_version = '1.4' - cur_version = __salt__['pip.version'](bin_env) - if not salt.utils.compare_versions(ver1=cur_version, oper='>=', - ver2=min_version): - ret['result'] = False - ret['comment'] = ('The \'no_use_wheel\' option is only supported in ' - 'pip {0} and newer. The version of pip detected ' - 'was {1}.').format(min_version, cur_version) - return ret - - # Deprecation warning for the repo option - if repo is not None: - msg = ('The \'repo\' argument to pip.installed is deprecated and will ' - 'be removed in Salt {version}. Please use \'name\' instead. ' - 'The current value for name, {0!r} will be replaced by the ' - 'value of repo, {1!r}'.format( - name, - repo, - version=_SaltStackVersion.from_name('Lithium').formatted_version - )) - salt.utils.warn_until('Lithium', msg) - ret.setdefault('warnings', []).append(msg) - name = repo - - # Get the packages parsed name and version from the pip library. - # This only is done when there is no requirements parameter. - pkgs_details = [] - if pkgs and not requirements: - comments = [] - for pkg in iter(pkgs): - out = _check_pkg_version_format(pkg) - if out['result'] is False: - ret['result'] = False - comments.append(out['comment']) - elif out['result'] is True: - pkgs_details.append((out['prefix'], pkg, out['version_spec'])) - - if ret['result'] is False: - ret['comment'] = '\n'.join(comments) - return ret - - # If a requirements file is specified, only install the contents of the - # requirements file. Similarly, using the --editable flag with pip should - # also ignore the "name" and "pkgs" parameters. - target_pkgs = [] - already_installed_comments = [] - if requirements or editable: - name = '' - comments = [] - # Append comments if this is a dry run. - if __opts__['test']: - ret['result'] = None - if requirements: - # TODO: Check requirements file against currently-installed - # packages to provide more accurate state output. - comments.append('Requirements file {0!r} will be ' - 'processed.'.format(requirements)) - if editable: - comments.append( - 'Package will be installed in editable mode (i.e. ' - 'setuptools "develop mode") from {0}.'.format(editable) - ) - ret['comment'] = ' '.join(comments) - return ret - - # No requirements case. - # Check pre-existence of the requested packages. - else: - for prefix, state_pkg_name, version_spec in pkgs_details: - - if prefix: - state_pkg_name = state_pkg_name - version_spec = version_spec - out = _check_if_installed(prefix, state_pkg_name, version_spec, - ignore_installed, force_reinstall, - upgrade, user, cwd, bin_env) - else: - out = {'result': False, 'comment': None} - - result = out['result'] - - # The package is not present. Add it to the pkgs to install. - if result is False: - # Replace commas (used for version ranges) with semicolons - # (which are not supported) in name so it does not treat - # them as multiple packages. - target_pkgs.append((prefix, state_pkg_name.replace(',', ';'))) - - # Append comments if this is a dry run. - if __opts__['test']: - msg = 'Python package {0} is set to be installed' - ret['result'] = None - ret['comment'] = msg.format(state_pkg_name) - return ret - - # The package is already present and will not be reinstalled. - elif result is True: - # Append comment stating its presence - already_installed_comments.append(out['comment']) - - # The command pip.list failed. Abort. - elif result is None: - ret['result'] = None - ret['comment'] = out['comment'] - return ret - - # Construct the string that will get passed to the install call - pkgs_str = ','.join([state_name for _, state_name in target_pkgs]) - - # Call to install the package. Actual installation takes place here - pip_install_call = __salt__['pip.install']( - pkgs='{0}'.format(pkgs_str) if pkgs_str else '', - requirements=requirements, - bin_env=bin_env, - use_wheel=use_wheel, - no_use_wheel=no_use_wheel, - log=log, - proxy=proxy, - timeout=timeout, - editable=editable, - find_links=find_links, - index_url=index_url, - extra_index_url=extra_index_url, - no_index=no_index, - mirrors=mirrors, - build=build, - target=target, - download=download, - download_cache=download_cache, - source=source, - upgrade=upgrade, - force_reinstall=force_reinstall, - ignore_installed=ignore_installed, - exists_action=exists_action, - no_deps=no_deps, - no_install=no_install, - no_download=no_download, - install_options=install_options, - global_options=global_options, - user=user, - no_chown=no_chown, - cwd=cwd, - activate=activate, - pre_releases=pre_releases, - cert=cert, - allow_all_external=allow_all_external, - allow_external=allow_external, - allow_unverified=allow_unverified, - process_dependency_links=process_dependency_links, - saltenv=__env__, - env_vars=env_vars, - use_vt=use_vt - ) - - # Check the retcode for success, but don't fail if using pip1 and the package is - # already present. Pip1 returns a retcode of 1 (instead of 0 for pip2) if you run - # "pip install" without any arguments. See issue #21845. - if pip_install_call and \ - (pip_install_call.get('retcode', 1) == 0 or pip_install_call.get('stdout', '').startswith( - 'You must give at least one requirement to install')): - ret['result'] = True - - if requirements or editable: - comments = [] - if requirements: - for line in pip_install_call.get('stdout', '').split('\n'): - if not line.startswith('Requirement already satisfied') \ - and line != 'Cleaning up...': - ret['changes']['requirements'] = True - if ret['changes'].get('requirements'): - comments.append('Successfully processed requirements file ' - '{0}.'.format(requirements)) - else: - comments.append('Requirements were already installed.') - - if editable: - comments.append('Package successfully installed from VCS ' - 'checkout {0}.'.format(editable)) - ret['changes']['editable'] = True - ret['comment'] = ' '.join(comments) - else: - - # Check that the packages set to be installed were installed. - # Create comments reporting success and failures - pkg_404_comms = [] - - for prefix, state_name in target_pkgs: - - # Case for packages that are not an URL - if prefix: - pipsearch = __salt__['pip.list'](prefix, bin_env, - user=user, cwd=cwd) - - # If we didnt find the package in the system after - # installing it report it - if not pipsearch: - msg = ( - 'There was no error installing package \'{0}\' ' - 'although it does not show when calling ' - '\'pip.freeze\'.'.format(pkg) - ) - pkg_404_comms.append(msg) - else: - pkg_name = _find_key(prefix, pipsearch) - ver = pipsearch[pkg_name] - ret['changes']['{0}=={1}'.format(pkg_name, - ver)] = 'Installed' - # Case for packages that are an URL - else: - ret['changes']['{0}==???'.format(state_name)] = 'Installed' - - # Set comments - aicomms = '\n'.join(already_installed_comments) - succ_comm = 'All packages were successfully installed'\ - if not pkg_404_comms else '\n'.join(pkg_404_comms) - ret['comment'] = aicomms + ('\n' if aicomms else '') + succ_comm - - return ret - - elif pip_install_call: - ret['result'] = False - if 'stdout' in pip_install_call: - error = 'Error: {0} {1}'.format(pip_install_call['stdout'], - pip_install_call['stderr']) - else: - error = 'Error: {0}'.format(pip_install_call['comment']) - - if requirements or editable: - comments = [] - if requirements: - comments.append('Unable to process requirements file ' - '{0}.'.format(requirements)) - if editable: - comments.append('Unable to install from VCS checkout' - '{0}.'.format(editable)) - comments.append(error) - ret['comment'] = ' '.join(comments) - else: - pkgs_str = ', '.join([state_name for _, state_name in target_pkgs]) - aicomms = '\n'.join(already_installed_comments) - error_comm = ('Failed to install packages: {0}. ' - '{1}'.format(pkgs_str, error)) - ret['comment'] = aicomms + ('\n' if aicomms else '') + error_comm - else: - ret['result'] = False - ret['comment'] = 'Could not install package' - - return ret - - -def removed(name, - requirements=None, - bin_env=None, - log=None, - proxy=None, - timeout=None, - user=None, - cwd=None, - use_vt=False): - ''' - Make sure that a package is not installed. - - name - The name of the package to uninstall - user - The user under which to run pip - bin_env : None - the pip executable or virtualenenv to use - use_vt - Use VT terminal emulation (see ouptut while installing) - ''' - ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} - - try: - pip_list = __salt__['pip.list'](bin_env=bin_env, user=user, cwd=cwd) - except (CommandExecutionError, CommandNotFoundError) as err: - ret['result'] = False - ret['comment'] = 'Error uninstalling \'{0}\': {1}'.format(name, err) - return ret - - if name not in pip_list: - ret['result'] = True - ret['comment'] = 'Package is not installed.' - return ret - - if __opts__['test']: - ret['result'] = None - ret['comment'] = 'Package {0} is set to be removed'.format(name) - return ret - - if __salt__['pip.uninstall'](pkgs=name, - requirements=requirements, - bin_env=bin_env, - log=log, - proxy=proxy, - timeout=timeout, - user=user, - cwd=cwd, - use_vt=use_vt): - ret['result'] = True - ret['changes'][name] = 'Removed' - ret['comment'] = 'Package was successfully removed.' - else: - ret['result'] = False - ret['comment'] = 'Could not remove package.' - return ret - - -def uptodate(name, - bin_env=None, - user=None, - cwd=None, - use_vt=False): - ''' - .. versionadded:: 2015.5.0 - - Verify that the system is completely up to date. - - name - The name has no functional value and is only used as a tracking - reference - user - The user under which to run pip - bin_env - the pip executable or virtualenenv to use - use_vt - Use VT terminal emulation (see ouptut while installing) - ''' - ret = {'name': name, - 'changes': {}, - 'result': False, - 'comment': 'Failed to update.'} - - try: - packages = __salt__['pip.list_upgrades'](bin_env=bin_env, user=user, cwd=cwd) - except Exception as e: - ret['comment'] = str(e) - return ret - - if not packages: - ret['comment'] = 'System is already up-to-date.' - ret['result'] = True - return ret - elif __opts__['test']: - ret['comment'] = 'System update will be performed' - ret['result'] = None - return ret - - updated = __salt__['pip.upgrade'](bin_env=bin_env, user=user, cwd=cwd, use_vt=use_vt) - - if updated.get('result') is False: - ret.update(updated) - elif updated: - ret['changes'] = updated - ret['comment'] = 'Upgrade successful.' - ret['result'] = True - else: - ret['comment'] = 'Upgrade failed.' - - return ret From db005cb0d6cb13aef6d250b84bf3afc832bd13d1 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Tue, 27 Sep 2016 17:42:48 -0400 Subject: [PATCH 2/5] Ensure Salt is linked by Homebrew after install If the previous version is the same as the new version, then we will unlink Salt, but Homebrew will not relink it when (re-)installing, so make sure to relink manually afterwards. --- .travis/install_salt.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis/install_salt.sh b/.travis/install_salt.sh index abdc0b5eb..6526171a0 100755 --- a/.travis/install_salt.sh +++ b/.travis/install_salt.sh @@ -28,6 +28,8 @@ install_salt () { brew unlink saltstack fi brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/9e3a66b6b7ca978bfea86897dcc3391c37f9f0ef/Formula/saltstack.rb + # In case we had the same version previously, we need to relink + brew link saltstack else printf >&2 "$0: unknown operating system ${OS_NAME}\n" exit 1 From d42a3948f06392c1046eae99dce551e564004901 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 30 Sep 2016 11:35:02 -0400 Subject: [PATCH 3/5] Fix homebrew autoconf install/linking script We need to unlink before relinking or else homebrew complains. Be more strict in the grep for each package to avoid false positives. --- .../files/install-homebrew-autoconf213.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/servo-build-dependencies/files/install-homebrew-autoconf213.sh b/servo-build-dependencies/files/install-homebrew-autoconf213.sh index c1437b0ee..5a37a961b 100644 --- a/servo-build-dependencies/files/install-homebrew-autoconf213.sh +++ b/servo-build-dependencies/files/install-homebrew-autoconf213.sh @@ -17,7 +17,12 @@ set -o pipefail # only via editing the Formula to add `keg_only`. # Hence, just ignore errors for now, and double-check everything at the end. brew_install() { set +o errexit; brew install "$@"; set -o errexit; } -brew_link() { set +o errexit; brew link "$@"; set -o errexit; } +brew_link() { + brew unlink "${1}" + set +o errexit + brew link --overwrite "${1}" + set -o errexit +} # Use "yes"/"no" for conditionals (not "true"/"false") # to avoid confusion with the true/false commands @@ -28,7 +33,7 @@ autoconf213_linked="no" set_autoconf_vars() { - if brew list | grep 'autoconf' >/dev/null; then + if brew list | grep '^autoconf$' >/dev/null; then autoconf_installed="yes" if readlink '/usr/local/share/info/autoconf.info' \ | grep 'Cellar/autoconf/' >/dev/null; then @@ -39,7 +44,7 @@ set_autoconf_vars() { set_autoconf213_vars() { - if brew list | grep 'autoconf213' >/dev/null; then + if brew list | grep '^autoconf213$' >/dev/null; then autoconf213_installed="yes" if readlink '/usr/local/bin/autoconf213' \ | grep 'Cellar/autoconf213/' >/dev/null; then @@ -65,7 +70,7 @@ check() { else if [[ "${verbose}" == "yes" ]]; then printf "%s\n" "autoconf/autoconf213 check failed:" - printf "%s %s\n" "autoconf 213 installed?" \ + printf "%s %s\n" "autoconf213 installed?" \ "${autoconf213_installed}" printf "%s %s\n" "autoconf installed?" \ "${autoconf_installed}" @@ -92,7 +97,7 @@ main() { fi set_autoconf213_vars if [[ "${autoconf213_linked}" == "no" ]]; then - brew_link --overwrite autoconf213 + brew_link autoconf213 fi set_autoconf_vars @@ -101,7 +106,7 @@ main() { fi set_autoconf_vars if [[ "${autoconf_fully_linked}" == "no" ]]; then - brew_link --overwrite autoconf + brew_link autoconf fi check 'verbose' # errexit will handle return in failure case From ef5c44769a670740f64ef05b929eb4302ee55a66 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 30 Sep 2016 11:36:28 -0400 Subject: [PATCH 4/5] Switch back to XCode 7.3 Homebrew 1.0.4 (https://github.com/Homebrew/brew/releases/tag/1.0.4) relaxed the XCode 8 requirement to only apply on Sierra on above. Because Servo does not yet build with XCode 8, and our builders and Travis are all using 10.11, we can continue to use XCode 7.3 for now. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2aa811894..d6430ca0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ matrix: - SALT_NODE_ID=servo-mac1 - SALT_FROM_SCRATCH=true os: osx - osx_image: xcode8 + osx_image: xcode7.3 - env: - SALT_NODE_ID=servo-linux1 - SALT_FROM_SCRATCH=true @@ -44,7 +44,7 @@ matrix: - SALT_NODE_ID=servo-mac1 - SALT_FROM_SCRATCH=false os: osx - osx_image: xcode8 + osx_image: xcode7.3 - env: - SALT_NODE_ID=servo-linux1 - SALT_FROM_SCRATCH=false From ab3d5f0c8d378c6e9b45cbdb25e238d263b5d3a7 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 30 Sep 2016 11:38:23 -0400 Subject: [PATCH 5/5] Print brew version when installing for debugging --- .travis/install_salt.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis/install_salt.sh b/.travis/install_salt.sh index 6526171a0..3323c46ed 100755 --- a/.travis/install_salt.sh +++ b/.travis/install_salt.sh @@ -22,6 +22,9 @@ install_salt () { elif [[ "${OS_NAME}" == "osx" ]]; then printf "$0: installing salt for Mac OS X\n" brew update + printf "\nhomebrew --version output:\n" + brew --version # For debugging + printf "\n" # Unlink allows switching versions, # I wish Homebrew had an atomic operation for pinned upgrades if brew list | grep 'saltstack' >/dev/null; then