From fa029bddef8a0947db7019756c48be3e2fe1dd82 Mon Sep 17 00:00:00 2001 From: Samuel Marks Date: Sun, 26 Jan 2014 19:25:45 +1100 Subject: [PATCH 1/2] Attempting to add an `--upgrade-all` option. Also some minor refactoring. --- pip/basecommand.py | 13 +---- pip/cmdoptions.py | 31 +++++----- pip/commands/install.py | 124 +++++++++++++++++++--------------------- 3 files changed, 76 insertions(+), 92 deletions(-) diff --git a/pip/basecommand.py b/pip/basecommand.py index e4670191293..04128488616 100644 --- a/pip/basecommand.py +++ b/pip/basecommand.py @@ -130,23 +130,12 @@ def main(self, args): logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = PREVIOUS_BUILD_DIR_ERROR - except (InstallationError, UninstallationError): + except (InstallationError, UninstallationError, BadCommand, CommandError): e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR - except BadCommand: - e = sys.exc_info()[1] - logger.fatal(str(e)) - logger.info('Exception information:\n%s' % format_exc()) - store_log = True - exit = ERROR - except CommandError: - e = sys.exc_info()[1] - logger.fatal('ERROR: %s' % e) - logger.info('Exception information:\n%s' % format_exc()) - exit = ERROR except KeyboardInterrupt: logger.fatal('Operation cancelled by user') logger.info('Exception information:\n%s' % format_exc()) diff --git a/pip/cmdoptions.py b/pip/cmdoptions.py index 8ed3d91fc6e..d0b8daac703 100644 --- a/pip/cmdoptions.py +++ b/pip/cmdoptions.py @@ -23,12 +23,15 @@ def make_option_group(group, parser): option_group.add_option(option.make()) return option_group + class OptionMaker(object): """Class that stores the args/kwargs that would be used to make an Option, for making them later, and uses deepcopy's to reset state.""" + def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs + def make(self): args_copy = copy.deepcopy(self.args) kwargs_copy = copy.deepcopy(self.kwargs) @@ -143,7 +146,7 @@ def make(self): action='append', metavar='action', help="Default action when a path already exists: " - "(s)witch, (i)gnore, (w)ipe, (b)ackup.") + "(s)witch, (i)gnore, (w)ipe, (b)ackup.") cert = OptionMaker( '--cert', @@ -151,7 +154,7 @@ def make(self): type='str', default='', metavar='path', - help = "Path to alternate CA bundle.") + help="Path to alternate CA bundle.") index_url = OptionMaker( '-i', '--index-url', '--pypi-url', @@ -175,7 +178,7 @@ def make(self): default=False, help='Ignore package index (only looking at --find-links URLs instead).') -find_links = OptionMaker( +find_links = OptionMaker( '-f', '--find-links', dest='find_links', action='append', @@ -261,7 +264,7 @@ def make(self): default=[], metavar='file', help='Install from the given requirements file. ' - 'This option can be used multiple times.') + 'This option can be used multiple times.') use_wheel = OptionMaker( '--use-wheel', @@ -299,8 +302,8 @@ def make(self): metavar='dir', default=build_prefix, help='Directory to unpack packages into and build in. ' - 'The default in a virtualenv is "/build". ' - 'The default for global installs is "/pip_build_".') + 'The default in a virtualenv is "/build". ' + 'The default for global installs is "/pip_build_".') install_options = OptionMaker( '--install-option', @@ -308,9 +311,9 @@ def make(self): action='append', metavar='options', help="Extra arguments to be supplied to the setup.py install " - "command (use like --install-option=\"--install-scripts=/usr/local/bin\"). " - "Use multiple --install-option options to pass multiple options to setup.py install. " - "If you are using an option with a directory path, be sure to use absolute path.") + "command (use like --install-option=\"--install-scripts=/usr/local/bin\"). " + "Use multiple --install-option options to pass multiple options to setup.py install. " + "If you are using an option with a directory path, be sure to use absolute path.") global_options = OptionMaker( '--global-option', @@ -318,7 +321,7 @@ def make(self): action='append', metavar='options', help="Extra global options to be supplied to the setup.py " - "call before the install command.") + "call before the install command.") no_clean = OptionMaker( '--no-clean', @@ -349,8 +352,8 @@ def make(self): skip_requirements_regex, exists_action, cert, - ] - } + ] +} index_group = { 'name': 'Package Index Options', @@ -367,5 +370,5 @@ def make(self): allow_unsafe, no_allow_unsafe, process_dependency_links, - ] - } + ] +} diff --git a/pip/commands/install.py b/pip/commands/install.py index f799366d99c..31c9f5c8b71 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -8,6 +8,7 @@ from pip.basecommand import Command from pip.index import PackageFinder from pip.exceptions import InstallationError, CommandError, PreviousBuildDirError +from pip import get_installed_distributions from pip import cmdoptions @@ -38,134 +39,121 @@ class InstallCommand(Command): summary = 'Install packages.' bundle = False - def __init__(self, *args, **kw): - super(InstallCommand, self).__init__(*args, **kw) - - cmd_opts = self.cmd_opts - - cmd_opts.add_option( + def populate_cmd_opts(self): + self.cmd_opts.add_option( '-e', '--editable', dest='editables', action='append', default=[], metavar='path/url', help='Install a project in editable mode (i.e. setuptools "develop mode") from a local project path or a VCS url.') - - cmd_opts.add_option(cmdoptions.requirements.make()) - cmd_opts.add_option(cmdoptions.build_dir.make()) - - cmd_opts.add_option( + self.cmd_opts.add_option(cmdoptions.requirements.make()) + self.cmd_opts.add_option(cmdoptions.build_dir.make()) + self.cmd_opts.add_option( '-t', '--target', dest='target_dir', metavar='dir', default=None, help='Install packages into .') - - cmd_opts.add_option( + self.cmd_opts.add_option( '-d', '--download', '--download-dir', '--download-directory', dest='download_dir', metavar='dir', default=None, help="Download packages into instead of installing them, regardless of what's already installed.") - - cmd_opts.add_option(cmdoptions.download_cache.make()) - - cmd_opts.add_option( + self.cmd_opts.add_option(cmdoptions.download_cache.make()) + self.cmd_opts.add_option( '--src', '--source', '--source-dir', '--source-directory', dest='src_dir', metavar='dir', default=src_prefix, help='Directory to check out editable projects into. ' - 'The default in a virtualenv is "/src". ' - 'The default for global installs is "/src".') - - cmd_opts.add_option( + 'The default in a virtualenv is "/src". ' + 'The default for global installs is "/src".') + self.cmd_opts.add_option( '-U', '--upgrade', dest='upgrade', action='store_true', help='Upgrade all packages to the newest available version. ' - 'This process is recursive regardless of whether a dependency is already satisfied.') - - cmd_opts.add_option( + 'This process is recursive regardless of whether a dependency is already satisfied.') + self.cmd_opts.add_option( + '--upgrade_all', + dest='upgrade_all', + action='store_true', + help='Upgrades all packages to the newest available version. ' + 'This process is recursive regardless of whether a dependency is already satisfied.') + self.cmd_opts.add_option( '--force-reinstall', dest='force_reinstall', action='store_true', help='When upgrading, reinstall all packages even if they are ' 'already up-to-date.') - - cmd_opts.add_option( + self.cmd_opts.add_option( '-I', '--ignore-installed', dest='ignore_installed', action='store_true', help='Ignore the installed packages (reinstalling instead).') - - cmd_opts.add_option(cmdoptions.no_deps.make()) - - cmd_opts.add_option( + self.cmd_opts.add_option(cmdoptions.no_deps.make()) + self.cmd_opts.add_option( '--no-install', dest='no_install', action='store_true', help="DEPRECATED. Download and unpack all packages, but don't actually install them.") - - cmd_opts.add_option( + self.cmd_opts.add_option( '--no-download', dest='no_download', action="store_true", help="DEPRECATED. Don't download any packages, just install the ones already downloaded " - "(completes an install run with --no-install).") - - cmd_opts.add_option(cmdoptions.install_options.make()) - cmd_opts.add_option(cmdoptions.global_options.make()) - - cmd_opts.add_option( + "(completes an install run with --no-install).") + self.cmd_opts.add_option(cmdoptions.install_options.make()) + self.cmd_opts.add_option(cmdoptions.global_options.make()) + self.cmd_opts.add_option( '--user', dest='use_user_site', action='store_true', help='Install using the user scheme.') - - cmd_opts.add_option( + self.cmd_opts.add_option( '--egg', dest='as_egg', action='store_true', help="Install packages as eggs, not 'flat', like pip normally does. This option is not about installing *from* eggs. (WARNING: Because this option overrides pip's normal install logic, requirements files may not behave as expected.)") - - cmd_opts.add_option( + self.cmd_opts.add_option( '--root', dest='root_path', metavar='dir', default=None, help="Install everything relative to this alternate root directory.") - - cmd_opts.add_option( + self.cmd_opts.add_option( "--compile", action="store_true", dest="compile", default=True, help="Compile py files to pyc", ) - - cmd_opts.add_option( + self.cmd_opts.add_option( "--no-compile", action="store_false", dest="compile", help="Do not compile py files to pyc", ) - - cmd_opts.add_option(cmdoptions.use_wheel.make()) - cmd_opts.add_option(cmdoptions.no_use_wheel.make()) - - cmd_opts.add_option( + self.cmd_opts.add_option(cmdoptions.use_wheel.make()) + self.cmd_opts.add_option(cmdoptions.no_use_wheel.make()) + self.cmd_opts.add_option( '--pre', action='store_true', default=False, help="Include pre-release and development versions. By default, pip only finds stable versions.") + self.cmd_opts.add_option(cmdoptions.no_clean.make()) + + def __init__(self, *args, **kw): + super(InstallCommand, self).__init__(*args, **kw) - cmd_opts.add_option(cmdoptions.no_clean.make()) + self.populate_cmd_opts() index_opts = cmdoptions.make_option_group(cmdoptions.index_group, self.parser) self.parser.insert_option_group(0, index_opts) - self.parser.insert_option_group(0, cmd_opts) + self.parser.insert_option_group(0, self.cmd_opts) def _build_package_finder(self, options, index_urls, session): """ @@ -180,15 +168,18 @@ def _build_package_finder(self, options, index_urls, session): allow_unverified=options.allow_unverified, allow_all_external=options.allow_all_external, allow_all_prereleases=options.pre, - process_dependency_links= - options.process_dependency_links, - session=session, - ) + process_dependency_links=options.process_dependency_links, + session=session) def run(self, options, args): + if options.upgrade_all: + args += [package.split(' ', 1)[0] for package in + get_installed_distributions(local_only=bool(options.use_user_site))] + logger.notify('Attempting to upgrade: %s' % args) if options.no_install or options.no_download: - logger.deprecated('1.7', "DEPRECATION: '--no-install' and '--no-download` are deprecated. See https://github.com/pypa/pip/issues/906.") + logger.deprecated('1.7', + "DEPRECATION: '--no-install' and '--no-download` are deprecated. See https://github.com/pypa/pip/issues/906.") if options.download_dir: options.no_install = True @@ -198,7 +189,8 @@ def run(self, options, args): install_options = options.install_options or [] if options.use_user_site: if virtualenv_no_global(): - raise InstallationError("Can not perform a '--user' install. User site-packages are not visible in this virtualenv.") + raise InstallationError( + "Can not perform a '--user' install. User site-packages are not visible in this virtualenv.") install_options.append('--user') temp_target_dir = None @@ -218,15 +210,15 @@ def run(self, options, args): if options.use_mirrors: logger.deprecated("1.7", - "--use-mirrors has been deprecated and will be removed" - " in the future. Explicit uses of --index-url and/or " - "--extra-index-url is suggested.") + "--use-mirrors has been deprecated and will be removed" + " in the future. Explicit uses of --index-url and/or " + "--extra-index-url is suggested.") if options.mirrors: logger.deprecated("1.7", - "--mirrors has been deprecated and will be removed in " - " the future. Explicit uses of --index-url and/or " - "--extra-index-url is suggested.") + "--mirrors has been deprecated and will be removed in " + " the future. Explicit uses of --index-url and/or " + "--extra-index-url is suggested.") index_urls += options.mirrors session = self._build_session(options) @@ -305,6 +297,6 @@ def run(self, options, args): shutil.move( os.path.join(lib_dir, item), os.path.join(options.target_dir, item) - ) + ) shutil.rmtree(temp_target_dir) return requirement_set From 85d8c0e9a0a452675dd9622574fdcc3ba34baada Mon Sep 17 00:00:00 2001 From: Samuel Marks Date: Sun, 26 Jan 2014 19:59:07 +1100 Subject: [PATCH 2/2] Added + tested an `--upgrade_all` option. --- pip/commands/install.py | 7 ++-- pip/util.py | 74 ++++++++++++++++++++--------------------- 2 files changed, 38 insertions(+), 43 deletions(-) diff --git a/pip/commands/install.py b/pip/commands/install.py index 31c9f5c8b71..f101a38406f 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -1,5 +1,4 @@ import os -import sys import tempfile import shutil from pip.req import InstallRequirement, RequirementSet, parse_requirements @@ -8,8 +7,7 @@ from pip.basecommand import Command from pip.index import PackageFinder from pip.exceptions import InstallationError, CommandError, PreviousBuildDirError -from pip import get_installed_distributions -from pip import cmdoptions +from pip import cmdoptions, get_installed_distributions class InstallCommand(Command): @@ -173,9 +171,8 @@ def _build_package_finder(self, options, index_urls, session): def run(self, options, args): if options.upgrade_all: - args += [package.split(' ', 1)[0] for package in + args += [package.project_name for package in get_installed_distributions(local_only=bool(options.use_user_site))] - logger.notify('Attempting to upgrade: %s' % args) if options.no_install or options.no_download: logger.deprecated('1.7', diff --git a/pip/util.py b/pip/util.py index 9ea46b98539..7a61c70a2c8 100644 --- a/pip/util.py +++ b/pip/util.py @@ -11,7 +11,7 @@ import textwrap from pip.exceptions import InstallationError, BadCommand, PipError -from pip.backwardcompat import(WindowsError, string_types, raw_input, +from pip.backwardcompat import (WindowsError, string_types, raw_input, console_to_str, user_site, PermissionError) from pip.locations import (site_packages, running_under_virtualenv, virtualenv_no_global, write_delete_marker_file) @@ -51,14 +51,14 @@ def rmtree_errorhandler(func, path, exc_info): read-only attribute, and hopefully continue without problems.""" exctype, value = exc_info[:2] if not ((exctype is WindowsError and value.args[0] == 5) or #others - (exctype is OSError and value.args[0] == 13) or #python2.4 - (exctype is PermissionError and value.args[3] == 5) #python3.3 - ): + (exctype is OSError and value.args[0] == 13) or #python2.4 + (exctype is PermissionError and value.args[3] == 5) #python3.3 + ): raise - # file type should currently be read only + # file type should currently be read only if ((os.stat(path).st_mode & stat.S_IREAD) != stat.S_IREAD): raise - # convert to read/write + # convert to read/write os.chmod(path, stat.S_IWRITE) # use the original function to repeat the operation func(path) @@ -90,14 +90,14 @@ def find_command(cmd, paths=None, pathext=None): paths = os.environ.get('PATH', '').split(os.pathsep) if isinstance(paths, string_types): paths = [paths] - # check if there are funny path extensions for executables, e.g. Windows + # check if there are funny path extensions for executables, e.g. Windows if pathext is None: pathext = get_pathext() pathext = [ext for ext in pathext.lower().split(os.pathsep) if len(ext)] # don't use extensions if the command ends with one of them if os.path.splitext(cmd)[1].lower() in pathext: pathext = [''] - # check if we find the command on PATH + # check if we find the command on PATH for path in paths: # try without extension first cmd_path = os.path.join(path, cmd) @@ -171,7 +171,6 @@ def __repr__(self): Inf = _Inf() #this object is not currently used as a sortable in our code del _Inf - _normalize_re = re.compile(r'[^a-z]', re.I) @@ -180,12 +179,12 @@ def normalize_name(name): def format_size(bytes): - if bytes > 1000*1000: - return '%.1fMB' % (bytes/1000.0/1000) - elif bytes > 10*1000: - return '%ikB' % (bytes/1000) + if bytes > 1000 * 1000: + return '%.1fMB' % (bytes / 1000.0 / 1000) + elif bytes > 10 * 1000: + return '%ikB' % (bytes / 1000) elif bytes > 1000: - return '%.1fkB' % (bytes/1000.0) + return '%.1fkB' % (bytes / 1000.0) else: return '%ibytes' % bytes @@ -264,7 +263,7 @@ def make_path_relative(path, rel_to): while path_parts and rel_to_parts and path_parts[0] == rel_to_parts[0]: path_parts.pop(0) rel_to_parts.pop(0) - full_parts = ['..']*len(rel_to_parts) + path_parts + [path_filename] + full_parts = ['..'] * len(rel_to_parts) + path_parts + [path_filename] if full_parts == ['']: return '.' + os.path.sep return os.path.sep.join(full_parts) @@ -336,6 +335,7 @@ def dist_in_usersite(dist): else: return False + def dist_in_site_packages(dist): """ Return True if given Distribution is installed in distutils.sysconfig.get_python_lib(). @@ -347,9 +347,11 @@ def dist_is_editable(dist): """Is distribution an editable install?""" #TODO: factor out determining editableness out of FrozenRequirement from pip import FrozenRequirement + req = FrozenRequirement.from_dist(dist, []) return req.editable + def get_installed_distributions(local_only=True, skip=('setuptools', 'pip', 'python', 'distribute', 'wsgiref'), include_editables=True, @@ -369,27 +371,16 @@ def get_installed_distributions(local_only=True, If ``editables_only`` is True , only report editables. """ - if local_only: - local_test = dist_is_local - else: - local_test = lambda d: True - - if include_editables: - editable_test = lambda d: True - else: - editable_test = lambda d: not dist_is_editable(d) + local_test = dist_is_local if local_only else lambda _: True - if editables_only: - editables_only_test = lambda d: dist_is_editable(d) - else: - editables_only_test = lambda d: True + editable_test = lambda d: True if include_editables else not dist_is_editable(d) + editables_only_test = lambda d: dist_is_editable(d) if editables_only else True return [d for d in pkg_resources.working_set - if local_test(d) - and d.key not in skip - and editable_test(d) - and editables_only_test(d) - ] + if (local_test(d) + and d.key not in skip + and editable_test(d) + and editables_only_test(d))] def egg_link_path(dist): @@ -408,6 +399,7 @@ def egg_link_path(dist): This method will just return the first one found. """ sites = [] + if running_under_virtualenv(): if virtualenv_no_global(): sites.append(site_packages) @@ -443,13 +435,15 @@ def dist_location(dist): def get_terminal_size(): """Returns a tuple (x, y) representing the width(x) and the height(x) in characters of the terminal window.""" + def ioctl_GWINSZ(fd): try: import fcntl import termios import struct + cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, - '1234')) + '1234')) except: return None if cr == (0, 0): @@ -457,6 +451,7 @@ def ioctl_GWINSZ(fd): if cr == (0, 0): return None return cr + cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) if not cr: try: @@ -514,10 +509,10 @@ def unzip_file(filename, location, flatten=True): fp.close() mode = info.external_attr >> 16 # if mode and regular file and any execute permissions for user/group/world? - if mode and stat.S_ISREG(mode) and mode & 0o111: + if mode and stat.S_ISREG(mode) and mode & 0o111: # make dest file have execute for user/group/world (chmod +x) # no-op on windows per python docs - os.chmod(fn, (0o777-current_umask() | 0o111)) + os.chmod(fn, (0o777 - current_umask() | 0o111)) finally: zipfp.close() @@ -593,7 +588,7 @@ def untar_file(filename, location): if member.mode & 0o111: # make dest file have execute for user/group/world # no-op on windows per python docs - os.chmod(path, (0o777-current_umask() | 0o111)) + os.chmod(path, (0o777 - current_umask() | 0o111)) finally: tar.close() @@ -608,7 +603,7 @@ def create_download_cache_folder(folder): def cache_download(target_file, temp_location, content_type): logger.notify('Storing download in cache at %s' % display_path(target_file)) shutil.copyfile(temp_location, target_file) - fp = open(target_file+'.content-type', 'w') + fp = open(target_file + '.content-type', 'w') fp.write(content_type) fp.close() @@ -629,6 +624,7 @@ def unpack_file(filename, location, content_type, link): and is_svn_page(file_contents(filename))): # We don't really care about this from pip.vcs.subversion import Subversion + Subversion('svn+' + link.url).unpack(location) else: ## FIXME: handle? @@ -721,6 +717,7 @@ def is_prerelease(vers): parsed = version._normalized_key(normalized) return any([any([y in set(["a", "b", "c", "rc", "dev"]) for y in x]) for x in parsed]) + def read_text_file(filename): """Return the contents of *filename*. @@ -753,6 +750,7 @@ def _make_build_dir(build_dir): class FakeFile(object): """Wrap a list of lines in an object with readline() to make ConfigParser happy.""" + def __init__(self, lines): self._gen = (l for l in lines)