Add vendor library management support to pip

docs/configuration.txt

@@ -52,7 +52,7 @@ global setting with the same name will be overridden; e.g. decreasing the
 
     [global]
     timeout = 60
-    
+
     [freeze]
     timeout = 10
 
@@ -88,6 +88,9 @@ On Unix and Mac OS X the configuration file is: :file:`$HOME/.pip/pip.conf`
 
 And on Windows, the configuration file is: :file:`%HOME%\\pip\\pip.ini`
 
+Also pip will search in the current and parent directories for a file
+called ``pip.conf`` or ``.pip.conf``, or just ``pip.ini`` on Windows.
+
 Environment variables
 -----------------------
 
@@ -139,3 +142,18 @@ configuration file::
     mirrors =
         http://d.pypi.python.org
         http://b.pypi.python.org
+
+Path management
+***************
+
+You can have entries added to ``sys.path`` in your config file,
+generally to support cases when you are installing libraries somewhere
+not normally on the path, but you want to be able to uninstall these
+libraries, or have pip detect them when upgrading/etc.  To do this
+use::
+
+    [global]
+    sys.path = %(here)s/vendor/
+
+You can include multiple paths, each on their own line.  These are
+added using ``site.addsitedir()`` so ``.pth`` files are respected.

docs/news.txt

@@ -39,6 +39,20 @@ develop (unreleased)
 * Fixed issue #427 - clearer error message on a malformed VCS url. Thanks
   Thomas Fenzl.
 
+* Search for config files (``pip.ini`` on Windows, ``pip.conf`` or
+  ``.pip.conf`` on other platforms) in the current and parent
+  directories.
+
+* Substitute ``%(here)s``, ``%(cwd)s`` and ``%(file)s`` in config file
+  values.
+
+* Add ``-f/--force`` option to ``pip uninstall``, which allows
+  uninstallation of files that aren't inside the virtualenv.
+
+* Add ``--script-fixup``, a hook on installation that allows you to rewrite
+  any scripts that were installed.  This can be helpful if you want to make
+  sure the environment is initialized before any scripts run (this is
+  implicitly what virtualenv does, with just the ``#!`` line).
 
 1.1 (2012-02-16)
 ----------------

pip/basecommand.py

@@ -62,6 +62,7 @@ def setup_logging(self):
     def main(self, args, initial_options):
         options, args = self.parser.parse_args(args)
         self.merge_options(initial_options, options)
+        self.parser.update_sys_path()
 
         level = 1 # Notify
         level += options.verbose
@@ -190,4 +191,3 @@ def load_all_commands():
 def command_names():
     names = set((pkg[1] for pkg in walk_packages(path=commands.__path__)))
     return list(names)
-

pip/baseparser.py

@@ -6,7 +6,7 @@
 import os
 from distutils.util import strtobool
 from pip.backwardcompat import ConfigParser, string_types
-from pip.locations import default_config_file, default_log_file
+from pip.locations import default_config_file, default_log_file, default_config_file_name, add_explicit_path
 
 
 class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter):
@@ -28,15 +28,58 @@ def __init__(self, *args, **kwargs):
         self.config = ConfigParser.RawConfigParser()
         self.name = kwargs.pop('name')
         self.files = self.get_config_files()
-        self.config.read(self.files)
+        self.read_config_files(self.files)
         assert self.name
         optparse.OptionParser.__init__(self, *args, **kwargs)
 
     def get_config_files(self):
         config_file = os.environ.get('PIP_CONFIG_FILE', False)
         if config_file and os.path.exists(config_file):
             return [config_file]
-        return [default_config_file]
+        files = []
+        search_path = os.path.abspath(os.getcwd())
+        last_search_path = None
+        while search_path != last_search_path:
+            path = os.path.join(search_path, default_config_file_name)
+            if os.path.exists(path):
+                files.append(path)
+            if sys.platform != 'win32':
+                path = os.path.join(search_path, '.' + default_config_file_name)
+                if os.path.exists(path):
+                    files.append(path)
+            last_search_path = search_path
+            search_path = os.path.dirname(search_path)
+        files.append(default_config_file)
+        # Put local files at the end of the list, where they get highest priority:
+        files.reverse()
+        return files
+
+    def read_config_files(self, files):
+        existing = {}
+        for file in files:
+            if not self.config.read([file]):
+                # This means nothing was actually read
+                continue
+            for section in self.config.sections():
+                for name in self.config.options(section):
+                    value = self.config.get(section, name)
+                    if existing.get(section, {}).get(name) == value:
+                        continue
+                    # It's a new option, and we need to do substitution
+                    value = self.substitute_config_value(value, file)
+                    existing.setdefault(section, {})[name] = value
+                    self.config.set(section, name, value)
+
+    def substitute_config_value(self, value, file):
+        file = os.path.abspath(file)
+        subs = [
+            ('%(file)s', file),
+            ('%(here)s', os.path.dirname(file)),
+            ('%(cwd)s', os.getcwd()),
+            ]
+        for var, sub in subs:
+            value = value.replace(var, sub)
+        return value
 
     def update_defaults(self, defaults):
         """Updates the given defaults with values from the config files and
@@ -58,7 +101,10 @@ def update_defaults(self, defaults):
                     continue
                 # handle multiline configs
                 if option.action == 'append':
-                    val = val.split()
+                    if '\n' in val:
+                        val = [v.strip() for v in val.splitlines() if v.strip()]
+                    else:
+                        val = val.split()
                 else:
                     option.nargs = 1
                 if option.action in ('store_true', 'store_false', 'count'):
@@ -111,6 +157,21 @@ def get_default_values(self):
                 defaults[option.dest] = option.check_value(opt_str, default)
         return optparse.Values(defaults)
 
+    def update_sys_path(self):
+        import pkg_resources
+        prev_sys_path = list(sys.path)
+        if self.config.has_option('global', 'sys.path'):
+            value = self.config.get('global', 'sys.path')
+            value = [v.strip() for v in value.splitlines() if v.strip()]
+            for path in value:
+                import site
+                site.addsitedir(path)
+        for path in sys.path:
+            if path not in prev_sys_path:
+                add_explicit_path(path)
+                pkg_resources.working_set.add_entry(path)
+
+
 try:
     pip_dist = pkg_resources.get_distribution('pip')
     version = '%s from %s (python %s)' % (

pip/commands/install.py

@@ -2,6 +2,7 @@
 import sys
 import tempfile
 import shutil
+import re
 from pip.req import InstallRequirement, RequirementSet
 from pip.req import parse_requirements
 from pip.log import logger
@@ -89,7 +90,7 @@ def __init__(self):
             dest='target_dir',
             metavar='DIR',
             default=None,
-            help='Install packages into DIR.')
+            help='Install packages into DIR (any scripts are thrown away!)')
         self.parser.add_option(
             '-d', '--download', '--download-dir', '--download-directory',
             dest='download_dir',
@@ -165,6 +166,13 @@ def __init__(self):
             action='store_true',
             help='Install to user-site')
 
+        self.parser.add_option(
+            '--script-fixup',
+            metavar="file.py:function or module.name:function",
+            dest='script_fixup',
+            help="Calls the given function with a list of all scripts created during installation, like "
+            "function([(req1, script1), (req2, script2)]).  You can use this to rewrite scripts.")
+
     def _build_package_finder(self, options, index_urls):
         """
         Create a package finder appropriate to this install command.
@@ -200,6 +208,11 @@ def run(self, options, args):
 
         finder = self._build_package_finder(options, index_urls)
 
+        if options.script_fixup:
+            script_fixup = ScriptFixup(options.script_fixup)
+        else:
+            script_fixup = None
+
         requirement_set = RequirementSet(
             build_dir=options.build_dir,
             src_dir=options.src_dir,
@@ -208,7 +221,8 @@ def run(self, options, args):
             upgrade=options.upgrade,
             ignore_installed=options.ignore_installed,
             ignore_dependencies=options.ignore_dependencies,
-            force_reinstall=options.force_reinstall)
+            force_reinstall=options.force_reinstall,
+            script_fixup=script_fixup)
         for name in args:
             requirement_set.add_requirement(
                 InstallRequirement.from_line(name, None))
@@ -275,5 +289,58 @@ def run(self, options, args):
             shutil.rmtree(temp_target_dir)
         return requirement_set
 
-
 InstallCommand()
+
+
+class ScriptFixup(object):
+
+    valid_function_re = re.compile(r'^[a-zA-Z_][a-zA-Z_0-9]*$')
+    valid_module_re = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_.]*$')
+
+    def __init__(self, script):
+        self.script = script
+        if ':' not in script:
+            raise CommandError("--script-fixup=%s must be in the form FILE:FUNCTION or MODULE:FUNCTION")
+        path, function = script.rsplit(':', 1)
+        if not self.valid_function_re.search(function):
+            raise CommandError("Function %s in --script-fixup=%s is not a valid function name" % (function, script))
+        if os.path.exists(path):
+            ns = {
+                '__file__': path,
+                '__name__': '__script_fixup__',
+                }
+            logger.debug('Execing %s' % path)
+            try:
+                execfile(path, ns)
+            except:
+                logger.error('Exception while running script %s from --script-fixup=%s' % (path, script))
+                raise
+            if function not in ns:
+                raise CommandError('File %s from --script-fixup=%s does not define a function %s'
+                                   % (path, script, function))
+            self.function = ns[function]
+        else:
+            if not self.valid_module_re.search(path):
+                raise CommandError("--script-fixup=%s refers to a file that does not exist, or an invalid module name: %s"
+                                   % (script, path))
+            logger.debug('Importing %s' % path)
+            try:
+                __import__(path)
+            except:
+                logger.error('Exception importing module %s from --script-fixup=%s' % (path, script))
+                raise
+            mod = sys.modules[path]
+            try:
+                self.function = getattr(mod, function)
+            except NameError:
+                raise CommandError(
+                    "Module %s (in %s) from --script-fixup=%s does not define a function or object named %s"
+                    % (path, mod.__file__, script, function))
+
+    def __call__(self, args):
+        try:
+            self.function(args)
+        except:
+            logger.error(
+                "Exception when calling --script-fixup=%s" % self.script)
+            raise

pip/commands/uninstall.py

@@ -23,6 +23,12 @@ def __init__(self):
             dest='yes',
             action='store_true',
             help="Don't ask for confirmation of uninstall deletions.")
+        self.parser.add_option(
+            '-f', '--force',
+            dest='force',
+            action='store_true',
+            help="Uninstall (or try to uninstall) a package even when it does not "
+            "appear to be in the current virtual environment")
 
     def run(self, options, args):
         requirement_set = RequirementSet(
@@ -38,6 +44,6 @@ def run(self, options, args):
         if not requirement_set.has_requirements:
             raise InstallationError('You must give at least one requirement '
                 'to %(name)s (see "pip help %(name)s")' % dict(name=self.name))
-        requirement_set.uninstall(auto_confirm=options.yes)
+        requirement_set.uninstall(auto_confirm=options.yes, force=options.force)
 
 UninstallCommand()

pip/locations.py

@@ -12,6 +12,10 @@ def running_under_virtualenv():
     """
     return hasattr(sys, 'real_prefix')
 
+explicit_paths = []
+
+def add_explicit_path(path):
+    explicit_paths.append(path)
 
 if running_under_virtualenv():
     ## FIXME: is build/ a good name?
@@ -36,14 +40,16 @@ def running_under_virtualenv():
     # buildout uses 'bin' on Windows too?
     if not os.path.exists(bin_py):
         bin_py = os.path.join(sys.prefix, 'bin')
+    default_config_file_name = 'pip.ini'
     user_dir = os.environ.get('APPDATA', user_dir) # Use %APPDATA% for roaming
     default_storage_dir = os.path.join(user_dir, 'pip')
-    default_config_file = os.path.join(default_storage_dir, 'pip.ini')
+    default_config_file = os.path.join(default_storage_dir, default_config_file_name)
     default_log_file = os.path.join(default_storage_dir, 'pip.log')
 else:
     bin_py = os.path.join(sys.prefix, 'bin')
+    default_config_file_name = 'pip.conf'
     default_storage_dir = os.path.join(user_dir, '.pip')
-    default_config_file = os.path.join(default_storage_dir, 'pip.conf')
+    default_config_file = os.path.join(default_storage_dir, default_config_file_name)
     default_log_file = os.path.join(default_storage_dir, 'pip.log')
     # Forcing to use /usr/local/bin for standard Mac OS X framework installs
     # Also log to ~/Library/Logs/ for use with the Console.app log viewer