diff --git a/ah_bootstrap.py b/ah_bootstrap.py index 6888c3ed2c..7e145e3d09 100644 --- a/ah_bootstrap.py +++ b/ah_bootstrap.py @@ -56,6 +56,12 @@ _text_type = str PY3 = True + +# What follows are several import statements meant to deal with install-time +# issues with either missing or misbehaving pacakges (including making sure +# setuptools itself is installed): + + # Some pre-setuptools checks to ensure that either distribute or setuptools >= # 0.7 is used (over pre-distribute setuptools) if it is available on the path; # otherwise the latest setuptools will be downloaded and bootstrapped with @@ -84,15 +90,6 @@ from ez_setup import use_setuptools use_setuptools() -from distutils import log -from distutils.debug import DEBUG - -# In case it didn't successfully import before the ez_setup checks -import pkg_resources - -from setuptools import Distribution -from setuptools.package_index import PackageIndex -from setuptools.sandbox import run_setup # Note: The following import is required as a workaround to # https://github.com/astropy/astropy-helpers/issues/89; if we don't import this @@ -104,6 +101,37 @@ except ImportError: pass + +# matplotlib can cause problems if it is imported from within a call of +# run_setup(), because in some circumstances it will try to write to the user's +# home directory, resulting in a SandboxViolation. See +# https://github.com/matplotlib/matplotlib/pull/4165 +# Making sure matplotlib, if it is available, is imported early in the setup +# process can mitigate this (note importing matplotlib.pyplot has the same +# issue) +try: + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot +except: + # Ignore if this fails for *any* reason* + pass + + +# End compatibility imports... + + +# In case it didn't successfully import before the ez_setup checks +import pkg_resources + +from setuptools import Distribution +from setuptools.package_index import PackageIndex +from setuptools.sandbox import run_setup + +from distutils import log +from distutils.debug import DEBUG + + # TODO: Maybe enable checking for a specific version of astropy_helpers? DIST_NAME = 'astropy-helpers' PACKAGE_NAME = 'astropy_helpers' @@ -112,199 +140,340 @@ DOWNLOAD_IF_NEEDED = True INDEX_URL = 'https://pypi.python.org/simple' USE_GIT = True +OFFLINE = False AUTO_UPGRADE = True +# A list of all the configuration options and their required types +CFG_OPTIONS = [ + ('auto_use', bool), ('path', str), ('download_if_needed', bool), + ('index_url', str), ('use_git', bool), ('offline', bool), + ('auto_upgrade', bool) +] -def use_astropy_helpers(path=None, download_if_needed=None, index_url=None, - use_git=None, auto_upgrade=None): + +class _Bootstrapper(object): + """ + Bootstrapper implementation. See ``use_astropy_helpers`` for parameter + documentation. """ - Ensure that the `astropy_helpers` module is available and is importable. - This supports automatic submodule initialization if astropy_helpers is - included in a project as a git submodule, or will download it from PyPI if - necessary. - Parameters - ---------- + def __init__(self, path=None, index_url=None, use_git=None, offline=None, + download_if_needed=None, auto_upgrade=None): - path : str or None, optional - A filesystem path relative to the root of the project's source code - that should be added to `sys.path` so that `astropy_helpers` can be - imported from that path. + if path is None: + path = PACKAGE_NAME - If the path is a git submodule it will automatically be initialzed - and/or updated. + if not (isinstance(path, _str_types) or path is False): + raise TypeError('path must be a string or False') - The path may also be to a ``.tar.gz`` archive of the astropy_helpers - source distribution. In this case the archive is automatically - unpacked and made temporarily available on `sys.path` as a ``.egg`` - archive. + if PY3 and not isinstance(path, _text_type): + fs_encoding = sys.getfilesystemencoding() + path = path.decode(fs_encoding) # path to unicode - If `None` skip straight to downloading. + self.path = path - download_if_needed : bool, optional - If the provided filesystem path is not found an attempt will be made to - download astropy_helpers from PyPI. It will then be made temporarily - available on `sys.path` as a ``.egg`` archive (using the - ``setup_requires`` feature of setuptools. If the ``--offline`` option - is given at the command line the value of this argument is overridden - to `False`. + # Set other option attributes, using defaults where necessary + self.index_url = index_url if index_url is not None else INDEX_URL + self.offline = offline if offline is not None else OFFLINE - index_url : str, optional - If provided, use a different URL for the Python package index than the - main PyPI server. + # If offline=True, override download and auto-upgrade + if self.offline: + download_if_needed = False + auto_upgrade = False - use_git : bool, optional - If `False` no git commands will be used--this effectively disables - support for git submodules. If the ``--no-git`` option is given at the - command line the value of this argument is overridden to `False`. + self.download = (download_if_needed + if download_if_needed is not None + else DOWNLOAD_IF_NEEDED) + self.auto_upgrade = (auto_upgrade + if auto_upgrade is not None else AUTO_UPGRADE) - auto_upgrade : bool, optional - By default, when installing a package from a non-development source - distribution ah_boostrap will try to automatically check for patch - releases to astropy-helpers on PyPI and use the patched version over - any bundled versions. Setting this to `False` will disable that - functionality. If the ``--offline`` option is given at the command line - the value of this argument is overridden to `False`. - """ + # If this is a release then the .git directory will not exist so we + # should not use git. + git_dir_exists = os.path.exists(os.path.join(os.path.dirname(__file__), '.git')) + if use_git is None and not git_dir_exists: + use_git = False - # True by default, unless the --offline option was provided on the command - # line - if '--offline' in sys.argv: - download_if_needed = False - auto_upgrade = False - offline = True - sys.argv.remove('--offline') - else: - offline = False + self.use_git = use_git if use_git is not None else USE_GIT + # Declared as False by default--later we check if astropy-helpers can be + # upgraded from PyPI, but only if not using a source distribution (as in + # the case of import from a git submodule) + self.is_submodule = False - if '--no-git' in sys.argv: - use_git = False - sys.argv.remove('--no-git') + @classmethod + def main(cls, argv=None): + if argv is None: + argv = sys.argv - if path is None: - path = PACKAGE_NAME + config = cls.parse_config() + config.update(cls.parse_command_line(argv)) - if download_if_needed is None: - download_if_needed = DOWNLOAD_IF_NEEDED + auto_use = config.pop('auto_use', False) + bootstrapper = cls(**config) - if index_url is None: - index_url = INDEX_URL + if auto_use: + # Run the bootstrapper, otherwise the setup.py is using the old + # use_astropy_helpers() interface, in which case it will run the + # bootstrapper manually after reconfiguring it. + bootstrapper.run() - # If this is a release then the .git directory will not exist so we - # should not use git. - git_dir_exists = os.path.exists(os.path.join(os.path.dirname(__file__), '.git')) - if use_git is None and not git_dir_exists: - use_git = False + return bootstrapper - if use_git is None: - use_git = USE_GIT + @classmethod + def parse_config(cls): + if not os.path.exists('setup.cfg'): + return {} - if auto_upgrade is None: - auto_upgrade = AUTO_UPGRADE + cfg = ConfigParser() - # Declared as False by default--later we check if astropy-helpers can be - # upgraded from PyPI, but only if not using a source distribution (as in - # the case of import from a git submodule) - is_submodule = False + try: + cfg.read('setup.cfg') + except Exception as e: + if DEBUG: + raise - if not isinstance(path, _str_types): - if path is not None: - raise TypeError('path must be a string or None') + log.error( + "Error reading setup.cfg: {0!r}\n{1} will not be " + "automatically bootstrapped and package installation may fail." + "\n{2}".format(e, PACKAGE_NAME, _err_help_msg)) + return {} - if not download_if_needed: - log.debug('a path was not given and download from PyPI was not ' - 'allowed so this is effectively a no-op') - return - elif not os.path.exists(path) or os.path.isdir(path): - # Even if the given path does not exist on the filesystem, if it *is* a - # submodule, `git submodule init` will create it - is_submodule = _check_submodule(path, use_git=use_git, - offline=offline) - - if is_submodule or os.path.isdir(path): - log.info( - 'Attempting to import astropy_helpers from {0} {1!r}'.format( - 'submodule' if is_submodule else 'directory', path)) - dist = _directory_import(path) + if not cfg.has_section('ah_bootstrap'): + return {} + + config = {} + + for option, type_ in CFG_OPTIONS: + if not cfg.has_option('ah_bootstrap', option): + continue + + if type_ is bool: + value = cfg.getboolean('ah_bootstrap', option) + else: + value = cfg.get('ah_bootstrap', option) + + config[option] = value + + return config + + @classmethod + def parse_command_line(cls, argv=None): + if argv is None: + argv = sys.argv + + config = {} + + # For now we just pop recognized ah_bootstrap options out of the + # arg list. This is imperfect; in the unlikely case that a setup.py + # custom command or even custom Distribution class defines an argument + # of the same name then we will break that. However there's a catch22 + # here that we can't just do full argument parsing right here, because + # we don't yet know *how* to parse all possible command-line arguments. + if '--no-git' in argv: + config['use_git'] = False + argv.remove('--no-git') + + if '--offline' in argv: + config['offline'] = True + argv.remove('--offline') + + return config + + def run(self): + strategies = ['local_directory', 'local_file', 'index'] + dist = None + + # First, remove any previously imported versions of astropy_helpers; + # this is necessary for nested installs where one package's installer + # is installing another package via setuptools.sandbox.run_setup, as in + # the case of setup_requires + for key in list(sys.modules): + try: + if key == PACKAGE_NAME or key.startswith(PACKAGE_NAME + '.'): + del sys.modules[key] + except AttributeError: + # Sometimes mysterious non-string things can turn up in + # sys.modules + continue + + # Check to see if the path is a submodule + self.is_submodule = self._check_submodule() + + for strategy in strategies: + method = getattr(self, 'get_{0}_dist'.format(strategy)) + dist = method() + if dist is not None: + break else: - dist = None + raise _AHBootstrapSystemExit( + "No source found for the {0!r} package; {0} must be " + "available and importable as a prerequisite to building " + "or installing this package.".format(PACKAGE_NAME)) + + # This is a bit hacky, but if astropy_helpers was loaded from a + # directory/submodule its Distribution object gets a "precedence" of + # "DEVELOP_DIST". However, in other cases it gets a precedence of + # "EGG_DIST". However, when activing the distribution it will only be + # placed early on sys.path if it is treated as an EGG_DIST, so always + # do that + dist = dist.clone(precedence=pkg_resources.EGG_DIST) + + # Otherwise we found a version of astropy-helpers, so we're done + # Just active the found distribution on sys.path--if we did a + # download this usually happens automatically but it doesn't hurt to + # do it again + # Note: Adding the dist to the global working set also activates it + # (makes it importable on sys.path) by default. + + try: + pkg_resources.working_set.add(dist, replace=True) + except TypeError: + # Some (much) older versions of setuptools do not have the + # replace=True option here. These versions are old enough that all + # bets may be off anyways, but it's easy enough to work around just + # in case... + if dist.key in pkg_resources.working_set.by_key: + del pkg_resources.working_set.by_key[dist.key] + pkg_resources.working_set.add(dist) + + @property + def config(self): + """ + A `dict` containing the options this `_Bootstrapper` was configured + with. + """ + + return dict((optname, getattr(self, optname)) + for optname, _ in CFG_OPTIONS if hasattr(self, optname)) + + def get_local_directory_dist(self): + """ + Handle importing a vendored package from a subdirectory of the source + distribution. + """ + + if not os.path.isdir(self.path): + return + + log.info('Attempting to import astropy_helpers from {0} {1!r}'.format( + 'submodule' if self.is_submodule else 'directory', + self.path)) + + dist = self._directory_import() if dist is None: - msg = ( + log.warn( 'The requested path {0!r} for importing {1} does not ' - 'exist, or does not contain a copy of the {1} package. ' - 'Attempting download instead.'.format(path, PACKAGE_NAME)) - if download_if_needed: - log.warn(msg) - else: - raise _AHBootstrapSystemExit(msg) - elif os.path.isfile(path): - # Handle importing from a source archive; this also uses setup_requires - # but points easy_install directly to the source archive + 'exist, or does not contain a copy of the {1} ' + 'package.'.format(self.path, PACKAGE_NAME)) + elif self.auto_upgrade and not self.is_submodule: + # A version of astropy-helpers was found on the available path, but + # check to see if a bugfix release is available on PyPI + upgrade = self._do_upgrade(dist) + if upgrade is not None: + dist = upgrade + + return dist + + def get_local_file_dist(self): + """ + Handle importing from a source archive; this also uses setup_requires + but points easy_install directly to the source archive. + """ + + if not os.path.isfile(self.path): + return + + log.info('Attempting to unpack and import astropy_helpers from ' + '{0!r}'.format(self.path)) + try: - dist = _do_download(find_links=[path]) + dist = self._do_download(find_links=[self.path]) except Exception as e: - if download_if_needed: - log.warn('{0}\nWill attempt to download astropy_helpers from ' - 'PyPI instead.'.format(str(e))) - dist = None - else: - raise _AHBootstrapSystemExit(e.args[0]) - else: - msg = ('{0!r} is not a valid file or directory (it could be a ' - 'symlink?)'.format(path)) - if download_if_needed: - log.warn(msg) + if DEBUG: + raise + + log.warn( + 'Failed to import {0} from the specified archive {1!r}: ' + '{2}'.format(PACKAGE_NAME, self.path, str(e))) dist = None - else: - raise _AHBootstrapSystemExit(msg) - if dist is not None and auto_upgrade and not is_submodule: - # A version of astropy-helpers was found on the available path, but - # check to see if a bugfix release is available on PyPI - upgrade = _do_upgrade(dist, index_url) - if upgrade is not None: - dist = upgrade - elif dist is None: - # Last resort--go ahead and try to download the latest version from - # PyPI + if dist is not None and self.auto_upgrade: + # A version of astropy-helpers was found on the available path, but + # check to see if a bugfix release is available on PyPI + upgrade = self._do_upgrade(dist) + if upgrade is not None: + dist = upgrade + + return dist + + def get_index_dist(self): + if not self.download: + log.warn('Downloading {0!r} disabled.'.format(DIST_NAME)) + return False + + log.warn( + "Downloading {0!r}; run setup.py with the --offline option to " + "force offline installation.".format(DIST_NAME)) + try: - if download_if_needed: - log.warn( - "Downloading astropy_helpers; run setup.py with the " - "--offline option to force offline installation.") - dist = _do_download(index_url=index_url) - else: - raise _AHBootstrapSystemExit( - "No source for the astropy_helpers package; " - "astropy_helpers must be available as a prerequisite to " - "installing this package.") + dist = self._do_download() except Exception as e: if DEBUG: raise - else: - raise _AHBootstrapSystemExit(e.args[0]) + log.warn( + 'Failed to download and/or install {0!r} from {1!r}:\n' + '{2}'.format(DIST_NAME, self.index_url, str(e))) + dist = None - if dist is not None: - # Otherwise we found a version of astropy-helpers so we're done - # Just activate the found distribibution on sys.path--if we did a - # download this usually happens automatically but do it again just to - # be sure - # Note: Adding the dist to the global working set also activates it by - # default - pkg_resources.working_set.add(dist) + # No need to run auto-upgrade here since we've already presumably + # gotten the most up-to-date version from the package index + return dist + def _directory_import(self): + """ + Import astropy_helpers from the given path, which will be added to + sys.path. -def _do_download(version='', find_links=None, index_url=None): - try: + Must return True if the import succeeded, and False otherwise. + """ + + # Return True on success, False on failure but download is allowed, and + # otherwise raise SystemExit + path = os.path.abspath(self.path) + + # Use an empty WorkingSet rather than the man + # pkg_resources.working_set, since on older versions of setuptools this + # will invoke a VersionConflict when trying to install an upgrade + ws = pkg_resources.WorkingSet([]) + ws.add_entry(path) + dist = ws.by_key.get(DIST_NAME) + + if dist is None: + # We didn't find an egg-info/dist-info in the given path, but if a + # setup.py exists we can generate it + setup_py = os.path.join(path, 'setup.py') + if os.path.isfile(setup_py): + with _silence(): + run_setup(os.path.join(path, 'setup.py'), + ['egg_info']) + + for dist in pkg_resources.find_distributions(path, True): + # There should be only one... + return dist + + return dist + + def _do_download(self, version='', find_links=None): if find_links: allow_hosts = '' index_url = None else: allow_hosts = None + index_url = self.index_url + # Annoyingly, setuptools will not handle other arguments to # Distribution (such as options) before handling setup_requires, so it - # is not straightfoward to programmatically augment the arguments which + # is not straightforward to programmatically augment the arguments which # are passed to easy_install class _Distribution(Distribution): def get_option_dict(self, command_name): @@ -324,129 +493,279 @@ def get_option_dict(self, command_name): req = DIST_NAME attrs = {'setup_requires': [req]} - if DEBUG: - dist = _Distribution(attrs=attrs) + + try: + if DEBUG: + _Distribution(attrs=attrs) + else: + with _silence(): + _Distribution(attrs=attrs) + + # If the setup_requires succeeded it will have added the new dist to + # the main working_set + return pkg_resources.working_set.by_key.get(DIST_NAME) + except Exception as e: + if DEBUG: + raise + + msg = 'Error retrieving {0} from {1}:\n{2}' + if find_links: + source = find_links[0] + elif index_url != INDEX_URL: + source = index_url + else: + source = 'PyPI' + + raise Exception(msg.format(DIST_NAME, source, repr(e))) + + def _do_upgrade(self, dist): + # Build up a requirement for a higher bugfix release but a lower minor + # release (so API compatibility is guaranteed) + next_version = _next_version(dist.parsed_version) + + req = pkg_resources.Requirement.parse( + '{0}>{1},<{2}'.format(DIST_NAME, dist.version, next_version)) + + package_index = PackageIndex(index_url=self.index_url) + + upgrade = package_index.obtain(req) + + if upgrade is not None: + return self._do_download(version=upgrade.version) + + def _check_submodule(self): + """ + Check if the given path is a git submodule. + + See the docstrings for ``_check_submodule_using_git`` and + ``_check_submodule_no_git`` for further details. + """ + + if (self.path is None or + (os.path.exists(self.path) and not os.path.isdir(self.path))): + return False + + if self.use_git: + return self._check_submodule_using_git() else: - with _silence(): - dist = _Distribution(attrs=attrs) + return self._check_submodule_no_git() - # If the setup_requires succeeded it will have added the new dist to - # the main working_set - return pkg_resources.working_set.by_key.get(DIST_NAME) - except Exception as e: - if DEBUG: - raise + def _check_submodule_using_git(self): + """ + Check if the given path is a git submodule. If so, attempt to initialize + and/or update the submodule if needed. - msg = 'Error retrieving astropy helpers from {0}:\n{1}' - if find_links: - source = find_links[0] - elif index_url: - source = index_url + This function makes calls to the ``git`` command in subprocesses. The + ``_check_submodule_no_git`` option uses pure Python to check if the given + path looks like a git submodule, but it cannot perform updates. + """ + + cmd = ['git', 'submodule', 'status', '--', self.path] + + try: + log.info('Running `{0}`; use the --no-git option to disable git ' + 'commands'.format(' '.join(cmd))) + returncode, stdout, stderr = run_cmd(cmd) + except _CommandNotFound: + # The git command simply wasn't found; this is most likely the + # case on user systems that don't have git and are simply + # trying to install the package from PyPI or a source + # distribution. Silently ignore this case and simply don't try + # to use submodules + return False + + stderr = stderr.strip() + + if returncode != 0 and stderr: + # Unfortunately the return code alone cannot be relied on, as + # earlier versions of git returned 0 even if the requested submodule + # does not exist + + # This is a warning that occurs in perl (from running git submodule) + # which only occurs with a malformatted locale setting which can + # happen sometimes on OSX. See again + # https://github.com/astropy/astropy/issues/2749 + perl_warning = ('perl: warning: Falling back to the standard locale ' + '("C").') + if not stderr.strip().endswith(perl_warning): + # Some other unknown error condition occurred + log.warn('git submodule command failed ' + 'unexpectedly:\n{0}'.format(stderr)) + return False + + # Output of `git submodule status` is as follows: + # + # 1: Status indicator: '-' for submodule is uninitialized, '+' if + # submodule is initialized but is not at the commit currently indicated + # in .gitmodules (and thus needs to be updated), or 'U' if the + # submodule is in an unstable state (i.e. has merge conflicts) + # + # 2. SHA-1 hash of the current commit of the submodule (we don't really + # need this information but it's useful for checking that the output is + # correct) + # + # 3. The output of `git describe` for the submodule's current commit + # hash (this includes for example what branches the commit is on) but + # only if the submodule is initialized. We ignore this information for + # now + _git_submodule_status_re = re.compile( + '^(?P[+-U ])(?P[0-9a-f]{40}) ' + '(?P\S+)( .*)?$') + + # The stdout should only contain one line--the status of the + # requested submodule + m = _git_submodule_status_re.match(stdout) + if m: + # Yes, the path *is* a git submodule + self._update_submodule(m.group('submodule'), m.group('status')) + return True else: - source = 'PyPI' + log.warn( + 'Unexpected output from `git submodule status`:\n{0}\n' + 'Will attempt import from {1!r} regardless.'.format( + stdout, self.path)) + return False - raise Exception(msg.format(source, repr(e))) + def _check_submodule_no_git(self): + """ + Like ``_check_submodule_using_git``, but simply parses the .gitmodules file + to determine if the supplied path is a git submodule, and does not exec any + subprocesses. + This can only determine if a path is a submodule--it does not perform + updates, etc. This function may need to be updated if the format of the + .gitmodules file is changed between git versions. + """ -def _do_upgrade(dist, index_url): - # Build up a requirement for a higher bugfix release but a lower minor - # release (so API compatibility is guaranteed) - # sketchy version parsing--maybe come up with something a bit more - # robust for this - major, minor = (int(part) for part in dist.parsed_version[:2]) - next_minor = '.'.join([str(major), str(minor + 1), '0']) - req = pkg_resources.Requirement.parse( - '{0}>{1},<{2}'.format(DIST_NAME, dist.version, next_minor)) + gitmodules_path = os.path.abspath('.gitmodules') - package_index = PackageIndex(index_url=index_url) + if not os.path.isfile(gitmodules_path): + return False - upgrade = package_index.obtain(req) + # This is a minimal reader for gitconfig-style files. It handles a few of + # the quirks that make gitconfig files incompatible with ConfigParser-style + # files, but does not support the full gitconfig syntax (just enough + # needed to read a .gitmodules file). + gitmodules_fileobj = io.StringIO() - if upgrade is not None: - return _do_download(version=upgrade.version, index_url=index_url) + # Must use io.open for cross-Python-compatible behavior wrt unicode + with io.open(gitmodules_path) as f: + for line in f: + # gitconfig files are more flexible with leading whitespace; just + # go ahead and remove it + line = line.lstrip() + # comments can start with either # or ; + if line and line[0] in (':', ';'): + continue -def _directory_import(path): - """ - Import astropy_helpers from the given path, which will be added to - sys.path. + gitmodules_fileobj.write(line) - Must return True if the import succeeded, and False otherwise. - """ + gitmodules_fileobj.seek(0) - # Return True on success, False on failure but download is allowed, and - # otherwise raise SystemExit - path = os.path.abspath(path) + cfg = RawConfigParser() - # Use an empty WorkingSet rather than the man pkg_resources.working_set, - # since on older versions of setuptools this will invoke a VersionConflict - # when trying to install an upgrade - ws = pkg_resources.WorkingSet([]) - ws.add_entry(path) - dist = ws.by_key.get(DIST_NAME) + try: + cfg.readfp(gitmodules_fileobj) + except Exception as exc: + log.warn('Malformatted .gitmodules file: {0}\n' + '{1} cannot be assumed to be a git submodule.'.format( + exc, self.path)) + return False - if dist is None: - # We didn't find an egg-info/dist-info in the given path, but if a - # setup.py exists we can generate it - setup_py = os.path.join(path, 'setup.py') - if os.path.isfile(setup_py): - with _silence(): - run_setup(os.path.join(path, 'setup.py'), ['egg_info']) + for section in cfg.sections(): + if not cfg.has_option(section, 'path'): + continue - for dist in pkg_resources.find_distributions(path, True): - # There should be only one... - return dist + submodule_path = cfg.get(section, 'path').rstrip(os.sep) - return dist + if submodule_path == self.path.rstrip(os.sep): + return True + return False -def _check_submodule(path, use_git=True, offline=False): - """ - Check if the given path is a git submodule. + def _update_submodule(self, submodule, status): + if status == ' ': + # The submodule is up to date; no action necessary + return + elif status == '-': + if self.offline: + raise _AHBootstrapSystemExit( + "Cannot initialize the {0} submodule in --offline mode; " + "this requires being able to clone the submodule from an " + "online repository.".format(submodule)) + cmd = ['update', '--init'] + action = 'Initializing' + elif status == '+': + cmd = ['update'] + action = 'Updating' + if self.offline: + cmd.append('--no-fetch') + elif status == 'U': + raise _AHBoostrapSystemExit( + 'Error: Submodule {0} contains unresolved merge conflicts. ' + 'Please complete or abandon any changes in the submodule so that ' + 'it is in a usable state, then try again.'.format(submodule)) + else: + log.warn('Unknown status {0!r} for git submodule {1!r}. Will ' + 'attempt to use the submodule as-is, but try to ensure ' + 'that the submodule is in a clean state and contains no ' + 'conflicts or errors.\n{2}'.format(status, submodule, + _err_help_msg)) + return - See the docstrings for ``_check_submodule_using_git`` and - ``_check_submodule_no_git`` for futher details. - """ + err_msg = None + cmd = ['git', 'submodule'] + cmd + ['--', submodule] + log.warn('{0} {1} submodule with: `{2}`'.format( + action, submodule, ' '.join(cmd))) - if use_git: - return _check_submodule_using_git(path, offline) - else: - return _check_submodule_no_git(path) + try: + log.info('Running `{0}`; use the --no-git option to disable git ' + 'commands'.format(' '.join(cmd))) + returncode, stdout, stderr = run_cmd(cmd) + except OSError as e: + err_msg = str(e) + else: + if returncode != 0: + err_msg = stderr + if err_msg is not None: + log.warn('An unexpected error occurred updating the git submodule ' + '{0!r}:\n{1}\n{2}'.format(submodule, err_msg, + _err_help_msg)) -def _check_submodule_using_git(path, offline): +class _CommandNotFound(OSError): + """ + An exception raised when a command run with run_cmd is not found on the + system. """ - Check if the given path is a git submodule. If so, attempt to initialize - and/or update the submodule if needed. - This function makes calls to the ``git`` command in subprocesses. The - ``_check_submodule_no_git`` option uses pure Python to check if the given - path looks like a git submodule, but it cannot perform updates. + +def run_cmd(cmd): """ + Run a command in a subprocess, given as a list of command-line + arguments. - if PY3 and not isinstance(path, _text_type): - fs_encoding = sys.getfilesystemencoding() - path = path.decode(fs_encoding) + Returns a ``(returncode, stdout, stderr)`` tuple. + """ try: - p = sp.Popen(['git', 'submodule', 'status', '--', path], - stdout=sp.PIPE, stderr=sp.PIPE) + p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) + # XXX: May block if either stdout or stderr fill their buffers; + # however for the commands this is currently used for that is + # unlikely (they should have very brief output) stdout, stderr = p.communicate() except OSError as e: if DEBUG: raise if e.errno == errno.ENOENT: - # The git command simply wasn't found; this is most likely the - # case on user systems that don't have git and are simply - # trying to install the package from PyPI or a source - # distribution. Silently ignore this case and simply don't try - # to use submodules - return False + msg = 'Command not found: `{0}`'.format(' '.join(cmd)) + raise _CommandNotFound(msg, cmd) else: raise _AHBoostrapSystemExit( 'An unexpected error occurred when running the ' - '`git submodule status` command:\n{0}'.format(str(e))) + '`{0}` command:\n{1}'.format(' '.join(cmd), str(e))) # Can fail of the default locale is not configured properly. See @@ -460,148 +779,47 @@ def _check_submodule_using_git(path, offline): # http://bugs.python.org/issue18378 stdio_encoding = 'latin1' - if p.returncode != 0 or stderr: - # Unfortunately the return code alone cannot be relied on, as - # earlier versions of git returned 0 even if the requested submodule - # does not exist - stderr = stderr.decode(stdio_encoding) - - # This is a warning that occurs in perl (from running git submodule) - # which only occurs with a malformatted locale setting which can - # happen sometimes on OSX. See again - # https://github.com/astropy/astropy/issues/2749 - perl_warning = ('perl: warning: Falling back to the standard locale ' - '("C").') - if not stderr.strip().endswith(perl_warning): - # Some other uknown error condition occurred - log.warn('git submodule command failed ' - 'unexpectedly:\n{0}'.format(stderr)) - return False + # Unlikely to fail at this point but even then let's be flexible + if not isinstance(stdout, _text_type): + stdout = stdout.decode(stdio_encoding, 'replace') + if not isinstance(stderr, _text_type): + stderr = stderr.decode(stdio_encoding, 'replace') - stdout = stdout.decode(stdio_encoding) - # The stdout should only contain one line--the status of the - # requested submodule - m = _git_submodule_status_re.match(stdout) - if m: - # Yes, the path *is* a git submodule - _update_submodule(m.group('submodule'), m.group('status'), offline) - return True - else: - log.warn( - 'Unexpected output from `git submodule status`:\n{0}\n' - 'Will attempt import from {1!r} regardless.'.format( - stdout, path)) - return False + return (p.returncode, stdout, stderr) -def _check_submodule_no_git(path): +def _next_version(version): """ - Like ``_check_submodule_using_git``, but simply parses the .gitmodules file - to determine if the supplied path is a git submodule, and does not exec any - subprocesses. + Given a parsed version from pkg_resources.parse_version, returns a new + version string with the next minor version. - This can only determine if a path is a submodule--it does not perform - updates, etc. This function may need to be updated if the format of the - .gitmodules file is changed between git versions. + Examples + ======== + >>> _next_version(pkg_resources.parse_version('1.2.3')) + '1.3.0' """ - gitmodules_path = os.path.abspath('.gitmodules') - - if not os.path.isfile(gitmodules_path): - return False - - # This is a minimal reader for gitconfig-style files. It handles a few of - # the quirks that make gitconfig files incompatible with ConfigParser-style - # files, but does not support the full gitconfig syntaix (just enough - # needed to read a .gitmodules file). - gitmodules_fileobj = io.StringIO() - - # Must use io.open for cross-Python-compatible behavior wrt unicode - with io.open(gitmodules_path) as f: - for line in f: - # gitconfig files are more flexible with leading whitespace; just - # go ahead and remove it - line = line.lstrip() - - # comments can start with either # or ; - if line and line[0] in (':', ';'): - continue - - gitmodules_fileobj.write(line) - - gitmodules_fileobj.seek(0) - - cfg = RawConfigParser() - - try: - cfg.readfp(gitmodules_fileobj) - except Exception as exc: - log.warn('Malformatted .gitmodules file: {0}\n' - '{1} cannot be assumed to be a git submodule.'.format( - exc, path)) - return False - - for section in cfg.sections(): - if not cfg.has_option(section, 'path'): - continue - - submodule_path = cfg.get(section, 'path').rstrip(os.sep) - - if submodule_path == path.rstrip(os.sep): - return True - - return False - - -def _update_submodule(submodule, status, offline): - if status == ' ': - # The submodule is up to date; no action necessary - return - elif status == '-': - if offline: - raise _AHBootstrapSystemExit( - "Cannot initialize the {0} submodule in --offline mode; this " - "requires being able to clone the submodule from an online " - "repository.".format(submodule)) - cmd = ['update', '--init'] - action = 'Initializing' - elif status == '+': - cmd = ['update'] - action = 'Updating' - if offline: - cmd.append('--no-fetch') - elif status == 'U': - raise _AHBoostrapSystemExit( - 'Error: Submodule {0} contains unresolved merge conflicts. ' - 'Please complete or abandon any changes in the submodule so that ' - 'it is in a usable state, then try again.'.format(submodule)) + if hasattr(version, 'base_version'): + # New version parsing from setuptools >= 8.0 + if version.base_version: + parts = version.base_version.split('.') + else: + parts = [] else: - log.warn('Unknown status {0!r} for git submodule {1!r}. Will ' - 'attempt to use the submodule as-is, but try to ensure ' - 'that the submodule is in a clean state and contains no ' - 'conflicts or errors.\n{2}'.format(status, submodule, - _err_help_msg)) - return + parts = [] + for part in version: + if part.startswith('*'): + break + parts.append(part) - err_msg = None + parts = [int(p) for p in parts] - cmd = ['git', 'submodule'] + cmd + ['--', submodule] - log.warn('{0} {1} submodule with: `{2}`'.format( - action, submodule, ' '.join(cmd))) + if len(parts) < 3: + parts += [0] * (3 - len(parts)) - try: - p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) - stdout, stderr = p.communicate() - except OSError as e: - err_msg = str(e) - else: - if p.returncode != 0: - stderr_encoding = locale.getdefaultlocale()[1] - err_msg = stderr.decode(stderr_encoding) + major, minor, micro = parts[:3] - if err_msg: - log.warn('An unexpected error occurred updating the git submodule ' - '{0!r}:\n{1}\n{2}'.format(submodule, err_msg, _err_help_msg)) + return '{0}.{1}.{2}'.format(major, minor + 1, 0) class _DummyFile(object): @@ -699,66 +917,71 @@ def _log_to_stderr(self, level, msg, *args): log = log() -# Output of `git submodule status` is as follows: -# -# 1: Status indicator: '-' for submodule is uninitialized, '+' if submodule is -# initialized but is not at the commit currently indicated in .gitmodules (and -# thus needs to be updated), or 'U' if the submodule is in an unstable state -# (i.e. has merge conflicts) -# -# 2. SHA-1 hash of the current commit of the submodule (we don't really need -# this information but it's useful for checking that the output is correct) -# -# 3. The output of `git describe` for the submodule's current commit hash (this -# includes for example what branches the commit is on) but only if the -# submodule is initialized. We ignore this information for now -_git_submodule_status_re = re.compile( - '^(?P[+-U ])(?P[0-9a-f]{40}) (?P\S+)( .*)?$') - - -# Implement the auto-use feature; this allows use_astropy_helpers() to be used -# at import-time automatically so long as the correct options are specified in -# setup.cfg -_CFG_OPTIONS = [('auto_use', bool), ('path', str), - ('download_if_needed', bool), ('index_url', str), - ('use_git', bool), ('auto_upgrade', bool)] - -def _main(): - if not os.path.exists('setup.cfg'): - return - - cfg = ConfigParser() - try: - cfg.read('setup.cfg') - except Exception as e: - if DEBUG: - raise +BOOTSTRAPPER = _Bootstrapper.main() - log.error( - "Error reading setup.cfg: {0!r}\nastropy_helpers will not be " - "automatically bootstrapped and package installation may fail." - "\n{1}".format(e, _err_help_msg)) - return - if not cfg.has_section('ah_bootstrap'): - return +def use_astropy_helpers(**kwargs): + """ + Ensure that the `astropy_helpers` module is available and is importable. + This supports automatic submodule initialization if astropy_helpers is + included in a project as a git submodule, or will download it from PyPI if + necessary. - kwargs = {} + Parameters + ---------- - for option, type_ in _CFG_OPTIONS: - if not cfg.has_option('ah_bootstrap', option): - continue + path : str or None, optional + A filesystem path relative to the root of the project's source code + that should be added to `sys.path` so that `astropy_helpers` can be + imported from that path. - if type_ is bool: - value = cfg.getboolean('ah_bootstrap', option) - else: - value = cfg.get('ah_bootstrap', option) + If the path is a git submodule it will automatically be initialized + and/or updated. - kwargs[option] = value + The path may also be to a ``.tar.gz`` archive of the astropy_helpers + source distribution. In this case the archive is automatically + unpacked and made temporarily available on `sys.path` as a ``.egg`` + archive. + + If `None` skip straight to downloading. + + download_if_needed : bool, optional + If the provided filesystem path is not found an attempt will be made to + download astropy_helpers from PyPI. It will then be made temporarily + available on `sys.path` as a ``.egg`` archive (using the + ``setup_requires`` feature of setuptools. If the ``--offline`` option + is given at the command line the value of this argument is overridden + to `False`. + + index_url : str, optional + If provided, use a different URL for the Python package index than the + main PyPI server. + + use_git : bool, optional + If `False` no git commands will be used--this effectively disables + support for git submodules. If the ``--no-git`` option is given at the + command line the value of this argument is overridden to `False`. + + auto_upgrade : bool, optional + By default, when installing a package from a non-development source + distribution ah_boostrap will try to automatically check for patch + releases to astropy-helpers on PyPI and use the patched version over + any bundled versions. Setting this to `False` will disable that + functionality. If the ``--offline`` option is given at the command line + the value of this argument is overridden to `False`. + + offline : bool, optional + If `False` disable all actions that require an internet connection, + including downloading packages from the package index and fetching + updates to any git submodule. Defaults to `True`. + """ - if kwargs.pop('auto_use', False): - use_astropy_helpers(**kwargs) + global BOOTSTRAPPER + config = BOOTSTRAPPER.config + config.update(**kwargs) -_main() + # Create a new bootstrapper with the updated configuration and run it + BOOTSTRAPPER = _Bootstrapper(**config) + BOOTSTRAPPER.run() diff --git a/astropy_helpers b/astropy_helpers index 5fd32d0edc..161773fa72 160000 --- a/astropy_helpers +++ b/astropy_helpers @@ -1 +1 @@ -Subproject commit 5fd32d0edc34f94de9640fd20865cfe5d605e499 +Subproject commit 161773fa72d916c498e0a2a513ecc24460244ac8