diff --git a/MANIFEST.in b/MANIFEST.in index 001bc294d..cedaf8db2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,10 +9,12 @@ recursive-include licenses * recursive-include cextern * recursive-include scripts * -exclude *.pyc *.o prune docs/_build prune build recursive-include astropy_helpers * exclude astropy_helpers/.git exclude astropy_helpers/.gitignore + +exclude *.pyc *.o +prune docs/api diff --git a/ah_bootstrap.py b/ah_bootstrap.py index 623f11d55..6888c3ed2 100644 --- a/ah_bootstrap.py +++ b/ah_bootstrap.py @@ -1,21 +1,60 @@ +""" +This bootstrap module contains code for ensuring that the astropy_helpers +package will be importable by the time the setup.py script runs. It also +includes some workarounds to ensure that a recent-enough version of setuptools +is being used for the installation. + +This module should be the first thing imported in the setup.py of distributions +that make use of the utilities in astropy_helpers. If the distribution ships +with its own copy of astropy_helpers, this module will first attempt to import +from the shipped copy. However, it will also check PyPI to see if there are +any bug-fix releases on top of the current version that may be useful to get +past platform-specific bugs that have been fixed. When running setup.py, use +the ``--offline`` command-line option to disable the auto-upgrade checks. + +When this module is imported or otherwise executed it automatically calls a +main function that attempts to read the project's setup.cfg file, which it +checks for a configuration section called ``[ah_bootstrap]`` the presences of +that section, and options therein, determine the next step taken: If it +contains an option called ``auto_use`` with a value of ``True``, it will +automatically call the main function of this module called +`use_astropy_helpers` (see that function's docstring for full details). +Otherwise no further action is taken (however, +``ah_bootstrap.use_astropy_helpers`` may be called manually from within the +setup.py script). + +Additional options in the ``[ah_boostrap]`` section of setup.cfg have the same +names as the arguments to `use_astropy_helpers`, and can be used to configure +the bootstrap script when ``auto_use = True``. + +See https://github.com/astropy/astropy-helpers for more details, and for the +latest version of this module. +""" + import contextlib import errno import imp +import io +import locale import os import re import subprocess as sp import sys try: - from ConfigParser import ConfigParser + from ConfigParser import ConfigParser, RawConfigParser except ImportError: - from configparser import ConfigParser + from configparser import ConfigParser, RawConfigParser if sys.version_info[0] < 3: _str_types = (str, unicode) + _text_type = unicode + PY3 = False else: _str_types = (str, bytes) + _text_type = str + PY3 = True # Some pre-setuptools checks to ensure that either distribute or setuptools >= # 0.7 is used (over pre-distribute setuptools) if it is available on the path; @@ -26,7 +65,7 @@ import pkg_resources _setuptools_req = pkg_resources.Requirement.parse('setuptools>=0.7') # This may raise a DistributionNotFound in which case no version of - # setuptools or distribute is properly instlaled + # setuptools or distribute is properly installed _setuptools = pkg_resources.get_distribution('setuptools') if _setuptools not in _setuptools_req: # Older version of setuptools; check if we have distribute; again if @@ -55,6 +94,16 @@ from setuptools.package_index import PackageIndex from setuptools.sandbox import run_setup +# Note: The following import is required as a workaround to +# https://github.com/astropy/astropy-helpers/issues/89; if we don't import this +# module now, it will get cleaned up after `run_setup` is called, but that will +# later cause the TemporaryDirectory class defined in it to stop working when +# used later on by setuptools +try: + import setuptools.py31compat +except ImportError: + pass + # TODO: Maybe enable checking for a specific version of astropy_helpers? DIST_NAME = 'astropy-helpers' PACKAGE_NAME = 'astropy_helpers' @@ -96,7 +145,9 @@ def use_astropy_helpers(path=None, download_if_needed=None, index_url=None, If the provided filesystem path is not found an attempt will be made to download astropy_helpers from PyPI. It will then be made temporarily available on `sys.path` as a ``.egg`` archive (using the - ``setup_requires`` feature of setuptools. + ``setup_requires`` feature of setuptools. If the ``--offline`` option + is given at the command line the value of this argument is overridden + to `False`. index_url : str, optional If provided, use a different URL for the Python package index than the @@ -104,14 +155,16 @@ def use_astropy_helpers(path=None, download_if_needed=None, index_url=None, use_git : bool, optional If `False` no git commands will be used--this effectively disables - support for git submodules. + support for git submodules. If the ``--no-git`` option is given at the + command line the value of this argument is overridden to `False`. auto_upgrade : bool, optional By default, when installing a package from a non-development source distribution ah_boostrap will try to automatically check for patch releases to astropy-helpers on PyPI and use the patched version over any bundled versions. Setting this to `False` will disable that - functionality. + functionality. If the ``--offline`` option is given at the command line + the value of this argument is overridden to `False`. """ # True by default, unless the --offline option was provided on the command @@ -119,7 +172,14 @@ def use_astropy_helpers(path=None, download_if_needed=None, index_url=None, if '--offline' in sys.argv: download_if_needed = False auto_upgrade = False + offline = True sys.argv.remove('--offline') + else: + offline = False + + if '--no-git' in sys.argv: + use_git = False + sys.argv.remove('--no-git') if path is None: path = PACKAGE_NAME @@ -130,6 +190,12 @@ def use_astropy_helpers(path=None, download_if_needed=None, index_url=None, if index_url is None: index_url = INDEX_URL + # If this is a release then the .git directory will not exist so we + # should not use git. + git_dir_exists = os.path.exists(os.path.join(os.path.dirname(__file__), '.git')) + if use_git is None and not git_dir_exists: + use_git = False + if use_git is None: use_git = USE_GIT @@ -152,7 +218,8 @@ def use_astropy_helpers(path=None, download_if_needed=None, index_url=None, elif not os.path.exists(path) or os.path.isdir(path): # Even if the given path does not exist on the filesystem, if it *is* a # submodule, `git submodule init` will create it - is_submodule = use_git and _check_submodule(path) + is_submodule = _check_submodule(path, use_git=use_git, + offline=offline) if is_submodule or os.path.isdir(path): log.info( @@ -165,7 +232,7 @@ def use_astropy_helpers(path=None, download_if_needed=None, index_url=None, if dist is None: msg = ( 'The requested path {0!r} for importing {1} does not ' - 'exist, or does not contain a copy of the {1} pacakge. ' + 'exist, or does not contain a copy of the {1} package. ' 'Attempting download instead.'.format(path, PACKAGE_NAME)) if download_if_needed: log.warn(msg) @@ -223,7 +290,9 @@ def use_astropy_helpers(path=None, download_if_needed=None, index_url=None, # Just activate the found distribibution on sys.path--if we did a # download this usually happens automatically but do it again just to # be sure - return dist.activate() + # Note: Adding the dist to the global working set also activates it by + # default + pkg_resources.working_set.add(dist) def _do_download(version='', find_links=None, index_url=None): @@ -261,6 +330,8 @@ def get_option_dict(self, command_name): with _silence(): dist = _Distribution(attrs=attrs) + # If the setup_requires succeeded it will have added the new dist to + # the main working_set return pkg_resources.working_set.by_key.get(DIST_NAME) except Exception as e: if DEBUG: @@ -306,8 +377,13 @@ def _directory_import(path): # Return True on success, False on failure but download is allowed, and # otherwise raise SystemExit path = os.path.abspath(path) - pkg_resources.working_set.add_entry(path) - dist = pkg_resources.working_set.by_key.get(DIST_NAME) + + # Use an empty WorkingSet rather than the man pkg_resources.working_set, + # since on older versions of setuptools this will invoke a VersionConflict + # when trying to install an upgrade + ws = pkg_resources.WorkingSet([]) + ws.add_entry(path) + dist = ws.by_key.get(DIST_NAME) if dist is None: # We didn't find an egg-info/dist-info in the given path, but if a @@ -319,13 +395,39 @@ def _directory_import(path): for dist in pkg_resources.find_distributions(path, True): # There should be only one... - pkg_resources.working_set.add(dist, path, False) - break + return dist return dist -def _check_submodule(path): +def _check_submodule(path, use_git=True, offline=False): + """ + Check if the given path is a git submodule. + + See the docstrings for ``_check_submodule_using_git`` and + ``_check_submodule_no_git`` for futher details. + """ + + if use_git: + return _check_submodule_using_git(path, offline) + else: + return _check_submodule_no_git(path) + + +def _check_submodule_using_git(path, offline): + """ + Check if the given path is a git submodule. If so, attempt to initialize + and/or update the submodule if needed. + + This function makes calls to the ``git`` command in subprocesses. The + ``_check_submodule_no_git`` option uses pure Python to check if the given + path looks like a git submodule, but it cannot perform updates. + """ + + if PY3 and not isinstance(path, _text_type): + fs_encoding = sys.getfilesystemencoding() + path = path.decode(fs_encoding) + try: p = sp.Popen(['git', 'submodule', 'status', '--', path], stdout=sp.PIPE, stderr=sp.PIPE) @@ -347,40 +449,128 @@ def _check_submodule(path): '`git submodule status` command:\n{0}'.format(str(e))) + # Can fail of the default locale is not configured properly. See + # https://github.com/astropy/astropy/issues/2749. For the purposes under + # consideration 'latin1' is an acceptable fallback. + try: + stdio_encoding = locale.getdefaultlocale()[1] or 'latin1' + except ValueError: + # Due to an OSX oddity locale.getdefaultlocale() can also crash + # depending on the user's locale/language settings. See: + # http://bugs.python.org/issue18378 + stdio_encoding = 'latin1' + if p.returncode != 0 or stderr: # Unfortunately the return code alone cannot be relied on, as - # earler versions of git returned 0 even if the requested submodule + # earlier versions of git returned 0 even if the requested submodule # does not exist - log.debug('git submodule command failed ' - 'unexpectedly:\n{0}'.format(stderr)) - return False + stderr = stderr.decode(stdio_encoding) + + # This is a warning that occurs in perl (from running git submodule) + # which only occurs with a malformatted locale setting which can + # happen sometimes on OSX. See again + # https://github.com/astropy/astropy/issues/2749 + perl_warning = ('perl: warning: Falling back to the standard locale ' + '("C").') + if not stderr.strip().endswith(perl_warning): + # Some other uknown error condition occurred + log.warn('git submodule command failed ' + 'unexpectedly:\n{0}'.format(stderr)) + return False + + stdout = stdout.decode(stdio_encoding) + # The stdout should only contain one line--the status of the + # requested submodule + m = _git_submodule_status_re.match(stdout) + if m: + # Yes, the path *is* a git submodule + _update_submodule(m.group('submodule'), m.group('status'), offline) + return True else: - # The stdout should only contain one line--the status of the - # requested submodule - m = _git_submodule_status_re.match(stdout) - if m: - # Yes, the path *is* a git submodule - _update_submodule(m.group('submodule'), m.group('status')) + log.warn( + 'Unexpected output from `git submodule status`:\n{0}\n' + 'Will attempt import from {1!r} regardless.'.format( + stdout, path)) + return False + + +def _check_submodule_no_git(path): + """ + Like ``_check_submodule_using_git``, but simply parses the .gitmodules file + to determine if the supplied path is a git submodule, and does not exec any + subprocesses. + + This can only determine if a path is a submodule--it does not perform + updates, etc. This function may need to be updated if the format of the + .gitmodules file is changed between git versions. + """ + + gitmodules_path = os.path.abspath('.gitmodules') + + if not os.path.isfile(gitmodules_path): + return False + + # This is a minimal reader for gitconfig-style files. It handles a few of + # the quirks that make gitconfig files incompatible with ConfigParser-style + # files, but does not support the full gitconfig syntaix (just enough + # needed to read a .gitmodules file). + gitmodules_fileobj = io.StringIO() + + # Must use io.open for cross-Python-compatible behavior wrt unicode + with io.open(gitmodules_path) as f: + for line in f: + # gitconfig files are more flexible with leading whitespace; just + # go ahead and remove it + line = line.lstrip() + + # comments can start with either # or ; + if line and line[0] in (':', ';'): + continue + + gitmodules_fileobj.write(line) + + gitmodules_fileobj.seek(0) + + cfg = RawConfigParser() + + try: + cfg.readfp(gitmodules_fileobj) + except Exception as exc: + log.warn('Malformatted .gitmodules file: {0}\n' + '{1} cannot be assumed to be a git submodule.'.format( + exc, path)) + return False + + for section in cfg.sections(): + if not cfg.has_option(section, 'path'): + continue + + submodule_path = cfg.get(section, 'path').rstrip(os.sep) + + if submodule_path == path.rstrip(os.sep): return True - else: - log.warn( - 'Unexected output from `git submodule status`:\n{0}\n' - 'Will attempt import from {1!r} regardless.'.format( - stdout, path)) - return False + + return False -def _update_submodule(submodule, status): - if status == b' ': +def _update_submodule(submodule, status, offline): + if status == ' ': # The submodule is up to date; no action necessary return - elif status == b'-': + elif status == '-': + if offline: + raise _AHBootstrapSystemExit( + "Cannot initialize the {0} submodule in --offline mode; this " + "requires being able to clone the submodule from an online " + "repository.".format(submodule)) cmd = ['update', '--init'] - log.info('Initializing submodule {0!r}'.format(submodule)) - elif status == b'+': + action = 'Initializing' + elif status == '+': cmd = ['update'] - log.info('Updating submodule {0!r}'.format(submodule)) - elif status == b'U': + action = 'Updating' + if offline: + cmd.append('--no-fetch') + elif status == 'U': raise _AHBoostrapSystemExit( 'Error: Submodule {0} contains unresolved merge conflicts. ' 'Please complete or abandon any changes in the submodule so that ' @@ -395,15 +585,19 @@ def _update_submodule(submodule, status): err_msg = None + cmd = ['git', 'submodule'] + cmd + ['--', submodule] + log.warn('{0} {1} submodule with: `{2}`'.format( + action, submodule, ' '.join(cmd))) + try: - p = sp.Popen(['git', 'submodule'] + cmd + ['--', submodule], - stdout=sp.PIPE, stderr=sp.PIPE) + p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) stdout, stderr = p.communicate() except OSError as e: err_msg = str(e) else: if p.returncode != 0: - err_msg = stderr + stderr_encoding = locale.getdefaultlocale()[1] + err_msg = stderr.decode(stderr_encoding) if err_msg: log.warn('An unexpected error occurred updating the git submodule ' @@ -414,6 +608,7 @@ class _DummyFile(object): """A noop writeable object.""" errors = '' # Required for Python 3.x + encoding = 'utf-8' def write(self, s): pass @@ -465,6 +660,45 @@ def __init__(self, *args): super(_AHBootstrapSystemExit, self).__init__(msg, *args[1:]) +if sys.version_info[:2] < (2, 7): + # In Python 2.6 the distutils log does not log warnings, errors, etc. to + # stderr so we have to wrap it to ensure consistency at least in this + # module + import distutils + + class log(object): + def __getattr__(self, attr): + return getattr(distutils.log, attr) + + def warn(self, msg, *args): + self._log_to_stderr(distutils.log.WARN, msg, *args) + + def error(self, msg): + self._log_to_stderr(distutils.log.ERROR, msg, *args) + + def fatal(self, msg): + self._log_to_stderr(distutils.log.FATAL, msg, *args) + + def log(self, level, msg, *args): + if level in (distutils.log.WARN, distutils.log.ERROR, + distutils.log.FATAL): + self._log_to_stderr(level, msg, *args) + else: + distutils.log.log(level, msg, *args) + + def _log_to_stderr(self, level, msg, *args): + # This is the only truly 'public' way to get the current threshold + # of the log + current_threshold = distutils.log.set_threshold(distutils.log.WARN) + distutils.log.set_threshold(current_threshold) + if level >= current_threshold: + if args: + msg = msg % args + sys.stderr.write('%s\n' % msg) + sys.stderr.flush() + + log = log() + # Output of `git submodule status` is as follows: # # 1: Status indicator: '-' for submodule is uninitialized, '+' if submodule is @@ -479,14 +713,14 @@ def __init__(self, *args): # includes for example what branches the commit is on) but only if the # submodule is initialized. We ignore this information for now _git_submodule_status_re = re.compile( - b'^(?P[+-U ])(?P[0-9a-f]{40}) (?P\S+)( .*)?$') + '^(?P[+-U ])(?P[0-9a-f]{40}) (?P\S+)( .*)?$') # Implement the auto-use feature; this allows use_astropy_helpers() to be used # at import-time automatically so long as the correct options are specified in # setup.cfg _CFG_OPTIONS = [('auto_use', bool), ('path', str), - ('download_if_needed', bool), ('index_ur', str), + ('download_if_needed', bool), ('index_url', str), ('use_git', bool), ('auto_upgrade', bool)] def _main(): diff --git a/astropy_helpers b/astropy_helpers index a93515f3f..5fd32d0ed 160000 --- a/astropy_helpers +++ b/astropy_helpers @@ -1 +1 @@ -Subproject commit a93515f3fad840e363d84ca1cc094aba7420f80b +Subproject commit 5fd32d0edc34f94de9640fd20865cfe5d605e499