From 78c14351706531f3ee898d1a8108a759715e4695 Mon Sep 17 00:00:00 2001 From: Brigitta Sipocz Date: Fri, 3 Jun 2016 12:16:21 +0100 Subject: [PATCH] Updating to package template 1.1.1 --- .gitignore | 16 +- MANIFEST.in | 27 ++- astroquery/conftest.py | 23 ++- docs/Makefile | 7 +- ez_setup.py | 435 ++++++++++++++++++++++------------------- setup.cfg | 4 +- setup.py | 19 +- 7 files changed, 300 insertions(+), 231 deletions(-) diff --git a/.gitignore b/.gitignore index 15fc7d39a0..70dff2eadb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # Compiled files -*.py[co] +*.py[cod] *.a *.o *.so @@ -12,7 +12,6 @@ __pycache__ # Other generated files */version.py */cython_version.py -.cache htmlcov .coverage MANIFEST @@ -34,7 +33,6 @@ docs/_build # Packages/installer info *.egg *.egg-info -.eggs dist build eggs @@ -47,16 +45,10 @@ develop-eggs distribute-*.tar.gz # Other -.*.swp +.cache +.tox +.*.sw[op] *~ # Mac OSX .DS_Store - -# Eclipse -.settings -.project -.pydevproject - -# Pycharm -.idea diff --git a/MANIFEST.in b/MANIFEST.in index e7f223706b..fa6cec4248 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -18,9 +18,26 @@ prune build prune docs/_build prune docs/api -recursive-include astropy_helpers * -exclude astropy_helpers/.git -exclude astropy_helpers/.gitignore -global-exclude *.pyc -global-exclude *.o +# the next few stanzas are for astropy_helpers. It's derived from the +# astropy_helpers/MANIFEST.in, but requires additional includes for the actual +# package directory and egg-info. + +include astropy_helpers/README.rst +include astropy_helpers/CHANGES.rst +include astropy_helpers/LICENSE.rst +recursive-include astropy_helpers/licenses * + +include astropy_helpers/ez_setup.py +include astropy_helpers/ah_bootstrap.py + +recursive-include astropy_helpers/astropy_helpers *.py *.pyx *.c *.h +recursive-include astropy_helpers/astropy_helpers.egg-info * +# include the sphinx stuff with "*" because there are css/html/rst/etc. +recursive-include astropy_helpers/astropy_helpers/sphinx * + +prune astropy_helpers/build +prune astropy_helpers/astropy_helpers/tests + + +global-exclude *.pyc *.o diff --git a/astroquery/conftest.py b/astroquery/conftest.py index cc8f2cc6e0..6a021aecdb 100644 --- a/astroquery/conftest.py +++ b/astroquery/conftest.py @@ -1,12 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from __future__ import print_function import os - -# This is to figure out the astroquery version, rather than using Astropy's -from . import version - - # this contains imports plugins that configure py.test for astropy tests. # by importing them here in conftest.py they are discoverable by py.test # no matter how it is invoked within the source tree. @@ -28,5 +22,20 @@ PYTEST_HEADER_MODULES['pyregion'] = 'pyregion' del PYTEST_HEADER_MODULES['h5py'] del PYTEST_HEADER_MODULES['Scipy'] -except NameError: +except (NameError, KeyError): pass + +## Uncomment the following line to treat all DeprecationWarnings as +## exceptions +# enable_deprecations_as_exceptions() + + +# This is to figure out the affiliated package version, rather than +# using Astropy's +try: + from .version import version +except ImportError: + version = 'dev' + +packagename = os.path.basename(os.path.dirname(__file__)) +TESTED_VERSIONS[packagename] = version diff --git a/docs/Makefile b/docs/Makefile index c7979506d6..fb03f26e6b 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -34,11 +34,11 @@ help: @echo " man to make manual pages" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR) -rm -rf api + -rm -rf generated html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @@ -129,6 +129,5 @@ linkcheck: "or in $(BUILDDIR)/linkcheck/output.txt." doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." + @echo "Run 'python setup.py test' in the root directory to run doctests " \ + @echo "in the documentation." diff --git a/ez_setup.py b/ez_setup.py index 9dc2c8729b..fc22046bd4 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -1,66 +1,59 @@ -#!python -"""Bootstrap setuptools installation +#!/usr/bin/env python -If you want to use setuptools in your package's setup.py, just include this -file in the same directory with it, and add this to the top of your setup.py:: - - from ez_setup import use_setuptools - use_setuptools() +""" +Setuptools bootstrapping installer. -If you want to require a specific version of setuptools, set a download -mirror, or use an alternate download directory, you can do so by supplying -the appropriate options to ``use_setuptools()``. +Maintained at https://github.com/pypa/setuptools/tree/bootstrap. -This file can also be run as a script to install or upgrade setuptools. +Run this script to install or upgrade setuptools. """ + import os import shutil import sys import tempfile -import tarfile +import zipfile import optparse import subprocess import platform +import textwrap +import contextlib +import json +import codecs from distutils import log +try: + from urllib.request import urlopen + from urllib.parse import urljoin +except ImportError: + from urllib2 import urlopen + from urlparse import urljoin + try: from site import USER_SITE except ImportError: USER_SITE = None -DEFAULT_VERSION = "1.4.2" -DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" +LATEST = object() +DEFAULT_VERSION = LATEST +DEFAULT_URL = "https://pypi.io/packages/source/s/setuptools/" +DEFAULT_SAVE_DIR = os.curdir + def _python_cmd(*args): + """ + Execute a command. + + Return True if the command succeeded. + """ args = (sys.executable,) + args return subprocess.call(args) == 0 -def _check_call_py24(cmd, *args, **kwargs): - res = subprocess.call(cmd, *args, **kwargs) - class CalledProcessError(Exception): - pass - if not res == 0: - msg = "Command '%s' return non-zero exit status %d" % (cmd, res) - raise CalledProcessError(msg) -vars(subprocess).setdefault('check_call', _check_call_py24) - -def _install(tarball, install_args=()): - # extracting the tarball - tmpdir = tempfile.mkdtemp() - log.warn('Extracting in %s', tmpdir) - old_wd = os.getcwd() - try: - os.chdir(tmpdir) - tar = tarfile.open(tarball) - _extractall(tar) - tar.close() - - # going in the directory - subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) - os.chdir(subdir) - log.warn('Now working in %s', subdir) +def _install(archive_filename, install_args=()): + """Install Setuptools.""" + with archive_context(archive_filename): # installing log.warn('Installing Setuptools') if not _python_cmd('setup.py', 'install', *install_args): @@ -68,93 +61,159 @@ def _install(tarball, install_args=()): log.warn('See the error message above.') # exitcode will be 2 return 2 - finally: - os.chdir(old_wd) - shutil.rmtree(tmpdir) -def _build_egg(egg, tarball, to_dir): - # extracting the tarball +def _build_egg(egg, archive_filename, to_dir): + """Build Setuptools egg.""" + with archive_context(archive_filename): + # building an egg + log.warn('Building a Setuptools egg in %s', to_dir) + _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) + # returning the result + log.warn(egg) + if not os.path.exists(egg): + raise IOError('Could not build the egg.') + + +class ContextualZipFile(zipfile.ZipFile): + + """Supplement ZipFile class to support context manager for Python 2.6.""" + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def __new__(cls, *args, **kwargs): + """Construct a ZipFile or ContextualZipFile as appropriate.""" + if hasattr(zipfile.ZipFile, '__exit__'): + return zipfile.ZipFile(*args, **kwargs) + return super(ContextualZipFile, cls).__new__(cls) + + +@contextlib.contextmanager +def archive_context(filename): + """ + Unzip filename to a temporary directory, set to the cwd. + + The unzipped target is cleaned up after. + """ tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) - tar = tarfile.open(tarball) - _extractall(tar) - tar.close() + with ContextualZipFile(filename) as archive: + archive.extractall() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) - - # building an egg - log.warn('Building a Setuptools egg in %s', to_dir) - _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) + yield finally: os.chdir(old_wd) shutil.rmtree(tmpdir) - # returning the result - log.warn(egg) - if not os.path.exists(egg): - raise IOError('Could not build the egg.') def _do_download(version, download_base, to_dir, download_delay): + """Download Setuptools.""" egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' % (version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): - tarball = download_setuptools(version, download_base, + archive = download_setuptools(version, download_base, to_dir, download_delay) - _build_egg(egg, tarball, to_dir) + _build_egg(egg, archive, to_dir) sys.path.insert(0, egg) # Remove previously-imported pkg_resources if present (see # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). if 'pkg_resources' in sys.modules: - del sys.modules['pkg_resources'] + _unload_pkg_resources() import setuptools setuptools.bootstrap_install_from = egg -def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, download_delay=15): - # making sure we use the absolute path +def use_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=DEFAULT_SAVE_DIR, download_delay=15): + """ + Ensure that a setuptools version is installed. + + Return None. Raise SystemExit if the requested version + or later cannot be installed. + """ + version = _resolve_version(version) to_dir = os.path.abspath(to_dir) - was_imported = 'pkg_resources' in sys.modules or \ - 'setuptools' in sys.modules + + # prior to importing, capture the module state for + # representative modules. + rep_modules = 'pkg_resources', 'setuptools' + imported = set(sys.modules).intersection(rep_modules) + try: import pkg_resources - except ImportError: - return _do_download(version, download_base, to_dir, download_delay) - try: pkg_resources.require("setuptools>=" + version) + # a suitable version is already installed return - except pkg_resources.VersionConflict: - e = sys.exc_info()[1] - if was_imported: - sys.stderr.write( - "The required version of setuptools (>=%s) is not available,\n" - "and can't be installed while this script is running. Please\n" - "install a more recent version first, using\n" - "'easy_install -U setuptools'." - "\n\n(Currently using %r)\n" % (version, e.args[0])) - sys.exit(2) - else: - del pkg_resources, sys.modules['pkg_resources'] # reload ok - return _do_download(version, download_base, to_dir, - download_delay) + except ImportError: + # pkg_resources not available; setuptools is not installed; download + pass except pkg_resources.DistributionNotFound: - return _do_download(version, download_base, to_dir, - download_delay) + # no version of setuptools was found; allow download + pass + except pkg_resources.VersionConflict as VC_err: + if imported: + _conflict_bail(VC_err, version) + + # otherwise, unload pkg_resources to allow the downloaded version to + # take precedence. + del pkg_resources + _unload_pkg_resources() + + return _do_download(version, download_base, to_dir, download_delay) + + +def _conflict_bail(VC_err, version): + """ + Setuptools was imported prior to invocation, so it is + unsafe to unload it. Bail out. + """ + conflict_tmpl = textwrap.dedent(""" + The required version of setuptools (>={version}) is not available, + and can't be installed while this script is running. Please + install a more recent version first, using + 'easy_install -U setuptools'. + + (Currently using {VC_err.args[0]!r}) + """) + msg = conflict_tmpl.format(**locals()) + sys.stderr.write(msg) + sys.exit(2) + + +def _unload_pkg_resources(): + sys.meta_path = [ + importer + for importer in sys.meta_path + if importer.__class__.__module__ != 'pkg_resources.extern' + ] + del_modules = [ + name for name in sys.modules + if name.startswith('pkg_resources') + ] + for mod_name in del_modules: + del sys.modules[mod_name] + def _clean_check(cmd, target): """ - Run the command to download target. If the command fails, clean up before - re-raising the error. + Run the command to download target. + + If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) @@ -163,115 +222,110 @@ def _clean_check(cmd, target): os.unlink(target) raise + def download_file_powershell(url, target): """ - Download the file at url to target using Powershell (which will validate - trust). Raise an exception if the command cannot complete. + Download the file at url to target using Powershell. + + Powershell will validate trust. + Raise an exception if the command cannot complete. """ target = os.path.abspath(target) + ps_cmd = ( + "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " + "[System.Net.CredentialCache]::DefaultCredentials; " + '(new-object System.Net.WebClient).DownloadFile("%(url)s", "%(target)s")' + % locals() + ) cmd = [ 'powershell', '-Command', - "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars(), + ps_cmd, ] _clean_check(cmd, target) + def has_powershell(): + """Determine if Powershell is available.""" if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except: + except Exception: return False - finally: - devnull.close() return True - download_file_powershell.viable = has_powershell + def download_file_curl(url, target): - cmd = ['curl', url, '--silent', '--output', target] + cmd = ['curl', url, '--location', '--silent', '--output', target] _clean_check(cmd, target) + def has_curl(): cmd = ['curl', '--version'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except: + except Exception: return False - finally: - devnull.close() return True - download_file_curl.viable = has_curl + def download_file_wget(url, target): cmd = ['wget', url, '--quiet', '--output-document', target] _clean_check(cmd, target) + def has_wget(): cmd = ['wget', '--version'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except: + except Exception: return False - finally: - devnull.close() return True - download_file_wget.viable = has_wget + def download_file_insecure(url, target): - """ - Use Python to download the file, even though it cannot authenticate the - connection. - """ - try: - from urllib.request import urlopen - except ImportError: - from urllib2 import urlopen - src = dst = None + """Use Python to download the file, without connection authentication.""" + src = urlopen(url) try: - src = urlopen(url) - # Read/write all in one block, so we don't create a corrupt file - # if the download is interrupted. + # Read all the data in one block. data = src.read() - dst = open(target, "wb") - dst.write(data) finally: - if src: - src.close() - if dst: - dst.close() + src.close() + # Write all the data in one block to avoid creating a partial file. + with open(target, "wb") as dst: + dst.write(data) download_file_insecure.viable = lambda: True + def get_best_downloader(): - downloaders = [ + downloaders = ( download_file_powershell, download_file_curl, download_file_wget, download_file_insecure, - ] + ) + viable_downloaders = (dl for dl in downloaders if dl.viable()) + return next(viable_downloaders, None) - for dl in downloaders: - if dl.viable(): - return dl -def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, delay=15, - downloader_factory=get_best_downloader): - """Download setuptools from a specified location and return its filename +def download_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=DEFAULT_SAVE_DIR, delay=15, + downloader_factory=get_best_downloader): + """ + Download setuptools from a specified location and return its filename. `version` should be a valid setuptools version number that is available - as an egg for download under the `download_base` URL (which should end + as an sdist for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. @@ -279,11 +333,12 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, ``downloader_factory`` should be a function taking no arguments and returning a function for downloading a URL to a target. """ + version = _resolve_version(version) # making sure we use the absolute path to_dir = os.path.abspath(to_dir) - tgz_name = "setuptools-%s.tar.gz" % version - url = download_base + tgz_name - saveto = os.path.join(to_dir, tgz_name) + zip_name = "setuptools-%s.zip" % version + url = download_base + zip_name + saveto = os.path.join(to_dir, zip_name) if not os.path.exists(saveto): # Avoid repeated downloads log.warn("Downloading %s", url) downloader = downloader_factory() @@ -291,73 +346,42 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, return os.path.realpath(saveto) -def _extractall(self, path=".", members=None): - """Extract all members from the archive to the current working - directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). +def _resolve_version(version): + """ + Resolve LATEST version """ - import copy - import operator - from tarfile import ExtractError - directories = [] - - if members is None: - members = self - - for tarinfo in members: - if tarinfo.isdir(): - # Extract directories with a safe mode. - directories.append(tarinfo) - tarinfo = copy.copy(tarinfo) - tarinfo.mode = 448 # decimal for oct 0700 - self.extract(tarinfo, path) - - # Reverse sort directories. - if sys.version_info < (2, 4): - def sorter(dir1, dir2): - return cmp(dir1.name, dir2.name) - directories.sort(sorter) - directories.reverse() - else: - directories.sort(key=operator.attrgetter('name'), reverse=True) - - # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) + if version is not LATEST: + return version + + meta_url = urljoin(DEFAULT_URL, '/pypi/setuptools/json') + resp = urlopen(meta_url) + with contextlib.closing(resp): try: - self.chown(tarinfo, dirpath) - self.utime(tarinfo, dirpath) - self.chmod(tarinfo, dirpath) - except ExtractError: - e = sys.exc_info()[1] - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) + charset = resp.info().get_content_charset() + except Exception: + # Python 2 compat; assume UTF-8 + charset = 'UTF-8' + reader = codecs.getreader(charset) + doc = json.load(reader(resp)) + + return str(doc['info']['version']) def _build_install_args(options): """ - Build the arguments to 'python setup.py install' on the setuptools package + Build the arguments to 'python setup.py install' on the setuptools package. + + Returns list of command line arguments. """ - install_args = [] - if options.user_install: - if sys.version_info < (2, 6): - log.warn("--user requires Python 2.6 or later") - raise SystemExit(1) - install_args.append('--user') - return install_args + return ['--user'] if options.user_install else [] + def _parse_args(): - """ - Parse the command line for options - """ + """Parse the command line for options.""" parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, - help='install in user site package (requires Python 2.6 or later)') + help='install in user site package') parser.add_option( '--download-base', dest='download_base', metavar="URL", default=DEFAULT_URL, @@ -367,16 +391,35 @@ def _parse_args(): const=lambda: download_file_insecure, default=get_best_downloader, help='Use internal, non-validating downloader' ) + parser.add_option( + '--version', help="Specify which version to download", + default=DEFAULT_VERSION, + ) + parser.add_option( + '--to-dir', + help="Directory to save (and re-use) package", + default=DEFAULT_SAVE_DIR, + ) options, args = parser.parse_args() # positional arguments are ignored return options -def main(version=DEFAULT_VERSION): - """Install or upgrade setuptools and EasyInstall""" + +def _download_args(options): + """Return args for download_setuptools function from cmdline args.""" + return dict( + version=options.version, + download_base=options.download_base, + downloader_factory=options.downloader_factory, + to_dir=options.to_dir, + ) + + +def main(): + """Install or upgrade setuptools and EasyInstall.""" options = _parse_args() - tarball = download_setuptools(download_base=options.download_base, - downloader_factory=options.downloader_factory) - return _install(tarball, _build_install_args(options)) + archive = download_setuptools(**_download_args(options)) + return _install(archive, _build_install_args(options)) if __name__ == '__main__': sys.exit(main()) diff --git a/setup.cfg b/setup.cfg index cd016c59e5..e2c4d37afd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,4 +30,6 @@ github_project = astropy/astroquery # E402 module level import not at top of file # We need to ignore this as the .conf has to be set in the __init__.py # before the modules are imported. -ignore = E402 \ No newline at end of file +ignore = E402 + +[entry_points] diff --git a/setup.py b/setup.py index d01e6e767c..11811127cb 100755 --- a/setup.py +++ b/setup.py @@ -21,8 +21,12 @@ from astropy_helpers.version_helpers import generate_version_py # Get some values from the setup.cfg -from distutils import config -conf = config.ConfigParser() +try: + from ConfigParser import ConfigParser +except ImportError: + from configparser import ConfigParser + +conf = ConfigParser() conf.read(['setup.cfg']) metadata = dict(conf.items('metadata')) @@ -77,10 +81,7 @@ # These lines are only needed if astroquery acquires command line scripts # # Define entry points for command-line scripts # entry_points = {} -# entry_points['console_scripts'] = [ -# 'astropy-package-template-example = packagename.example_mod:main', -# ] - +# entry_points = {'console_scripts': []} # Currently the only entry points installed by Astropy are hooks to # zest.releaser for doing Astropy's releases @@ -92,6 +93,11 @@ hook_func = 'astropy.utils.release:' + '_'.join(hook) entry_points[hook_ep] = ['%s = %s' % (hook_name, hook_func)] +entry_point_list = conf.items('entry_points') +for entry_point in entry_point_list: + entry_points['console_scripts'].append('{0} = {1}'.format(entry_point[0], + entry_point[1])) + # Include all .c files, recursively, including those generated by # Cython, since we can not do this in MANIFEST.in with a "dynamic" # directory name. @@ -120,5 +126,6 @@ cmdclass=cmdclassd, zip_safe=False, use_2to3=False, + entry_points=entry_points, **package_info )