diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 4868fc2bc88..baf4230528b 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -18,6 +18,7 @@ Changes: line arguments (see #2489) * removed calls to deprecated NumPy functionality (see #2949) * cleaned the documentation, build process, and docstrings (see #2662) + * refactored and modernized setup.py (see #2422) - obspy.core: * read_inventory(): add "level" option to read files faster when less level of detail is needed. currently only implemented for StationXML reading diff --git a/MANIFEST.in b/MANIFEST.in index 88e10fc33f9..98eed089693 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,11 +1,3 @@ -# Unfortunately setuptools and numpy.distutils do not like each other and lot's -# of small incompatibilities are around. One manifestation of this is that the -# source code and data files included in the setup.py are included in binary -# distributions but not in source distributions... -# Therefore the MANIFEST.in files appears to be necessary. -# See http://scipy-user.10969.n7.nabble.com/SciPy-User-setuptools-messing-with-sdists-using-numpy-distutils-and-Fortran-libraries-td19023.html -# for more details. - # Include all files in top-level and obspy-top-level directories (e.g. CHANGELOG, RELEASE-VERSION, ...) include * # seem to catch only files, so ./misc and ./debian are not catched.. good! recursive-include obspy * # includes all files in any subdirs, so it also catches *all* subdirs diff --git a/obspy/conftest.py b/obspy/conftest.py index e8a99299555..bf95b26f244 100644 --- a/obspy/conftest.py +++ b/obspy/conftest.py @@ -15,12 +15,15 @@ import obspy from obspy.core.util import NETWORK_MODULES -from obspy.core.util.requirements import SOFT_DEPENDENCIES OBSPY_PATH = os.path.dirname(obspy.__file__) +# Soft dependencies to include in ObsPy test report. +SOFT_DEPENDENCIES = ['cartopy', 'flake8', 'geographiclib', 'pyproj', + 'shapefile'] + # --- ObsPy fixtures diff --git a/obspy/core/tests/__init__.py b/obspy/core/tests/__init__.py new file mode 100644 index 00000000000..9d44cfb382a --- /dev/null +++ b/obspy/core/tests/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +MODULE_NAME = "obspy.core" diff --git a/obspy/core/tests/test_util_misc.py b/obspy/core/tests/test_util_misc.py index be95151f7b5..94ad1f0aa99 100644 --- a/obspy/core/tests/test_util_misc.py +++ b/obspy/core/tests/test_util_misc.py @@ -80,7 +80,7 @@ def test_no_obspy_imports(self): Check files that are used at install time for obspy imports. """ from obspy.core import util - files = ["libnames.py", "version.py", "requirements.py"] + files = ["version.py"] for file_ in files: file_ = os.path.join(os.path.dirname(util.__file__), file_) diff --git a/obspy/core/util/libnames.py b/obspy/core/util/libnames.py index fdc3735a0cd..2af3b60a667 100644 --- a/obspy/core/util/libnames.py +++ b/obspy/core/util/libnames.py @@ -8,14 +8,10 @@ GNU Lesser General Public License, Version 3 (https://www.gnu.org/copyleft/lesser.html) """ -# NO IMPORTS FROM OBSPY OR FUTURE IN THIS FILE! (file gets used at -# installation time) import ctypes +import importlib.machinery from pathlib import Path -import platform import re -import warnings -from distutils import sysconfig def cleanse_pymodule_filename(filename): @@ -39,43 +35,6 @@ def cleanse_pymodule_filename(filename): return filename -def _get_lib_name(lib, add_extension_suffix): - """ - Helper function to get an architecture and Python version specific library - filename. - - :type add_extension_suffix: bool - :param add_extension_suffix: NumPy distutils adds a suffix to - the filename we specify to build internally (as specified by Python - builtin `sysconfig.get_config_var("EXT_SUFFIX")`. So when loading the - file we have to add this suffix, but not during building. - """ - # our custom defined part of the extension file name - libname = "lib%s_%s_%s_py%s" % ( - lib, platform.system(), platform.architecture()[0], - ''.join([str(i) for i in platform.python_version_tuple()[:2]])) - libname = cleanse_pymodule_filename(libname) - # NumPy distutils adds extension suffix by itself during build (#771, #755) - if add_extension_suffix: - # append any extension suffix defined by Python for current platform - ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") - # in principle "EXT_SUFFIX" is what we want. - # "SO" seems to be deprecated on newer python - # but: older python seems to have empty "EXT_SUFFIX", so we fall back - if not ext_suffix: - try: - ext_suffix = sysconfig.get_config_var("SO") - except Exception as e: - msg = ("Empty 'EXT_SUFFIX' encountered while building CDLL " - "filename and fallback to 'SO' variable failed " - "(%s)." % str(e)) - warnings.warn(msg) - pass - if ext_suffix: - libname = libname + ext_suffix - return libname - - def _load_cdll(name): """ Helper function to load a shared library built during ObsPy installation @@ -85,25 +44,24 @@ def _load_cdll(name): :param name: Name of the library to load (e.g. 'mseed'). :rtype: :class:`ctypes.CDLL` """ - # our custom defined part of the extension file name - libname = _get_lib_name(name, add_extension_suffix=True) + errors = [] libdir = Path(__file__).parent.parent.parent / 'lib' - libpath = (libdir / libname).resolve() - try: - cdll = ctypes.CDLL(str(libpath)) - except Exception as e: - dirlisting = sorted(libpath.parent.iterdir()) - dirlisting = ' \n'.join(map(str, dirlisting)) - msg = ['Could not load shared library "%s"' % libname, - 'Path: %s' % libpath, - 'Current directory: %s' % Path().resolve(), - 'ctypes error message: %s' % str(e), - 'Directory listing of lib directory:', - ' %s' % dirlisting, - ] - msg = '\n '.join(msg) - raise ImportError(msg) - return cdll + for ext in importlib.machinery.EXTENSION_SUFFIXES: + libpath = (libdir / (name + ext)).resolve() + try: + cdll = ctypes.CDLL(str(libpath)) + except Exception as e: + errors.append(f' {str(e)}') + else: + return cdll + # If we got here, then none of the attempted extensions worked. + raise ImportError('\n '.join([ + f'Could not load shared library "{name}"', + *errors, + 'Current directory: %s' % Path().resolve(), + 'Directory listing of lib directory:', + *(f' {str(d)}' for d in sorted(libpath.parent.iterdir())), + ])) if __name__ == '__main__': diff --git a/obspy/core/util/requirements.py b/obspy/core/util/requirements.py deleted file mode 100644 index 861aa64711c..00000000000 --- a/obspy/core/util/requirements.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -A python module for ObsPy's requirements. - -These are kept in this module so they are accessible both from the setup.py -and from the ObsPy namespace. -""" -# Hard dependencies needed to install/run ObsPy. -INSTALL_REQUIRES = [ - 'numpy>=1.15.0', - 'scipy>=1.0.0', - 'matplotlib>=3.2.0', - 'lxml', - 'setuptools', - 'sqlalchemy', - 'decorator', - 'requests'] - -# The modules needed for running ObsPy's test suite. -PYTEST_REQUIRES = [ - 'packaging', - 'pytest', - 'pytest-cov', - 'pytest-json-report', -] - -# Extra dependencies -EXTRAS_REQUIRES = { - 'tests': ['pyproj'] + PYTEST_REQUIRES, - # arclink decryption also works with: pycrypto, m2crypto, pycryptodome - 'arclink': ['cryptography'], - 'io.shapefile': ['pyshp'], - } - -# Soft dependencies to include in ObsPy test report. -SOFT_DEPENDENCIES = ['cartopy', 'flake8', 'geographiclib', 'pyproj', - 'shapefile'] diff --git a/obspy/geodetics/tests/__init__.py b/obspy/geodetics/tests/__init__.py new file mode 100644 index 00000000000..c859c91a82c --- /dev/null +++ b/obspy/geodetics/tests/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +MODULE_NAME = "obspy.geodetics" diff --git a/obspy/imaging/tests/__init__.py b/obspy/imaging/tests/__init__.py new file mode 100644 index 00000000000..b9e930d6570 --- /dev/null +++ b/obspy/imaging/tests/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +MODULE_NAME = "obspy.imaging" diff --git a/obspy/io/nordic/tests/__init__.py b/obspy/io/nordic/tests/__init__.py new file mode 100644 index 00000000000..9d9ef0f82e8 --- /dev/null +++ b/obspy/io/nordic/tests/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +MODULE_NAME = "obspy.io.nordic" diff --git a/obspy/scripts/runtests.py b/obspy/scripts/runtests.py index 74d1eb0876c..9332a687105 100644 --- a/obspy/scripts/runtests.py +++ b/obspy/scripts/runtests.py @@ -45,7 +45,6 @@ GNU Lesser General Public License, Version 3 (https://www.gnu.org/copyleft/lesser.html) """ -import re import sys from pathlib import Path @@ -54,7 +53,6 @@ import obspy from obspy.core.util.misc import change_directory -from obspy.core.util.requirements import PYTEST_REQUIRES # URL to upload json report @@ -67,16 +65,13 @@ def _ensure_tests_requirements_installed(): This function is intended to help less experienced users run the tests. """ - delimiters = (" ", "=", "<", ">", "!") - patterns = '|'.join(map(re.escape, delimiters)) - msg = (f"\nNot all ObsPy's test requirements are installed. You need to " - f"install them before using obspy-runtest. Example with pip: \n" - f"\t$ pip install {' '.join(PYTEST_REQUIRES)}") - for package_req in PYTEST_REQUIRES: - # strip off any requirements, just get pkg_name - pkg_name = re.split(patterns, package_req, maxsplit=1)[0] + msg = ("\nNot all ObsPy's test requirements are installed. You need to " + "install them before using obspy-runtest. Example with pip: \n" + "\t$ pip install obspy[tests]") + dist = pkg_resources.get_distribution('obspy') + for package_req in dist.requires(['tests']): try: - pkg_resources.get_distribution(pkg_name).version + pkg_resources.get_distribution(package_req).version except pkg_resources.DistributionNotFound: raise ImportError(msg) diff --git a/obspy/taup/tests/__init__.py b/obspy/taup/tests/__init__.py new file mode 100644 index 00000000000..fec612f38a2 --- /dev/null +++ b/obspy/taup/tests/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +MODULE_NAME = "obspy.taup" diff --git a/pyproject.toml b/pyproject.toml index 622788b1f03..974ce5f83f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,6 @@ # Minimum requirements for the build system to execute. # see PEP518: https://www.python.org/dev/peps/pep-0518/ requires = [ - 'numpy>=1.6.1', 'setuptools' ] build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.py b/setup.py index e826fc233de..1a63de10e88 100644 --- a/setup.py +++ b/setup.py @@ -20,41 +20,28 @@ GNU Lesser General Public License, Version 3 (https://www.gnu.org/copyleft/lesser.html) """ -# Importing setuptools monkeypatches some of distutils commands so things like -# 'python setup.py develop' work. Wrap in try/except so it is not an actual -# dependency. Inplace installation with pip works also without importing -# setuptools. -try: - import setuptools # @UnusedImport # NOQA -except ImportError: - pass - -try: - import numpy # @UnusedImport # NOQA -except ImportError: - msg = ("No module named numpy. " - "Please install numpy first, it is needed before installing ObsPy.") - raise ImportError(msg) - -import fnmatch + import glob import inspect import os -import sys import platform -from distutils.util import change_root +import shutil +import subprocess +import sys + +import setuptools + +from distutils.ccompiler import get_default_compiler +from distutils.command.build import build +from distutils.command.install import install from distutils.errors import DistutilsSetupError +from distutils.util import change_root -from numpy.distutils.core import setup -from numpy.distutils.ccompiler import get_default_compiler -from numpy.distutils.command.build import build -from numpy.distutils.command.install import install -from numpy.distutils.exec_command import exec_command, find_executable -from numpy.distutils.misc_util import Configuration +from setuptools import Extension, find_packages, setup # The minimum python version which can be used to run ObsPy -MIN_PYTHON_VERSION = (3, 6) +MIN_PYTHON_VERSION = (3, 7) # Fail fast if the user is on an unsupported version of python. if sys.version_info < MIN_PYTHON_VERSION: @@ -74,8 +61,6 @@ UTIL_PATH = os.path.join(SETUP_DIRECTORY, "obspy", "core", "util") sys.path.insert(0, UTIL_PATH) from version import get_git_version # @UnresolvedImport -from libnames import _get_lib_name # @UnresolvedImport -from requirements import INSTALL_REQUIRES, EXTRAS_REQUIRES sys.path.pop(0) LOCAL_PATH = os.path.join(SETUP_DIRECTORY, "setup.py") @@ -94,6 +79,32 @@ EXTERNAL_EVALRESP = False EXTERNAL_LIBMSEED = False +# Hard dependencies needed to install/run ObsPy. +INSTALL_REQUIRES = [ + 'numpy>=1.15.0', + 'scipy>=1.0.0', + 'matplotlib>=3.2.0', + 'lxml', + 'setuptools', + 'sqlalchemy', + 'decorator', + 'requests', +] + +# Extra dependencies +EXTRAS_REQUIRES = { + 'tests': [ + 'packaging', + 'pyproj', + 'pytest', + 'pytest-cov', + 'pytest-json-report', + ], + # arclink decryption also works with: pycrypto, m2crypto, pycryptodome + 'arclink': ['cryptography'], + 'io.shapefile': ['pyshp'], +} + # package specific settings KEYWORDS = [ 'ArcLink', 'array', 'array analysis', 'ASC', 'beachball', @@ -520,18 +531,6 @@ } -def find_packages(): - """ - Simple function to find all modules under the current folder. - """ - modules = [] - for dirpath, _, filenames in os.walk(os.path.join(SETUP_DIRECTORY, - "obspy")): - if "__init__.py" in filenames: - modules.append(os.path.relpath(dirpath, SETUP_DIRECTORY)) - return [_i.replace(os.sep, ".") for _i in modules] - - # monkey patches for MS Visual Studio if IS_MSVC: # remove 'init' entry in exported symbols @@ -549,7 +548,7 @@ def export_symbols(*path): # adds --with-system-libs command-line option if possible def add_features(): - if 'setuptools' not in sys.modules or not hasattr(setuptools, 'Feature'): + if not hasattr(setuptools, 'Feature'): return {} class ExternalLibFeature(setuptools.Feature): @@ -582,11 +581,11 @@ def exclude_from(self, dist): } -def configuration(parent_package="", top_path=None): +def get_extensions(): """ Config function mainly used to compile C code. """ - config = Configuration("", parent_package, top_path) + extensions = [] # GSE2 path = os.path.join("obspy", "io", "gse2", "src", "GSE_UTI") @@ -596,8 +595,7 @@ def configuration(parent_package="", top_path=None): if IS_MSVC: # get export symbols kwargs['export_symbols'] = export_symbols(path, 'gse_functions.def') - config.add_extension(_get_lib_name("gse2", add_extension_suffix=False), - files, **kwargs) + extensions.append(Extension("gse2", files, **kwargs)) # LIBMSEED path = os.path.join("obspy", "io", "mseed", "src") @@ -616,8 +614,7 @@ def configuration(parent_package="", top_path=None): export_symbols(path, 'obspy-readbuffer.def') if EXTERNAL_LIBMSEED: kwargs['libraries'] = ['mseed'] - config.add_extension(_get_lib_name("mseed", add_extension_suffix=False), - files, **kwargs) + extensions.append(Extension("mseed", files, **kwargs)) # SEGY path = os.path.join("obspy", "io", "segy", "src") @@ -627,8 +624,7 @@ def configuration(parent_package="", top_path=None): if IS_MSVC: # get export symbols kwargs['export_symbols'] = export_symbols(path, 'libsegy.def') - config.add_extension(_get_lib_name("segy", add_extension_suffix=False), - files, **kwargs) + extensions.append(Extension("segy", files, **kwargs)) # SIGNAL path = os.path.join("obspy", "signal", "src") @@ -638,8 +634,7 @@ def configuration(parent_package="", top_path=None): if IS_MSVC: # get export symbols kwargs['export_symbols'] = export_symbols(path, 'libsignal.def') - config.add_extension(_get_lib_name("signal", add_extension_suffix=False), - files, **kwargs) + extensions.append(Extension("signal", files, **kwargs)) # EVALRESP path = os.path.join("obspy", "signal", "src") @@ -656,8 +651,7 @@ def configuration(parent_package="", top_path=None): kwargs['export_symbols'] = export_symbols(path, 'libevresp.def') if EXTERNAL_EVALRESP: kwargs['libraries'] = ['evresp'] - config.add_extension(_get_lib_name("evresp", add_extension_suffix=False), - files, **kwargs) + extensions.append(Extension("evresp", files, **kwargs)) # TAU path = os.path.join("obspy", "taup", "src") @@ -667,44 +661,9 @@ def configuration(parent_package="", top_path=None): if IS_MSVC: # get export symbols kwargs['export_symbols'] = export_symbols(path, 'libtau.def') - config.add_extension(_get_lib_name("tau", add_extension_suffix=False), - files, **kwargs) + extensions.append(Extension("tau", files, **kwargs)) - add_data_files(config) - - return config - - -def add_data_files(config): - """ - Recursively include all non python files - """ - # python files are included per default, we only include data files - # here - EXCLUDE_WILDCARDS = ['*.py', '*.pyc', '*.pyo', '*.pdf', '.git*'] - EXCLUDE_DIRS = ['src', '__pycache__'] - common_prefix = SETUP_DIRECTORY + os.path.sep - for root, dirs, files in os.walk(os.path.join(SETUP_DIRECTORY, 'obspy')): - root = root.replace(common_prefix, '') - for name in files: - if any(fnmatch.fnmatch(name, w) for w in EXCLUDE_WILDCARDS): - continue - config.add_data_files(os.path.join(root, name)) - for folder in EXCLUDE_DIRS: - if folder in dirs: - dirs.remove(folder) - - # Force include the contents of some directories. - FORCE_INCLUDE_DIRS = [ - os.path.join(SETUP_DIRECTORY, 'obspy', 'io', 'mseed', 'src', - 'libmseed', 'test')] - - for folder in FORCE_INCLUDE_DIRS: - for root, _, files in os.walk(folder): - for filename in files: - config.add_data_files( - os.path.relpath(os.path.join(root, filename), - SETUP_DIRECTORY)) + return extensions # Auto-generate man pages from --help output @@ -713,8 +672,8 @@ class Help2ManBuild(build): def finalize_options(self): build.finalize_options(self) - self.help2man = find_executable('help2man') - if not self.help2man: + self.help2man = shutil.which('help2man') + if self.help2man is None: raise DistutilsSetupError('Building man pages requires help2man.') def run(self): @@ -729,11 +688,11 @@ def run(self): output = os.path.join(mandir, ep.name + '.1') print('Generating %s ...' % (output)) - exec_command([self.help2man, - '--no-info', '--no-discard-stderr', - '--output', output, - '"%s -m %s"' % (sys.executable, - ep.module_name)]) + subprocess.call([self.help2man, + '--no-info', '--no-discard-stderr', + '--output', output, + '"%s -m %s"' % (sys.executable, + ep.module_name)]) class Help2ManInstall(install): @@ -771,6 +730,11 @@ def setupPackage(): description=DOCSTRING[1], long_description="\n".join(DOCSTRING[3:]), url="https://www.obspy.org", + project_urls={ + "Bug Tracker": "https://github.com/obspy/obspy/issues", + "Documentation": "https://docs.obspy.org/", + "Source Code": "https://github.com/obspy/obspy", + }, author='The ObsPy Development Team', author_email='devs@obspy.org', license='GNU Lesser General Public License, Version 3 (LGPLv3)', @@ -780,8 +744,8 @@ def setupPackage(): 'Environment :: Console', 'Intended Audience :: Science/Research', 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU Library or ' + - 'Lesser General Public License (LGPL)', + 'License :: OSI Approved :: ' + 'GNU Lesser General Public License v3 (LGPLv3)', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', @@ -791,6 +755,32 @@ def setupPackage(): 'Topic :: Scientific/Engineering :: Physics'], keywords=KEYWORDS, packages=find_packages(), + include_package_data=True, + exclude_package_data={ + 'obspy.io.css': ['contrib/*'], + # NOTE: If the libmseed test data wasn't used in our tests, we + # could just ignore src/* everywhere. + 'obspy.io.gse2': ['src/*'], + 'obspy.io.mseed': [ + # Only keep src/libmseed/test/* except for the C files. + 'src/*.c', + 'src/*.def', + 'src/libmseed/.clang-format', + 'src/libmseed/ChangeLog', + 'src/libmseed/Makefile*', + 'src/libmseed/README.byteorder', + 'src/libmseed/doc/*', + 'src/libmseed/example/*', + 'src/libmseed/test/Makefile', + 'src/libmseed/*.h', + 'src/libmseed/*.in', + 'src/libmseed/*.map', + 'src/libmseed/*.md', + ], + 'obspy.io.segy': ['src/*'], + 'obspy.signal': ['src/*'], + 'obspy.taup': ['src/*'], + }, namespace_packages=[], zip_safe=False, python_requires=f'>={MIN_PYTHON_VERSION[0]}.{MIN_PYTHON_VERSION[1]}', @@ -798,23 +788,19 @@ def setupPackage(): tests_require=EXTRAS_REQUIRES['tests'], extras_require=EXTRAS_REQUIRES, features=add_features(), - # this is needed for "easy_install obspy==dev" - download_url=("https://github.com/obspy/obspy/zipball/master" - "#egg=obspy=dev"), - include_package_data=True, entry_points=ENTRY_POINTS, + ext_modules=get_extensions(), ext_package='obspy.lib', cmdclass={ 'build_man': Help2ManBuild, 'install_man': Help2ManInstall }, - configuration=configuration) + ) if __name__ == '__main__': # clean --all does not remove extensions automatically if 'clean' in sys.argv and '--all' in sys.argv: - import shutil # delete complete build directory path = os.path.join(SETUP_DIRECTORY, 'build') try: