diff --git a/.travis.yml b/.travis.yml index e66190e0..96f0cfdd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,24 @@ matrix: env: PYTHON_VERSION=3.6 PIP_DEPENDENCIES='git+https://github.com/sphinx-doc/sphinx.git#egg=sphinx coveralls' CONDA_DEPENDENCIES="setuptools cython numpy pytest-cov" + # Test conda's clang + - os: osx + env: + - PYTHON_VERSION=3.5 + - CONDA_DEPENDENCIES="setuptools sphinx cython numpy pytest-cov clang llvm-openmp" + before_install: + - export TEST_OPENMP="True" + - export CC=/Users/travis/miniconda/envs/test/bin/clang + + # Test gcc on OSX + - os: osx + env: + - PYTHON_VERSION=3.5 + - CONDA_DEPENDENCIES="setuptools sphinx cython numpy pytest-cov gcc" + before_install: + - export TEST_OPENMP="True" + - export CC=/Users/travis/miniconda/envs/test/bin/gcc + # Uncomment the following if there are issues in setuptools that we # can't work around quickly - otherwise leave uncommented so that # we notice when things go wrong. @@ -45,6 +63,16 @@ matrix: # CONDA_DEPENDENCIES='sphinx cython numpy pytest-cov' # EVENT_TYPE='push pull_request cron' +before_install: + + # Test OSX without OpenMP support + # Since the matrix OSX tests use the OS shipped version of clang, they double up + # as exploratory tests for when the shipped version has automatic OpenMP support. + # These tests will then fail and at such a time a new one should be added + # to explicitly remove OpenMP support. + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export TEST_OPENMP="False"; + else export TEST_OPENMP="True"; fi + install: - git clone git://github.com/astropy/ci-helpers.git diff --git a/CHANGES.rst b/CHANGES.rst index 6147c70d..fece8a55 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,6 +12,15 @@ astropy-helpers Changelog by the ``sphinx-astropy`` package in conjunction with the ``astropy-theme-sphinx``, ``sphinx-automodapi``, and ``numpydoc`` packages. [#368] +- openmp_helpers.py: Make add_openmp_flags_if_available() work for clang. + The small openmp C test code, used to determine if openmp works, fails to + build with clang due to missing include and library paths for openmp. + When run, it then fails due to rpath (runtime path) issues. + Adds the necessary include, library, and runtime paths to the build. + Autogenerator utility added to gen openmp_enabled.py::is_openmp_enabled() + which can be called post build to determine state of OpenMP support. + [#382] + 3.0.2 (unreleased) ------------------ diff --git a/astropy_helpers/openmp_helpers.py b/astropy_helpers/openmp_helpers.py index 06823c1b..18518330 100644 --- a/astropy_helpers/openmp_helpers.py +++ b/astropy_helpers/openmp_helpers.py @@ -13,15 +13,17 @@ from __future__ import absolute_import, print_function +import datetime +import glob import os +import subprocess import sys -import glob import tempfile -import subprocess +import time from distutils import log from distutils.ccompiler import new_compiler -from distutils.sysconfig import customize_compiler +from distutils.sysconfig import customize_compiler, get_config_var from distutils.errors import CompileError, LinkError from .setup_helpers import get_compiler_option @@ -39,69 +41,222 @@ } """ +def _get_flag_value_from_var(flag, var, delim=' '): + """ + Utility to extract `flag` value from `os.environ[`var`]` or, if not present, + from `distutils.sysconfig.get_config_var(`var`)`. + E.g. to get include path: _get_flag_value_from_var('-I', 'CFLAGS') + might return "/usr/local/include". -def add_openmp_flags_if_available(extension): + Notes + ----- + Not yet tested and therefore not yet supported on Windows """ - Add OpenMP compilation flags, if available (if not a warning will be - printed to the console and no flags will be added) - Returns `True` if the flags were added, `False` otherwise. + if sys.platform.startswith('win'): + return None + + # Simple input validation + if not var or not flag: + return None + l = len(flag) + if not l: + return None + + # Look for var in os.eviron + try: + flags = os.environ[var] + except KeyError: + flags = None + + # If os.environ was unsuccesful try in sysconfig + if not flags: + try: + flags = get_config_var(var) + except KeyError: + return None + + # Extract flag from {var:value} + if flags: + for item in flags.split(delim): + if flag in item[:l]: + return item[l:] + + return None + +def _get_include_path(): + return _get_flag_value_from_var('-I', 'CFLAGS') + +def _get_library_path(): + return _get_flag_value_from_var('-L', 'LDFLAGS') + +def get_openmp_flags(): """ + Utility for returning compiler and linker flags possibly needed for + OpenMP support. - ccompiler = new_compiler() - customize_compiler(ccompiler) + Returns + ------- + result : `{'compiler_flags':, 'linker_flags':}` - tmp_dir = tempfile.mkdtemp() + Notes + ----- + The flags returned are not tested for validity, use + `test_openmp_support(openmp_flags=get_openmp_flags())` to do so. + """ - start_dir = os.path.abspath('.') + compile_flags = [] + include_path = _get_include_path() + if include_path: + compile_flags.append('-I' + include_path) + + link_flags = [] + lib_path = _get_library_path() + if lib_path: + link_flags.append('-L' + lib_path) if get_compiler_option() == 'msvc': - compile_flag = '-openmp' - link_flag = '' + compile_flags.append('-openmp') else: - compile_flag = '-fopenmp' - link_flag = '-fopenmp' + compile_flags.append('-fopenmp') + link_flags.append('-fopenmp') + if lib_path: + link_flags.append('-Wl,-rpath,' + lib_path) - try: + return {'compiler_flags':compile_flags, 'linker_flags':link_flags} + +def test_openmp_support(openmp_flags=None, silent=False): + """ + Compile and run OpenMP test code to determine viable support. + + Parameters + ---------- + openmp_flags : dictionary, optional + Expecting `{'compiler_flags':, 'linker_flags':}`. + These are passed as `extra_postargs` to `compile()` and + `link_executable()` respectively. + silent : bool, optional + silence log warnings + + Returns + ------- + result : bool + `True` if the test passed, `False` otherwise. + """ + + ccompiler = new_compiler() + customize_compiler(ccompiler) + if not openmp_flags: + # customize_compiler() extracts info from os.environ. If certain keys + # exist it uses these plus those from sysconfig.get_config_vars(). + # If the key is missing in os.environ it is not extracted from + # sysconfig.get_config_var(). E.g. 'LDFLAGS' get left out, preventing + # clang from finding libomp.dylib because -L is not passed to linker. + # Call get_openmp_flags() to get flags missed by customize_compiler(). + openmp_flags = get_openmp_flags() + compile_flags = openmp_flags['compiler_flags'] if 'compiler_flags' in openmp_flags else None + link_flags = openmp_flags['linker_flags'] if 'linker_flags' in openmp_flags else None + + tmp_dir = tempfile.mkdtemp() + start_dir = os.path.abspath('.') + + try: os.chdir(tmp_dir) + # Write test program with open('test_openmp.c', 'w') as f: f.write(CCODE) os.mkdir('objects') # Compile, link, and run test program - ccompiler.compile(['test_openmp.c'], output_dir='objects', extra_postargs=[compile_flag]) - ccompiler.link_executable(glob.glob(os.path.join('objects', '*' + ccompiler.obj_extension)), 'test_openmp', extra_postargs=[link_flag]) + ccompiler.compile(['test_openmp.c'], output_dir='objects', extra_postargs=compile_flags) + ccompiler.link_executable(glob.glob(os.path.join('objects', '*' + ccompiler.obj_extension)), 'test_openmp', extra_postargs=link_flags) output = subprocess.check_output('./test_openmp').decode(sys.stdout.encoding or 'utf-8').splitlines() if 'nthreads=' in output[0]: nthreads = int(output[0].strip().split('=')[1]) if len(output) == nthreads: - using_openmp = True + is_openmp_supported = True else: - log.warn("Unexpected number of lines from output of test OpenMP " - "program (output was {0})".format(output)) - using_openmp = False + if not silent: + log.warn("Unexpected number of lines from output of test OpenMP " + "program (output was {0})".format(output)) + is_openmp_supported = False else: - log.warn("Unexpected output from test OpenMP " - "program (output was {0})".format(output)) - using_openmp = False - + if not silent: + log.warn("Unexpected output from test OpenMP " + "program (output was {0})".format(output)) + is_openmp_supported = False except (CompileError, LinkError): - - using_openmp = False + is_openmp_supported = False finally: - os.chdir(start_dir) + + return is_openmp_supported + +def is_openmp_supported(): + """ + Utility to determine whether the build compiler + has OpenMP support. + """ + return test_openmp_support(silent=True) + +def add_openmp_flags_if_available(extension): + """ + Add OpenMP compilation flags, if supported (if not a warning will be + printed to the console and no flags will be added.) + + Returns `True` if the flags were added, `False` otherwise. + """ + + openmp_flags = get_openmp_flags() + using_openmp = test_openmp_support(openmp_flags=openmp_flags, silent=False) if using_openmp: - log.info("Compiling Cython extension with OpenMP support") - extension.extra_compile_args.append(compile_flag) - extension.extra_link_args.append(link_flag) + compile_flags = openmp_flags['compiler_flags'] if 'compiler_flags' in openmp_flags else None + link_flags = openmp_flags['linker_flags'] if 'linker_flags' in openmp_flags else None + log.info("Compiling Cython/C/C++ extension with OpenMP support") + extension.extra_compile_args.extend(compile_flags) + extension.extra_link_args.extend(link_flags) else: log.warn("Cannot compile Cython extension with OpenMP, reverting to non-parallel code") return using_openmp + +_IS_OPENMP_ENABLED_SRC = """ +# Autogenerated by {packagetitle}'s setup.py on {timestamp!s} + +def is_openmp_enabled(): + \'\'\' + Autogenerated utility to determine, post build, whether the package + was built with or without OpenMP support. + \'\'\' + return {return_bool} +"""[1:] + +def generate_openmp_enabled_py(packagename, srcdir='.'): + """ + Utility for creating openmp_enabled.py::is_openmp_enabled() + used to determine, post build, whether the package was built + with or without OpenMP support. + """ + + if packagename.lower() == 'astropy': + packagetitle = 'Astropy' + else: + packagetitle = packagename + + epoch = int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) + timestamp = datetime.datetime.utcfromtimestamp(epoch) + + src = _IS_OPENMP_ENABLED_SRC.format(packagetitle=packagetitle, + timestamp=timestamp, + return_bool=is_openmp_supported()) + + package_srcdir = os.path.join(srcdir, *packagename.split('.')) + is_openmp_enabled_py = os.path.join(package_srcdir, 'openmp_enabled.py') + with open(is_openmp_enabled_py, 'w') as f: + f.write(src) diff --git a/astropy_helpers/tests/test_openmp_helpers.py b/astropy_helpers/tests/test_openmp_helpers.py index 04d8a39d..f6ffd33f 100644 --- a/astropy_helpers/tests/test_openmp_helpers.py +++ b/astropy_helpers/tests/test_openmp_helpers.py @@ -3,15 +3,23 @@ from copy import deepcopy from distutils.core import Extension -from ..openmp_helpers import add_openmp_flags_if_available +from ..openmp_helpers import add_openmp_flags_if_available, generate_openmp_enabled_py from ..setup_helpers import _module_state, register_commands IS_TRAVIS_LINUX = os.environ.get('TRAVIS_OS_NAME', None) == 'linux' +IS_TRAVIS_OSX = os.environ.get('TRAVIS_OS_NAME', None) == 'osx' IS_APPVEYOR = os.environ.get('APPVEYOR', None) == 'True' PY3_LT_35 = sys.version_info[0] == 3 and sys.version_info[1] < 5 _state = None +try: + TEST_OPENMP = 'True' == os.environ['TEST_OPENMP'] +except KeyError: + if IS_APPVEYOR: + TEST_OPENMP = True + else: + raise def setup_function(function): global state @@ -34,5 +42,37 @@ def test_add_openmp_flags_if_available(): # Having this is useful because we'll find out if OpenMP no longer works # for any reason on platforms on which it does work at the time of writing. # OpenMP doesn't work on Python 3.x where x<5 on AppVeyor though. - if IS_TRAVIS_LINUX or (IS_APPVEYOR and not PY3_LT_35): - assert using_openmp + if IS_TRAVIS_LINUX or IS_TRAVIS_OSX or (IS_APPVEYOR and not PY3_LT_35): + if TEST_OPENMP: + assert using_openmp + else: + assert not using_openmp + +def test_generate_openmp_enabled_py(): + + register_commands('openmp_autogeneration_testing', '0.0', False) + + # Test file generation + generate_openmp_enabled_py('') + assert os.path.isfile('openmp_enabled.py') + + with open('openmp_enabled.py', 'r') as fid: + contents = fid.read() + print(contents) + + # Travis OSX tests experience an unstable module import error. + # Sometimes finding and importing the module, `openmp_enabled`, + # more often not. Adding '.' to the path seems to stabilize the + # issue, even though it should not be needed. + sys.path.append('.') + from openmp_enabled import is_openmp_enabled + + is_openmp_enabled = is_openmp_enabled() + + # Test is_openmp_enabled() + assert isinstance(is_openmp_enabled, bool) + + if TEST_OPENMP: + assert is_openmp_enabled + else: + assert not is_openmp_enabled