diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..4c1808108 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.11.0) +project(arrayfire-python) +find_package(PythonExtensions REQUIRED) +include(FetchContent) + +set(CMAKE_MODULE_PATH_OLD ${CMAKE_MODULE_PATH}) +set(CMAKE_MODULE_PATH "") +set(NO_SONAME) + +FetchContent_Declare( + arrayfire + GIT_REPOSITORY https://github.com/arrayfire/arrayfire.git + GIT_TAG v3.8 +) +FetchContent_MakeAvailable(arrayfire) +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH_OLD}) + +set(ignoreWarning "${SKBUILD}") diff --git a/README.md b/README.md index fb3a1b8a1..02af7fe26 100644 --- a/README.md +++ b/README.md @@ -25,30 +25,29 @@ def calc_pi_device(samples): Choosing a particular backend can be done using `af.set_backend(name)` where name is either "_cuda_", "_opencl_", or "_cpu_". The default device is chosen in the same order of preference. -## Requirements - -Currently, this project is tested only on Linux and OSX. You also need to have the ArrayFire C/C++ library installed on your machine. You can get it from the following sources. +## Getting started +ArrayFire can be installed from a variety of sources. [Pre-built wheels](https://repo.arrayfire.com/python/wheels/3.8.0/) are available for a number of systems and toolkits. Wheels for some systems are available on PyPI. Unsupported systems will require separate installation of the ArrayFire C/C++ libraries and only the python wrapper will be installed in that case. +You can get the ArrayFire C/C++ library from the following sources: - [Download and install binaries](https://arrayfire.com/download) - [Build and install from source](https://github.com/arrayfire/arrayfire) -Please check the following links for dependencies. - -- [Linux dependencies](http://www.arrayfire.com/docs/using_on_linux.htm) -- [OSX dependencies](http://www.arrayfire.com/docs/using_on_osx.htm) - -## Getting started - -**Install the last stable version:** +**Install the last stable version:** ``` pip install arrayfire ``` -**Install the development version:** +**Install a pre-built wheel for a specific CUDA toolkit version:** +``` +pip install arrayfire==3.8.0+cu112 -f https://repo.arrayfire.com/python/wheels/3.8.0/ +# Replace the +cu112 local version with the desired toolkit +``` + +**Install the development source distribution:** ``` -pip install git+git://github.com/arrayfire/arrayfire-python.git@devel +pip install git+git://github.com/arrayfire/arrayfire-python.git@master ``` **Installing offline:** @@ -57,10 +56,11 @@ pip install git+git://github.com/arrayfire/arrayfire-python.git@devel cd path/to/arrayfire-python python setup.py install ``` +Rather than installing and building ArrayFire elsewhere in the system, you can also build directly through python by first setting the `AF_BUILD_LOCAL_LIBS=1` environment variable. Additional setup will be required to build ArrayFire, including satisfying dependencies and further CMake configuration. Details on how to pass additional arguments to the build systems can be found in the [scikit-build documentation.](https://scikit-build.readthedocs.io/en/latest/) **Post Installation:** -Please follow [these instructions](https://github.com/arrayfire/arrayfire-python/wiki) to ensure the arrayfire-python can find the arrayfire libraries. +If you are not using one of the pre-built wheels, you may need to ensure arrayfire-python can find the installed arrayfire libraries. Please follow [these instructions](https://github.com/arrayfire/arrayfire-python/wiki) to ensure that arrayfire-python can find the arrayfire libraries. To run arrayfire tests, you can run the following command from command line. diff --git a/arrayfire/library.py b/arrayfire/library.py index 009a8e122..27ca4f4d9 100644 --- a/arrayfire/library.py +++ b/arrayfire/library.py @@ -15,6 +15,7 @@ import ctypes as ct import traceback import os +import sys c_float_t = ct.c_float c_double_t = ct.c_double @@ -492,8 +493,6 @@ class VARIANCE(_Enum): _VER_MAJOR_PLACEHOLDER = "__VER_MAJOR__" def _setup(): - import platform - platform_name = platform.system() try: @@ -527,7 +526,7 @@ def _setup(): ct.windll.kernel32.SetErrorMode(0x0001 | 0x0002) if AF_SEARCH_PATH is None: - AF_SEARCH_PATH="C:/Program Files/ArrayFire/v" + AF_VER_MAJOR +"/" + AF_SEARCH_PATH = "C:/Program Files/ArrayFire/v" + AF_VER_MAJOR +"/" if CUDA_PATH is not None: CUDA_FOUND = os.path.isdir(CUDA_PATH + '/bin') and os.path.isdir(CUDA_PATH + '/nvvm/bin/') @@ -554,7 +553,12 @@ def _setup(): post = '.so.' + _VER_MAJOR_PLACEHOLDER if AF_SEARCH_PATH is None: - AF_SEARCH_PATH='/opt/arrayfire-' + AF_VER_MAJOR + '/' + if os.path.exists('/opt/arrayfire-' + AF_VER_MAJOR + '/'): + AF_SEARCH_PATH = '/opt/arrayfire-' + AF_VER_MAJOR + '/' + elif os.path.exists('/opt/arrayfire/'): + AF_SEARCH_PATH = '/opt/arrayfire/' + else: + AF_SEARCH_PATH = '/usr/local/' if CUDA_PATH is None: CUDA_PATH='/usr/local/cuda/' @@ -566,21 +570,46 @@ def _setup(): else: raise OSError(platform_name + ' not supported') - if AF_PATH is None: - os.environ['AF_PATH'] = AF_SEARCH_PATH - - return pre, post, AF_SEARCH_PATH, CUDA_FOUND + return pre, post, AF_PATH, AF_SEARCH_PATH, CUDA_FOUND class _clibrary(object): + def __find_nvrtc_builtins_libname(self, search_path): + filelist = os.listdir(search_path) + for f in filelist: + if 'nvrtc-builtins' in f: + return f + return None + def __libname(self, name, head='af', ver_major=AF_VER_MAJOR): post = self.__post.replace(_VER_MAJOR_PLACEHOLDER, ver_major) libname = self.__pre + head + name + post - if os.path.isdir(self.AF_PATH + '/lib64'): - libname_full = self.AF_PATH + '/lib64/' + libname + + if self.AF_PATH: + if os.path.isdir(self.AF_PATH + '/lib64'): + path_search = self.AF_PATH + '/lib64/' + else: + path_search = self.AF_PATH + '/lib/' + else: + if os.path.isdir(self.AF_SEARCH_PATH + '/lib64'): + path_search = self.AF_SEARCH_PATH + '/lib64/' + else: + path_search = self.AF_SEARCH_PATH + '/lib/' + + if platform.architecture()[0][:2] == '64': + path_site = sys.prefix + '/lib64/' + else: + path_site = sys.prefix + '/lib/' + + path_local = self.AF_PYMODULE_PATH + libpaths = [('', libname), + (path_site, libname), + (path_local,libname)] + if self.AF_PATH: #prefer specified AF_PATH if exists + libpaths.append((path_search, libname)) else: - libname_full = self.AF_PATH + '/lib/' + libname - return (libname, libname_full) + libpaths.insert(2, (path_search, libname)) + return libpaths def set_unsafe(self, name): lib = self.__clibs[name] @@ -592,13 +621,19 @@ def __init__(self): more_info_str = "Please look at https://github.com/arrayfire/arrayfire-python/wiki for more information." - pre, post, AF_PATH, CUDA_FOUND = _setup() + pre, post, AF_PATH, AF_SEARCH_PATH, CUDA_FOUND = _setup() self.__pre = pre self.__post = post self.AF_PATH = AF_PATH + self.AF_SEARCH_PATH = AF_SEARCH_PATH self.CUDA_FOUND = CUDA_FOUND + # prefer locally packaged arrayfire libraries if they exist + af_module = __import__(__name__) + self.AF_PYMODULE_PATH = af_module.__path__[0] + '/' if af_module.__path__ else None + + self.__name = None self.__clibs = {'cuda' : None, @@ -618,7 +653,7 @@ def __init__(self): 'opencl' : 4} # Try to pre-load forge library if it exists - libnames = self.__libname('forge', head='', ver_major=FORGE_VER_MAJOR) + libnames = reversed(self.__libname('forge', head='', ver_major=FORGE_VER_MAJOR)) try: VERBOSE_LOADS = os.environ['AF_VERBOSE_LOADS'] == '1' @@ -628,14 +663,15 @@ def __init__(self): for libname in libnames: try: - ct.cdll.LoadLibrary(libname) + full_libname = libname[0] + libname[1] + ct.cdll.LoadLibrary(full_libname) if VERBOSE_LOADS: - print('Loaded ' + libname) + print('Loaded ' + full_libname) break except OSError: if VERBOSE_LOADS: traceback.print_exc() - print('Unable to load ' + libname) + print('Unable to load ' + full_libname) pass c_dim4 = c_dim_t*4 @@ -644,24 +680,39 @@ def __init__(self): # Iterate in reverse order of preference for name in ('cpu', 'opencl', 'cuda', ''): - libnames = self.__libname(name) + libnames = reversed(self.__libname(name)) for libname in libnames: try: - ct.cdll.LoadLibrary(libname) + full_libname = libname[0] + libname[1] + + ct.cdll.LoadLibrary(full_libname) __name = 'unified' if name == '' else name - clib = ct.CDLL(libname) + clib = ct.CDLL(full_libname) self.__clibs[__name] = clib err = clib.af_randu(c_pointer(out), 4, c_pointer(dims), Dtype.f32.value) if (err == ERR.NONE.value): self.__name = __name clib.af_release_array(out) if VERBOSE_LOADS: - print('Loaded ' + libname) + print('Loaded ' + full_libname) + + # load nvrtc-builtins library if using cuda + if name == 'cuda': + nvrtc_name = self.__find_nvrtc_builtins_libname(libname[0]) + if nvrtc_name: + ct.cdll.LoadLibrary(libname[0] + nvrtc_name) + + if VERBOSE_LOADS: + print('Loaded ' + libname[0] + nvrtc_name) + else: + if VERBOSE_LOADS: + print('Could not find local nvrtc-builtins libarary') + break; except OSError: if VERBOSE_LOADS: traceback.print_exc() - print('Unable to load ' + libname) + print('Unable to load ' + full_libname) pass if (self.__name is None): @@ -689,6 +740,7 @@ def parse(self, res): lst.append(key) return tuple(lst) + backend = _clibrary() def set_backend(name, unsafe=False): diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..75335283e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "wheel", "scikit-build", "cmake", "ninja"] diff --git a/setup.cfg b/setup.cfg index 831f2063b..e5414158b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,10 @@ [metadata] name = arrayfire -version = 3.7.20200213 +version = 3.8.0 description = Python bindings for ArrayFire licence = BSD long_description = file: README.md +long_description_content_type = text/markdown maintainer = ArrayFire maintainer_email = technical@arrayfire.com url = http://arrayfire.com @@ -15,6 +16,8 @@ classifiers = [options] packages = find: +install_requires= + scikit-build [options.packages.find] include = arrayfire diff --git a/setup.py b/setup.py index e4d485c30..b6d37bb86 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,98 @@ # http://arrayfire.com/licenses/BSD-3-Clause ######################################################## -# TODO: Look for af libraries during setup +import os +import re -from setuptools import setup +# package can be distributed with arrayfire binaries or +# just with python wrapper files, the AF_BUILD_LOCAL +# environment var determines whether to build the arrayfire +# binaries locally rather than searching in a system install + +AF_BUILD_LOCAL_LIBS = os.environ.get('AF_BUILD_LOCAL_LIBS') +print(f'AF_BUILD_LOCAL_LIBS={AF_BUILD_LOCAL_LIBS}') +if AF_BUILD_LOCAL_LIBS: + print('Proceeding to build ArrayFire libraries') +else: + print('Skipping binaries installation, only python files will be installed') + +AF_BUILD_CPU = os.environ.get('AF_BUILD_CPU') +AF_BUILD_CPU = 1 if AF_BUILD_CPU is None else int(AF_BUILD_CPU) +AF_BUILD_CPU_CMAKE_STR = '-DAF_BUILD_CPU:BOOL=ON' if (AF_BUILD_CPU == 1) else '-DAF_BUILD_CPU:BOOL=OFF' + +AF_BUILD_CUDA = os.environ.get('AF_BUILD_CUDA') +AF_BUILD_CUDA = 1 if AF_BUILD_CUDA is None else int(AF_BUILD_CUDA) +AF_BUILD_CUDA_CMAKE_STR = '-DAF_BUILD_CUDA:BOOL=ON' if (AF_BUILD_CUDA == 1) else '-DAF_BUILD_CUDA:BOOL=OFF' + +AF_BUILD_OPENCL = os.environ.get('AF_BUILD_OPENCL') +AF_BUILD_OPENCL = 1 if AF_BUILD_OPENCL is None else int(AF_BUILD_OPENCL) +AF_BUILD_OPENCL_CMAKE_STR = '-DAF_BUILD_OPENCL:BOOL=ON' if (AF_BUILD_OPENCL == 1) else '-DAF_BUILD_OPENCL:BOOL=OFF' + +AF_BUILD_UNIFIED = os.environ.get('AF_BUILD_UNIFIED') +AF_BUILD_UNIFIED = 1 if AF_BUILD_UNIFIED is None else int(AF_BUILD_UNIFIED) +AF_BUILD_UNIFIED_CMAKE_STR = '-DAF_BUILD_UNIFIED:BOOL=ON' if (AF_BUILD_UNIFIED == 1) else '-DAF_BUILD_UNIFIED:BOOL=OFF' + +if AF_BUILD_LOCAL_LIBS: + # invoke cmake and build arrayfire libraries to install locally in package + from skbuild import setup + + def filter_af_files(cmake_manifest): + cmake_manifest = list(filter(lambda name: not (name.endswith('.h') + or name.endswith('.cpp') + or name.endswith('.hpp') + or name.endswith('.cmake') + or name.endswith('jpg') + or name.endswith('png') + or name.endswith('libaf.so') #avoids duplicates due to symlinks + or re.match('.*libaf\.so\.3\..*', name) is not None + or name.endswith('libafcpu.so') + or re.match('.*libafcpu\.so\.3\..*', name) is not None + or name.endswith('libafcuda.so') + or re.match('.*libafcuda\.so\.3\..*', name) is not None + or name.endswith('libafopencl.so') + or re.match('.*libafopencl\.so\.3\..*', name) is not None + or name.endswith('libforge.so') + or re.match('.*libforge\.so\.1\..*', name) is not None + or 'examples' in name), cmake_manifest)) + return cmake_manifest + + print('Building CMAKE with following configurable variables: ') + print(AF_BUILD_CPU_CMAKE_STR) + print(AF_BUILD_CUDA_CMAKE_STR) + print(AF_BUILD_OPENCL_CMAKE_STR) + print(AF_BUILD_UNIFIED_CMAKE_STR) + + + setup( + packages=['arrayfire'], + cmake_install_dir='', + cmake_process_manifest_hook=filter_af_files, + include_package_data=False, + cmake_args=[AF_BUILD_CPU_CMAKE_STR, + AF_BUILD_CUDA_CMAKE_STR, + AF_BUILD_OPENCL_CMAKE_STR, + AF_BUILD_UNIFIED_CMAKE_STR, + # todo: pass additional args from environ + '-DCMAKE_BUILD_TYPE:STRING="RelWithDebInfo"', + '-DFG_USE_STATIC_CPPFLAGS:BOOL=OFF', + '-DFG_WITH_FREEIMAGE:BOOL=OFF', + '-DCUDA_architecture_build_targets:STRING=All', + '-DAF_BUILD_DOCS:BOOL=OFF', + '-DAF_BUILD_EXAMPLES:BOOL=OFF', + '-DAF_INSTALL_STANDALONE:BOOL=ON', + '-DAF_WITH_IMAGEIO:BOOL=ON', + '-DAF_WITH_LOGGING:BOOL=ON', + '-DBUILD_TESTING:BOOL=OFF', + '-DAF_BUILD_FORGE:BOOL=ON', + '-DAF_INSTALL_LIB_DIR:STRING=arrayfire', + '-DAF_INSTALL_BIN_DIR:STRING=arrayfire', + '-DFG_INSTALL_LIB_DIR:STRING=arrayfire', + '-DAF_WITH_STATIC_MKL=ON', + ] + ) + +else: + # ignores local arrayfire libraries, will search system instead + from setuptools import setup + setup() -setup()