From 005e004f6c3e0bbc12a22b420a17b03aaa6a5510 Mon Sep 17 00:00:00 2001 From: syurkevi Date: Wed, 2 Dec 2020 21:53:35 -0500 Subject: [PATCH 01/10] adds initial skbuild configuration load from local lib directory loads local binaries if they are available --- CMakeLists.txt | 18 +++++++++++ arrayfire/library.py | 23 ++++++++++---- pyproject.toml | 2 ++ setup.cfg | 4 ++- setup.py | 72 ++++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 109 insertions(+), 10 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 pyproject.toml diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..0c740dd61 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.11.0) +project(arrayfire-python) +find_package(PythonExtensions REQUIRED) +include(FetchContent) + +set(CMAKE_MODULE_PATH_OLD ${CMAKE_MODULE_PATH}) +set(CMAKE_MODULE_PATH "") + +FetchContent_Declare( + arrayfire + GIT_REPOSITORY https://github.com/arrayfire/arrayfire.git + GIT_TAG v3.8.rc +) +FetchContent_MakeAvailable(arrayfire) +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH_OLD}) + + +set(ignoreWarning "${SKBUILD}") diff --git a/arrayfire/library.py b/arrayfire/library.py index 009a8e122..785cd7608 100644 --- a/arrayfire/library.py +++ b/arrayfire/library.py @@ -493,9 +493,12 @@ class VARIANCE(_Enum): def _setup(): import platform - platform_name = platform.system() + # prefer locally packaged arrayfire libraries if they exist + af_module = __import__(__name__) + af_module_path = af_module.__path__[0] if af_module.__path__ else None + try: AF_PATH = os.environ['AF_PATH'] except KeyError: @@ -527,7 +530,10 @@ def _setup(): ct.windll.kernel32.SetErrorMode(0x0001 | 0x0002) if AF_SEARCH_PATH is None: - AF_SEARCH_PATH="C:/Program Files/ArrayFire/v" + AF_VER_MAJOR +"/" + if af_module_path and os.path.exists(af_module_path + '/lib/'): + AF_SEARCH_PATH = af_module_path + else: + AF_SEARCH_PATH = "C:/Program Files/ArrayFire/v" + AF_VER_MAJOR +"/" if CUDA_PATH is not None: CUDA_FOUND = os.path.isdir(CUDA_PATH + '/bin') and os.path.isdir(CUDA_PATH + '/nvvm/bin/') @@ -539,7 +545,9 @@ def _setup(): post = '.' + _VER_MAJOR_PLACEHOLDER + '.dylib' if AF_SEARCH_PATH is None: - if os.path.exists('/opt/arrayfire'): + if af_module_path and os.path.exists(af_module_path + '/lib/'): + AF_SEARCH_PATH = af_module_path + elif os.path.exists('/opt/arrayfire'): AF_SEARCH_PATH = '/opt/arrayfire/' else: AF_SEARCH_PATH = '/usr/local/' @@ -554,7 +562,10 @@ def _setup(): post = '.so.' + _VER_MAJOR_PLACEHOLDER if AF_SEARCH_PATH is None: - AF_SEARCH_PATH='/opt/arrayfire-' + AF_VER_MAJOR + '/' + if af_module_path and os.path.exists(af_module_path + '/lib/'): + AF_SEARCH_PATH = af_module_path + else: + AF_SEARCH_PATH = '/opt/arrayfire-' + AF_VER_MAJOR + '/' if CUDA_PATH is None: CUDA_PATH='/usr/local/cuda/' @@ -618,7 +629,7 @@ def __init__(self): 'opencl' : 4} # Try to pre-load forge library if it exists - libnames = self.__libname('forge', head='', ver_major=FORGE_VER_MAJOR) + libnames = reversed(self.__libname('forge', head='', ver_major=FORGE_VER_MAJOR)) try: VERBOSE_LOADS = os.environ['AF_VERBOSE_LOADS'] == '1' @@ -644,7 +655,7 @@ def __init__(self): # Iterate in reverse order of preference for name in ('cpu', 'opencl', 'cuda', ''): - libnames = self.__libname(name) + libnames = reversed(self.__libname(name)) for libname in libnames: try: ct.cdll.LoadLibrary(libname) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..10a3f3773 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "wheel", "scikit-build", "cmake", "ninja", "git"] diff --git a/setup.cfg b/setup.cfg index 831f2063b..45fb875c3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = arrayfire -version = 3.7.20200213 +version = 3.8.0 description = Python bindings for ArrayFire licence = BSD long_description = file: README.md @@ -15,6 +15,8 @@ classifiers = [options] packages = find: +install_requires= + scikit-build [options.packages.find] include = arrayfire diff --git a/setup.py b/setup.py index e4d485c30..3bce9b9f4 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,74 @@ # http://arrayfire.com/licenses/BSD-3-Clause ######################################################## -# TODO: Look for af libraries during setup +import os -from setuptools import setup +# package can be distributed with arrayfire binaries or +# just with python wrapper files, the AF_BUILD_LOCAL +# environment var determines whether to build the arrayfire +# binaries locally rather than searching in a system install + +AF_BUILD_LOCAL_LIBS = os.environ.get('AF_BUILD_LOCAL_LIBS') +print(f'AF_BUILD_LOCAL_LIBS={AF_BUILD_LOCAL_LIBS}') +if AF_BUILD_LOCAL_LIBS: + print('Proceeding to build ArrayFire libraries') +else: + print('Skipping binaries installation, only python files will be installed') + +AF_BUILD_CPU = os.environ.get('AF_BUILD_CPU') +AF_BUILD_CPU = 1 if AF_BUILD_CPU is None else int(AF_BUILD_CPU) +AF_BUILD_CPU_CMAKE_STR = '-DAF_BUILD_CPU:BOOL=ON' if (AF_BUILD_CPU == 1) else '-DAF_BUILD_CPU:BOOL=OFF' + +AF_BUILD_CUDA = os.environ.get('AF_BUILD_CUDA') +AF_BUILD_CUDA = 1 if AF_BUILD_CUDA is None else int(AF_BUILD_CUDA) +AF_BUILD_CUDA_CMAKE_STR = '-DAF_BUILD_CUDA:BOOL=ON' if (AF_BUILD_CUDA == 1) else '-DAF_BUILD_CUDA:BOOL=OFF' + +AF_BUILD_OPENCL = os.environ.get('AF_BUILD_OPENCL') +AF_BUILD_OPENCL = 1 if AF_BUILD_OPENCL is None else int(AF_BUILD_OPENCL) +AF_BUILD_OPENCL_CMAKE_STR = '-DAF_BUILD_OPENCL:BOOL=ON' if (AF_BUILD_OPENCL == 1) else '-DAF_BUILD_OPENCL:BOOL=OFF' + +AF_BUILD_UNIFIED = os.environ.get('AF_BUILD_UNIFIED') +AF_BUILD_UNIFIED = 1 if AF_BUILD_UNIFIED is None else int(AF_BUILD_UNIFIED) +AF_BUILD_UNIFIED_CMAKE_STR = '-DAF_BUILD_UNIFIED:BOOL=ON' if (AF_BUILD_UNIFIED == 1) else '-DAF_BUILD_UNIFIED:BOOL=OFF' + +if AF_BUILD_LOCAL_LIBS: + # invoke cmake and build arrayfire libraries to install locally in package + from skbuild import setup + + def filter_af_files(cmake_manifest): + cmake_manifest = list(filter(lambda name: not (name.endswith('.h') + or name.endswith('.cpp') + or name.endswith('.hpp') + or name.endswith('.cmake') + or name.endswith('jpg') + or name.endswith('png') + or 'examples' in name), cmake_manifest)) + return cmake_manifest + + print('Building CMAKE with following configurable variables: ') + print(AF_BUILD_CPU_CMAKE_STR) + print(AF_BUILD_CUDA_CMAKE_STR) + print(AF_BUILD_OPENCL_CMAKE_STR) + print(AF_BUILD_UNIFIED_CMAKE_STR) + + + setup( + cmake_install_dir='arrayfire', + cmake_process_manifest_hook=filter_af_files, + cmake_args=[AF_BUILD_CPU_CMAKE_STR, + AF_BUILD_CUDA_CMAKE_STR, + AF_BUILD_OPENCL_CMAKE_STR, + AF_BUILD_UNIFIED_CMAKE_STR, + '-DCUDA_architecture_build_targets:STRING=All' # todo: pass from environ + '-DAF_BUILD_DOCS:BOOL=OFF', + '-DAF_BUILD_EXAMPLES:BOOL=OFF', + '-DAF_INSTALL_STANDALONE:BOOL=ON', + '-DBUILD_TESTING:BOOL=OFF', + ] + ) + +else: + # ignores local arrayfire libraries, will search system instead + from setuptools import setup + setup() -setup() From a9be9de13d0ea72c27a8ea5bb273a1abf9d1e3e7 Mon Sep 17 00:00:00 2001 From: syurkevi Date: Fri, 18 Dec 2020 09:32:01 -0500 Subject: [PATCH 02/10] remove git from toml dependencies Co-authored-by: pradeep --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 10a3f3773..75335283e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires = ["setuptools", "wheel", "scikit-build", "cmake", "ninja", "git"] +requires = ["setuptools", "wheel", "scikit-build", "cmake", "ninja"] From 8b7d69c13b70ab977d26d97d2322c4d5bfd45a78 Mon Sep 17 00:00:00 2001 From: syurkevi Date: Fri, 22 Jan 2021 16:28:12 -0500 Subject: [PATCH 03/10] additional cmake args in setup.py --- setup.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3bce9b9f4..7236fa9f7 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,14 @@ def filter_af_files(cmake_manifest): or name.endswith('.cmake') or name.endswith('jpg') or name.endswith('png') + or name.endswith('libaf.so') #avoids duplicates due to symlinks + or name.endswith('libaf.so.3') + or name.endswith('libafcpu.so') + or name.endswith('libafcpu.so.3') + or name.endswith('libafcuda.so') + or name.endswith('libafcuda.so.3') + or name.endswith('libafopencl.so') + or name.endswith('libafopencl.so.3') or 'examples' in name), cmake_manifest)) return cmake_manifest @@ -67,11 +75,17 @@ def filter_af_files(cmake_manifest): AF_BUILD_CUDA_CMAKE_STR, AF_BUILD_OPENCL_CMAKE_STR, AF_BUILD_UNIFIED_CMAKE_STR, - '-DCUDA_architecture_build_targets:STRING=All' # todo: pass from environ + '-DCMAKE_BUILD_TYPE:STRING="RelWithDebInfo"', + '-DFG_USE_STATIC_CPPFLAGS:BOOL=ON', + '-DFG_WITH_FREEIMAGE:BOOL=OFF', + '-DCUDA_architecture_build_targets:STRING=All', # todo: pass from environ '-DAF_BUILD_DOCS:BOOL=OFF', '-DAF_BUILD_EXAMPLES:BOOL=OFF', '-DAF_INSTALL_STANDALONE:BOOL=ON', + '-DAF_WITH_IMAGEIO:BOOL=ON', + '-DAF_WITH_LOGGING:BOOL=ON', '-DBUILD_TESTING:BOOL=OFF', + '-DAF_BUILD_FORGE:BOOL=ON', ] ) From 570bf95936c6866eef9f365324941246265be10f Mon Sep 17 00:00:00 2001 From: syurkevi Date: Tue, 26 Jan 2021 18:43:23 -0500 Subject: [PATCH 04/10] add description format for pypi --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 45fb875c3..e5414158b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,6 +4,7 @@ version = 3.8.0 description = Python bindings for ArrayFire licence = BSD long_description = file: README.md +long_description_content_type = text/markdown maintainer = ArrayFire maintainer_email = technical@arrayfire.com url = http://arrayfire.com From c8ed9135cd4c433e6b095f4fc56501d566308ab2 Mon Sep 17 00:00:00 2001 From: syurkevi Date: Wed, 17 Feb 2021 13:15:30 -0500 Subject: [PATCH 05/10] directory corrections for skbuild --- CMakeLists.txt | 4 ++-- arrayfire/library.py | 37 ++++++++++++++++++++++--------------- setup.py | 23 ++++++++++++++++------- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c740dd61..4c1808108 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,14 +5,14 @@ include(FetchContent) set(CMAKE_MODULE_PATH_OLD ${CMAKE_MODULE_PATH}) set(CMAKE_MODULE_PATH "") +set(NO_SONAME) FetchContent_Declare( arrayfire GIT_REPOSITORY https://github.com/arrayfire/arrayfire.git - GIT_TAG v3.8.rc + GIT_TAG v3.8 ) FetchContent_MakeAvailable(arrayfire) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH_OLD}) - set(ignoreWarning "${SKBUILD}") diff --git a/arrayfire/library.py b/arrayfire/library.py index 785cd7608..74fe9872f 100644 --- a/arrayfire/library.py +++ b/arrayfire/library.py @@ -15,6 +15,7 @@ import ctypes as ct import traceback import os +import sys c_float_t = ct.c_float c_double_t = ct.c_double @@ -495,10 +496,6 @@ def _setup(): import platform platform_name = platform.system() - # prefer locally packaged arrayfire libraries if they exist - af_module = __import__(__name__) - af_module_path = af_module.__path__[0] if af_module.__path__ else None - try: AF_PATH = os.environ['AF_PATH'] except KeyError: @@ -530,10 +527,7 @@ def _setup(): ct.windll.kernel32.SetErrorMode(0x0001 | 0x0002) if AF_SEARCH_PATH is None: - if af_module_path and os.path.exists(af_module_path + '/lib/'): - AF_SEARCH_PATH = af_module_path - else: - AF_SEARCH_PATH = "C:/Program Files/ArrayFire/v" + AF_VER_MAJOR +"/" + AF_SEARCH_PATH = "C:/Program Files/ArrayFire/v" + AF_VER_MAJOR +"/" if CUDA_PATH is not None: CUDA_FOUND = os.path.isdir(CUDA_PATH + '/bin') and os.path.isdir(CUDA_PATH + '/nvvm/bin/') @@ -545,9 +539,7 @@ def _setup(): post = '.' + _VER_MAJOR_PLACEHOLDER + '.dylib' if AF_SEARCH_PATH is None: - if af_module_path and os.path.exists(af_module_path + '/lib/'): - AF_SEARCH_PATH = af_module_path - elif os.path.exists('/opt/arrayfire'): + if os.path.exists('/opt/arrayfire'): AF_SEARCH_PATH = '/opt/arrayfire/' else: AF_SEARCH_PATH = '/usr/local/' @@ -562,10 +554,12 @@ def _setup(): post = '.so.' + _VER_MAJOR_PLACEHOLDER if AF_SEARCH_PATH is None: - if af_module_path and os.path.exists(af_module_path + '/lib/'): - AF_SEARCH_PATH = af_module_path - else: + if os.path.exists('/opt/arrayfire-' + AF_VER_MAJOR + '/'): AF_SEARCH_PATH = '/opt/arrayfire-' + AF_VER_MAJOR + '/' + elif os.path.exists('/opt/arrayfire/'): + AF_SEARCH_PATH = '/opt/arrayfire/' + else: + AF_SEARCH_PATH = '/usr/local/' if CUDA_PATH is None: CUDA_PATH='/usr/local/cuda/' @@ -587,11 +581,19 @@ class _clibrary(object): def __libname(self, name, head='af', ver_major=AF_VER_MAJOR): post = self.__post.replace(_VER_MAJOR_PLACEHOLDER, ver_major) libname = self.__pre + head + name + post + if os.path.isdir(self.AF_PATH + '/lib64'): libname_full = self.AF_PATH + '/lib64/' + libname else: libname_full = self.AF_PATH + '/lib/' + libname - return (libname, libname_full) + + if platform.architecture()[0][:2] == '64': + libname_site = sys.prefix + '/lib64/' + libname + else: + libname_site = sys.prefix + '/lib/' + libname + + libname_local = self.AF_PYMODULE_PATH + libname + return (libname, libname_full, libname_site, libname_local) def set_unsafe(self, name): lib = self.__clibs[name] @@ -610,6 +612,11 @@ def __init__(self): self.AF_PATH = AF_PATH self.CUDA_FOUND = CUDA_FOUND + # prefer locally packaged arrayfire libraries if they exist + af_module = __import__(__name__) + self.AF_PYMODULE_PATH = af_module.__path__[0] + '/' if af_module.__path__ else None + + self.__name = None self.__clibs = {'cuda' : None, diff --git a/setup.py b/setup.py index 7236fa9f7..c7c604436 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ ######################################################## import os +import re # package can be distributed with arrayfire binaries or # just with python wrapper files, the AF_BUILD_LOCAL @@ -51,14 +52,17 @@ def filter_af_files(cmake_manifest): or name.endswith('jpg') or name.endswith('png') or name.endswith('libaf.so') #avoids duplicates due to symlinks - or name.endswith('libaf.so.3') + or re.match('.*libaf\.so\.3\..*', name) is not None or name.endswith('libafcpu.so') - or name.endswith('libafcpu.so.3') + or re.match('.*libafcpu\.so\.3\..*', name) is not None or name.endswith('libafcuda.so') - or name.endswith('libafcuda.so.3') + or re.match('.*libafcuda\.so\.3\..*', name) is not None or name.endswith('libafopencl.so') - or name.endswith('libafopencl.so.3') + or re.match('.*libafopencl\.so\.3\..*', name) is not None + or name.endswith('libforge.so') + or re.match('.*libforge\.so\.1\..*', name) is not None or 'examples' in name), cmake_manifest)) + print(cmake_manifest) return cmake_manifest print('Building CMAKE with following configurable variables: ') @@ -69,16 +73,19 @@ def filter_af_files(cmake_manifest): setup( - cmake_install_dir='arrayfire', + packages=['arrayfire'], + cmake_install_dir='', cmake_process_manifest_hook=filter_af_files, + include_package_data=False, cmake_args=[AF_BUILD_CPU_CMAKE_STR, AF_BUILD_CUDA_CMAKE_STR, AF_BUILD_OPENCL_CMAKE_STR, AF_BUILD_UNIFIED_CMAKE_STR, + # todo: pass additional args from environ '-DCMAKE_BUILD_TYPE:STRING="RelWithDebInfo"', - '-DFG_USE_STATIC_CPPFLAGS:BOOL=ON', + '-DFG_USE_STATIC_CPPFLAGS:BOOL=OFF', '-DFG_WITH_FREEIMAGE:BOOL=OFF', - '-DCUDA_architecture_build_targets:STRING=All', # todo: pass from environ + '-DCUDA_architecture_build_targets:STRING=All', '-DAF_BUILD_DOCS:BOOL=OFF', '-DAF_BUILD_EXAMPLES:BOOL=OFF', '-DAF_INSTALL_STANDALONE:BOOL=ON', @@ -86,6 +93,8 @@ def filter_af_files(cmake_manifest): '-DAF_WITH_LOGGING:BOOL=ON', '-DBUILD_TESTING:BOOL=OFF', '-DAF_BUILD_FORGE:BOOL=ON', + '-DAF_INSTALL_LIB_DIR:STRING=arrayfire', + '-DFG_INSTALL_LIB_DIR:STRING=arrayfire', ] ) From 6f04bb0dd11b0bd4b99a95e5de0a812fb09cdee4 Mon Sep 17 00:00:00 2001 From: syurkevi Date: Fri, 19 Feb 2021 01:34:02 -0500 Subject: [PATCH 06/10] load local nvrtc-builtins when possible --- arrayfire/library.py | 53 +++++++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/arrayfire/library.py b/arrayfire/library.py index 74fe9872f..82e776e59 100644 --- a/arrayfire/library.py +++ b/arrayfire/library.py @@ -578,22 +578,32 @@ def _setup(): class _clibrary(object): + def __find_nvrtc_builtins_libname(self, search_path): + filelist = os.listdir(search_path) + for f in filelist: + if 'nvrtc-builtins' in f: + return f + return None + def __libname(self, name, head='af', ver_major=AF_VER_MAJOR): post = self.__post.replace(_VER_MAJOR_PLACEHOLDER, ver_major) libname = self.__pre + head + name + post if os.path.isdir(self.AF_PATH + '/lib64'): - libname_full = self.AF_PATH + '/lib64/' + libname + path_search = self.AF_PATH + '/lib64/' else: - libname_full = self.AF_PATH + '/lib/' + libname + path_search = self.AF_PATH + '/lib/' if platform.architecture()[0][:2] == '64': - libname_site = sys.prefix + '/lib64/' + libname + path_site = sys.prefix + '/lib64/' else: - libname_site = sys.prefix + '/lib/' + libname + path_site = sys.prefix + '/lib/' - libname_local = self.AF_PYMODULE_PATH + libname - return (libname, libname_full, libname_site, libname_local) + path_local = self.AF_PYMODULE_PATH + return [('', libname), + (path_search, libname), + (path_site, libname), + (path_local,libname)] def set_unsafe(self, name): lib = self.__clibs[name] @@ -646,14 +656,15 @@ def __init__(self): for libname in libnames: try: - ct.cdll.LoadLibrary(libname) + full_libname = libname[0] + libname[1] + ct.cdll.LoadLibrary(full_libname) if VERBOSE_LOADS: - print('Loaded ' + libname) + print('Loaded ' + full_libname) break except OSError: if VERBOSE_LOADS: traceback.print_exc() - print('Unable to load ' + libname) + print('Unable to load ' + full_libname) pass c_dim4 = c_dim_t*4 @@ -665,21 +676,36 @@ def __init__(self): libnames = reversed(self.__libname(name)) for libname in libnames: try: - ct.cdll.LoadLibrary(libname) + full_libname = libname[0] + libname[1] + + ct.cdll.LoadLibrary(full_libname) __name = 'unified' if name == '' else name - clib = ct.CDLL(libname) + clib = ct.CDLL(full_libname) self.__clibs[__name] = clib err = clib.af_randu(c_pointer(out), 4, c_pointer(dims), Dtype.f32.value) if (err == ERR.NONE.value): self.__name = __name clib.af_release_array(out) if VERBOSE_LOADS: - print('Loaded ' + libname) + print('Loaded ' + full_libname) + + # load nvrtc-builtins library if using cuda + if name == 'cuda': + nvrtc_name = self.__find_nvrtc_builtins_libname(libname[0]) + if nvrtc_name: + ct.cdll.LoadLibrary(libname[0] + nvrtc_name) + + if VERBOSE_LOADS: + print('Loaded ' + libname[0] + nvrtc_name) + else: + if VERBOSE_LOADS: + print('Could not find local nvrtc-builtins libarary') + break; except OSError: if VERBOSE_LOADS: traceback.print_exc() - print('Unable to load ' + libname) + print('Unable to load ' + full_libname) pass if (self.__name is None): @@ -707,6 +733,7 @@ def parse(self, res): lst.append(key) return tuple(lst) + backend = _clibrary() def set_backend(name, unsafe=False): From 14c90e2d9a2e68946fb937905185db297e893301 Mon Sep 17 00:00:00 2001 From: syurkevi Date: Tue, 23 Feb 2021 14:39:06 -0500 Subject: [PATCH 07/10] make mkl libs static for distribution --- arrayfire/library.py | 32 ++++++++++++++++++++------------ setup.py | 3 ++- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/arrayfire/library.py b/arrayfire/library.py index 82e776e59..dbcee0697 100644 --- a/arrayfire/library.py +++ b/arrayfire/library.py @@ -571,10 +571,7 @@ def _setup(): else: raise OSError(platform_name + ' not supported') - if AF_PATH is None: - os.environ['AF_PATH'] = AF_SEARCH_PATH - - return pre, post, AF_SEARCH_PATH, CUDA_FOUND + return pre, post, AF_PATH, AF_SEARCH_PATH, CUDA_FOUND class _clibrary(object): @@ -589,10 +586,16 @@ def __libname(self, name, head='af', ver_major=AF_VER_MAJOR): post = self.__post.replace(_VER_MAJOR_PLACEHOLDER, ver_major) libname = self.__pre + head + name + post - if os.path.isdir(self.AF_PATH + '/lib64'): - path_search = self.AF_PATH + '/lib64/' + if self.AF_PATH: + if os.path.isdir(self.AF_PATH + '/lib64'): + path_search = self.AF_PATH + '/lib64/' + else: + path_search = self.AF_PATH + '/lib/' else: - path_search = self.AF_PATH + '/lib/' + if os.path.isdir(self.AF_SEARCH_PATH + '/lib64'): + path_search = self.AF_SEARCH_PATH + '/lib64/' + else: + path_search = self.AF_SEARCH_PATH + '/lib/' if platform.architecture()[0][:2] == '64': path_site = sys.prefix + '/lib64/' @@ -600,10 +603,14 @@ def __libname(self, name, head='af', ver_major=AF_VER_MAJOR): path_site = sys.prefix + '/lib/' path_local = self.AF_PYMODULE_PATH - return [('', libname), - (path_search, libname), - (path_site, libname), - (path_local,libname)] + libpaths = [('', libname), + (path_site, libname), + (path_local,libname)] + if self.AF_PATH: #prefer specified AF_PATH if exists + libpaths.append((path_search, libname)) + else: + libpaths.insert(2, (path_search, libname)) + return libpaths def set_unsafe(self, name): lib = self.__clibs[name] @@ -615,11 +622,12 @@ def __init__(self): more_info_str = "Please look at https://github.com/arrayfire/arrayfire-python/wiki for more information." - pre, post, AF_PATH, CUDA_FOUND = _setup() + pre, post, AF_PATH, AF_SEARCH_PATH, CUDA_FOUND = _setup() self.__pre = pre self.__post = post self.AF_PATH = AF_PATH + self.AF_SEARCH_PATH = AF_SEARCH_PATH self.CUDA_FOUND = CUDA_FOUND # prefer locally packaged arrayfire libraries if they exist diff --git a/setup.py b/setup.py index c7c604436..b6d37bb86 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,6 @@ def filter_af_files(cmake_manifest): or name.endswith('libforge.so') or re.match('.*libforge\.so\.1\..*', name) is not None or 'examples' in name), cmake_manifest)) - print(cmake_manifest) return cmake_manifest print('Building CMAKE with following configurable variables: ') @@ -94,7 +93,9 @@ def filter_af_files(cmake_manifest): '-DBUILD_TESTING:BOOL=OFF', '-DAF_BUILD_FORGE:BOOL=ON', '-DAF_INSTALL_LIB_DIR:STRING=arrayfire', + '-DAF_INSTALL_BIN_DIR:STRING=arrayfire', '-DFG_INSTALL_LIB_DIR:STRING=arrayfire', + '-DAF_WITH_STATIC_MKL=ON', ] ) From 6b8c7cfc965a3e18d203d7d2cadd9f247ade7eeb Mon Sep 17 00:00:00 2001 From: syurkevi Date: Mon, 1 Mar 2021 23:38:23 -0500 Subject: [PATCH 08/10] Update README to include information about wheels --- README.md | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index fb3a1b8a1..0b24084ba 100644 --- a/README.md +++ b/README.md @@ -27,28 +27,30 @@ Choosing a particular backend can be done using `af.set_backend(name)` where na ## Requirements -Currently, this project is tested only on Linux and OSX. You also need to have the ArrayFire C/C++ library installed on your machine. You can get it from the following sources. +You will need to have the ArrayFire C/C++ library on your machine. You can get it from the following sources: - [Download and install binaries](https://arrayfire.com/download) - [Build and install from source](https://github.com/arrayfire/arrayfire) - -Please check the following links for dependencies. - -- [Linux dependencies](http://www.arrayfire.com/docs/using_on_linux.htm) -- [OSX dependencies](http://www.arrayfire.com/docs/using_on_osx.htm) +- [Pre-built wheel distributions](https://repo.arrayfire.com/python/wheels/3.8.0/) ## Getting started -**Install the last stable version:** - +**Install the last stable version:** ``` pip install arrayfire ``` +Wheels for some systems are available on PyPI. Unsupported systems will require separate installation of the ArrayFire C/C++ libraries and only the python wrapper will be installed in that case. + +**Install a pre-built wheel for a specific CUDA toolkit version:** +``` +pip install arrayfire==3.8.0+cu112 -f https://repo.arrayfire.com/python/wheels/3.8.0/ +# Replace the +cu112 local version with the desired toolkit +``` -**Install the development version:** +**Install the development source distribution:** ``` -pip install git+git://github.com/arrayfire/arrayfire-python.git@devel +pip install git+git://github.com/arrayfire/arrayfire-python.git@master ``` **Installing offline:** @@ -57,10 +59,11 @@ pip install git+git://github.com/arrayfire/arrayfire-python.git@devel cd path/to/arrayfire-python python setup.py install ``` +Rather than installing and building ArrayFire elsewhere in the system, you can also build directly through python by first setting the `AF_BUILD_LOCAL_LIBS=1` environment variable. Additional setup will be required to build ArrayFire, including satisfying dependencies and further CMake configuration. Details on how to pass additional arguments to the build systems can be found in the [scikit-build documentation.](https://scikit-build.readthedocs.io/en/latest/) **Post Installation:** -Please follow [these instructions](https://github.com/arrayfire/arrayfire-python/wiki) to ensure the arrayfire-python can find the arrayfire libraries. +If you are not using one of the pre-built wheels, you may need to ensure arrayfire-python can find the installed arrayfire libraries. Please follow [these instructions](https://github.com/arrayfire/arrayfire-python/wiki) to ensure that arrayfire-python can find the arrayfire libraries. To run arrayfire tests, you can run the following command from command line. From 0ed907c5f046dcf000ffcd5553dde997805c1146 Mon Sep 17 00:00:00 2001 From: syurkevi Date: Tue, 2 Mar 2021 01:19:56 -0500 Subject: [PATCH 09/10] remove extra import platform --- arrayfire/library.py | 1 - 1 file changed, 1 deletion(-) diff --git a/arrayfire/library.py b/arrayfire/library.py index dbcee0697..27ca4f4d9 100644 --- a/arrayfire/library.py +++ b/arrayfire/library.py @@ -493,7 +493,6 @@ class VARIANCE(_Enum): _VER_MAJOR_PLACEHOLDER = "__VER_MAJOR__" def _setup(): - import platform platform_name = platform.system() try: From fdcbb1ca528b7943a15833d41062e37707a4d109 Mon Sep 17 00:00:00 2001 From: syurkevi Date: Tue, 2 Mar 2021 23:51:57 -0500 Subject: [PATCH 10/10] Update README.md --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0b24084ba..02af7fe26 100644 --- a/README.md +++ b/README.md @@ -25,21 +25,18 @@ def calc_pi_device(samples): Choosing a particular backend can be done using `af.set_backend(name)` where name is either "_cuda_", "_opencl_", or "_cpu_". The default device is chosen in the same order of preference. -## Requirements - -You will need to have the ArrayFire C/C++ library on your machine. You can get it from the following sources: +## Getting started +ArrayFire can be installed from a variety of sources. [Pre-built wheels](https://repo.arrayfire.com/python/wheels/3.8.0/) are available for a number of systems and toolkits. Wheels for some systems are available on PyPI. Unsupported systems will require separate installation of the ArrayFire C/C++ libraries and only the python wrapper will be installed in that case. +You can get the ArrayFire C/C++ library from the following sources: - [Download and install binaries](https://arrayfire.com/download) - [Build and install from source](https://github.com/arrayfire/arrayfire) -- [Pre-built wheel distributions](https://repo.arrayfire.com/python/wheels/3.8.0/) -## Getting started **Install the last stable version:** ``` pip install arrayfire ``` -Wheels for some systems are available on PyPI. Unsupported systems will require separate installation of the ArrayFire C/C++ libraries and only the python wrapper will be installed in that case. **Install a pre-built wheel for a specific CUDA toolkit version:** ```