Skip to content

Commit

Permalink
improve python setup script
Browse files Browse the repository at this point in the history
  • Loading branch information
@david-cortes
david-cortes committed Mar 19, 2023
1 parent ce9e93d commit 77cc185
Show file tree
Hide file tree
Showing 3 changed files with 200 additions and 131 deletions.
2 changes: 0 additions & 2 deletions README.md
View file
Expand Up @@ -37,8 +37,6 @@ The variants implemented here are based on multiple oracle calls (building a ser

```pip install costsensitive```

**Note: Python 2 is not supported and package will fail to run on Python 2.7**

** *
**IMPORTANT:** the setup script will try to add compilation flag `-march=native`. This instructs the compiler to tune the package for the CPU in which it is being installed (by e.g. using AVX instructions if available), but the result might not be usable in other computers. If building a binary wheel of this package or putting it into a docker image which will be used in different machines, this can be overriden either by (a) defining an environment variable `DONT_SET_MARCH=1`, or by (b) manually supplying compilation `CFLAGS` as an environment variable with something related to architecture. For maximum compatibility (but slowest speed), it's possible to do something like this:

Expand Down
1 change: 1 addition & 0 deletions costsensitive/vwrapper.pyx
View file
@@ -1,3 +1,4 @@
#cython: language_level=3
import numpy as np
cimport numpy as np

Expand Down
328 changes: 199 additions & 129 deletions setup.py
View file
@@ -1,146 +1,216 @@
try:
from setuptools import setup
from setuptools import Extension
from setuptools import setup
from setuptools import Extension
except ImportError:
from distutils.core import setup
from distutils.extension import Extension
from distutils.core import setup
from distutils.extension import Extension
import numpy as np
import os, warnings
import os, sys, warnings
import subprocess
from sys import platform

found_omp = True
def set_omp_false():
global found_omp
found_omp = False
global found_omp
found_omp = False

## Modify this to make the output of the compilation tests more verbose
silent_tests = not (("verbose" in sys.argv)
or ("-verbose" in sys.argv)
or ("--verbose" in sys.argv))

## Workaround for python<=3.9 on windows
try:
EXIT_SUCCESS = os.EX_OK
except AttributeError:
EXIT_SUCCESS = 0

from Cython.Distutils import build_ext
## https://stackoverflow.com/questions/724664/python-distutils-how-to-get-a-compiler-that-is-going-to-be-used
class build_ext_subclass( build_ext ):
def build_extensions(self):
if self.compiler.compiler_type == 'msvc':
for e in self.extensions:
e.extra_compile_args = ['/O2']
else:
if not self.check_for_variable_dont_set_march() and not self.check_cflags_contain_arch():
self.add_march_native()
self.add_openmp_linkage()

for e in self.extensions:
# e.extra_compile_args = ['-fopenmp', '-O2', '-march=native', '-std=c99']
# e.extra_link_args = ['-fopenmp']
e.extra_compile_args += ['-O2', '-std=c99']

build_ext.build_extensions(self)

def check_cflags_contain_arch(self):
if "CFLAGS" in os.environ:
arch_list = ["-march", "-mcpu", "-mtune", "-msse", "-msse2", "-msse3", "-mssse3", "-msse4", "-msse4a", "-msse4.1", "-msse4.2", "-mavx", "-mavx2"]
for flag in arch_list:
if flag in os.environ["CFLAGS"]:
return True
return False

def check_for_variable_dont_set_march(self):
return "DONT_SET_MARCH" in os.environ

def add_march_native(self):
arg_march_native = "-march=native"
arg_mcpu_native = "-mcpu=native"
if self.test_supports_compile_arg(arg_march_native):
for e in self.extensions:
e.extra_compile_args.append(arg_march_native)
elif self.test_supports_compile_arg(arg_mcpu_native):
for e in self.extensions:
e.extra_compile_args.append(arg_mcpu_native)

def add_openmp_linkage(self):
arg_omp1 = "-fopenmp"
arg_omp2 = "-qopenmp"
arg_omp3 = "-xopenmp"
args_apple_omp = ["-Xclang", "-fopenmp", "-lomp"]
args_apple_omp2 = ["-Xclang", "-fopenmp", "-L/usr/local/lib", "-lomp", "-I/usr/local/include"]
if self.test_supports_compile_arg(arg_omp1, with_omp=True):
for e in self.extensions:
e.extra_compile_args.append(arg_omp1)
e.extra_link_args.append(arg_omp1)
elif (sys.platform[:3].lower() == "dar") and self.test_supports_compile_arg(args_apple_omp, with_omp=True):
for e in self.extensions:
e.extra_compile_args += ["-Xclang", "-fopenmp"]
e.extra_link_args += ["-lomp"]
elif (sys.platform[:3].lower() == "dar") and self.test_supports_compile_arg(args_apple_omp2, with_omp=True):
for e in self.extensions:
e.extra_compile_args += ["-Xclang", "-fopenmp"]
e.extra_link_args += ["-L/usr/local/lib", "-lomp"]
e.include_dirs += ["/usr/local/include"]
elif self.test_supports_compile_arg(arg_omp2, with_omp=True):
for e in self.extensions:
e.extra_compile_args.append(arg_omp2)
e.extra_link_args.append(arg_omp2)
elif self.test_supports_compile_arg(arg_omp3, with_omp=True):
for e in self.extensions:
e.extra_compile_args.append(arg_omp3)
e.extra_link_args.append(arg_omp3)
else:
set_omp_false()


def test_supports_compile_arg(self, comm, with_omp=False):
is_supported = False
try:
if not hasattr(self.compiler, "compiler"):
return False
if not isinstance(comm, list):
comm = [comm]
comm_str = " ".join(comm)
fname = "costsensitive_compiler_testing.c"
with open(fname, "w") as ftest:
ftest.write(u"int main(int argc, char**argv) {return 0;}\n")
try:
if not isinstance(self.compiler.compiler, list):
cmd = list(self.compiler.compiler)
else:
cmd = self.compiler.compiler
except Exception:
cmd = self.compiler.compiler
val_good = subprocess.call(cmd + [fname])
if with_omp:
with open(fname, "w") as ftest:
ftest.write(u"#include <omp.h>\nint main(int argc, char**argv) {return 0;}\n")
try:
val = subprocess.call(cmd + comm + [fname])
is_supported = (val == val_good)
except Exception:
is_supported = False
except Exception:
pass
try:
os.remove(fname)
except Exception:
pass
return is_supported
def build_extensions(self):
if self.compiler.compiler_type == 'msvc':
for e in self.extensions:
e.extra_compile_args = ['/O2', '/openmp', '/GL']
else:
if not self.check_for_variable_dont_set_march() and not self.check_cflags_contain_arch():
self.add_march_native()
self.add_openmp_linkage()
self.add_O2()
self.add_std_c99()
self.add_link_time_optimization()

# for e in self.extensions:
# e.extra_compile_args = ['-fopenmp', '-O2', '-march=native', '-std=c99']
# e.extra_link_args = ['-fopenmp']
# e.extra_compile_args += ['-O2', '-std=c99']

build_ext.build_extensions(self)

def check_cflags_contain_arch(self):
if "CFLAGS" in os.environ:
arch_list = [
"-march", "-mcpu", "-mtune", "-msse", "-msse2", "-msse3",
"-mssse3", "-msse4", "-msse4a", "-msse4.1", "-msse4.2",
"-mavx", "-mavx2", "-mavx512"
]
for flag in arch_list:
if flag in os.environ["CFLAGS"]:
return True
return False

def check_for_variable_dont_set_march(self):
return "DONT_SET_MARCH" in os.environ

def add_march_native(self):
args_march_native = ["-march=native", "-mcpu=native"]
for arg_march_native in args_march_native:
if self.test_supports_compile_arg(arg_march_native):
for e in self.extensions:
e.extra_compile_args.append(arg_march_native)
break

def add_link_time_optimization(self):
args_lto = ["-flto=auto", "-flto"]
for arg_lto in args_lto:
if self.test_supports_compile_arg(arg_lto):
for e in self.extensions:
e.extra_compile_args.append(arg_lto)
e.extra_link_args.append(arg_lto)
break

def add_O2(self):
arg_O2 = "-O2"
if self.test_supports_compile_arg(arg_O2):
for e in self.extensions:
e.extra_compile_args.append(arg_O2)
e.extra_link_args.append(arg_O2)

def add_std_c99(self):
arg_std_c99 = "-std=c99"
if self.test_supports_compile_arg(arg_std_c99):
for e in self.extensions:
e.extra_compile_args.append(arg_std_c99)
e.extra_link_args.append(arg_std_c99)

def add_openmp_linkage(self):
arg_omp1 = "-fopenmp"
arg_omp2 = "-fopenmp=libomp"
args_omp3 = ["-fopenmp=libomp", "-lomp"]
arg_omp4 = "-qopenmp"
arg_omp5 = "-xopenmp"
is_apple = sys.platform[:3].lower() == "dar"
args_apple_omp = ["-Xclang", "-fopenmp", "-lomp"]
args_apple_omp2 = ["-Xclang", "-fopenmp", "-L/usr/local/lib", "-lomp", "-I/usr/local/include"]
has_brew_omp = False
if is_apple:
res_brew_pref = subprocess.run(["brew", "--prefix", "libomp"], capture_output=silent_tests)
if res_brew_pref.returncode == EXIT_SUCCESS:
has_brew_omp = True
brew_omp_prefix = res_brew_pref.stdout.decode().strip()
args_apple_omp3 = ["-Xclang", "-fopenmp", f"-L{brew_omp_prefix}/lib", "-lomp", f"-I{brew_omp_prefix}/include"]


if self.test_supports_compile_arg(arg_omp1, with_omp=True):
for e in self.extensions:
e.extra_compile_args.append(arg_omp1)
e.extra_link_args.append(arg_omp1)
elif is_apple and self.test_supports_compile_arg(args_apple_omp, with_omp=True):
for e in self.extensions:
e.extra_compile_args += ["-Xclang", "-fopenmp"]
e.extra_link_args += ["-lomp"]
elif is_apple and self.test_supports_compile_arg(args_apple_omp2, with_omp=True):
for e in self.extensions:
e.extra_compile_args += ["-Xclang", "-fopenmp"]
e.extra_link_args += ["-L/usr/local/lib", "-lomp"]
e.include_dirs += ["/usr/local/include"]
elif is_apple and has_brew_omp and self.test_supports_compile_arg(args_apple_omp3, with_omp=True):
for e in self.extensions:
e.extra_compile_args += ["-Xclang", "-fopenmp"]
e.extra_link_args += [f"-L{brew_omp_prefix}/lib", "-lomp"]
e.include_dirs += [f"{brew_omp_prefix}/include"]
elif self.test_supports_compile_arg(arg_omp2, with_omp=True):
for e in self.extensions:
e.extra_compile_args += ["-fopenmp=libomp"]
e.extra_link_args += ["-fopenmp"]
elif self.test_supports_compile_arg(args_omp3, with_omp=True):
for e in self.extensions:
e.extra_compile_args += ["-fopenmp=libomp"]
e.extra_link_args += ["-fopenmp", "-lomp"]
elif self.test_supports_compile_arg(arg_omp4, with_omp=True):
for e in self.extensions:
e.extra_compile_args.append(arg_omp4)
e.extra_link_args.append(arg_omp4)
elif self.test_supports_compile_arg(arg_omp5, with_omp=True):
for e in self.extensions:
e.extra_compile_args.append(arg_omp5)
e.extra_link_args.append(arg_omp5)
else:
set_omp_false()


def test_supports_compile_arg(self, comm, with_omp=False):
is_supported = False
try:
if not hasattr(self.compiler, "compiler"):
return False
if not isinstance(comm, list):
comm = [comm]
print("--- Checking compiler support for option '%s'" % " ".join(comm))
fname = "costsensitive_compiler_testing.c"
with open(fname, "w") as ftest:
ftest.write(u"int main(int argc, char**argv) {return 0;}\n")
try:
if not isinstance(self.compiler.compiler, list):
cmd = list(self.compiler.compiler)
else:
cmd = self.compiler.compiler
except Exception:
cmd = self.compiler.compiler
if with_omp:
with open(fname, "w") as ftest:
ftest.write(u"#include <omp.h>\nint main(int argc, char**argv) {return 0;}\n")
try:
val = subprocess.run(cmd + comm + [fname], capture_output=silent_tests).returncode
is_supported = (val == EXIT_SUCCESS)
except Exception:
is_supported = False
except Exception:
pass
try:
os.remove(fname)
except Exception:
pass
return is_supported

setup(
name = 'costsensitive',
packages = ['costsensitive'],
install_requires=[
'numpy>=1.17',
'scipy',
'joblib>=0.13',
'cython'
],
python_requires = ">=3",
version = '0.1.2.13-6',
description = 'Reductions for Cost-Sensitive Multi-Class Classification',
author = 'David Cortes',
author_email = 'david.cortes.rivera@gmail.com',
url = 'https://github.com/david-cortes/costsensitive',
keywords = ['cost sensitive multi class', 'cost-sensitive multi-class classification', 'weighted all pairs', 'filter tree'],
classifiers = [],

cmdclass = {'build_ext': build_ext_subclass},
ext_modules = [Extension("costsensitive._vwrapper", sources=["costsensitive/vwrapper.pyx"], include_dirs=[np.get_include()])]
name = 'costsensitive',
packages = ['costsensitive'],
install_requires=[
'numpy>=1.17',
'scipy',
'joblib>=0.13',
'cython'
],
python_requires = ">=3",
version = '0.1.2.13-8',
description = 'Reductions for Cost-Sensitive Multi-Class Classification',
author = 'David Cortes',
url = 'https://github.com/david-cortes/costsensitive',
keywords = ['cost sensitive multi class', 'cost-sensitive multi-class classification', 'weighted all pairs', 'filter tree'],
classifiers = [],

cmdclass = {'build_ext': build_ext_subclass},
ext_modules = [Extension("costsensitive._vwrapper", sources=["costsensitive/vwrapper.pyx"], include_dirs=[np.get_include()])]
)

if not found_omp:
warnings.warn("\n\n\nCould not find OpenMP. Package will be built without multi-threading capabilities.")
omp_msg = "\n\n\nCould not detect OpenMP. Package will be built without multi-threading capabilities. "
omp_msg += " To enable multi-threading, first install OpenMP"
if (sys.platform[:3] == "dar"):
omp_msg += " - for macOS: 'brew install libomp'\n"
else:
omp_msg += " modules for your compiler. "

omp_msg += "Then reinstall this package from scratch: 'pip install --upgrade --no-deps --force-reinstall costsensitive'.\n"
warnings.warn(omp_msg)

0 comments on commit 77cc185

Please sign in to comment.