From 2358618683e99542a317b0d9d8434250c65fa7b7 Mon Sep 17 00:00:00 2001 From: Tianqi Chen Date: Mon, 28 Sep 2015 19:56:37 -0700 Subject: [PATCH 1/2] Make python package install more easily --- doc/env_var.md | 11 +++++++++-- python/.gitignore | 3 +++ python/README.md | 9 +++++++++ python/mxnet/__init__.py | 2 +- python/mxnet/base.py | 36 ++++-------------------------------- python/mxnet/libinfo.py | 35 +++++++++++++++++++++++++++++++++++ python/setup.py | 21 ++++++++++++++------- 7 files changed, 75 insertions(+), 42 deletions(-) create mode 100644 python/.gitignore create mode 100644 python/README.md create mode 100644 python/mxnet/libinfo.py diff --git a/doc/env_var.md b/doc/env_var.md index ccaf7f2b61d8..d1467d535f14 100644 --- a/doc/env_var.md +++ b/doc/env_var.md @@ -3,13 +3,15 @@ Environment Variables MXNet have several settings that can be changed via environment variable. Usually you do not need to change these settings, but they are listed here for reference. -* MXNET_GPU_WORKER_NTHREADS (default=1 when number of cpu<8, otherwise 2) +* MXNET_GPU_WORKER_NTHREADS (default=1) - Maximum number of threads that do the computation job on each GPU. * MXNET_GPU_COPY_NTHREADS (default=1) - Maximum number of threads that do memory copy job on each GPU. * MXNET_CPU_WORKER_NTHREADS (default=1) - Maximum number of threads that do the CPU computation job. -* MXNET_EXEC_ENABLE_INPLACE (default=true) +* MXNET_CPU_PRIORITY_NTHREADS (default=4) + - Number of threads given to prioritized CPU jobs. + * MXNET_EXEC_ENABLE_INPLACE (default=true) - Whether to enable inplace optimization in symbolic execution. * MXNET_EXEC_MATCH_RANGE (default=10) - The rough matching scale in symbolic execution memory allocator. @@ -20,3 +22,8 @@ Usually you do not need to change these settings, but they are listed here for r - NaiveEngine: very simple engine that use master thread to do computation. - ThreadedEngine: a threaded engine that uses global thread pool to schedule jobs. - ThreadedEnginePerDevice: a threaded engine that allocates thread per GPU. +* MXNET_KVSTORE_REDUCTION_NTHREADS (default=4) + - Number of threads used for summing of big arrays. +* MXNET_KVSTORE_BIGARRAY_BOUND (default=1e6) + - The minimum size of "big array". + - When the array size is bigger than this threshold, MXNET_KVSTORE_REDUCTION_NTHREADS threads will be used for reduction. diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 000000000000..3a34b6464274 --- /dev/null +++ b/python/.gitignore @@ -0,0 +1,3 @@ +dist +*.egg-info +build diff --git a/python/README.md b/python/README.md new file mode 100644 index 000000000000..cbb2dbb4961d --- /dev/null +++ b/python/README.md @@ -0,0 +1,9 @@ +MXNet Python Package +==================== +MXNet is a deep learning framework designed for both *efficiency* and *flexibility*. +It allows you to mix the flavours of deep learning programs together to maximize the efficiency and your productivity. + + +Installation +------------ +To install, check [Build Instruction](http://mxnet.readthedocs.org/en/latest/build.html) diff --git a/python/mxnet/__init__.py b/python/mxnet/__init__.py index 60691cad5b45..b0e9be22c65d 100644 --- a/python/mxnet/__init__.py +++ b/python/mxnet/__init__.py @@ -28,4 +28,4 @@ from . import callback from . import misc -__version__ = "0.1.0" +__version__ = base.__version__ diff --git a/python/mxnet/base.py b/python/mxnet/base.py index 7a5606066ac6..7b0ce9d5a9bd 100644 --- a/python/mxnet/base.py +++ b/python/mxnet/base.py @@ -3,12 +3,11 @@ """ ctypes library of mxnet and helper functions """ from __future__ import absolute_import -import os import sys import ctypes -import platform import numpy as np import atexit +from . import libinfo __all__ = ['MXNetError'] #---------------------------- @@ -30,43 +29,16 @@ class MXNetError(Exception): """Error that will be throwed by all mxnet functions""" pass - -def find_lib_path(): - """Find MXNet dynamic library files. - - Returns - ------- - lib_path : list(string) - List of all found path to the libraries - """ - curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) - api_path = os.path.join(curr_path, '../../lib/') - dll_path = [curr_path, api_path] - if os.name == 'nt': - if platform.architecture()[0] == '64bit': - dll_path.append(os.path.join(api_path, '../windows/x64/Release/')) - else: - dll_path.append(os.path.join(api_path, '../windows/Release/')) - if os.name == 'nt': - dll_path = [os.path.join(p, 'mxnet.dll') for p in dll_path] - else: - dll_path = [os.path.join(p, 'libmxnet.so') for p in dll_path] - lib_path = [p for p in dll_path if os.path.exists(p) and os.path.isfile(p)] - if len(lib_path) == 0: - raise MXNetError('Cannot find find the files.\n' + - 'List of candidates:\n' + str('\n'.join(dll_path))) - return lib_path - - def _load_lib(): """Load libary by searching possible path.""" - lib_path = find_lib_path() + lib_path = libinfo.find_lib_path() lib = ctypes.cdll.LoadLibrary(lib_path[0]) # DMatrix functions lib.MXGetLastError.restype = ctypes.c_char_p return lib - +# version number +__version__ = libinfo.__version__ # library instance of mxnet _LIB = _load_lib() diff --git a/python/mxnet/libinfo.py b/python/mxnet/libinfo.py new file mode 100644 index 000000000000..821d33e5bce2 --- /dev/null +++ b/python/mxnet/libinfo.py @@ -0,0 +1,35 @@ +# coding: utf-8 +"""Information about mxnet.""" +from __future__ import absolute_import +import os +import platform + +def find_lib_path(): + """Find MXNet dynamic library files. + + Returns + ------- + lib_path : list(string) + List of all found path to the libraries + """ + curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) + api_path = os.path.join(curr_path, '../../lib/') + dll_path = [curr_path, api_path] + if os.name == 'nt': + if platform.architecture()[0] == '64bit': + dll_path.append(os.path.join(api_path, '../../windows/x64/Release/')) + else: + dll_path.append(os.path.join(api_path, '../../windows/Release/')) + if os.name == 'nt': + dll_path = [os.path.join(p, 'mxnet.dll') for p in dll_path] + else: + dll_path = [os.path.join(p, 'libmxnet.so') for p in dll_path] + lib_path = [p for p in dll_path if os.path.exists(p) and os.path.isfile(p)] + if len(lib_path) == 0: + raise RuntimeError('Cannot find find the files.\n' + + 'List of candidates:\n' + str('\n'.join(dll_path))) + return lib_path + + +# current version +__version__ = "0.5.0" diff --git a/python/setup.py b/python/setup.py index 1d1196717292..a41b2f0bd3d4 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,15 +1,22 @@ -"""Setup script for mxnet.""" +# pylint: disable=invalid-name, exec-used +"""Setup mxnet package.""" from __future__ import absolute_import -import sys +import os from setuptools import setup -sys.path.insert(0, '.') -import mxnet -LIB_PATH = mxnet.base.find_lib_path() +# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py +# Will be invoked which introduces dependences +CURRENT_DIR = os.path.dirname(__file__) +libinfo_py = os.path.join(CURRENT_DIR, 'mxnet/libinfo.py') +libinfo = {'__file__': libinfo_py} +exec(compile(open(libinfo_py, "rb").read(), libinfo_py, 'exec'), libinfo, libinfo) + +LIB_PATH = libinfo['find_lib_path']() +__version__ = libinfo['__version__'] setup(name='mxnet', - version=mxnet.__version__, - description=mxnet.__doc__, + version=__version__, + description=open(os.path.join(CURRENT_DIR, 'README.md')).read(), install_requires=[ 'numpy', ], From 516a90a68f2d19812be8199a499668833330fe45 Mon Sep 17 00:00:00 2001 From: tqchen Date: Mon, 28 Sep 2015 20:41:41 -0700 Subject: [PATCH 2/2] Change NDArray function registry to function pointers --- include/mxnet/ndarray.h | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index 587a698c9616..1685cee2cbb4 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -467,10 +467,10 @@ struct NDArrayFunctionReg * \param fsetvalue function body to set * \return ref to the registered entry, used to set properties */ - inline NDArrayFunctionReg &set_function(void fsetvalue(const real_t &rhs, - NDArray *out)) { + inline NDArrayFunctionReg &set_function(void (*fsetvalue)(const real_t &rhs, + NDArray *out)) { body = [fsetvalue] (NDArray **used_vars, real_t *s, NDArray **mutate_vars) { - fsetvalue(s[0], mutate_vars[0]); + (*fsetvalue)(s[0], mutate_vars[0]); }; num_mutate_vars = 1; num_scalars = 1; this->add_argument("src", "real_t", "Source input to the function."); @@ -482,12 +482,12 @@ struct NDArrayFunctionReg * \param fbinary function body to set * \return ref to the registered entry, used to set properties */ - inline NDArrayFunctionReg &set_function(void fbinary(const NDArray &lhs, - const NDArray &rhs, - NDArray *out)) { + inline NDArrayFunctionReg &set_function(void (*fbinary)(const NDArray &lhs, + const NDArray &rhs, + NDArray *out)) { body = [fbinary] (NDArray **used_vars, real_t *s, NDArray **mutate_vars) { - fbinary(*used_vars[0], *used_vars[1], mutate_vars[0]); + (*fbinary)(*used_vars[0], *used_vars[1], mutate_vars[0]); }; num_use_vars = 2; num_mutate_vars = 1; type_mask = kNDArrayArgBeforeScalar | kAcceptEmptyMutateTarget; @@ -501,12 +501,12 @@ struct NDArrayFunctionReg * \param fscalar function body to set * \return ref to the registered entry, used to set properties */ - inline NDArrayFunctionReg &set_function(void fscalar(const NDArray &lhs, - const real_t &rhs, - NDArray *out)) { + inline NDArrayFunctionReg &set_function(void (*fscalar)(const NDArray &lhs, + const real_t &rhs, + NDArray *out)) { body = [fscalar] (NDArray **used_vars, real_t *s, NDArray **mutate_vars) { - fscalar(*used_vars[0], s[0], mutate_vars[0]); + (*fscalar)(*used_vars[0], s[0], mutate_vars[0]); }; num_use_vars = 1; num_mutate_vars = 1; num_scalars = 1; type_mask = kNDArrayArgBeforeScalar | kAcceptEmptyMutateTarget; @@ -520,11 +520,11 @@ struct NDArrayFunctionReg * \param funary function body to set * \return ref to the registered entry, used to set properties */ - inline NDArrayFunctionReg &set_function(void funary(const NDArray &src, - NDArray *out)) { + inline NDArrayFunctionReg &set_function(void (*funary)(const NDArray &src, + NDArray *out)) { body = [funary] (NDArray **used_vars, real_t *s, NDArray **mutate_vars) { - funary(*used_vars[0], mutate_vars[0]); + (*funary)(*used_vars[0], mutate_vars[0]); }; num_use_vars = 1; num_mutate_vars = 1; type_mask = kNDArrayArgBeforeScalar | kAcceptEmptyMutateTarget;