Build standalone Python wheels

.travis.yml

@@ -48,7 +48,7 @@ env:
 before_install:
   # we need latest pip to work with only-binary option
   - pip install -U pip
-  - pip install -U pytest pep8
+  - pip install -U pytest pep8 wheel
   - pip install -U numpy scipy pandas tqdm --only-binary numpy scipy pandas
   - pip install protobuf==3.0.0
   # configure ccache

CMakeLists.txt

@@ -35,11 +35,7 @@ option(BUILD_TESTS "Indicates whether to build artm_tests" ON)
 option(BUILD_BIGARTM_CLI "Indicates whether to build bigartm-CLI executable" ON)
 option(BUILD_INTERNAL_PYTHON_API "Indicates whether to build Python API" ON)
 
-if (MSVC OR APPLE)
 set(PYTHON python CACHE INTERNAL "Python command")
-else (MSVC OR APPLE)
-set(PYTHON python2 CACHE INTERNAL "Python command")
-endif (MSVC OR APPLE)
 
 if (BUILD_BIGARTM_CLI AND UNIX AND NOT APPLE)
   option(BUILD_BIGARTM_CLI_STATIC "Request build of static executable bigartm (for Linux only)" ON)

docs/installation/linux.txt

@@ -27,19 +27,13 @@ Scroll further below for more OS-specific insructinos.
 
     # Step 1. Update and install dependencies
     apt-get --yes update
-    apt-get --yes install git
-    apt-get --yes install make
-    apt-get --yes install cmake
-    apt-get --yes install build-essential
-    apt-get --yes install libboost-all-dev
+    apt-get --yes install git make cmake build-essential libboost-all-dev
 
     # Step 2. Insall python packages
-    apt-get --yes install python-numpy
-    apt-get --yes install python-pandas
+    apt-get --yes install python-numpy python-pandas
     wget https://bootstrap.pypa.io/get-pip.py
     python get-pip.py
-    pip install protobuf
-    pip install tqdm
+    pip install protobuf tqdm wheel
 
     # Step 3. Clone repository and build
     git clone --branch=stable https://github.com/bigartm/bigartm.git
@@ -51,6 +45,9 @@ Scroll further below for more OS-specific insructinos.
     # Step 4. Install BigARTM
     make install
     export ARTM_SHARED_LIBRARY=/usr/local/lib/libartm.so
+    
+    # Alternative step 4 - installing only the Python package
+    pip install python/bigartm*.whl
 
 Now you should be able to use BigARTM command line utility (try ``bigartm --help``),
 or run BigARTM from python, like this: ``import artm; print(artm.version()); print(artm.ARTM(num_topics=10).info)``.
@@ -132,8 +129,8 @@ Then install required python packages:
 
 .. code-block:: bash
 
-   sudo pip2 install -U numpy pandas protobuf==3.0.0 tqdm # Python 2
-   sudo pip3 install -U numpy pandas protobuf==3.0.0 tqdm # Python 3
+   sudo pip2 install -U numpy pandas protobuf==3.0.0 tqdm wheel # Python 2
+   sudo pip3 install -U numpy pandas protobuf==3.0.0 tqdm wheel # Python 3
 
 Step 3. Build and install BigARTM library
 -----------------------------------------

python/CMakeLists.txt

@@ -13,6 +13,11 @@ if (BUILD_INTERNAL_PYTHON_API)
     COMMENT "Building python package bigartm"
     WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
   add_dependencies(python_bigartm_build protoc)
+  add_custom_target(python_bigartm_wheel ALL
+    ${CMAKE_COMMAND} -E env ARTM_SHARED_LIBRARY=$<TARGET_FILE:artm> ${PYTHON} setup.py bdist_wheel -d ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Building wheel bigartm"
+    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
+  add_dependencies(python_bigartm_wheel python_bigartm_build artm)
 
   if (MSVC)
     FILE(GLOB PYTHON_ARTM artm/*.py)
@@ -25,7 +30,7 @@ if (BUILD_INTERNAL_PYTHON_API)
   elseif (UNIX)
     install(CODE "message(\"Installing python package bigartm\")")
     install(CODE "execute_process(COMMAND ${PYTHON} setup.py install
-      WORKING_DIRECTORY \"${CMAKE_CURRENT_LIST_DIR}\")")
+                                  WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})")
   endif (MSVC)
 endif (BUILD_INTERNAL_PYTHON_API)
 

python/artm/wrapper/api.py

@@ -42,22 +42,35 @@ def _load_cdll(self, lib_name):
         if sys.platform.startswith('darwin'):
             default_lib_name = 'libartm.dylib'
 
-        if lib_name is None:
-            # try to get library path from environment variable
-            lib_name = os.environ.get('ARTM_SHARED_LIBRARY')
-
-        if lib_name is None:
-            # set the default library name
-            lib_name = default_lib_name
-
-        try:
-            cdll = ctypes.CDLL(lib_name)
-        except OSError as e:
+        lib_names = []
+
+        if lib_name is not None:
+            lib_names.append(lib_name)
+
+        env_lib_name = os.environ.get('ARTM_SHARED_LIBRARY')
+        if env_lib_name is not None:
+            lib_names.append(env_lib_name)
+
+        lib_names.append(default_lib_name)
+        lib_names.append(os.path.join(os.path.dirname(__file__), "..", default_lib_name))
+
+        # We look into 4 places: lib_name, ARTM_SHARED_LIBRARY, default_lib_name and
+        # default_lib_name in the python package root
+        cdll = None
+        for ln in lib_names:
+            try:
+                cdll = ctypes.CDLL(ln)
+                break
+            except OSError as e:
+                if ln == default_lib_name:
+                    exc = e
+                continue
+        if cdll is None:
             exception_message = (
-                '{e}\n'
-                'Failed to load artm shared library. '
+                '{exc}\n'
+                'Failed to load artm shared library from `{lib_names}`. '
                 'Try to add the location of `{default_lib_name}` file into your PATH '
-                'system variable, or to set ARTM_SHARED_LIBRARY - a specific system variable '
+                'system variable, or to set ARTM_SHARED_LIBRARY - the specific system variable '
                 'which may point to `{default_lib_name}` file, including the full path.'
             ).format(**locals())
             raise OSError(exception_message)

python/setup.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 
-from setuptools import setup, find_packages
+from setuptools import setup, find_packages, Distribution
+from setuptools.command.build_py import build_py
 from distutils.spawn import find_executable
 
 # parse arguments
@@ -91,7 +92,47 @@ def run(self):
         _build.run(self)
 
 
-setup(
+class AddLibraryBuild(build_py):
+    """
+    This hacky inheritor adds the shared library into the binary distribution.
+    We pretend that we generated our library and place it into the temporary
+    build directory.
+    """
+    def run(self):
+        if not self.dry_run:
+            self.copy_library()
+        build_py.run(self)
+
+    def get_outputs(self, *args, **kwargs):
+        outputs = build_py.get_outputs(*args, **kwargs)
+        outputs.extend(self._library_paths)
+        return outputs
+
+    def copy_library(self, builddir=None):
+        self._library_paths = []
+        library = os.getenv("ARTM_SHARED_LIBRARY", None)
+        if library is None:
+            return
+        destdir = os.path.join(self.build_lib, 'artm')
+        self.mkpath(destdir)
+        dest = os.path.join(destdir, os.path.basename(library))
+        shutil.copy(library, dest)
+        self._library_paths = [dest]
+
+
+class BinaryDistribution(Distribution):
+    """
+    This inheritor forces setuptools to include the "built" shared library into
+    the binary distribution.
+    """
+    def has_ext_modules(self):
+        return True
+
+    def is_pure(self):
+        return False
+
+
+setup_kwargs = dict(
     name='bigartm',
     version='0.8.3',
     packages=find_packages(),
@@ -109,3 +150,11 @@ def run(self):
     ],
     cmdclass={'build': build},
 )
+
+if sys.argv[1] == "bdist_wheel":
+    # we only mess up with those hacks if we are building a wheel
+    setup_kwargs['distclass'] = BinaryDistribution
+    setup_kwargs['cmdclass']['build_py'] = AddLibraryBuild
+
+setup(**setup_kwargs)
+