pip install: discussion

.travis.yml

@@ -1,72 +1,64 @@
-sudo: false
+language: python
 
-language:
-  - python
-# Currently Travis doesn't support multiple values for language variable
-#  - cpp
-
-#compiler:
-#  - gcc
+addons:
+  artifacts:
+    # warning: works only with some regions
+    # see https://github.com/travis-ci/artifacts/issues/57
+    # see https://github.com/travis-ci/travis-ci/issues/5318
+    s3_region: us-west-1
+    working_dir: wheelhouse
+    # Below 3 values are configured as environmental variables in Travis web console
+    access_key_id: $ARTIFACTS_KEY
+    secret_access_key: $ARTIFACTS_SECRET
+    bucket: $ARTIFACTS_BUCKET
 
-# not sure whether the following configuration will be available
-# with non-C/C++ language (e.g python)
-#cache: ccache
 
-cache:
-  directories:
-    - $HOME/.ccache
+jobs:
+  include:
+    # perform a linux build
+    - os: linux
+      dist: trusty
+      env:
+        - CIBW_MANYLINUX_X86_64_IMAGE="manylinux1"
+        - CIBW_BEFORE_BUILD="tools/prepare-cibuildwheel-linux.sh"
+        - CIBW_BUILD=cp3*
+        - CIBW_SKIP="cp35-* *-manylinux_i686"
+      services: docker
 
-python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
+    # and a mac build
+    - os: osx
+      language: shell
+      env:
+        - CIBW_REPAIR_WHEEL_COMMAND="delocate-listdeps {wheel} && delocate-wheel -v --require-archs x86_64 -w {dest_dir} {wheel} && ls {dest_dir}"
+        - CIBW_BEFORE_BUILD="tools/prepare-cibuildwheel-macosx.sh"
+        - CIBW_BUILD=cp3*
+        - CIBW_SKIP="cp35-*"
+    # and a windows build
+    # - os: windows
+    #  language: shell
+    #  before_install:
+    #    - choco install python --version 3.8.0
+    #    - export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
+    #    # make sure it's on PATH as 'python3'
+    #    - ln -s /c/Python38/python.exe /c/Python38/python3.exe
+    #- CIBW_SKIP="*-win32 *-manylinux_i686"
 
-addons:
-  apt:
-    sources:
-      - boost-latest                       # for boost 1.55
-      - ubuntu-toolchain-r-test            # for g++-4.9
-      - george-edison55-precise-backports  # for cmake 3
-    packages:
-      - cmake
-      - cmake-data
-      - libboost1.55-all-dev
-      - g++-4.9
-      - gcc-4.9
+install:
+  - python3 -m pip install cibuildwheel==1.3.0
 
-env:
-  global:
-    # variables for caching
-    - CCACHE_DIR=$HOME/.ccache
-    - CCACHE_COMPILERCHECK=content
-    - CCACHE_COMPRESS=true
-    - CCACHE_NODISABLE=true
-    - CCACHE_MAXSIZE=500M
-  matrix:
-    - GCC_VER=4.9
+script:
+  # build the wheels, put them into './wheelhouse'
 
-before_install:
-  # we need latest pip to work with only-binary option
-  - pip install -U pip
-  - pip install -U pytest pep8 wheel
-  - pip install -U numpy scipy pandas tqdm --only-binary numpy scipy pandas
-  - pip install protobuf==3.0.0
-  # configure ccache
-  # code from https://github.com/urho3d/Urho3D/blob/master/.travis.yml
-  - export PATH=$(whereis -b ccache |grep -o '\S*lib\S*'):$PATH
-  - export CXX=g++-$GCC_VER CC=gcc-$GCC_VER
-  - for compiler in gcc g++; do ln -s $(which ccache) $HOME/$compiler-$GCC_VER; done && export PATH=$HOME:$PATH
+  # quick fix for
+  # "Invalid binary wheel, found the following shared library/libraries in purelib folder:"
+  # see github.com/pypa/auditwheel/pull/28#issuecomment-212082647
+  # TODO: maybe this is no longer needed because BigARTM is compiled as extension now
+  - |
+    if [[ $TRAVIS_OS_NAME == linux ]]; then
+       echo $CIBW_MANYLINUX_X86_64_IMAGE
+       echo -e "[install]\ninstall_lib=" >> setup.cfg
+    fi
 
-install:
-  - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then ./codestyle_checks.sh; fi
-  - mkdir build
-  - pushd build && cmake -DBUILD_BIGARTM_CLI_STATIC=OFF .. && make -j2 && file ./bin/bigartm && popd
-  - pushd python && python setup.py install && popd
+  - python3 -m cibuildwheel --output-dir wheelhouse
 
-before_script:
-  - pushd test_data && ./download_datasets.sh && popd
 
-script:
-  - export ARTM_SHARED_LIBRARY=`pwd`/build/lib/libartm.so
-  - export BIGARTM_UNITTEST_DATA=`pwd`/test_data
-  - pushd build && make test ARGS="-V" && popd

.travis_old.yml

@@ -0,0 +1,61 @@
+language:
+  - python
+# Currently Travis doesn't support multiple values for language variable
+#  - cpp
+
+#compiler:
+#  - gcc
+
+# not sure whether the following configuration will be available
+# with non-C/C++ language (e.g python)
+#cache: ccache
+
+cache:
+  directories:
+    - $HOME/.ccache
+
+python:
+  - "2.7"
+  - "3.4"
+  - "3.5"
+  - "3.6"
+
+env:
+  global:
+    # variables for caching
+    - CCACHE_DIR=$HOME/.ccache
+    - CCACHE_COMPILERCHECK=content
+    - CCACHE_COMPRESS=true
+    - CCACHE_NODISABLE=true
+    - CCACHE_MAXSIZE=500M
+  matrix:
+    - GCC_VER=4.9
+
+before_install:
+  # we need latest pip to work with only-binary option
+  - pip install -U pip
+  - pip install -U pytest pep8 wheel pytest-forked
+  - pip install -U numpy scipy pandas tqdm --only-binary numpy scipy pandas
+  - pip install protobuf
+  # install boost
+  - sudo add-apt-repository universe
+  - sudo apt-get update
+  - sudo apt-get install libboost-all-dev
+  # configure ccache
+  # code from https://github.com/urho3d/Urho3D/blob/master/.travis.yml
+  - export PATH=$(whereis -b ccache |grep -o '\S*lib\S*'):$PATH
+  - export CXX=g++ CC=gcc PATH=$HOME:$PATH
+
+install:
+  - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then ./codestyle_checks.sh; fi
+  - mkdir build
+  - pushd build && cmake -DBUILD_BIGARTM_CLI_STATIC=OFF .. && make -j2 && file ./bin/bigartm && popd
+  - pushd python && python setup.py install && popd
+
+before_script:
+  - pushd test_data && ./download_datasets.sh && popd
+
+script:
+  - export ARTM_SHARED_LIBRARY=`pwd`/build/lib/libartm.so
+  - export BIGARTM_UNITTEST_DATA=`pwd`/test_data
+  - pushd build && make test ARGS="-V" && popd

3rdparty/glog/CMakeLists.txt

@@ -120,7 +120,7 @@ else()
         endif()
 endif()
 
-mz_add_flag(GCC -Wall -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare)
+mz_add_flag(GCC -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare)
 add_library(google-glog STATIC ${SRCS} ${HDRS})
 target_link_libraries(google-glog ${GFLAGS_LIBRARY} ${GLOG_3LIBS})
 set_property(TARGET google-glog PROPERTY VERSION 0.3.3)

3rdparty/glog/compiler.cmake

@@ -316,7 +316,7 @@ endif()
 
 # compiler flags
 mz_add_definition(${CMAKE_SYSTEM_PROCESSOR}=1)
-mz_add_flag(GCC -Wall -Werror -Wno-unused-function)
+mz_add_flag(GCC -Werror -Wno-unused-function)
 if(WINDOWS)
     mz_add_definition(WIN32=1 WINDOWS=1)
 elseif(IOS)

3rdparty/protobuf-3.0.0/BUILD

@@ -8,7 +8,6 @@ licenses(["notice"])
 
 COPTS = [
     "-DHAVE_PTHREAD",
-    "-Wall",
     "-Wwrite-strings",
     "-Woverloaded-virtual",
     "-Wno-sign-compare",

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 2.8.12)
 
 project(BigARTM)
 
@@ -7,7 +7,9 @@ set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules)
 enable_testing()
 
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY)
+    set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+endif()
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 
 set(3RD_PARTY_DIR ${CMAKE_SOURCE_DIR}/3rdparty)
@@ -31,12 +33,22 @@ else (MSVC)
   message("-- Build type: ${CMAKE_BUILD_TYPE}")
 endif (MSVC)
 
-option(BUILD_TESTS "Indicates whether to build artm_tests" ON)
-option(BUILD_BIGARTM_CLI "Indicates whether to build bigartm-CLI executable" ON)
-option(BUILD_INTERNAL_PYTHON_API "Indicates whether to build Python API" ON)
-
 set(PYTHON python CACHE INTERNAL "Python command")
 
+if (BUILD_PIP_DIST)
+  message("BUILD_PIP_DIST")
+  add_custom_target(python_generate_proto ALL
+    ${PYTHON} ${CMAKE_CURRENT_LIST_DIR}/python/gen_proto.py --protoc_executable=$<TARGET_FILE:protoc>
+    COMMENT "Generating python protobuf files"
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+  add_dependencies(python_generate_proto protoc)
+else (BUILD_PIP_DIST)
+  option(BUILD_TESTS "Indicates whether to build artm_tests" ON)
+  option(BUILD_BIGARTM_CLI "Indicates whether to build bigartm-CLI executable" ON)
+  option(BUILD_INTERNAL_PYTHON_API "Indicates whether to build Python API" ON)
+endif (BUILD_PIP_DIST)
+
+
 if (BUILD_BIGARTM_CLI AND UNIX AND NOT APPLE)
   option(BUILD_BIGARTM_CLI_STATIC "Request build of static executable bigartm (for Linux only)" ON)
 endif (BUILD_BIGARTM_CLI AND UNIX AND NOT APPLE)
@@ -59,7 +71,7 @@ check_include_files(unistd.h HAVE_UNISTD_H)
 # set compiler flags
 if (("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") OR
     ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU"))
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -pthread -fPIC")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -fPIC")
   set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -g -Wall")
   set(GFLAGS_INTTYPES_FORMAT "C99")
 elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")

appveyor.yml

@@ -27,9 +27,6 @@ environment:
     - PYTHON_VERSION: 3.6
       MINICONDA: C:\Miniconda36-x64
 
-    - PYTHON_VERSION: 2.7
-      MINICONDA: C:\Miniconda-x64
-
 # scripts that are called at very beginning, before repo cloning
 init:
   - cmd: cmake --version

python/CMakeLists.txt

@@ -6,13 +6,20 @@ if (BUILD_INTERNAL_PYTHON_API)
   if (BUILD_TESTS)
     add_subdirectory(tests)
   endif (BUILD_TESTS)
+  add_custom_target(python_generate_proto ALL
+    ${PYTHON} ${CMAKE_CURRENT_LIST_DIR}/gen_proto.py --protoc_executable=$<TARGET_FILE:protoc>
+    COMMENT "Generating python protobuf files"
+    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
+  add_dependencies(python_generate_proto protoc)
+
+
 
   # perform "python setup.py build" during executing make
   add_custom_target(python_bigartm_build ALL
-    ${PYTHON} ${CMAKE_CURRENT_LIST_DIR}/setup.py build --protoc_executable=$<TARGET_FILE:protoc>
+    ${PYTHON} ${CMAKE_CURRENT_LIST_DIR}/setup.py build
     COMMENT "Building python package bigartm"
     WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
-  add_dependencies(python_bigartm_build protoc)
+  add_dependencies(python_bigartm_build python_generate_proto)
   add_custom_target(python_bigartm_wheel ALL
     ${CMAKE_COMMAND} -E env ARTM_SHARED_LIBRARY=$<TARGET_FILE:artm> ${PYTHON} setup.py bdist_wheel -d ${CMAKE_CURRENT_BINARY_DIR}
     COMMENT "Building wheel bigartm"

python/artm/artm_model.py

@@ -51,11 +51,11 @@
 PARAMETERS_FILENAME_BIN = 'parameters.bin'
 
 
-def _run_from_ipython():
+def _run_from_notebook():
     try:
-        get_ipython().config
-        return True
-    except:
+        shell = get_ipython().__class__.__name__
+        return shell == 'ZMQInteractiveShell'
+    except:  # noqa
         return False
 
 
@@ -92,8 +92,8 @@ def _topic_selection_regularizer_func(self, regularizers):
 
 
 class ArtmThreadPool(object):
-    def __init__(self, async=True):
-        self._pool = ThreadPool(processes=1) if async else None
+    def __init__(self, asynchronous=True):
+        self._pool = ThreadPool(processes=1) if asynchronous else None
 
     def apply_async(self, func, args):
         return self._pool.apply_async(func, args) if self._pool else func(*args)
@@ -188,7 +188,7 @@ def __init__(self, num_topics=None, topic_names=None, num_processors=None, class
         self._theta_columns_naming = 'id'
         self._seed = -1
         self._show_progress_bars = show_progress_bars
-        self._pool = ArtmThreadPool(async=show_progress_bars)
+        self._pool = ArtmThreadPool(asynchronous=show_progress_bars)
 
         if topic_names is not None:
             self._topic_names = topic_names
@@ -511,7 +511,7 @@ def _wait_for_batches_processed(self, async_result, num_batches):
         import warnings
         with warnings.catch_warnings():
             warnings.filterwarnings("ignore", category=DeprecationWarning)
-            progress = tqdm.tqdm_notebook if _run_from_ipython() else tqdm.tqdm
+            progress = tqdm.tqdm_notebook if _run_from_notebook() else tqdm.tqdm
             with progress(total=num_batches, desc='Batch', leave=False,
                           disable=not self._show_progress_bars) as batch_tqdm:
                 previous_num_batches = 0
@@ -544,7 +544,7 @@ def fit_offline(self, batch_vectorizer=None, num_collection_passes=1, reset_nwt=
         import warnings
         with warnings.catch_warnings():
             warnings.filterwarnings("ignore", category=DeprecationWarning)
-            progress = tqdm.tnrange if _run_from_ipython() else tqdm.trange
+            progress = tqdm.tnrange if _run_from_notebook() else tqdm.trange
             for _ in progress(num_collection_passes, desc='Pass',
                               disable=not self._show_progress_bars):
                 # temp code for easy using of TopicSelectionThetaRegularizer from Python
@@ -565,7 +565,7 @@ def fit_offline(self, batch_vectorizer=None, num_collection_passes=1, reset_nwt=
         self._phi_cached = None
 
     def fit_online(self, batch_vectorizer=None, tau0=1024.0, kappa=0.7, update_every=1,
-                   apply_weight=None, decay_weight=None, update_after=None, async=False):
+                   apply_weight=None, decay_weight=None, update_after=None, asynchronous=False):
         """
         :Description: proceeds the learning of topic model in online mode
 
@@ -579,10 +579,10 @@ def fit_online(self, batch_vectorizer=None, tau0=1024.0, kappa=0.7, update_every
         :type apply_weight: list of float
         :param decay_weight: weight of applying old counters
         :type decay_weight: list of float
-        :param bool async: use or not the async implementation of the EM-algorithm
+        :param bool asynchronous: use or not the asynchronous implementation of the EM-algorithm
 
         :Note:
-          async=True leads to impossibility of score extraction via score_tracker.\
+          asynchronous=True leads to impossibility of score extraction via score_tracker.\
           Use get_score() instead.
 
         :Update formulas:
@@ -625,7 +625,7 @@ def fit_online(self, batch_vectorizer=None, tau0=1024.0, kappa=0.7, update_every
             self._pool.apply_async(func=self.master.fit_online,
                                    args=(batch_vectorizer.batches_ids, batch_vectorizer.weights,
                                          update_after_final, apply_weight_final,
-                                         decay_weight_final, async)),
+                                         decay_weight_final, asynchronous)),
             batch_vectorizer.num_batches)
 
         for name in self.scores.data.keys():
@@ -1090,7 +1090,10 @@ def reshape(self, topic_names=None, dictionary=None):
             return
 
     def __repr__(self):
-        num_tokens = next((x.num_tokens for x in self.info.model if x.name == self._model_pwt), None)
+        if self.master is not None:
+            num_tokens = next((x.num_tokens for x in self.info.model if x.name == self._model_pwt), None)
+        else:
+            num_tokens = "<UNKNOWN>"
         class_ids = ', class_ids={0}'.format(list(self.class_ids.keys())) if self.class_ids else ''
         return 'artm.ARTM(num_topics={0}, num_tokens={1}{2})'.format(
             self.num_topics, num_tokens, class_ids)