[python] [setup] improving installation

.travis.yml

@@ -5,46 +5,46 @@ dist: trusty
 compiler:
   - gcc
 
+os:
+  - linux
+  - osx
+
+env:
+  matrix:
+    - TASK=regular PYTHON_VERSION=3.6
+    - TASK=mpi PYTHON_VERSION=3.6
+    - TASK=pylint PYTHON_VERSION=3.6
+    - TASK=if-else PYTHON_VERSION=3.6
+    - TASK=sdist PYTHON_VERSION=3.6
+    - TASK=bdist PYTHON_VERSION=3.6
+    - TASK=gpu METHOD=source PYTHON_VERSION=3.6
+    - TASK=gpu METHOD=pip PYTHON_VERSION=3.6
+    - TASK=regular PYTHON_VERSION=2.7
+    - TASK=regular PYTHON_VERSION=3.4
+    - TASK=regular PYTHON_VERSION=3.5
+
 matrix:
-  include:
-    - os: linux
-      env: TASK=regular
-    - os: linux
-      env: TASK=mpi
-    - os: linux
-      env: TASK=gpu METHOD=source
-    - os: linux
-      env: TASK=sdist
-    - os: linux
-      env: TASK=gpu METHOD=pip
-    - os: linux
-      env: TASK=bdist
-    - os: linux
-      env: TASK=pylint
-    - os: linux
-      env: TASK=if-else
-    - os: osx
-      env: TASK=regular
+  exclude:
     - os: osx
-      env: TASK=mpi
+      env: TASK=gpu METHOD=source PYTHON_VERSION=3.6
     - os: osx
-      env: TASK=sdist
+      env: TASK=gpu METHOD=pip PYTHON_VERSION=3.6
     - os: osx
-      env: TASK=bdist
+      env: TASK=pylint PYTHON_VERSION=3.6
 
 before_install:
-- test -n $CC  && unset CC
-- test -n $CXX && unset CXX
+  - test -n $CC  && unset CC
+  - test -n $CXX && unset CXX
 
 install:
-- bash .travis/setup.sh
-- bash conda.sh -b -p $HOME/miniconda
-- export PATH="$HOME/miniconda/bin:$PATH"
-- conda config --set always_yes yes --set changeps1 no
-- conda update -q conda
+  - bash .travis/setup.sh
+  - bash conda.sh -b -p $HOME/miniconda
+  - export PATH="$HOME/miniconda/bin:$PATH"
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
 
 script:
-- bash .travis/test.sh
+  - bash .travis/test.sh
 
 notifications:
   email: false

.travis/setup.sh

@@ -2,13 +2,13 @@
 
 if [[ $TRAVIS_OS_NAME == "osx" ]]; then
     brew install openmpi  # use openmpi to install gcc
-    wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+    wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh
 else
     if [[ ${TASK} != "pylint" ]]; then
         sudo add-apt-repository ppa:george-edison55/cmake-3.x -y
         sudo apt-get update -q
         sudo apt-get install -y cmake
         sudo apt-get install -y libopenmpi-dev openmpi-bin build-essential
     fi
-    wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+    wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh
 fi

.travis/test.sh

@@ -1,4 +1,4 @@
-if [[ ${TASK} == "gpu" ]]; then 
+if [[ ${TASK} == "gpu" ]]; then
     bash .travis/amd_sdk.sh;
     tar -xjf AMD-SDK.tar.bz2;
     AMDAPPSDK=${HOME}/AMDAPPSDK;
@@ -9,9 +9,9 @@ if [[ ${TASK} == "gpu" ]]; then
     export LD_LIBRARY_PATH=${AMDAPPSDK}/lib/x86_64:${LD_LIBRARY_PATH};
     chmod +x ${AMDAPPSDK}/bin/x86_64/clinfo;
     ${AMDAPPSDK}/bin/x86_64/clinfo;
-    export LIBRARY_PATH="$HOME/miniconda/lib:$LIBRARY_PATH"
-    export LD_RUN_PATH="$HOME/miniconda/lib:$LD_RUN_PATH"
-    export CPLUS_INCLUDE_PATH="$HOME/miniconda/include:$AMDAPPSDK/include/:$CPLUS_INCLUDE_PATH"
+    export LIBRARY_PATH="$HOME/miniconda/envs/test-env/lib:$LIBRARY_PATH"
+    export LD_RUN_PATH="$HOME/miniconda/envs/test-env/lib:$LD_RUN_PATH"
+    export CPLUS_INCLUDE_PATH="$HOME/miniconda/envs/test-env/include:$AMDAPPSDK/include/:$CPLUS_INCLUDE_PATH"
 fi
 
 case ${TRAVIS_OS_NAME} in
@@ -32,16 +32,17 @@ if [[ ${TASK} == "pylint" ]]; then
 fi
 
 if [[ ${TASK} == "if-else" ]]; then
-    conda install --yes numpy
+    conda create -q -n test-env python=$PYTHON_VERSION numpy
+    source activate test-env
     mkdir build && cd build && cmake .. && make lightgbm || exit -1
     cd $TRAVIS_BUILD_DIR/tests/cpp_test && ../../lightgbm config=train.conf && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
     cd $TRAVIS_BUILD_DIR/build && make lightgbm || exit -1
     cd $TRAVIS_BUILD_DIR/tests/cpp_test && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
     exit 0
 fi
 
-conda install --yes numpy nose scipy scikit-learn pandas matplotlib
-pip install pytest
+conda create -q -n test-env python=$PYTHON_VERSION numpy nose scipy scikit-learn pandas matplotlib pytest
+source activate test-env
 
 if [[ ${TASK} == "sdist" ]]; then
     LGB_VER=$(head -n 1 VERSION.txt)
@@ -66,7 +67,7 @@ if [[ ${TASK} == "gpu" ]]; then
     conda install --yes -c conda-forge boost=1.63.0
     if [[ ${METHOD} == "pip" ]]; then
         export PATH="$AMDAPPSDK/include/:$PATH"
-        export BOOST_ROOT="$HOME/miniconda/"
+        export BOOST_ROOT="$HOME/miniconda/envs/test-env/"
         LGB_VER=$(head -n 1 VERSION.txt)
         sed -i 's/const std::string kDefaultDevice = "cpu";/const std::string kDefaultDevice = "gpu";/' ../include/LightGBM/config.h
         cd $TRAVIS_BUILD_DIR/python-package && python setup.py sdist || exit -1
@@ -81,7 +82,7 @@ mkdir build && cd build
 if [[ ${TASK} == "mpi" ]]; then
     cmake -DUSE_MPI=ON ..
 elif [[ ${TASK} == "gpu" ]]; then
-    cmake -DUSE_GPU=ON -DBOOST_ROOT="$HOME/miniconda/" -DOpenCL_INCLUDE_DIR=$AMDAPPSDK/include/ ..
+    cmake -DUSE_GPU=ON -DBOOST_ROOT="$HOME/miniconda/envs/test-env/" -DOpenCL_INCLUDE_DIR=$AMDAPPSDK/include/ ..
     sed -i 's/const std::string kDefaultDevice = "cpu";/const std::string kDefaultDevice = "gpu";/' ../include/LightGBM/config.h
 else
     cmake ..

README.md

@@ -8,6 +8,7 @@ LightGBM, Light Gradient Boosting Machine
 [![GitHub Issues](https://img.shields.io/github/issues/Microsoft/LightGBM.svg)](https://github.com/Microsoft/LightGBM/issues)
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/Microsoft/LightGBM/blob/master/LICENSE)
 [![PyPI Version](https://badge.fury.io/py/lightgbm.svg)](https://badge.fury.io/py/lightgbm)
+<!--- # Uncomment after updating PyPI [![Python Versions](https://img.shields.io/pypi/pyversions/lightgbm.svg)](https://pypi.python.org/pypi/lightgbm) -->
 
 LightGBM is a gradient boosting framework that uses tree based learning algorithms. It is designed to be distributed and efficient with the following advantages:
 

appveyor.yml

@@ -1,36 +1,56 @@
+version: 2.0.5.{build}
+
+configuration:  # A trick to construct a build matrix
+  - 3.5
+  - 3.6
 
-version: 1.0.{build}
 environment:
   matrix:
-    - PYTHON: "C:/Python36-x64"
-      PYTHON_VERSION: 3.6
-      MINICONDA: "C:/Miniconda36-x64"
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+      COMPILER: MSVC
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+      COMPILER: MINGW
 
 clone_depth: 50
 
-init:
-  - "ECHO %PYTHON_VERSION% %MINICONDA%"
+install:
+  - set PATH=%PATH:C:\Program Files\Git\usr\bin;=%  # Delete sh.exe from PATH (mingw32-make fix)
+  - set PATH=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin;%PATH%
+  - set PYTHON_VERSION=%CONFIGURATION%
+  - ps: >-
+      switch ($env:PYTHON_VERSION) {
+          "2.7" {$env:MINICONDA = """C:\Miniconda-x64"""}
+          "3.4" {$env:MINICONDA = """C:\Miniconda3-x64"""}
+          "3.5" {$env:MINICONDA = """C:\Miniconda35-x64"""}
+          "3.6" {$env:MINICONDA = """C:\Miniconda36-x64"""}
+          default {$env:MINICONDA = """C:\Miniconda36-x64"""}
+      }
+  - set PATH=%MINICONDA%;%MINICONDA%\Scripts;%PATH%
+  - ps: $env:LGB_VER = (Get-Content VERSION.txt).trim()
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
+  - conda create -q -n test-env python=%PYTHON_VERSION% numpy nose scipy scikit-learn pandas matplotlib pep8 pytest
+  - activate test-env
 
 build_script:
   - mkdir build && cd build
   - cmake -DCMAKE_GENERATOR_PLATFORM=x64 .. && cmake --build . --target ALL_BUILD --config Release
   - cd ..
 
 test_script:
-  - "set PATH=%MINICONDA%;%MINICONDA%/Scripts;%PATH%"
-  - conda config --set always_yes yes --set changeps1 no
-  - conda update -q conda
-  - conda info -a
-  - conda install --yes numpy nose scipy scikit-learn pandas matplotlib
-  - pip install pep8 pytest
   - pytest tests/c_api_test/test_.py
-  - "set /p LGB_VER=< VERSION.txt"
   - cd python-package && python setup.py sdist --formats gztar
-  - cd dist 
-  - "pip install lightgbm-%LGB_VER%.tar.gz -v"
+  - cd dist
+  - IF "%COMPILER%"=="MINGW" (
+    pip install lightgbm-%LGB_VER%.tar.gz --install-option=--mingw -v)
+    ELSE (
+    pip install lightgbm-%LGB_VER%.tar.gz -v)
   - cd ../..
   - pytest tests/python_package_test
+  - IF "%COMPILER%"=="MINGW" appveyor exit  # skip all further steps
   - cd python-package && python setup.py bdist_wheel --plat-name=win-amd64 --universal
+
+after_test:
   - cd ../.nuget
   - python create_nuget.py
   - nuget.exe pack LightGBM.nuspec

python-package/MANIFEST.in

@@ -5,5 +5,6 @@ recursive-include lightgbm *.py *.txt *.so
 recursive-include lightgbm/Release *.dll
 recursive-include lightgbm/include *
 recursive-include lightgbm/src *
-global-exclude *.pyo
-global-exclude *.pyc
+recursive-include lightgbm/windows LightGBM.sln LightGBM.vcxproj LightGBM.vcxproj.filters
+recursive-include lightgbm/windows/x64/DLL *.dll
+global-exclude *.py[co]

python-package/README.rst

@@ -1,8 +1,9 @@
 LightGBM Python Package
 =======================
 
-|PyPI version|
+|License| |PyPI Version|
 
+.. # Uncomment after updating PyPI |Python Versions|
 
 Installation
 ------------
@@ -16,63 +17,76 @@ For Mac OS X users, gcc with OpenMP support must be installed first. Refer to `w
 
 Note: 32-bit python is not supported. Please install 64-bit version.
 
-Install from pip
-''''''''''''''''
+Install from `PyPI <https://pypi.python.org/pypi/lightgbm>`_ using ``pip``
+''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
+For Windows users, `VC runtime <https://go.microsoft.com/fwlink/?LinkId=746572>`_ is needed if Visual Studio (2013, 2015 or 2017) is not installed.
 
-Install `wheel <http://pythonwheels.com>`_ via ``pip install wheel`` first. For Windows user, `VC runtime <https://go.microsoft.com/fwlink/?LinkId=746572>`_ is needed if Visual Studio (2013, 2015 or 2017) is not installed.
+Install `wheel <http://pythonwheels.com>`_ via ``pip install wheel`` first. After that download the wheel file and install from it:
 
+.. code:: sh
 
-``pip install lightgbm``
+    pip install lightgbm
 
+Build from sources
+******************
 
-Install source package from pip
-*******************************
+.. code:: sh
 
-``pip install --no-binary :all: lightgbm``
+    pip install --no-binary :all: lightgbm
 
+For Linux and Mac OS X users, installation from sources requires installed `CMake <https://cmake.org/>`_.
 
-Note: Installation from source package require installing `CMake <https://cmake.org/>`_ first.
+For Mac OS X users, you need to specify compilers by runnig ``export CXX=g++-7 CC=gcc-7`` first.
 
-For Windows user, Visual Studio (or `MS Build <https://www.visualstudio.com/downloads/#build-tools-for-visual-studio-2017>`_) is needed, and `CMake <https://cmake.org/>`_ must be version 3.8 or higher.
+For Windows users, Visual Studio (or `MS Build <https://www.visualstudio.com/downloads/#build-tools-for-visual-studio-2017>`_) is needed. If you get any errors during installation, you may need to install `CMake <https://cmake.org/>`_ (version 3.8 or higher).
 
-For OSX user, you need to run ```export CXX=g++-7 CC=gcc-7``` before running ```pip install ... ```.
+Build GPU version
+~~~~~~~~~~~~~~~~~
 
-Install GPU version:
+.. code:: sh
 
-``pip install lightgbm --install-option=--gpu``
+    pip install lightgbm --install-option=--gpu
 
-Note: Boost and OpenCL are needed: details for installation can be found in `gpu-support <https://github.com/Microsoft/LightGBM/wiki/Installation-Guide#with-gpu-support>`_. Need to add OpenCL_INCLUDE_DIR to PATH and export BOOST_ROOT before installation.
+For Windows users, `CMake <https://cmake.org/>`_ (version 3.8 or higher) is strongly required in this case.
 
-Install with MinGW on Windows:
+Note: Boost and OpenCL are needed: details for installation can be found in `gpu-support <https://github.com/Microsoft/LightGBM/wiki/Installation-Guide#with-gpu-support>`_. You need to add ``OpenCL_INCLUDE_DIR`` to the environmental variable **'PATH'** and export ``BOOST_ROOT`` before installation.
 
-``pip install lightgbm --install-option=--mingw``
+Build with MinGW-w64 on Windows
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: sh
+
+    pip install lightgbm --install-option=--mingw
+
+Note: `CMake <https://cmake.org/>`_ and `MinGW-w64 <https://mingw-w64.org/>`_ should be installed first.
 
 Install from GitHub
 '''''''''''''''''''
 
-Installation from GitHub require installing `CMake <https://cmake.org/>`_ first.
+For Linux and Mac OS X users, installation from GitHub requires installed `CMake <https://cmake.org/>`_.
 
-For Windows user, Visual Studio (or `MS Build <https://www.visualstudio.com/downloads/#build-tools-for-visual-studio-2017>`_) is needed, and `CMake <https://cmake.org/>`_ must be version 3.8 or higher.
+For Windows users, Visual Studio (or `MS Build <https://www.visualstudio.com/downloads/#build-tools-for-visual-studio-2017>`_) is needed. If you get any errors during installation and there is the warning ``WARNING:LightGBM:Compilation with MSBuild from existing solution file failed.`` in the log, you should install `CMake <https://cmake.org/>`_ (version 3.8 or higher).
 
 .. code:: sh
 
-    git clone --recursive https://github.com/Microsoft/LightGBM
+    git clone --recursive https://github.com/Microsoft/LightGBM.git
     cd LightGBM/python-package
-    # export CXX=g++-7 CC=gcc-7 # for OSX
+    # export CXX=g++-7 CC=gcc-7  # for Mac OS X users only
     python setup.py install
 
-``sudo`` (or administrator rights in Windows) may is needed to perform ``python setup.py install``.
+Note: ``sudo`` (or administrator rights in Windows) may be needed to perform the command.
 
-Use ``python setup.py install --mingw`` to use MinGW in Windows.
+Run ``python setup.py install --mingw`` if you want to use MinGW-w64 on Windows instead of Visual Studio. `CMake <https://cmake.org/>`_ and `MinGW-w64 <https://mingw-w64.org/>`_ should be installed first.
 
-Use ``python setup.py install --gpu`` to enable GPU support. Boost and OpenCL are needed: details for installation can be found in `gpu-support <https://github.com/Microsoft/LightGBM/wiki/Installation-Guide#with-gpu-support>`_.
+Run ``python setup.py install --gpu`` to enable GPU support. For Windows users, `CMake <https://cmake.org/>`_ (version 3.8 or higher) is strongly required in this case. Boost and OpenCL are needed: details for installation can be found in `gpu-support <https://github.com/Microsoft/LightGBM/wiki/Installation-Guide#with-gpu-support>`_.
+
+If you get any errors during installation or due to any other reason, you may want to build dynamic library from sources by any method you prefer (see `Installation-Guide <https://github.com/Microsoft/LightGBM/wiki/Installation-Guide>`_) and then run ``python setup.py install --precompile``.
 
 Examples
 --------
 
 Refer to the walk through examples in `python-guide folder <https://github.com/Microsoft/LightGBM/tree/master/examples/python-guide>`_.
 
-
 Troubleshooting
 ---------------
 
@@ -93,5 +107,9 @@ The code style of python package follows `pep8 <https://www.python.org/dev/peps/
 
 E501 can be ignored (line too long).
 
-.. |PyPI version| image:: https://badge.fury.io/py/lightgbm.svg
+.. |License| image:: https://img.shields.io/badge/license-MIT-blue.svg
+   :target: https://github.com/Microsoft/LightGBM/blob/master/LICENSE
+.. |Python Versions| image:: https://img.shields.io/pypi/pyversions/lightgbm.svg
+   :target: https://pypi.python.org/pypi/lightgbm
+.. |PyPI Version| image:: https://badge.fury.io/py/lightgbm.svg
     :target: https://badge.fury.io/py/lightgbm

python-package/lightgbm/libpath.py

@@ -19,6 +19,7 @@ def find_lib_path():
     dll_path = [curr_path, os.path.join(curr_path, '../../'), os.path.join(curr_path, '../../lib/')]
     if os.name == 'nt':
         dll_path.append(os.path.join(curr_path, './Release/'))
+        dll_path.append(os.path.join(curr_path, './windows/x64/DLL/'))
         dll_path.append(os.path.join(curr_path, '../../Release/'))
         dll_path.append(os.path.join(curr_path, '../../windows/x64/DLL/'))
         dll_path = [os.path.join(p, 'lib_lightgbm.dll') for p in dll_path]
@@ -27,5 +28,5 @@ def find_lib_path():
     lib_path = [p for p in dll_path if os.path.exists(p) and os.path.isfile(p)]
     if not lib_path:
         dll_path = [os.path.realpath(p) for p in dll_path]
-        raise Exception('Cannot find lightgbm Library in following paths: ' + ','.join(dll_path))
+        raise Exception('Cannot find lightgbm library in following paths: ' + '\n'.join(dll_path))
     return lib_path