[ci] merged CI test scripts into one (#1671)

microsoft · Sep 17, 2018 · dea7c78 · dea7c78
1 parent 6214657
commit dea7c78
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 162 deletions.
diff --git a/.ci/test.sh b/.ci/test.sh
@@ -1,14 +1,48 @@
 #!/bin/bash
 
-if [[ $AGENT_OS == "Darwin" ]] && [[ $COMPILER == "gcc" ]]; then
+if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then
     export CXX=g++-8
     export CC=gcc-8
-elif [[ $AGENT_OS == "Linux" ]] && [[ $COMPILER == "clang" ]]; then
+elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang" ]]; then
     export CXX=clang++
     export CC=clang
 fi
 
-cd $BUILD_SOURCESDIRECTORY
+if [[ $TRAVIS == "true" ]]; then
+    conda create -q -y -n $CONDA_ENV python=$PYTHON_VERSION
+    source activate $CONDA_ENV
+fi
+
+cd $BUILD_DIRECTORY
+
+if [[ $TRAVIS == "true" ]] && [[ $TASK == "check-docs" ]]; then
+    if [[ $PYTHON_VERSION == "2.7" ]]; then
+        conda -y -n $CONDA_ENV mock
+    fi
+    conda install -y -n $CONDA_ENV sphinx "sphinx_rtd_theme>=0.3"  # html5validator
+    pip install rstcheck
+    # check reStructuredText formatting
+    cd $BUILD_DIRECTORY/python-package
+    rstcheck --report warning `find . -type f -name "*.rst"` || exit -1
+    cd $BUILD_DIRECTORY/docs
+    rstcheck --report warning --ignore-directives=autoclass,autofunction `find . -type f -name "*.rst"` || exit -1
+    # build docs and check them for broken links
+    make html || exit -1
+    find ./_build/html/ -type f -name '*.html' -exec \
+    sed -i'.bak' -e 's;\(\.\/[^.]*\.\)rst\([^[:space:]]*\);\1html\2;g' {} \;  # emulate js function
+#    html5validator --root ./_build/html/ || exit -1
+    if [[ $OS_NAME == "linux" ]]; then
+        sudo apt-get install linkchecker
+        linkchecker --config=.linkcheckerrc ./_build/html/*.html || exit -1
+    fi
+    # check the consistency of parameters' descriptions and other stuff
+    cp $BUILD_DIRECTORY/docs/Parameters.rst $BUILD_DIRECTORY/docs/Parameters-backup.rst
+    cp $BUILD_DIRECTORY/src/io/config_auto.cpp $BUILD_DIRECTORY/src/io/config_auto-backup.cpp
+    python $BUILD_DIRECTORY/helper/parameter_generator.py || exit -1
+    diff $BUILD_DIRECTORY/docs/Parameters-backup.rst $BUILD_DIRECTORY/docs/Parameters.rst || exit -1
+    diff $BUILD_DIRECTORY/src/io/config_auto-backup.cpp $BUILD_DIRECTORY/src/io/config_auto.cpp || exit -1
+    exit 0
+fi
 
 if [[ $TASK == "pylint" ]]; then
     conda install -y -n $CONDA_ENV pycodestyle
@@ -18,78 +52,97 @@ fi
 
 if [[ $TASK == "if-else" ]]; then
     conda install -y -n $CONDA_ENV numpy
-    mkdir $BUILD_SOURCESDIRECTORY/build && cd $BUILD_SOURCESDIRECTORY/build && cmake .. && make lightgbm || exit -1
-    cd $BUILD_SOURCESDIRECTORY/tests/cpp_test && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
-    cd $BUILD_SOURCESDIRECTORY/build && make lightgbm || exit -1
-    cd $BUILD_SOURCESDIRECTORY/tests/cpp_test && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
+    mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build && cmake .. && make lightgbm || exit -1
+    cd $BUILD_DIRECTORY/tests/cpp_test && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
+    cd $BUILD_DIRECTORY/build && make lightgbm || exit -1
+    cd $BUILD_DIRECTORY/tests/cpp_test && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py || exit -1
     exit 0
 fi
 
 conda install -q -y -n $CONDA_ENV numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest
 
-if [[ $AGENT_OS == "Darwin" ]] && [[ $COMPILER == "clang" ]]; then
+if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
     ln -sf `ls -d "$(brew --cellar libomp)"/*/lib`/* $CONDA_PREFIX/lib || exit -1  # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
 fi
 
 if [[ $TASK == "sdist" ]]; then
-    cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py sdist || exit -1
-    pip install $BUILD_SOURCESDIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
-    cp $BUILD_SOURCESDIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
-    pytest $BUILD_SOURCESDIRECTORY/tests/python_package_test || exit -1
+    cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
+    pip install $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
+    if [[ $AZURE == "true" ]]; then
+        cp $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
+    fi
+    pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
     exit 0
 elif [[ $TASK == "bdist" ]]; then
-    if [[ $AGENT_OS == "Darwin" ]]; then
-        cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py bdist_wheel --plat-name=macdarwin --universal || exit -1
-        mv dist/lightgbm-$LGB_VER-py2.py3-none-macdarwin.whl dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl
-        cp dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
+    if [[ $OS_NAME == "macos" ]]; then
+        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=macosx --universal || exit -1
+        mv dist/lightgbm-$LGB_VER-py2.py3-none-macosx.whl dist/lightgbm-$LGB_VER-py2.py3-none-macosx_10_6_x86_64.macosx_10_7_x86_64.macosx_10_8_x86_64.macosx_10_9_x86_64.macosx_10_10_x86_64.macosx_10_11_x86_64.macosx_10_12_x86_64.macosx_10_13_x86_64.whl
+        if [[ $AZURE == "true" ]]; then
+            cp dist/lightgbm-$LGB_VER-py2.py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
+        fi
     else
-        cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py bdist_wheel --plat-name=manylinux1_x86_64 --universal || exit -1
-        cp dist/lightgbm-$LGB_VER-py2.py3-none-manylinux1_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
+        cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=manylinux1_x86_64 --universal || exit -1
+        if [[ $AZURE == "true" ]]; then
+            cp dist/lightgbm-$LGB_VER-py2.py3-none-manylinux1_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
+        fi
     fi
-    pip install $BUILD_SOURCESDIRECTORY/python-package/dist/*.whl || exit -1
-    pytest $BUILD_SOURCESDIRECTORY/tests || exit -1
+    pip install $BUILD_DIRECTORY/python-package/dist/*.whl || exit -1
+    pytest $BUILD_DIRECTORY/tests || exit -1
     exit 0
 fi
 
 if [[ $TASK == "gpu" ]]; then
-    sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_SOURCESDIRECTORY/include/LightGBM/config.h
-    grep -q 'std::string device_type = "gpu"' $BUILD_SOURCESDIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
+    if [[ $TRAVIS == "true" ]]; then
+        conda install -y -n $CONDA_ENV -c conda-forge boost
+    fi
+    sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h
+    grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1  # make sure that changes were really done
     if [[ $METHOD == "pip" ]]; then
-        cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py sdist || exit -1
-        pip install $BUILD_SOURCESDIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
-        pytest $BUILD_SOURCESDIRECTORY/tests/python_package_test || exit -1
+        cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
+        if [[ $AZURE == "true" ]]; then
+            pip install $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
+        else
+            pip install $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--boost-root=$CONDA_PREFIX" --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
+        fi
+        pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
         exit 0
     fi
 fi
 
-mkdir $BUILD_SOURCESDIRECTORY/build && cd $BUILD_SOURCESDIRECTORY/build
+mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
 
 if [[ $TASK == "mpi" ]]; then
     if [[ $METHOD == "pip" ]]; then
-        cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py sdist || exit -1
-        pip install $BUILD_SOURCESDIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
-        pytest $BUILD_SOURCESDIRECTORY/tests/python_package_test || exit -1
+        cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
+        pip install $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
+        pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
         exit 0
     fi
     cmake -DUSE_MPI=ON ..
 elif [[ $TASK == "gpu" ]]; then
-    cmake -DUSE_GPU=ON -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ ..
+    if [[ $AZURE == "true" ]]; then
+        cmake -DUSE_GPU=ON -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ ..
+    else
+        cmake -DUSE_GPU=ON -DBOOST_ROOT=$CONDA_PREFIX -DOpenCL_INCLUDE_DIR=$AMDAPPSDK_PATH/include/ ..
+    fi
 else
     cmake ..
 fi
 
 make _lightgbm || exit -1
 
-cd $BUILD_SOURCESDIRECTORY/python-package && python setup.py install --precompile || exit -1
-pytest $BUILD_SOURCESDIRECTORY/tests || exit -1
+cd $BUILD_DIRECTORY/python-package && python setup.py install --precompile || exit -1
+pytest $BUILD_DIRECTORY/tests || exit -1
 
 if [[ $TASK == "regular" ]]; then
-    if [[ $AGENT_OS == "Darwin" ]]; then
-        cp $BUILD_SOURCESDIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
-    else
-        cp $BUILD_SOURCESDIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
+    if [[ $AZURE == "true" ]]; then
+        if [[ $OS_NAME == "macos" ]]; then
+            cp $BUILD_DIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
+        else
+            cp $BUILD_DIRECTORY/lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
+        fi
     fi
-    cd $BUILD_SOURCESDIRECTORY/examples/python-guide
+    cd $BUILD_DIRECTORY/examples/python-guide
     sed -i'.bak' '/import lightgbm as lgb/a\
 import matplotlib\
 matplotlib.use\(\"Agg\"\)\

diff --git a/.travis.yml b/.travis.yml
@@ -39,12 +39,14 @@ before_install:
   - test -n $CC  && unset CC
   - test -n $CXX && unset CXX
   - export HOME_DIRECTORY="$HOME"
+  - export BUILD_DIRECTORY="$TRAVIS_BUILD_DIR"
   - if [[ $TRAVIS_OS_NAME == "osx" ]]; then
         export OS_NAME="macos";
     else
         export OS_NAME="linux";
     fi
   - export PATH="$HOME/miniconda/bin:$PATH"
+  - export CONDA_ENV="test-env"
   - export LGB_VER=$(head -n 1 VERSION.txt)
   - export AMDAPPSDK_PATH=$HOME/AMDAPPSDK
   - export LD_LIBRARY_PATH="$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH"
@@ -55,7 +57,7 @@ install:
   - bash .ci/setup.sh
 
 script:
-  - bash .travis/test.sh
+  - bash .ci/test.sh
 
 notifications:
   email: false
diff --git a/.travis/test.sh b/.travis/test.sh
diff --git a/.vsts-ci.yml b/.vsts-ci.yml
@@ -38,6 +38,7 @@ phases:
       createOptions: '-q'
   - script: |
       echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
+      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
       echo "##vso[task.setvariable variable=OS_NAME]linux"
       echo "##vso[task.setvariable variable=AZURE]true"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
@@ -88,6 +89,7 @@ phases:
       createOptions: '-q'
   - script: |
       echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
+      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
       echo "##vso[task.setvariable variable=OS_NAME]macos"
       echo "##vso[task.setvariable variable=AZURE]true"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"