From 73ccf13b27889111a1828992fb79b479635bb543 Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Fri, 14 Sep 2018 23:33:14 +0300 Subject: [PATCH 1/2] reorganized CI files --- {.vsts-ci => .ci}/setup.sh | 0 {.vsts-ci => .ci}/test.sh | 0 {.vsts-ci => .ci}/test_windows.ps1 | 0 .travis/setup.sh | 39 ------------------------------ 4 files changed, 39 deletions(-) rename {.vsts-ci => .ci}/setup.sh (100%) rename {.vsts-ci => .ci}/test.sh (100%) rename {.vsts-ci => .ci}/test_windows.ps1 (100%) delete mode 100644 .travis/setup.sh diff --git a/.vsts-ci/setup.sh b/.ci/setup.sh similarity index 100% rename from .vsts-ci/setup.sh rename to .ci/setup.sh diff --git a/.vsts-ci/test.sh b/.ci/test.sh similarity index 100% rename from .vsts-ci/test.sh rename to .ci/test.sh diff --git a/.vsts-ci/test_windows.ps1 b/.ci/test_windows.ps1 similarity index 100% rename from .vsts-ci/test_windows.ps1 rename to .ci/test_windows.ps1 diff --git a/.travis/setup.sh b/.travis/setup.sh deleted file mode 100644 index 773f3c60a9f3..000000000000 --- a/.travis/setup.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -if [[ $TRAVIS_OS_NAME == "osx" ]]; then - if [[ $COMPILER == "clang" ]]; then - brew install libomp - brew reinstall cmake # CMake >=3.12 is needed to find OpenMP at macOS - else - sudo softwareupdate -i "Command Line Tools (macOS High Sierra version 10.13) for Xcode-9.3" # fix "fatal error: _stdio.h: No such file or directory" - rm '/usr/local/include/c++' -# brew cask uninstall oclint # reserve variant to deal with conflict link - if [[ $TASK != "mpi" ]]; then - brew install gcc - fi -# brew link --overwrite gcc # previous variant to deal with conflict link - fi - if [[ $TASK == "mpi" ]]; then - brew install open-mpi - fi - wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh -else # Linux - if [[ $TASK == "mpi" ]]; then - sudo apt-get install -y libopenmpi-dev openmpi-bin - fi - if [[ $TASK == "gpu" ]]; then - sudo apt-get install -y ocl-icd-opencl-dev - wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2 - tar -xjf AMD-APP-SDK*.tar.bz2 - mkdir -p $OPENCL_VENDOR_PATH - mkdir -p $AMDAPPSDK_PATH - sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK_PATH - mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/ - echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd - fi - wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh -fi - -sh conda.sh -b -p $HOME/miniconda -conda config --set always_yes yes --set changeps1 no -conda update -q conda From 0f2c1d1711b084c741a14a4a37c7a3de06a3dd63 Mon Sep 17 00:00:00 2001 From: StrikerRUS Date: Sat, 15 Sep 2018 00:09:40 +0300 Subject: [PATCH 2/2] merged scripts into one --- .ci/setup.sh | 35 +++++++++++++++++++++++++++++------ .travis.yml | 8 +++++++- .vsts-ci.yml | 24 +++++++++++++++--------- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index c71de2cde7c5..18af055a53b3 100644 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -1,10 +1,16 @@ #!/bin/bash -if [[ $AGENT_OS == "Darwin" ]]; then +if [[ $OS_NAME == "macos" ]]; then if [[ $COMPILER == "clang" ]]; then brew install libomp brew reinstall cmake # CMake >=3.12 is needed to find OpenMP at macOS else + if [[ $TRAVIS == "true" ]]; then + sudo softwareupdate -i "Command Line Tools (macOS High Sierra version 10.13) for Xcode-9.3" # fix "fatal error: _stdio.h: No such file or directory" + rm '/usr/local/include/c++' +# brew cask uninstall oclint # reserve variant to deal with conflict link +# brew link --overwrite gcc # previous variant to deal with conflict link + fi if [[ $TASK != "mpi" ]]; then brew install gcc fi @@ -12,21 +18,29 @@ if [[ $AGENT_OS == "Darwin" ]]; then if [[ $TASK == "mpi" ]]; then brew install open-mpi fi + if [[ $TRAVIS == "true" ]]; then + wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh + fi else # Linux - if [[ $COMPILER == "clang" ]]; then + if [[ $AZURE == "true" ]] && [[ $COMPILER == "clang" ]]; then update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100 update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100 sudo apt-get update sudo apt-get install libomp-dev fi if [[ $TASK == "mpi" ]]; then - sudo apt-get update + if [[ $AZURE == "true" ]]; then + sudo apt-get update + fi sudo apt-get install -y libopenmpi-dev openmpi-bin fi if [[ $TASK == "gpu" ]]; then - sudo apt-get update - sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev libboost-dev libboost-system-dev libboost-filesystem-dev - cd $AGENT_HOMEDIRECTORY + if [[ $AZURE == "true" ]]; then + sudo apt-get update + sudo apt-get install --no-install-recommends -y libboost-dev libboost-system-dev libboost-filesystem-dev + fi + sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev + cd $HOME_DIRECTORY wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2 tar -xjf AMD-APP-SDK*.tar.bz2 mkdir -p $OPENCL_VENDOR_PATH @@ -35,4 +49,13 @@ else # Linux mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/ echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd fi + if [[ $TRAVIS == "true" ]]; then + wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh + fi +fi + +if [[ $TRAVIS == "true" ]]; then + sh conda.sh -b -p $HOME_DIRECTORY/miniconda + conda config --set always_yes yes --set changeps1 no + conda update -q conda fi diff --git a/.travis.yml b/.travis.yml index eee6000eceb0..0b2c51aef1e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,6 +38,12 @@ matrix: before_install: - test -n $CC && unset CC - test -n $CXX && unset CXX + - export HOME_DIRECTORY="$HOME" + - if [[ $TRAVIS_OS_NAME == "osx" ]]; then + export OS_NAME="macos"; + else + export OS_NAME="linux"; + fi - export PATH="$HOME/miniconda/bin:$PATH" - export LGB_VER=$(head -n 1 VERSION.txt) - export AMDAPPSDK_PATH=$HOME/AMDAPPSDK @@ -46,7 +52,7 @@ before_install: - export OPENCL_VENDOR_PATH=$AMDAPPSDK_PATH/etc/OpenCL/vendors install: - - bash .travis/setup.sh + - bash .ci/setup.sh script: - bash .travis/test.sh diff --git a/.vsts-ci.yml b/.vsts-ci.yml index a2a3fd27b32e..4c45a48f4480 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -37,18 +37,21 @@ phases: packageSpecs: 'python=$(PYTHON_VERSION)' createOptions: '-q' - script: | + echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" + echo "##vso[task.setvariable variable=OS_NAME]linux" + echo "##vso[task.setvariable variable=AZURE]true" echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" AMDAPPSDK_PATH=$AGENT_HOMEDIRECTORY/AMDAPPSDK echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" - chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/setup.sh - chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/test.sh + chmod +x $BUILD_SOURCESDIRECTORY/.ci/setup.sh + chmod +x $BUILD_SOURCESDIRECTORY/.ci/test.sh displayName: 'Set variables' - - bash: $(Build.SourcesDirectory)/.vsts-ci/setup.sh + - bash: $(Build.SourcesDirectory)/.ci/setup.sh displayName: Setup - - bash: $(Build.SourcesDirectory)/.vsts-ci/test.sh + - bash: $(Build.SourcesDirectory)/.ci/test.sh displayName: Test - task: PublishBuildArtifacts@1 condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) @@ -84,13 +87,16 @@ phases: packageSpecs: 'python=$(PYTHON_VERSION)' createOptions: '-q' - script: | + echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" + echo "##vso[task.setvariable variable=OS_NAME]macos" + echo "##vso[task.setvariable variable=AZURE]true" echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/setup.sh - chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/test.sh + chmod +x $BUILD_SOURCESDIRECTORY/.ci/setup.sh + chmod +x $BUILD_SOURCESDIRECTORY/.ci/test.sh displayName: 'Set variables' - - bash: $(Build.SourcesDirectory)/.vsts-ci/setup.sh + - bash: $(Build.SourcesDirectory)/.ci/setup.sh displayName: Setup - - bash: $(Build.SourcesDirectory)/.vsts-ci/test.sh + - bash: $(Build.SourcesDirectory)/.ci/test.sh displayName: Test - task: PublishBuildArtifacts@1 condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) @@ -121,7 +127,7 @@ phases: environmentName: $(CONDA_ENV) packageSpecs: 'python=$(PYTHON_VERSION) numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest' createOptions: '-q' - - powershell: $(Build.SourcesDirectory)/.vsts-ci/test_windows.ps1 + - powershell: $(Build.SourcesDirectory)/.ci/test_windows.ps1 displayName: Test - task: PublishBuildArtifacts@1 condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))