[ci] merged CI setup scripts into one (#1669)

* reorganized CI files * merged scripts into one
microsoft · Sep 15, 2018 · 73a9a19 · 73a9a19
1 parent c1320da
commit 73a9a19
Show file tree

Hide file tree

Showing 7 changed files with 83 additions and 87 deletions.
diff --git a/.ci/setup.sh b/.ci/setup.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+if [[ $OS_NAME == "macos" ]]; then
+    if  [[ $COMPILER == "clang" ]]; then
+        brew install libomp
+        brew reinstall cmake  # CMake >=3.12 is needed to find OpenMP at macOS
+    else
+        if [[ $TRAVIS == "true" ]]; then
+            sudo softwareupdate -i "Command Line Tools (macOS High Sierra version 10.13) for Xcode-9.3"  # fix "fatal error: _stdio.h: No such file or directory"
+            rm '/usr/local/include/c++'
+#            brew cask uninstall oclint  #  reserve variant to deal with conflict link
+#            brew link --overwrite gcc  # previous variant to deal with conflict link
+        fi
+        if [[ $TASK != "mpi" ]]; then
+            brew install gcc
+        fi
+    fi
+    if [[ $TASK == "mpi" ]]; then
+        brew install open-mpi
+    fi
+    if [[ $TRAVIS == "true" ]]; then
+        wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh
+    fi
+else  # Linux
+    if [[ $AZURE == "true" ]] && [[ $COMPILER == "clang" ]]; then
+        update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
+        update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
+        sudo apt-get update
+        sudo apt-get install libomp-dev
+    fi
+    if [[ $TASK == "mpi" ]]; then
+        if [[ $AZURE == "true" ]]; then
+            sudo apt-get update
+        fi
+        sudo apt-get install -y libopenmpi-dev openmpi-bin
+    fi
+    if [[ $TASK == "gpu" ]]; then
+        if [[ $AZURE == "true" ]]; then
+            sudo apt-get update
+            sudo apt-get install --no-install-recommends -y libboost-dev libboost-system-dev libboost-filesystem-dev
+        fi
+        sudo apt-get install --no-install-recommends -y ocl-icd-opencl-dev
+        cd $HOME_DIRECTORY
+        wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
+        tar -xjf AMD-APP-SDK*.tar.bz2
+        mkdir -p $OPENCL_VENDOR_PATH
+        mkdir -p $AMDAPPSDK_PATH
+        sh AMD-APP-SDK*.sh --tar -xf -C $AMDAPPSDK_PATH
+        mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
+        echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
+    fi
+    if [[ $TRAVIS == "true" ]]; then
+        wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh
+    fi
+fi
+
+if [[ $TRAVIS == "true" ]]; then
+    sh conda.sh -b -p $HOME_DIRECTORY/miniconda
+    conda config --set always_yes yes --set changeps1 no
+    conda update -q conda
+fi
diff --git a/.vsts-ci/test.sh → .ci/test.sh b/.vsts-ci/test.sh → .ci/test.sh
diff --git a/.vsts-ci/test_windows.ps1 → .ci/test_windows.ps1 b/.vsts-ci/test_windows.ps1 → .ci/test_windows.ps1
diff --git a/.travis.yml b/.travis.yml
@@ -38,6 +38,12 @@ matrix:
 before_install:
   - test -n $CC  && unset CC
   - test -n $CXX && unset CXX
+  - export HOME_DIRECTORY="$HOME"
+  - if [[ $TRAVIS_OS_NAME == "osx" ]]; then
+        export OS_NAME="macos";
+    else
+        export OS_NAME="linux";
+    fi
   - export PATH="$HOME/miniconda/bin:$PATH"
   - export LGB_VER=$(head -n 1 VERSION.txt)
   - export AMDAPPSDK_PATH=$HOME/AMDAPPSDK
@@ -46,7 +52,7 @@ before_install:
   - export OPENCL_VENDOR_PATH=$AMDAPPSDK_PATH/etc/OpenCL/vendors
 
 install:
-  - bash .travis/setup.sh
+  - bash .ci/setup.sh
 
 script:
   - bash .travis/test.sh

diff --git a/.travis/setup.sh b/.travis/setup.sh
diff --git a/.vsts-ci.yml b/.vsts-ci.yml
@@ -37,18 +37,21 @@ phases:
       packageSpecs: 'python=$(PYTHON_VERSION)'
       createOptions: '-q'
   - script: |
+      echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
+      echo "##vso[task.setvariable variable=OS_NAME]linux"
+      echo "##vso[task.setvariable variable=AZURE]true"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
       AMDAPPSDK_PATH=$AGENT_HOMEDIRECTORY/AMDAPPSDK
       echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
       LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
       echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
       echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
-      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/setup.sh
-      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/test.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/.ci/setup.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/.ci/test.sh
     displayName: 'Set variables'
-  - bash: $(Build.SourcesDirectory)/.vsts-ci/setup.sh
+  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
     displayName: Setup
-  - bash: $(Build.SourcesDirectory)/.vsts-ci/test.sh
+  - bash: $(Build.SourcesDirectory)/.ci/test.sh
     displayName: Test
   - task: PublishBuildArtifacts@1
     condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
@@ -84,13 +87,16 @@ phases:
       packageSpecs: 'python=$(PYTHON_VERSION)'
       createOptions: '-q'
   - script: |
+      echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
+      echo "##vso[task.setvariable variable=OS_NAME]macos"
+      echo "##vso[task.setvariable variable=AZURE]true"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
-      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/setup.sh
-      chmod +x $BUILD_SOURCESDIRECTORY/.vsts-ci/test.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/.ci/setup.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/.ci/test.sh
     displayName: 'Set variables'
-  - bash: $(Build.SourcesDirectory)/.vsts-ci/setup.sh
+  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
     displayName: Setup
-  - bash: $(Build.SourcesDirectory)/.vsts-ci/test.sh
+  - bash: $(Build.SourcesDirectory)/.ci/test.sh
     displayName: Test
   - task: PublishBuildArtifacts@1
     condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
@@ -121,7 +127,7 @@ phases:
       environmentName: $(CONDA_ENV)
       packageSpecs: 'python=$(PYTHON_VERSION) numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest'
       createOptions: '-q'
-  - powershell: $(Build.SourcesDirectory)/.vsts-ci/test_windows.ps1
+  - powershell: $(Build.SourcesDirectory)/.ci/test_windows.ps1
     displayName: Test
   - task: PublishBuildArtifacts@1
     condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))

diff --git a/.vsts-ci/setup.sh b/.vsts-ci/setup.sh