Skip to content
Permalink
Browse files

[ci] update CI stuff (#2079)

* updated boost submodule

* updated docker with new stable Clang and CMake

* switch to dev docker

* updated setup script

* updated MinGW on Appveyor

* updated Azure config to use docker for GPU task

* do not upgrade gcc - takes too long

* test: switch compilers

* switch compilers back

* get back to main docker
  • Loading branch information...
StrikerRUS authored and guolinke committed Apr 9, 2019
1 parent 7610228 commit 691b842832f511a72b4aaab715b3cb6b00de8f90
Showing with 24 additions and 53 deletions.
  1. +1 −1 .appveyor.yml
  2. +5 −4 .ci/dockers/ubuntu-14.04/Dockerfile
  3. +8 −13 .ci/setup.sh
  4. +9 −34 .vsts-ci.yml
  5. +1 −1 compute
@@ -15,7 +15,7 @@ clone_depth: 50
install:
- git submodule update --init --recursive # get `compute` folder
- set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix)
- set PATH=C:\mingw-w64\x86_64-7.3.0-posix-seh-rt_v5-rev0\mingw64\bin;%PATH%
- set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
- set PYTHON_VERSION=%CONFIGURATION%
- ps: >-
switch ($env:PYTHON_VERSION) {
@@ -24,18 +24,19 @@ RUN apt-get update \
build-essential \
&& rm -rf /var/lib/apt/lists/*

# Install clang 6.0
# Install clang 7.0
RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y \
&& cd /tmp \
&& wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
&& add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-6.0 main" -y \
&& add-apt-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-7 main" -y \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
clang-6.0 \
clang-7 \
libomp-7-dev \
&& rm -rf /var/lib/apt/lists/*

# Install CMake
RUN curl -sL https://cmake.org/files/v3.13/cmake-3.13.2-Linux-x86_64.sh -o cmake.sh \
RUN curl -sL https://cmake.org/files/v3.14/cmake-3.14.1-Linux-x86_64.sh -o cmake.sh \
&& chmod +x cmake.sh \
&& ./cmake.sh --prefix=/usr/local --exclude-subdir \
&& rm cmake.sh
@@ -3,11 +3,10 @@
if [[ $OS_NAME == "macos" ]]; then
if [[ $COMPILER == "clang" ]]; then
brew install libomp
brew upgrade cmake # CMake >=3.12 is needed to find OpenMP at macOS
if [[ $AZURE == "true" ]]; then
sudo xcode-select -s /Applications/Xcode_8.3.3.app/Contents/Developer
fi
else
else # gcc
if [[ $TRAVIS == "true" ]]; then
# rm '/usr/local/include/c++' # previous variant to deal with conflict link
# brew cask uninstall oclint # reserve variant to deal with conflict link
@@ -23,22 +22,18 @@ if [[ $OS_NAME == "macos" ]]; then
wget -q -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh
else # Linux
if [[ $AZURE == "true" ]] && [[ $COMPILER == "clang" ]]; then
sudo apt-get update
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
sudo apt-get install --no-install-recommends -y libomp-dev
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-7 100
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-7 100
fi
if [[ $TASK == "mpi" ]]; then
sudo apt-get update
sudo apt-get install --no-install-recommends -y libopenmpi-dev openmpi-bin
fi
if [[ $TASK == "gpu" ]]; then
if [[ $TRAVIS == "true" ]]; then
sudo add-apt-repository ppa:kzemek/boost -y
fi
sudo add-apt-repository ppa:mhier/libboost-latest -y
sudo apt-get update
sudo apt-get install --no-install-recommends -y libboost1.58-dev libboost-system1.58-dev libboost-filesystem1.58-dev ocl-icd-opencl-dev
cd $HOME_DIRECTORY
sudo apt-get install --no-install-recommends -y libboost1.68-dev ocl-icd-opencl-dev
cd $BUILD_DIRECTORY # to avoid permission errors
wget -q https://github.com/Microsoft/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
tar -xjf AMD-APP-SDK*.tar.bz2
mkdir -p $OPENCL_VENDOR_PATH
@@ -47,12 +42,12 @@ else # Linux
mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
fi
if [[ $TRAVIS == "true" ]] || [[ $TASK == "gpu" ]]; then
if [[ $TRAVIS == "true" ]]; then
wget -q -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh
fi
fi

if [[ $TRAVIS == "true" ]] || [[ $OS_NAME == "macos" ]] || [[ $TASK == "gpu" ]]; then
if [[ $TRAVIS == "true" ]] || [[ $OS_NAME == "macos" ]]; then
sh conda.sh -b -p $CONDA
fi
conda config --set always_yes yes --set changeps1 no
@@ -15,7 +15,7 @@ jobs:
vmImage: 'ubuntu-16.04'
container: ubuntu1404
strategy:
maxParallel: 5
maxParallel: 6
matrix:
regular:
TASK: regular
@@ -31,35 +31,6 @@ jobs:
TASK: mpi
METHOD: source
PYTHON_VERSION: 2.7
steps:
- script: |
echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
echo "##vso[task.setvariable variable=OS_NAME]linux"
echo "##vso[task.setvariable variable=AZURE]true"
echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
echo "##vso[task.prependpath]$CONDA/bin"
displayName: 'Set variables'
- bash: $(Build.SourcesDirectory)/.ci/setup.sh
displayName: Setup
- bash: $(Build.SourcesDirectory)/.ci/test.sh
displayName: Test
- task: PublishBuildArtifacts@1
condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
- job: Linux_gpu
###########################################
variables:
COMPILER: gcc
pool:
vmImage: 'ubuntu-16.04'
strategy:
maxParallel: 1
matrix:
gpu_source:
TASK: gpu
METHOD: source
@@ -71,19 +42,23 @@ jobs:
echo "##vso[task.setvariable variable=OS_NAME]linux"
echo "##vso[task.setvariable variable=AZURE]true"
echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
AMDAPPSDK_PATH=$AGENT_HOMEDIRECTORY/AMDAPPSDK
echo "##vso[task.prependpath]$CONDA/bin"
AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
CONDA=$AGENT_HOMEDIRECTORY/miniconda
echo "##vso[task.setvariable variable=CONDA]$CONDA"
echo "##vso[task.prependpath]$CONDA/bin"
displayName: 'Set variables'
- bash: $(Build.SourcesDirectory)/.ci/setup.sh
displayName: Setup
- bash: $(Build.SourcesDirectory)/.ci/test.sh
displayName: Test
- task: PublishBuildArtifacts@1
condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
- job: MacOS
###########################################

0 comments on commit 691b842

Please sign in to comment.
You can’t perform that action at this time.