[Azure Pipeline] switch to Hosted Ubuntu 1604 (#1740)

* use new keywords * fix conda install * try to fix conda * conda
microsoft · Oct 10, 2018 · e8395e9 · guolinke · Oct 11, 2018 · e8395e9
1 parent 496a07d
commit e8395e9
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 40 deletions.
diff --git a/.ci/setup.sh b/.ci/setup.sh
@@ -63,3 +63,5 @@ if [[ $TRAVIS == "true" ]]; then
     conda config --set always_yes yes --set changeps1 no
     conda update -q conda
 fi
+
+
diff --git a/.ci/test.sh b/.ci/test.sh
@@ -8,10 +8,8 @@ elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang" ]]; then
     export CC=clang
 fi
 
-if [[ $TRAVIS == "true" ]]; then
-    conda create -q -y -n $CONDA_ENV python=$PYTHON_VERSION
-    source activate $CONDA_ENV
-fi
+conda create -q -y -n $CONDA_ENV python=$PYTHON_VERSION
+source activate $CONDA_ENV
 
 cd $BUILD_DIRECTORY
 
@@ -21,7 +19,7 @@ if [[ $TRAVIS == "true" ]] && [[ $TASK == "check-docs" ]]; then
     fi
     # sphinx >=1.8 is incompatible with rstcheck
     conda install -y -n $CONDA_ENV "sphinx<1.8" "sphinx_rtd_theme>=0.3"  # html5validator
-    pip install rstcheck
+    pip install --user rstcheck
     # check reStructuredText formatting
     cd $BUILD_DIRECTORY/python-package
     rstcheck --report warning `find . -type f -name "*.rst"` || exit -1
@@ -64,12 +62,12 @@ fi
 conda install -q -y -n $CONDA_ENV numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest
 
 if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
-    ln -sf `ls -d "$(brew --cellar libomp)"/*/lib`/* $CONDA_PREFIX/lib || exit -1  # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
+    sudo ln -sf `ls -d "$(brew --cellar libomp)"/*/lib`/* $CONDA_PREFIX/lib || exit -1  # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
 fi
 
 if [[ $TASK == "sdist" ]]; then
     cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
-    pip install $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
+    pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
     if [[ $AZURE == "true" ]]; then
         cp $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
     fi
@@ -89,7 +87,7 @@ elif [[ $TASK == "bdist" ]]; then
             cp dist/lightgbm-$LGB_VER-py2.py3-none-manylinux1_x86_64.whl $BUILD_ARTIFACTSTAGINGDIRECTORY
         fi
     fi
-    pip install $BUILD_DIRECTORY/python-package/dist/*.whl || exit -1
+    pip install --user $BUILD_DIRECTORY/python-package/dist/*.whl || exit -1
     pytest $BUILD_DIRECTORY/tests || exit -1
     exit 0
 fi
@@ -103,9 +101,9 @@ if [[ $TASK == "gpu" ]]; then
     if [[ $METHOD == "pip" ]]; then
         cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
         if [[ $AZURE == "true" ]]; then
-            pip install $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
+            pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
         else
-            pip install $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--boost-root=$CONDA_PREFIX" --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
+            pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu --install-option="--boost-root=$CONDA_PREFIX" --install-option="--opencl-include-dir=$AMDAPPSDK_PATH/include/" || exit -1
         fi
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
         exit 0
@@ -117,7 +115,7 @@ mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
 if [[ $TASK == "mpi" ]]; then
     if [[ $METHOD == "pip" ]]; then
         cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1
-        pip install $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
+        pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1
         pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
         exit 0
     fi
@@ -134,7 +132,7 @@ fi
 
 make _lightgbm || exit -1
 
-cd $BUILD_DIRECTORY/python-package && python setup.py install --precompile || exit -1
+cd $BUILD_DIRECTORY/python-package && python setup.py install --precompile --user || exit -1
 pytest $BUILD_DIRECTORY/tests || exit -1
 
 if [[ $TASK == "regular" ]]; then

diff --git a/.vsts-ci.yml b/.vsts-ci.yml
@@ -1,15 +1,16 @@
 variables:
   PYTHON_VERSION: 3.7
   CONDA_ENV: test-env
-phases:
+jobs:
 ###########################################
-- phase: Linux
+- job: Linux
 ###########################################
   variables:
     COMPILER: gcc
-  queue:
-    name: 'Hosted Linux Preview'
-    parallel: 6
+  pool:
+    vmImage: 'ubuntu-16.04'
+  strategy:
+    maxParallel: 6
     matrix:
       regular:
         TASK: regular
@@ -30,12 +31,9 @@ phases:
         METHOD: source
         PYTHON_VERSION: 3.6
   steps:
-  - task: CondaEnvironment@0
+  - task: CondaEnvironment@1
     inputs:
       updateConda: true
-      environmentName: $(CONDA_ENV)
-      packageSpecs: 'python=$(PYTHON_VERSION)'
-      createOptions: '-q'
   - script: |
       echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
@@ -59,13 +57,14 @@ phases:
       artifactName: PackageAssets
       artifactType: container
 ###########################################
-- phase: MacOS
+- job: MacOS
 ###########################################
   variables:
     COMPILER: clang
-  queue:
-    name: 'Hosted macOS'
-    parallel: 3
+  pool:
+    vmImage: 'macOS-10.13'
+  strategy:
+    maxParallel: 3
     matrix:
       regular:
         TASK: regular
@@ -76,15 +75,9 @@ phases:
       bdist:
         TASK: bdist
   steps:
-  - script: |
-      sudo chmod -R 777 /usr/local/miniconda/envs
-    displayName: 'Fix for conda error on macOS'
-  - task: CondaEnvironment@0
+  - task: CondaEnvironment@1
     inputs:
       updateConda: true
-      environmentName: $(CONDA_ENV)
-      packageSpecs: 'python=$(PYTHON_VERSION)'
-      createOptions: '-q'
   - script: |
       echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
@@ -103,11 +96,12 @@ phases:
       artifactName: PackageAssets
       artifactType: container
 ###########################################
-- phase: Windows
+- job: Windows
 ###########################################
-  queue:
-    name: 'Hosted VS2017'
-    parallel: 3
+  pool:
+    vmImage: 'vs2017-win2016'
+  strategy:
+    maxParallel: 3
     matrix:
       regular:
         TASK: regular
@@ -119,8 +113,9 @@ phases:
         TASK: bdist
         PYTHON_VERSION: 3.6
   steps:
-  - task: CondaEnvironment@0
+  - task: CondaEnvironment@1
     inputs:
+      createCustomEnvironment: true
       updateConda: true
       environmentName: $(CONDA_ENV)
       packageSpecs: 'python=$(PYTHON_VERSION) numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest'
@@ -135,15 +130,15 @@ phases:
       artifactType: container
 
 ###########################################
-- phase: Package
+- job: Package
 ###########################################
   dependsOn:
   - Linux
   - MacOS
   - Windows
   condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
-  queue:
-    name: 'Hosted VS2017'
+  pool:
+    vmImage: 'vs2017-win2016'
   steps:
   # Download all agent packages from all previous phases
   - task: DownloadBuildArtifacts@0