[ci] use conda-forge in Linux and macOS CI jobs (#4953)

* [ci] use conda-forge in CI jobs (fixes #4948)

* comment out more jobs

* try reverting graphviz patch, running more cuda jobs

* get graphviz from PyPI and try removing some patches for r-lintr

* start running appveyor again

* use conda-forge if using conda

* fix commands

* conda install graphviz

* try newer openmp

* pin below openmp 11.x

* focus on gpu task

* trying to narrow down error

* maybe gcc11 is the issue

* start adding other tests back

* pin openmp too

* maybe need to pin to gcc less than 10.x

* pin libgfortran and libstdcxx as well

* pin to gcc 9.3.0

* move constraints up to initial environment

* add all CI jobs back

* try installing python-graphviz separately

* try new lightgbm/vsts-agent image

* fix typo

* test if pinning gcc for linux gpu_source build is still necessary

* ok yes, pinning gcc is necessary

* test if Linux gpu_source works with Python 3.9.6

* no special exception for Linux gpu_source job

* pin to Python 3.9.6 in Linux gpu_source

* try explicitly asking for libstdcxx-ng for every linux build

* swap compilers

* switch compilers back

* revert accidental whitespace change

* comment out CI

* try Linux gpu_source with different Python versions

* Revert "try Linux gpu_source with different Python versions"

This reverts commit f6f63cb.

* Revert "comment out CI"

This reverts commit ece191f.

* remove libxml2 install, change CONDA path

* avoid installing conda in rchk job

* empty commit 1

* empty commit 2

* empty commit 3

* empty commit 4

* add more verbose logging around installation of python-graphviz

* empty commit 1

* get mamba info

* get more conda info

* add another mamba info call

* allow for other macOS environments in GHA configuration

* Revert "allow for other macOS environments in GHA configuration"

This reverts commit a3c7a19.

* get more logs from mamba

* get Build.ArtifactsStagingDirectory

* get more logs and try to force re-installing everything

* clean cache after every step

* remove --update-all and make logs less verbose

* remove more print statements and uncomment jobs

* test if conda-clean issue fixes segfaults for gpu_source

* pin python version for gpu_source

* empty commit 1

* use miniforge instead

* empty commit 1

* Apply suggestions from code review

* bring workarounds back

* remove duplicated graphviz system-wide installation (reverts #4095, #4097, #4238)

* empty commit 1

* empty commit 2

* empty commit 3

* empty commit 4

* empty commit 5

* empty commit 6

* empty commit 7

* empty commit 8

* empty commit 9

* empty commit 10

* empty commit 10

* empty commit 10

* empty commit 10

* empty commit 11

* one more try

* try to downgrade Python version for Linux GPU job

* swap compilers

* Revert "swap compilers"

This reverts commit f04dc27.

Co-authored-by: Nikita Titov <nekit94-12@hotmail.com>
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

.ci/setup.sh

@@ -17,8 +17,10 @@ if [[ $OS_NAME == "macos" ]]; then
     if [[ $TASK == "swig" ]]; then
         brew install swig
     fi
-    brew install graphviz
-    curl -sL -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+    curl \
+        -sL \
+        -o miniforge.sh \
+        https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh
 else  # Linux
     if [[ $IN_UBUNTU_LATEST_CONTAINER == "true" ]]; then
         # fixes error "unable to initialize frontend: Dialog"
@@ -42,9 +44,6 @@ else  # Linux
             libicu66 \
             libssl1.1 \
             libunwind8 \
-            libxau6 \
-            libxext6 \
-            libxrender1 \
             locales \
             netcat \
             unzip \
@@ -81,16 +80,11 @@ else  # Linux
         mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
         echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
     fi
-    ARCH=$(uname -m)
     if [[ $TASK == "cuda" ]]; then
         echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
         apt-get update
         apt-get install --no-install-recommends -y \
             curl \
-            graphviz \
-            libxau6 \
-            libxext6 \
-            libxrender1 \
             lsb-release \
             software-properties-common
         if [[ $COMPILER == "clang" ]]; then
@@ -103,29 +97,19 @@ else  # Linux
         apt-get update
         apt-get install --no-install-recommends -y \
             cmake
-    else
-        if [[ $ARCH != "x86_64" ]]; then
-            yum update -y
-            yum install -y \
-                graphviz
-        else
-            sudo apt-get update
-            sudo apt-get install --no-install-recommends -y \
-                graphviz
-        fi
     fi
     if [[ $SETUP_CONDA != "false" ]]; then
-        if [[ $ARCH == "x86_64" ]]; then
-            curl -sL -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
-        else
-            curl -sL -o conda.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${ARCH}.sh
-        fi
+        ARCH=$(uname -m)
+        curl \
+            -sL \
+            -o miniforge.sh \
+            https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${ARCH}.sh
     fi
 fi
 
-if [[ "${TASK}" != "r-package" ]]; then
+if [[ "${TASK}" != "r-package" ]] && [[ "${TASK}" != "r-rchk" ]]; then
     if [[ $SETUP_CONDA != "false" ]]; then
-        sh conda.sh -b -p $CONDA
+        sh miniforge.sh -b -p $CONDA
     fi
     conda config --set always_yes yes --set changeps1 no
     conda update -q -y conda

.ci/test.sh

@@ -62,16 +62,13 @@ fi
 
 if [[ $TASK == "lint" ]]; then
     conda install -q -y -n $CONDA_ENV \
+        cmakelint \
+        cpplint \
+        isort \
+        mypy \
         pycodestyle \
         pydocstyle \
-        r-stringi  # stringi needs to be installed separate from r-lintr to avoid issues like 'unable to load shared object stringi.so'
-    # r-xfun below has to be upgraded because lintr requires > 0.19 for that package
-    conda install -q -y -n $CONDA_ENV \
-        -c conda-forge \
-            libxml2 \
-            "r-xfun>=0.19" \
-            "r-lintr>=2.0"
-    pip install --user cmakelint cpplint isort mypy
+        "r-lintr>=2.0"
     echo "Linting Python code"
     pycodestyle --ignore=E501,W503 --exclude=./.nuget,./external_libs . || exit -1
     pydocstyle --convention=numpy --add-ignore=D105 --match-dir="^(?!^external_libs|test|example).*" --match="(?!^test_|setup).*\.py" . || exit -1
@@ -114,8 +111,22 @@ if [[ $TASK == "swig" ]]; then
     exit 0
 fi
 
-conda install -q -y -n $CONDA_ENV cloudpickle "dask=2021.9.1" "distributed=2021.9.1" joblib matplotlib numpy pandas psutil pytest scikit-learn scipy
-pip install graphviz  # python-graphviz from Anaconda is not allowed to be installed with Python 3.9
+conda install -q -y -n $CONDA_ENV \
+    cloudpickle \
+    dask \
+    distributed \
+    joblib \
+    matplotlib \
+    numpy \
+    pandas \
+    psutil \
+    pytest \
+    scikit-learn \
+    scipy || exit -1
+
+# python-graphviz has to be installed separately to prevent conda from downgrading to pypy
+conda install -q -y -n $CONDA_ENV \
+    python-graphviz || exit -1
 
 if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
     # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
@@ -229,7 +240,11 @@ import matplotlib\
 matplotlib.use\(\"Agg\"\)\
 ' plot_example.py  # prevent interactive window mode
     sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
-    conda install -q -y -n $CONDA_ENV h5py ipywidgets notebook  # requirements for examples
+    # requirements for examples
+    conda install -q -y -n $CONDA_ENV \
+        h5py \
+        ipywidgets \
+        notebook
     for f in *.py **/*.py; do python $f || exit -1; done  # run all examples
     cd $BUILD_DIRECTORY/examples/python-guide/notebooks
     sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb

.github/workflows/cuda.yml

@@ -81,7 +81,7 @@ jobs:
             LGB_VER=$(head -n 1 VERSION.txt)
             EOF
             cat > docker-script.sh <<EOF
-            export CONDA=\$HOME/miniconda
+            export CONDA=\$HOME/miniforge
             export PATH=\$CONDA/bin:\$PATH
             nvidia-smi
             $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1

.github/workflows/linkchecker.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Setup and run tests
         run: |
           export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
-          export CONDA=${HOME}/miniconda
+          export CONDA=${HOME}/miniforge
           export PATH=${CONDA}/bin:${HOME}/.local/bin:${PATH}
           $GITHUB_WORKSPACE/.ci/setup.sh || exit -1
           $GITHUB_WORKSPACE/.ci/test.sh || exit -1

.github/workflows/python_package.yml

@@ -66,7 +66,7 @@ jobs:
           fi
           export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
           export LGB_VER=$(head -n 1 VERSION.txt)
-          export CONDA=${HOME}/miniconda
+          export CONDA=${HOME}/miniforge
           export PATH=${CONDA}/bin:${PATH}
           $GITHUB_WORKSPACE/.ci/setup.sh || exit -1
           $GITHUB_WORKSPACE/.ci/test.sh || exit -1

.github/workflows/static_analysis.yml

@@ -39,7 +39,7 @@ jobs:
         run: |
           export TASK="${{ matrix.task }}"
           export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
-          export CONDA=${HOME}/miniconda
+          export CONDA=${HOME}/miniforge
           export PATH=${CONDA}/bin:$HOME/.local/bin:${PATH}
           $GITHUB_WORKSPACE/.ci/setup.sh || exit -1
           $GITHUB_WORKSPACE/.ci/test.sh || exit -1

.vsts-ci.yml

@@ -14,7 +14,7 @@ variables:
 resources:
   containers:
   - container: ubuntu1404
-    image: lightgbm/vsts-agent:ubuntu-14.04
+    image: lightgbm/vsts-agent:ubuntu-14.04-dev
   - container: ubuntu-latest
     image: 'ubuntu:latest'
     options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
@@ -49,6 +49,9 @@ jobs:
       gpu_source:
         TASK: gpu
         METHOD: source
+        # on Ubuntu 14.04, gpu_source build segfaults with newer version of Python
+        # (and newer version of scipy as a consequence)
+        PYTHON_VERSION: 3.7
       swig:
         TASK: swig
   steps:
@@ -122,7 +125,7 @@ jobs:
   - script: |
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
-      CONDA=$HOME/miniconda
+      CONDA=$HOME/miniforge
       echo "##vso[task.setvariable variable=CONDA]$CONDA"
       echo "##vso[task.prependpath]$CONDA/bin"
       AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
@@ -183,7 +186,7 @@ jobs:
       BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
       EOF
       cat > docker-script.sh <<EOF
-      export CONDA=\$HOME/miniconda
+      export CONDA=\$HOME/miniforge
       export PATH=\$CONDA/bin:\$PATH
       $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
       $ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
@@ -236,7 +239,7 @@ jobs:
   - script: |
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
-      CONDA=$AGENT_HOMEDIRECTORY/miniconda
+      CONDA=$AGENT_HOMEDIRECTORY/miniforge
       echo "##vso[task.setvariable variable=CONDA]$CONDA"
       echo "##vso[task.prependpath]$CONDA/bin"
       echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"