Merge master

.appveyor.yml

@@ -1,4 +1,4 @@
-version: 4.1.0.99.{build}
+version: 4.2.0.99.{build}
 
 image: Visual Studio 2015
 platform: x64

.ci/check_python_dists.sh

@@ -25,7 +25,7 @@ if [ $PY_MINOR_VER -gt 7 ]; then
         pydistcheck \
             --inspect \
             --ignore 'compiled-objects-have-debug-symbols,distro-too-large-compressed' \
-            --max-allowed-size-uncompressed '60M' \
+            --max-allowed-size-uncompressed '100M' \
             --max-allowed-files 800 \
             ${DIST_DIR}/* || exit -1
     elif { test $(uname -m) = "aarch64"; }; then

.ci/lint-cpp.sh

@@ -18,3 +18,27 @@ cmakelint \
     ${cmake_files} \
 || exit -1
 echo "done running cmakelint"
+
+echo "checking that all OpenMP pragmas specify num_threads()"
+get_omp_pragmas_without_num_threads() {
+    grep \
+        -n \
+        -R \
+        --include='*.c' \
+        --include='*.cc' \
+        --include='*.cpp' \
+        --include='*.h' \
+        --include='*.hpp' \
+        'pragma omp parallel' \
+    | grep -v ' num_threads'
+}
+PROBLEMATIC_LINES=$(
+    get_omp_pragmas_without_num_threads
+)
+if test "${PROBLEMATIC_LINES}" != ""; then
+    get_omp_pragmas_without_num_threads
+    echo "Found '#pragma omp parallel' not using explicit num_threads() configuration. Fix those."
+    echo "For details, see https://www.openmp.org/spec-html/5.0/openmpse14.html#x54-800002.6"
+    exit -1
+fi
+echo "done checking OpenMP pragmas"

.ci/lint_r_code.R

@@ -1,5 +1,5 @@
 
-library(lintr)
+loadNamespace("lintr")
 
 args <- commandArgs(
     trailingOnly = TRUE
@@ -33,30 +33,34 @@ LINTERS_TO_USE <- list(
     , "any_duplicated"       = lintr::any_duplicated_linter()
     , "any_is_na"            = lintr::any_is_na_linter()
     , "assignment"           = lintr::assignment_linter()
+    , "backport"             = lintr::backport_linter()
     , "boolean_arithmetic"   = lintr::boolean_arithmetic_linter()
     , "braces"               = lintr::brace_linter()
     , "class_equals"         = lintr::class_equals_linter()
     , "commas"               = lintr::commas_linter()
+    , "conjunct_test"        = lintr::conjunct_test_linter()
     , "duplicate_argument"   = lintr::duplicate_argument_linter()
     , "empty_assignment"     = lintr::empty_assignment_linter()
     , "equals_na"            = lintr::equals_na_linter()
+    , "fixed_regex"          = lintr::fixed_regex_linter()
     , "for_loop_index"       = lintr::for_loop_index_linter()
     , "function_left"        = lintr::function_left_parentheses_linter()
+    , "function_return"      = lintr::function_return_linter()
+    , "implicit_assignment"  = lintr::implicit_assignment_linter()
     , "implicit_integers"    = lintr::implicit_integer_linter()
     , "infix_spaces"         = lintr::infix_spaces_linter()
     , "inner_combine"        = lintr::inner_combine_linter()
     , "is_numeric"           = lintr::is_numeric_linter()
-    , "fixed_regex"          = lintr::fixed_regex_linter()
-    , "function_return"      = lintr::function_return_linter()
     , "lengths"              = lintr::lengths_linter()
+    , "line_length"          = lintr::line_length_linter(length = 120L)
     , "literal_coercion"     = lintr::literal_coercion_linter()
-    , "long_lines"           = lintr::line_length_linter(length = 120L)
     , "matrix"               = lintr::matrix_apply_linter()
     , "missing_argument"     = lintr::missing_argument_linter()
     , "non_portable_path"    = lintr::nonportable_path_linter()
     , "numeric_leading_zero" = lintr::numeric_leading_zero_linter()
     , "outer_negation"       = lintr::outer_negation_linter()
     , "package_hooks"        = lintr::package_hooks_linter()
+    , "paren_body"           = lintr::paren_body_linter()
     , "paste"                = lintr::paste_linter()
     , "quotes"               = lintr::quotes_linter()
     , "redundant_equals"     = lintr::redundant_equals_linter()
@@ -74,8 +78,7 @@ LINTERS_TO_USE <- list(
     , "true_false"           = lintr::T_and_F_symbol_linter()
     , "undesirable_function" = lintr::undesirable_function_linter(
         fun = c(
-            "cat" = "CRAN forbids the use of cat() in packages except in special cases. Use message() or warning()."
-            , "cbind" = paste0(
+            "cbind" = paste0(
                 "cbind is an unsafe way to build up a data frame. merge() or direct "
                 , "column assignment is preferred."
             )
@@ -100,13 +103,15 @@ LINTERS_TO_USE <- list(
             "%>%" = pipe_text
             , "%.%" = pipe_text
             , "%..%" = pipe_text
+            , "|>" = pipe_text
             , "?" = interactive_text
             , "??" = interactive_text
         )
     )
     , "unnecessary_concatenation" = lintr::unnecessary_concatenation_linter()
     , "unnecessary_lambda"        = lintr::unnecessary_lambda_linter()
     , "unreachable_code"          = lintr::unreachable_code_linter()
+    , "unused_import"             = lintr::unused_import_linter()
     , "vector_logic"              = lintr::vector_logic_linter()
     , "whitespace"                = lintr::whitespace_linter()
 )

.ci/setup.sh

@@ -54,6 +54,14 @@ else  # Linux
             sudo apt-get install --no-install-recommends -y \
                 clang \
                 libomp-dev
+        elif [[ $COMPILER == "clang-17" ]]; then
+            sudo apt-get install wget
+            wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
+            sudo apt-add-repository deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
+            sudo apt-add-repository deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
+            sudo apt-get update
+            sudo apt-get install -y clang-17
+            sudo apt-get install --no-install-recommends -y libomp-17-dev
         fi
 
         export LANG="en_US.UTF-8"

.ci/test-python-oldest.sh

@@ -7,9 +7,11 @@
 #
 echo "installing lightgbm's dependencies"
 pip install \
+  'cffi==1.15.1' \
   'dataclasses' \
-  'numpy==1.12.0' \
+  'numpy==1.16.6' \
   'pandas==0.24.0' \
+  'pyarrow==6.0.1' \
   'scikit-learn==0.18.2' \
   'scipy==0.19.0' \
 || exit -1

.ci/test.sh

@@ -6,6 +6,9 @@ if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then
 elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang" ]]; then
     export CXX=clang++
     export CC=clang
+elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang-17" ]]; then
+    export CXX=clang++-17
+    export CC=clang-17
 fi
 
 if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
@@ -37,7 +40,7 @@ fi
 CONDA_PYTHON_REQUIREMENT="python=$PYTHON_VERSION[build=*cpython]"
 
 if [[ $TASK == "if-else" ]]; then
-    conda create -q -y -n $CONDA_ENV ${CONDA_PYTHON_REQUIREMENT} numpy
+    mamba create -q -y -n $CONDA_ENV ${CONDA_PYTHON_REQUIREMENT} numpy
     source activate $CONDA_ENV
     mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build && cmake .. && make lightgbm -j4 || exit -1
     cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
@@ -67,7 +70,7 @@ fi
 
 if [[ $TASK == "lint" ]]; then
     cd ${BUILD_DIRECTORY}
-    conda create -q -y -n $CONDA_ENV \
+    mamba create -q -y -n $CONDA_ENV \
         ${CONDA_PYTHON_REQUIREMENT} \
         cmakelint \
         cpplint \
@@ -87,10 +90,10 @@ fi
 
 if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
     cd $BUILD_DIRECTORY/docs
-    conda env create \
+    mamba env create \
         -n $CONDA_ENV \
         --file ./env.yml || exit -1
-    conda install \
+    mamba install \
         -q \
         -y \
         -n $CONDA_ENV \
@@ -128,13 +131,15 @@ if [[ $PYTHON_VERSION == "3.7" ]]; then
 fi
 
 # including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy
-conda create -q -y -n $CONDA_ENV \
+mamba create -q -y -n $CONDA_ENV \
     ${CONSTRAINED_DEPENDENCIES} \
+    cffi \
     cloudpickle \
     joblib \
     matplotlib \
     numpy \
     psutil \
+    pyarrow \
     pytest \
     ${CONDA_PYTHON_REQUIREMENT} \
     python-graphviz \
@@ -304,7 +309,7 @@ matplotlib.use\(\"Agg\"\)\
 ' plot_example.py  # prevent interactive window mode
     sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
     # requirements for examples
-    conda install -q -y -n $CONDA_ENV \
+    mamba install -q -y -n $CONDA_ENV \
         h5py \
         ipywidgets \
         notebook
@@ -315,11 +320,13 @@ matplotlib.use\(\"Agg\"\)\
 
     # importing the library should succeed even if all optional dependencies are not present
     conda uninstall --force --yes \
+        cffi \
         dask \
         distributed \
         joblib \
         matplotlib \
         psutil \
+        pyarrow \
         python-graphviz \
         scikit-learn || exit -1
     python -c "import lightgbm" || exit -1

.ci/test_r_package.sh

@@ -127,13 +127,13 @@ if [[ "${R_MAJOR_VERSION}" == "3" ]]; then
     Rscript --vanilla -e "install.packages('https://cran.r-project.org/src/contrib/Archive/lattice/lattice_0.20-41.tar.gz', repos = NULL, lib = '${R_LIB_PATH}')"
 fi
 
-# Manually install Depends and Imports libraries + 'knitr', 'RhpcBLASctl', 'rmarkdown', 'testthat'
+# Manually install Depends and Imports libraries + 'knitr', 'markdown', 'RhpcBLASctl', 'testthat'
 # to avoid a CI-time dependency on devtools (for devtools::install_deps())
 # NOTE: testthat is not required when running rchk
 if [[ "${TASK}" == "r-rchk" ]]; then
-    packages="c('data.table', 'jsonlite', 'knitr', 'Matrix', 'R6', 'RhpcBLASctl', 'rmarkdown')"
+    packages="c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'R6', 'RhpcBLASctl')"
 else
-    packages="c('data.table', 'jsonlite', 'knitr', 'Matrix', 'R6', 'RhpcBLASctl', 'rmarkdown', 'testthat')"
+    packages="c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'R6', 'RhpcBLASctl', 'testthat')"
 fi
 compile_from_source="both"
 if [[ $OS_NAME == "macos" ]]; then

.ci/test_r_package_valgrind.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
 
-RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" || exit -1
+RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com')" || exit -1
 sh build-cran-package.sh \
   --r-executable=RDvalgrind \
+  --no-build-vignettes \
   || exit -1
+
 RDvalgrind CMD INSTALL --preclean --install-tests lightgbm_*.tar.gz || exit -1
 
 cd R-package/tests
@@ -68,7 +70,7 @@ bytes_possibly_lost=$(
     | tr -d ","
 )
 echo "valgrind found ${bytes_possibly_lost} bytes possibly lost"
-if [[ ${bytes_possibly_lost} -gt 352 ]]; then
+if [[ ${bytes_possibly_lost} -gt 1056 ]]; then
     exit -1
 fi
 

.ci/test_r_package_windows.ps1

@@ -55,12 +55,14 @@ Remove-From-Path ".*\\R\\.*"
 Remove-From-Path ".*R Client.*"
 Remove-From-Path ".*rtools40.*"
 Remove-From-Path ".*rtools42.*"
+Remove-From-Path ".*rtools43.*"
 Remove-From-Path ".*shells.*"
 Remove-From-Path ".*Strawberry.*"
 Remove-From-Path ".*tools.*"
 
 Remove-Item C:\rtools40 -Force -Recurse -ErrorAction Ignore
 Remove-Item C:\rtools42 -Force -Recurse -ErrorAction Ignore
+Remove-Item C:\rtools43 -Force -Recurse -ErrorAction Ignore
 
 # Get details needed for installing R components
 #
@@ -76,11 +78,11 @@ if ($env:R_MAJOR_VERSION -eq "3") {
   $env:RTOOLS_EXE_FILE = "rtools35-x86_64.exe"
   $env:R_WINDOWS_VERSION = "3.6.3"
 } elseif ($env:R_MAJOR_VERSION -eq "4") {
-  $RTOOLS_INSTALL_PATH = "C:\rtools42"
+  $RTOOLS_INSTALL_PATH = "C:\rtools43"
   $env:RTOOLS_BIN = "$RTOOLS_INSTALL_PATH\usr\bin"
   $env:RTOOLS_MINGW_BIN = "$RTOOLS_INSTALL_PATH\x86_64-w64-mingw32.static.posix\bin"
-  $env:RTOOLS_EXE_FILE = "rtools42-5253-5107.exe"
-  $env:R_WINDOWS_VERSION = "4.2.2"
+  $env:RTOOLS_EXE_FILE = "rtools43-5550-5548.exe"
+  $env:R_WINDOWS_VERSION = "4.3.1"
 } else {
   Write-Output "[ERROR] Unrecognized R version: $env:R_VERSION"
   Check-Output $false
@@ -122,7 +124,7 @@ Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT
 Write-Output "Done installing Rtools"
 
 Write-Output "Installing dependencies"
-$packages = "c('data.table', 'jsonlite', 'knitr', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'rmarkdown', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')"
+$packages = "c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')"
 Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $?
 
 Write-Output "Building R package"
@@ -203,6 +205,19 @@ if ($env:COMPILER -ne "MSVC") {
   }
 }
 
+# Checking that the correct R version was used
+if ($env:TOOLCHAIN -ne "MSVC") {
+  $checks = Select-String -Path "${LOG_FILE_NAME}" -Pattern "using R version $env:R_WINDOWS_VERSION"
+  $checks_cnt = $checks.Matches.length
+} else {
+  $checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "R version passed into FindLibR.* $env:R_WINDOWS_VERSION"
+  $checks_cnt = $checks.Matches.length
+}
+if ($checks_cnt -eq 0) {
+  Write-Output "Wrong R version was found (expected '$env:R_WINDOWS_VERSION'). Check the build logs."
+  Check-Output $False
+}
+
 # Checking that we actually got the expected compiler. The R package has some logic
 # to fail back to MinGW if MSVC fails, but for CI builds we need to check that the correct
 # compiler was used.

.ci/test_windows.ps1

@@ -52,12 +52,14 @@ conda install brotlipy
 
 conda update -q -y conda
 conda create -q -y -n $env:CONDA_ENV `
+  cffi `
   cloudpickle `
   joblib `
   matplotlib `
   numpy `
   pandas `
   psutil `
+  pyarrow `
   pytest `
   "python=$env:PYTHON_VERSION[build=*cpython]" `
   python-graphviz `

.github/CODEOWNERS

@@ -7,4 +7,4 @@
 # offer a reasonable automatic best-guess
 
 # catch-all rule (this only gets matched if no rules below match)
-*    @guolinke @jameslamb @shiyu1994 @jmoralez
+*    @guolinke @jameslamb @shiyu1994 @jmoralez @borchero

.github/workflows/cuda.yml

@@ -31,17 +31,17 @@ jobs:
           - method: wheel
             compiler: gcc
             python_version: "3.11"
-            cuda_version: "11.7.1"
+            cuda_version: "11.8.0"
             task: cuda
           - method: source
             compiler: gcc
             python_version: "3.9"
-            cuda_version: "10.0"
+            cuda_version: "12.2.0"
             task: cuda
           - method: pip
             compiler: clang
             python_version: "3.10"
-            cuda_version: "11.7.1"
+            cuda_version: "11.8.0"
             task: cuda
     steps:
       - name: Setup or update software on host machine
@@ -98,8 +98,10 @@ jobs:
             cuda_version="${{ matrix.cuda_version }}"
             cuda_major=${cuda_version%%.*}
             docker_img="nvcr.io/nvidia/cuda:${cuda_version}-devel"
-            if [[ ${cuda_major} -gt 10 ]]; then
-                docker_img="${docker_img}-ubuntu$(lsb_release -rs)" 
+            if [[ ${cuda_major} -eq 11 ]]; then
+                docker_img="${docker_img}-ubuntu18.04"
+            elif [[ ${cuda_major} -ge 12 ]]; then
+                docker_img="${docker_img}-ubuntu20.04"
             fi
             docker run --env-file docker.env -v "$GITHUB_WORKSPACE":"$ROOT_DOCKER_FOLDER" --rm --gpus all "$docker_img" /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
   all-cuda-jobs-successful: