Skip to content

Commit

Permalink
[R] Remove dependency on gendef for Visual Studio builds (fixes #5608) (
Browse files Browse the repository at this point in the history
#5764)

* [R-package] Remove dependency on gendef for Visual Studio builds (fixes #5608)

* clarify docs

* removed debugging print statement

* Make R CMake install more robust

* Fix doc format; add ToC

* Update build.rst

* Fix AppVeyor

Co-authored-by: Hyunsu Cho <chohyu01@cs.washington.edu>
  • Loading branch information
jameslamb and hcho3 committed Jun 15, 2020
1 parent 529b5c2 commit d39da42
Show file tree
Hide file tree
Showing 10 changed files with 225 additions and 65 deletions.
1 change: 1 addition & 0 deletions .gitignore
Expand Up @@ -93,6 +93,7 @@ metastore_db
# files from R-package source install
**/config.status
R-package/src/Makevars
*.lib

# Visual Studio Code
/.vscode/
Expand Down
3 changes: 2 additions & 1 deletion CMakeLists.txt
Expand Up @@ -226,11 +226,12 @@ add_dependencies(xgboost runxgboost)

#-- Installing XGBoost
if (R_LIB)
include(cmake/RPackageInstallTargetSetup.cmake)
set_target_properties(xgboost PROPERTIES PREFIX "")
if (APPLE)
set_target_properties(xgboost PROPERTIES SUFFIX ".so")
endif (APPLE)
setup_rpackage_install_target(xgboost ${CMAKE_CURRENT_BINARY_DIR})
setup_rpackage_install_target(xgboost "${CMAKE_CURRENT_BINARY_DIR}/R-package-install")
set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/dummy_inst")
endif (R_LIB)
if (MINGW)
Expand Down
3 changes: 3 additions & 0 deletions R-package/CMakeLists.txt
Expand Up @@ -36,3 +36,6 @@ set(LINKED_LIBRARIES_PRIVATE ${LINKED_LIBRARIES_PRIVATE} ${LIBR_CORE_LIBRARY} PA
if (USE_OPENMP)
target_link_libraries(xgboost-r PRIVATE OpenMP::OpenMP_CXX)
endif ()

set(LIBR_HOME "${LIBR_HOME}" PARENT_SCOPE)
set(LIBR_EXECUTABLE "${LIBR_EXECUTABLE}" PARENT_SCOPE)
96 changes: 96 additions & 0 deletions R-package/inst/make-r-def.R
@@ -0,0 +1,96 @@
# [description]
# Create a definition file (.def) from a .dll file, using objdump. This
# is used by FindLibR.cmake when building the R package with MSVC.
#
# [usage]
#
# Rscript make-r-def.R something.dll something.def
#
# [references]
# * https://www.cs.colorado.edu/~main/cs1300/doc/mingwfaq.html

args <- commandArgs(trailingOnly = TRUE)

IN_DLL_FILE <- args[[1L]]
OUT_DEF_FILE <- args[[2L]]
DLL_BASE_NAME <- basename(IN_DLL_FILE)

message(sprintf("Creating '%s' from '%s'", OUT_DEF_FILE, IN_DLL_FILE))

# system() will not raise an R exception if the process called
# fails. Wrapping it here to get that behavior.
#
# system() introduces a lot of overhead, at least on Windows,
# so trying processx if it is available
.pipe_shell_command_to_stdout <- function(command, args, out_file) {
has_processx <- suppressMessages({
suppressWarnings({
require("processx") # nolint
})
})
if (has_processx) {
p <- processx::process$new(
command = command
, args = args
, stdout = out_file
, windows_verbatim_args = FALSE
)
invisible(p$wait())
} else {
message(paste0(
"Using system2() to run shell commands. Installing "
, "'processx' with install.packages('processx') might "
, "make this faster."
))
exit_code <- system2(
command = command
, args = shQuote(args)
, stdout = out_file
)
if (exit_code != 0L) {
stop(paste0("Command failed with exit code: ", exit_code))
}
}
return(invisible(NULL))
}

# use objdump to dump all the symbols
OBJDUMP_FILE <- "objdump-out.txt"
.pipe_shell_command_to_stdout(
command = "objdump"
, args = c("-p", IN_DLL_FILE)
, out_file = OBJDUMP_FILE
)

objdump_results <- readLines(OBJDUMP_FILE)
result <- file.remove(OBJDUMP_FILE)

# Only one table in the objdump results matters for our purposes,
# see https://www.cs.colorado.edu/~main/cs1300/doc/mingwfaq.html
start_index <- which(
grepl(
pattern = "[Ordinal/Name Pointer] Table"
, x = objdump_results
, fixed = TRUE
)
)
empty_lines <- which(objdump_results == "")
end_of_table <- empty_lines[empty_lines > start_index][1L]

# Read the contents of the table
exported_symbols <- objdump_results[(start_index + 1L):end_of_table]
exported_symbols <- gsub("\t", "", exported_symbols)
exported_symbols <- gsub(".*\\] ", "", exported_symbols)
exported_symbols <- gsub(" ", "", exported_symbols)

# Write R.def file
writeLines(
text = c(
paste0("LIBRARY \"", DLL_BASE_NAME, "\"")
, "EXPORTS"
, exported_symbols
)
, con = OUT_DEF_FILE
, sep = "\n"
)
message(sprintf("Successfully created '%s'", OUT_DEF_FILE))
2 changes: 1 addition & 1 deletion appveyor.yml
Expand Up @@ -107,7 +107,7 @@ test_script:
)
# MSVC R package: run only the unit tests
- if /i "%target%" == "rmsvc" (
cd build_rmsvc%ver%\R-package &&
cd R-package &&
R.exe -q -e "library(testthat); setwd('tests'); source('testthat.R')"
)

Expand Down
34 changes: 34 additions & 0 deletions cmake/RPackageInstall.cmake.in
@@ -0,0 +1,34 @@
# Commands to install the R package as a CMake install target

function(check_call)
set(cmd COMMAND)
cmake_parse_arguments(
PARSE_ARGV 0
CALL_ARG "" "" "${cmd}"
)
string(REPLACE ";" " " commands "${CALL_ARG_COMMAND}")
message("Command: ${commands}")
execute_process(COMMAND ${CALL_ARG_COMMAND}
OUTPUT_VARIABLE _out
ERROR_VARIABLE _err
RESULT_VARIABLE _res)
if(NOT "${_res}" EQUAL "0")
message(FATAL_ERROR "out: ${_out}, err: ${_err}, res: ${_res}")
endif()
endfunction()

# Important paths
set(build_dir "@build_dir@")
set(LIBR_EXECUTABLE "@LIBR_EXECUTABLE@")

# Back up cmake_install.cmake
file(WRITE "${build_dir}/R-package/src/Makevars" "all:")
file(WRITE "${build_dir}/R-package/src/Makevars.win" "all:")

# Install dependencies
set(XGB_DEPS_SCRIPT
"deps = setdiff(c('data.table', 'magrittr', 'stringi'), rownames(installed.packages())); if(length(deps)>0) install.packages(deps, repo = 'https://cloud.r-project.org/')")
check_call(COMMAND "${LIBR_EXECUTABLE}" -q -e "${XGB_DEPS_SCRIPT}")

# Install the XGBoost R package
check_call(COMMAND "${LIBR_EXECUTABLE}" CMD INSTALL --no-multiarch --build "${build_dir}/R-package")
16 changes: 16 additions & 0 deletions cmake/RPackageInstallTargetSetup.cmake
@@ -0,0 +1,16 @@
# Assembles the R-package files in build_dir;
# if necessary, installs the main R package dependencies;
# runs R CMD INSTALL.
function(setup_rpackage_install_target rlib_target build_dir)
configure_file(${PROJECT_SOURCE_DIR}/cmake/RPackageInstall.cmake.in ${PROJECT_BINARY_DIR}/RPackageInstall.cmake @ONLY)
install(
DIRECTORY "${xgboost_SOURCE_DIR}/R-package"
DESTINATION "${build_dir}"
REGEX "src/*" EXCLUDE
REGEX "R-package/configure" EXCLUDE
)
install(TARGETS ${rlib_target}
LIBRARY DESTINATION "${build_dir}/R-package/src/"
RUNTIME DESTINATION "${build_dir}/R-package/src/")
install(SCRIPT ${PROJECT_BINARY_DIR}/RPackageInstall.cmake)
endfunction()
32 changes: 0 additions & 32 deletions cmake/Utils.cmake
Expand Up @@ -110,38 +110,6 @@ function(format_gencode_flags flags out)
set(${out} "${${out}}" PARENT_SCOPE)
endfunction(format_gencode_flags flags)

# Assembles the R-package files in build_dir;
# if necessary, installs the main R package dependencies;
# runs R CMD INSTALL.
function(setup_rpackage_install_target rlib_target build_dir)
# backup cmake_install.cmake
install(CODE "file(COPY \"${build_dir}/R-package/cmake_install.cmake\"
DESTINATION \"${build_dir}/bak\")")

install(CODE "file(REMOVE_RECURSE \"${build_dir}/R-package\")")
install(
DIRECTORY "${xgboost_SOURCE_DIR}/R-package"
DESTINATION "${build_dir}"
REGEX "src/*" EXCLUDE
REGEX "R-package/configure" EXCLUDE
)
install(TARGETS ${rlib_target}
LIBRARY DESTINATION "${build_dir}/R-package/src/"
RUNTIME DESTINATION "${build_dir}/R-package/src/")
install(CODE "file(WRITE \"${build_dir}/R-package/src/Makevars\" \"all:\")")
install(CODE "file(WRITE \"${build_dir}/R-package/src/Makevars.win\" \"all:\")")
set(XGB_DEPS_SCRIPT
"deps = setdiff(c('data.table', 'magrittr', 'stringi'), rownames(installed.packages()));\
if(length(deps)>0) install.packages(deps, repo = 'https://cloud.r-project.org/')")
install(CODE "execute_process(COMMAND \"${LIBR_EXECUTABLE}\" \"-q\" \"-e\" \"${XGB_DEPS_SCRIPT}\")")
install(CODE "execute_process(COMMAND \"${LIBR_EXECUTABLE}\" CMD INSTALL\
\"--no-multiarch\" \"--build\" \"${build_dir}/R-package\")")

# restore cmake_install.cmake
install(CODE "file(RENAME \"${build_dir}/bak/cmake_install.cmake\"
\"${build_dir}/R-package/cmake_install.cmake\")")
endfunction(setup_rpackage_install_target)

macro(enable_nvtx target)
find_package(NVTX REQUIRED)
target_include_directories(${target} PRIVATE "${NVTX_INCLUDE_DIR}")
Expand Down
26 changes: 18 additions & 8 deletions cmake/modules/FindLibR.cmake
Expand Up @@ -23,7 +23,7 @@

# Windows users might want to change this to their R version:
if(NOT R_VERSION)
set(R_VERSION "3.4.1")
set(R_VERSION "4.0.0")
endif()
if(NOT R_ARCH)
if("${CMAKE_SIZEOF_VOID_P}" STREQUAL "4")
Expand All @@ -43,16 +43,26 @@ function(create_rlib_for_msvc)
if(NOT EXISTS "${LIBR_LIB_DIR}")
message(FATAL_ERROR "LIBR_LIB_DIR was not set!")
endif()
find_program(GENDEF_EXE gendef)
find_program(DLLTOOL_EXE dlltool)
if(NOT GENDEF_EXE OR NOT DLLTOOL_EXE)
message(FATAL_ERROR "\nEither gendef.exe or dlltool.exe not found!\
if(NOT DLLTOOL_EXE)
message(FATAL_ERROR "\ndlltool.exe not found!\
\nDo you have Rtools installed with its MinGW's bin/ in PATH?")
endif()
endif()

# extract symbols from R.dll into R.def and R.lib import library
execute_process(COMMAND ${GENDEF_EXE}
"-" "${LIBR_LIB_DIR}/R.dll"
OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/R.def")
get_filename_component(
LIBR_RSCRIPT_EXECUTABLE_DIR
${LIBR_EXECUTABLE}
DIRECTORY
)
set(LIBR_RSCRIPT_EXECUTABLE "${LIBR_RSCRIPT_EXECUTABLE_DIR}/Rscript")

execute_process(
COMMAND ${LIBR_RSCRIPT_EXECUTABLE}
"${CMAKE_CURRENT_BINARY_DIR}/../../R-package/inst/make-r-def.R"
"${LIBR_LIB_DIR}/R.dll" "${CMAKE_CURRENT_BINARY_DIR}/R.def"
)

execute_process(COMMAND ${DLLTOOL_EXE}
"--input-def" "${CMAKE_CURRENT_BINARY_DIR}/R.def"
"--output-lib" "${CMAKE_CURRENT_BINARY_DIR}/R.lib")
Expand Down
77 changes: 54 additions & 23 deletions doc/build.rst
Expand Up @@ -49,19 +49,7 @@ Please refer to `Trouble Shooting`_ section first if you have any problem
during installation. If the instructions do not work for you, please feel free
to ask questions at `the user forum <https://discuss.xgboost.ai>`_.

**Contents**

* `Building the Shared Library`_

- `Building on Linux Distributions`_
- `Building on OSX`_
- `Building on Windows`_
- `Building with GPU support`_

* `Python Package Installation`_
* `R Package Installation`_
* `Trouble Shooting`_
* `Building the documentation`_
.. contents:: Contents

.. _build_shared_lib:

Expand Down Expand Up @@ -365,12 +353,11 @@ You can install XGBoost from CRAN just like any other R package:
and then run ``install.packages("xgboost")``. Without OpenMP, XGBoost will only use a single CPU core, leading to suboptimal training speed.

Installing the development version
----------------------------------
Installing the development version (Linux / Mac OSX)
----------------------------------------------------

Make sure you have installed git and a recent C++ compiler supporting C++11 (See above
sections for requirements of building C++ core). On Windows, Rtools must be installed,
and its bin directory has to be added to ``PATH`` during the installation.
sections for requirements of building C++ core).

Due to the use of git-submodules, ``devtools::install_github`` can no longer be used to install the latest version of R package.
Thus, one has to run git to check out the code first:
Expand All @@ -390,6 +377,37 @@ Thus, one has to run git to check out the code first:
If all fails, try `Building the shared library`_ to see whether a problem is specific to R
package or not. Notice that the R package is installed by CMake directly.

Installing the development version with Visual Studio
-----------------------------------------------------

On Windows, CMake with Visual C++ Build Tools (or Visual Studio) can be used to build the R package.

While not required, this build can be faster if you install the R package ``processx`` with ``install.packages("processx")``.

.. note:: Setting correct PATH environment variable on Windows

If you are using Windows, make sure to include the right directories in the PATH environment variable.

* If you are using R 4.x with RTools 4.0:
- ``C:\rtools40\usr\bin``
- ``C:\rtools40\mingw64\bin``

* If you are using R 3.x with RTools 3.x:

- ``C:\Rtools\bin``
- ``C:\Rtools\mingw_64\bin``

Open the Command Prompt and navigate to the XGBoost directory, and then run the following commands. Make sure to specify the correct R version.

.. code-block:: bash
cd C:\path\to\xgboost
mkdir build
cd build
cmake .. -G"Visual Studio 16 2019" -A x64 -DR_LIB=ON -DR_VERSION=4.0.0
cmake --build . --target install --config Release
.. _r_gpu_support:

Installing R package with GPU support
Expand All @@ -409,19 +427,32 @@ On Linux, starting from the XGBoost directory type:
When default target is used, an R package shared library would be built in the ``build`` area.
The ``install`` target, in addition, assembles the package files with this shared library under ``build/R-package`` and runs ``R CMD INSTALL``.

On Windows, CMake with Visual C++ Build Tools (or Visual Studio) has to be used to build an R package with GPU support. Rtools must also be installed (perhaps, some other MinGW distributions with ``gendef.exe`` and ``dlltool.exe`` would work, but that was not tested).
On Windows, CMake with Visual Studio has to be used to build an R package with GPU support. Rtools must also be installed.

.. note:: Setting correct PATH environment variable on Windows

If you are using Windows, make sure to include the right directories in the PATH environment variable.

* If you are using R 4.x with RTools 4.0:

- ``C:\rtools40\usr\bin``
- ``C:\rtools40\mingw64\bin``
* If you are using R 3.x with RTools 3.x:

- ``C:\Rtools\bin``
- ``C:\Rtools\mingw_64\bin``

Open the Command Prompt and navigate to the XGBoost directory, and then run the following commands. Make sure to specify the correct R version.

.. code-block:: bash
cd C:\path\to\xgboost
mkdir build
cd build
cmake .. -G"Visual Studio 14 2015 Win64" -DUSE_CUDA=ON -DR_LIB=ON
cmake .. -G"Visual Studio 16 2019" -A x64 -DUSE_CUDA=ON -DR_LIB=ON -DR_VERSION=4.0.0
cmake --build . --target install --config Release
When ``--target xgboost`` is used, an R package DLL would be built under ``build/Release``.
The ``--target install``, in addition, assembles the package files with this dll under ``build/R-package`` and runs ``R CMD INSTALL``.

If cmake can't find your R during the configuration step, you might provide the location of its executable to cmake like this: ``-DLIBR_EXECUTABLE="C:/Program Files/R/R-3.4.1/bin/x64/R.exe"``.
If CMake can't find your R during the configuration step, you might provide the location of R to CMake like this: ``-DLIBR_HOME="C:\Program Files\R\R-4.0.0"``.

If on Windows you get a "permission denied" error when trying to write to ...Program Files/R/... during the package installation, create a ``.Rprofile`` file in your personal home directory (if you don't already have one in there), and add a line to it which specifies the location of your R packages user library, like the following:

Expand Down

0 comments on commit d39da42

Please sign in to comment.