Compile R package by custom tool chain. (#584)

* add R's library file to vs project and cmake.

* support using dll built by vs.

* better search for the library file.

* remove mingw related doc .

* update document.

* Let R handle the library compile.

* try fix build from github.

* Update README.md

* cleaner build.

* fix the install problem in linux.

* Update README.md

CMakeLists.txt

@@ -108,7 +108,7 @@ file(GLOB SOURCES
 )
 
 add_executable(lightgbm src/main.cpp ${SOURCES})
-add_library(_lightgbm SHARED src/c_api.cpp ${SOURCES})
+add_library(_lightgbm SHARED src/c_api.cpp src/lightgbm_R.cpp ${SOURCES})
 
 if(MSVC)
     set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")

R-package/NAMESPACE

@@ -40,4 +40,4 @@ importFrom(R6,R6Class)
 importFrom(data.table,":=")
 importFrom(magrittr,"%>%")
 importFrom(magrittr,"%T>%")
-useDynLib(lightgbm)
+useDynLib(lib_lightgbm)

R-package/R/utils.R

@@ -31,9 +31,9 @@ lgb.call <- function(fun_name, ret, ...) {
 
   # Check for a ret call
   if (!is.null(ret)) {
-    call_state <- .Call(fun_name, ..., ret, call_state, PACKAGE = "lightgbm") # Call with ret
+    call_state <- .Call(fun_name, ..., ret, call_state, PACKAGE = "lib_lightgbm") # Call with ret
   } else {
-    call_state <- .Call(fun_name, ..., call_state, PACKAGE = "lightgbm") # Call without ret
+    call_state <- .Call(fun_name, ..., call_state, PACKAGE = "lib_lightgbm") # Call without ret
   }
 
   # Check for call state value post call
@@ -43,7 +43,7 @@ lgb.call <- function(fun_name, ret, ...) {
     buf_len <- 200L
     act_len <- 0L
     err_msg <- raw(buf_len)
-    err_msg <- .Call("LGBM_GetLastError_R", buf_len, act_len, err_msg, PACKAGE = "lightgbm")
+    err_msg <- .Call("LGBM_GetLastError_R", buf_len, act_len, err_msg, PACKAGE = "lib_lightgbm")
 
     # Check error buffer
     if (act_len > buf_len) {
@@ -53,7 +53,7 @@ lgb.call <- function(fun_name, ret, ...) {
                         buf_len,
                         act_len,
                         err_msg,
-                        PACKAGE = "lightgbm")
+                        PACKAGE = "lib_lightgbm")
     }
 
     # Return error

R-package/README.md

@@ -4,30 +4,41 @@ LightGBM R Package
 Installation
 ------------
 
-Windows users may need to run with administrator rights (either R or the command prompt, depending on the way you are installing this package). Rtools must be installed for Windows. Linux users might require the appropriate user write permissions for packages.
+### Preparation
+You need to install *git* and [cmake](https://cmake.org/) first. 
+
+The default compiler is Visual Studio (or [MS Build](https://www.visualstudio.com/downloads/#build-tools-for-visual-studio-2017)) in Windows. You also can use MinGW64 to compile by set ```use_mingw <- TRUE``` in ```R-package/src/install.libs.R``` (We recommend *Visual Studio* for its better multi-threading efficency in Windows).
+
+For OSX user, gcc need to be installed first (refer to https://github.com/Microsoft/LightGBM/wiki/Installation-Guide#osx). 
 
-You can use a command prompt to install via command line:
+### Install
+Install LightGBM R-package by following command:
 
 ```
-cd R-package
-R CMD INSTALL --build  .
+git clone --recursive https://github.com/Microsoft/LightGBM
+cd LightGBM/R-package
+R CMD INSTALL --build .
+``` 
+Or build a self-contained R package then install:
 ```
+git clone --recursive https://github.com/Microsoft/LightGBM
+cd LightGBM/R-package
+Rscript build_package.R
+R CMD INSTALL lightgbm_0.1.tar.gz
+``` 
 
-You can also install directly from R using the repository with `devtools`:
 
-```r
-devtools::install_github("Microsoft/LightGBM", subdir = "R-package")
-```
+Windows users may need to run with administrator rights (either R or the command prompt, depending on the way you are installing this package). Rtools must be installed for Windows. Linux users might require the appropriate user write permissions for packages.
+
 
-For the `devtools` install scenario, you can safely ignore this message:
+Set ```use_gpu <- TRUE``` in ```R-package/src/install.libs.R``` can enable the build with GPU support (Need to install *Boost* and *OpenCL* first, details can be found in [gpu-support](https://github.com/Microsoft/LightGBM/wiki/Installation-Guide#with-gpu-support)).
+
+You can also install directly from R using the repository with `devtools`:
 
 ```r
-Warning message:
-GitHub repo contains submodules, may not function as expected! 
+devtools::install_github("Microsoft/LightGBM", subdir = "R-package")
 ```
 
-If you want to build the self-contained R package, you can run ```unix_build_package.sh```(for UNIX) or ```win_build_package.cmd ```(for Windows). Then use ```R CMD INSTALL lightgbm_0.1.tar.gz``` to install.
-
 When your package installation is done, you can check quickly if your LightGBM R package is working by running the following:
 
 ```r
@@ -38,50 +49,6 @@ dtrain <- lgb.Dataset(train$data, label=train$label)
 params <- list(objective="regression", metric="l2")
 model <- lgb.cv(params, dtrain, 10, nfold=5, min_data=1, learning_rate=1, early_stopping_rounds=10)
 ```
-### OSX installation 
-
-The default installation cannot successfully complete in OSX because clang doesn't support OpenMP.
-
-You can use the following script to change default compiler to gcc, then compile LightGBM R package:
-
-```bash
-brew install gcc --without-multilib
-mkdir -p ~/.R
-touch ~/.R/Makevars
-cat <<EOF >>~/.R/Makevars
-C=gcc-6
-CXX=g++-6
-CXX1X=g++-6
-LDFLAGS=-L/usr/local/Cellar/gcc/6.3.0/lib
-CPPFLAGS=-I/usr/local/Cellar/gcc/6.3.0/include
-SHLIB_OPENMP_CFLAGS = -fopenmp
-SHLIB_OPENMP_CXXFLAGS = -fopenmp
-SHLIB_OPENMP_FCFLAGS = -fopenmp
-SHLIB_OPENMP_FFLAGS = -fopenmp
-EOF
-```
-
-Note:
-
-* For `LDFLAGS=-L/usr/local/Cellar/gcc/6.3.0/lib` and `CPPFLAGS=-I/usr/local/Cellar/gcc/6.3.0/include`, you may need to change `6.3.0` to your gcc version.
-* For `gcc-6` and `g++-6`, you may need to change to your gcc version (like `gcc-7` and `g++7` if using gcc with version 7).
-* For `CXX1X`, if you are using R 3.4 or a more recent version, you must change it to `CXX11`.
-
-To check your LightGBM installation, the test is identical to Linux/Windows versions (check the test provided just before OSX Installation part)
-
-Performance note
-------------
-
-With `gcc`, it is recommended to use `-O3 -mtune=native` instead of the default `-O2 -mtune=core2` by modifying the appropriate file (`Makeconf` or `Makevars`) if you want to achieve maximum speed.
-
-Benchmark example using Intel Ivy Bridge CPU on 1M x 1K dataset:
-
-| Compilation Flag | Performance Index |
-| --- | ---: |
-| `-O2 -mtune=core2` | 100.00% |
-| `-O2 -mtune=native` | 100.90% |
-| `-O3 -mtune=native` | 102.78% |
-| `-O3 -ffast-math -mtune=native` | 100.64% |
 
 Examples
 ------------

R-package/build_package.R

@@ -0,0 +1,16 @@
+if(!file.copy("./../include", "src/", overwrite=TRUE, recursive = TRUE)){
+  stop("cannot find folder LightGBM/include")
+}
+if(!file.copy("./../src", "src/", overwrite=TRUE, recursive = TRUE)){
+  stop("cannot find folder LightGBM/src")
+}
+if(!file.copy("./../compute", "src/", overwrite=TRUE, recursive = TRUE)){
+  print("cannot find folder LightGBM/compute, will disable GPU build")
+}
+if(!file.copy("./../CMakeLists.txt", "src/", overwrite=TRUE, recursive=TRUE)){
+  stop("cannot find file LightGBM/CMakeLists.txt")
+}
+if(!file.exists("src/_IS_FULL_PACKAGE")){
+  file.create("src/_IS_FULL_PACKAGE")
+}
+system("R CMD build --no-build-vignettes .")

R-package/src/Makevars

@@ -1,12 +0,0 @@
-# package root
-PKGROOT=../../
-
-ENABLE_STD_THREAD=1
-CXX_STD = CXX11
-
-LGBM_RFLAGS = -DUSE_SOCKET
-
-PKG_CPPFLAGS= -I$(PKGROOT)/include $(LGBM_RFLAGS) -Wno-deprecated-declarations -Wno-unknown-pragmas
-PKG_CXXFLAGS= $(SHLIB_OPENMP_CFLAGS) $(SHLIB_PTHREAD_FLAGS) -std=c++11 
-PKG_LIBS = $(SHLIB_OPENMP_CFLAGS) $(SHLIB_PTHREAD_FLAGS)
-OBJECTS = ./lightgbm-all.o ./lightgbm_R.o

R-package/src/Makevars.win

@@ -1,12 +0,0 @@
-# package root
-PKGROOT=../../
-
-ENABLE_STD_THREAD=1
-CXX_STD = CXX11
-
-LGBM_RFLAGS = -DUSE_SOCKET 
-
-PKG_CPPFLAGS= -I$(PKGROOT)/include $(LGBM_RFLAGS) -Wno-unknown-pragmas
-PKG_CXXFLAGS= $(SHLIB_OPENMP_CFLAGS) $(SHLIB_PTHREAD_FLAGS) -std=c++11 
-PKG_LIBS = $(SHLIB_OPENMP_CFLAGS) $(SHLIB_PTHREAD_FLAGS) -lws2_32 -liphlpapi 
-OBJECTS = ./lightgbm-all.o ./lightgbm_R.o 

R-package/src/install.libs.R

@@ -0,0 +1,56 @@
+use_gpu <- FALSE
+use_mingw <- FALSE
+source_dir <- paste0(R_PACKAGE_SOURCE, '/src')
+setwd(source_dir)
+if(!file.exists("_IS_FULL_PACKAGE")){
+  if(!file.copy("./../../include", "./", overwrite=TRUE, recursive = TRUE)){
+    stop("cannot find folder LightGBM/include")
+  }
+  if(!file.copy("./../../src", "./", overwrite=TRUE, recursive = TRUE)){
+    stop("cannot find folder LightGBM/src")
+  }
+  if(use_gpu){
+    if(!file.copy("./../../compute", "./", overwrite=TRUE, recursive = TRUE)){
+      print("cannot find folder LightGBM/compute, will disable GPU build")
+      use_gpu <- FALSE
+    }
+  }
+  if(!file.copy("./../../CMakeLists.txt", "./", overwrite=TRUE, recursive=TRUE)){
+    stop("cannot find file LightGBM/CMakeLists.txt")
+  }
+}
+
+build_dir <- paste0(source_dir, "/build")
+dir.create(build_dir, recursive = TRUE, showWarnings = FALSE)
+setwd(build_dir)
+
+cmake_cmd <- "cmake"
+build_cmd <- "make -j"
+lib_folder <- paste0(R_PACKAGE_SOURCE, '/src')
+
+if (WINDOWS) {
+  if(use_mingw){
+    cmake_cmd <- paste0(cmake_cmd, " -G \"MinGW Makefiles\" ")
+    build_cmd <- "mingw32-make.exe -j"
+  } else{
+    cmake_cmd <- paste0(cmake_cmd, " -DCMAKE_GENERATOR_PLATFORM=x64 ")
+    build_cmd <- "cmake --build . --target _lightgbm  --config Release"
+    lib_folder <- paste0(R_PACKAGE_SOURCE, '/src/Release')
+  }
+}
+
+if(use_gpu) {
+  cmake_cmd <- paste0(cmake_cmd, " -DUSE_GPU=1 ")
+}
+
+system(paste0(cmake_cmd, " .."))
+system(build_cmd)
+dest <- file.path(R_PACKAGE_DIR, paste0('libs', R_ARCH))
+dir.create(dest, recursive = TRUE, showWarnings = FALSE)
+src <- paste0(lib_folder, '/lib_lightgbm', SHLIB_EXT)
+if(file.exists(src)){
+  print(paste0("find library file: ", src))
+  file.copy(src, dest, overwrite = TRUE)
+} else {
+  stop("cannot find lib_lightgbm.dll")
+}