Target and Count encodings for categorical features

R-package/R/lgb.Dataset.R

@@ -209,6 +209,7 @@ Dataset <- R6::R6Class(
             "LGBM_DatasetCreateFromMat_R"
             , ret = handle
             , private$raw_data
+            , self$getinfo("label")
             , nrow(private$raw_data)
             , ncol(private$raw_data)
             , params_str
@@ -226,6 +227,7 @@ Dataset <- R6::R6Class(
             , private$raw_data@p
             , private$raw_data@i
             , private$raw_data@x
+            , self$getinfo("label")
             , length(private$raw_data@p)
             , length(private$raw_data@x)
             , nrow(private$raw_data)

R-package/src/lightgbm_R.cpp

@@ -72,6 +72,7 @@ LGBM_SE LGBM_DatasetCreateFromFile_R(LGBM_SE filename,
 LGBM_SE LGBM_DatasetCreateFromCSC_R(LGBM_SE indptr,
   LGBM_SE indices,
   LGBM_SE data,
+  LGBM_SE label,
   LGBM_SE num_indptr,
   LGBM_SE nelem,
   LGBM_SE num_row,
@@ -83,13 +84,14 @@ LGBM_SE LGBM_DatasetCreateFromCSC_R(LGBM_SE indptr,
   const int* p_indptr = R_INT_PTR(indptr);
   const int* p_indices = R_INT_PTR(indices);
   const double* p_data = R_REAL_PTR(data);
+  const double* p_label = R_REAL_PTR(label);
 
   int64_t nindptr = static_cast<int64_t>(R_AS_INT(num_indptr));
   int64_t ndata = static_cast<int64_t>(R_AS_INT(nelem));
   int64_t nrow = static_cast<int64_t>(R_AS_INT(num_row));
   DatasetHandle handle = nullptr;
   CHECK_CALL(LGBM_DatasetCreateFromCSC(p_indptr, C_API_DTYPE_INT32, p_indices,
-    p_data, C_API_DTYPE_FLOAT64, nindptr, ndata,
+    p_data, p_label, C_API_DTYPE_FLOAT64, C_API_DTYPE_FLOAT64, nindptr, ndata,
     nrow, R_CHAR_PTR(parameters), R_GET_PTR(reference), &handle));
   R_SET_PTR(out, handle);
   R_API_END();
@@ -106,8 +108,9 @@ LGBM_SE LGBM_DatasetCreateFromMat_R(LGBM_SE data,
   int32_t nrow = static_cast<int32_t>(R_AS_INT(num_row));
   int32_t ncol = static_cast<int32_t>(R_AS_INT(num_col));
   double* p_mat = R_REAL_PTR(data);
+  const double* p_label = R_REAL_PTR(label);
   DatasetHandle handle = nullptr;
-  CHECK_CALL(LGBM_DatasetCreateFromMat(p_mat, C_API_DTYPE_FLOAT64, nrow, ncol, COL_MAJOR,
+  CHECK_CALL(LGBM_DatasetCreateFromMat(p_mat, p_label, C_API_DTYPE_FLOAT64, C_API_DTYPE_FLOAT64, nrow, ncol, COL_MAJOR,
     R_CHAR_PTR(parameters), R_GET_PTR(reference), &handle));
   R_SET_PTR(out, handle);
   R_API_END();

include/LightGBM/boosting.h

@@ -131,10 +131,10 @@ class LIGHTGBM_EXPORT Boosting {
   * \param output Prediction result for this record
   * \param early_stop Early stopping instance. If nullptr, no early stopping is applied and all models are evaluated.
   */
-  virtual void PredictRaw(const double* features, double* output,
+  virtual void PredictRaw(double* features, double* output,
                           const PredictionEarlyStopInstance* early_stop) const = 0;
 
-  virtual void PredictRawByMap(const std::unordered_map<int, double>& features, double* output,
+  virtual void PredictRawByMap(std::unordered_map<int, double>& features, double* output,
                                const PredictionEarlyStopInstance* early_stop) const = 0;
 
 
@@ -144,10 +144,10 @@ class LIGHTGBM_EXPORT Boosting {
   * \param output Prediction result for this record
   * \param early_stop Early stopping instance. If nullptr, no early stopping is applied and all models are evaluated.
   */
-  virtual void Predict(const double* features, double* output,
+  virtual void Predict(double* features, double* output,
                        const PredictionEarlyStopInstance* early_stop) const = 0;
 
-  virtual void PredictByMap(const std::unordered_map<int, double>& features, double* output,
+  virtual void PredictByMap(std::unordered_map<int, double>& features, double* output,
                             const PredictionEarlyStopInstance* early_stop) const = 0;
 
 
@@ -157,19 +157,19 @@ class LIGHTGBM_EXPORT Boosting {
   * \param output Prediction result for this record
   */
   virtual void PredictLeafIndex(
-    const double* features, double* output) const = 0;
+    double* features, double* output) const = 0;
 
   virtual void PredictLeafIndexByMap(
-    const std::unordered_map<int, double>& features, double* output) const = 0;
+    std::unordered_map<int, double>& features, double* output) const = 0;
 
   /*!
   * \brief Feature contributions for the model's prediction of one record
   * \param feature_values Feature value on this record
   * \param output Prediction result for this record
   */
-  virtual void PredictContrib(const double* features, double* output) const = 0;
+  virtual void PredictContrib(double* features, double* output) const = 0;
 
-  virtual void PredictContribByMap(const std::unordered_map<int, double>& features,
+  virtual void PredictContribByMap(std::unordered_map<int, double>& features,
                                    std::vector<std::unordered_map<int, double>>* output) const = 0;
 
   /*!
@@ -295,6 +295,8 @@ class LIGHTGBM_EXPORT Boosting {
   */
   virtual const char* SubModelName() const = 0;
 
+  virtual int num_extra_features() const { return 0; }
+
   Boosting() = default;
   /*! \brief Disable copy */
   Boosting& operator=(const Boosting&) = delete;

include/LightGBM/c_api.h

@@ -28,6 +28,7 @@ typedef void* FastConfigHandle; /*!< \brief Handle of FastConfig. */
 #define C_API_DTYPE_FLOAT64 (1)  /*!< \brief float64 (double precision float). */
 #define C_API_DTYPE_INT32   (2)  /*!< \brief int32. */
 #define C_API_DTYPE_INT64   (3)  /*!< \brief int64. */
+#define C_API_DTYPE_NONE   (4)  /*!< \brief None. */
 
 #define C_API_PREDICT_NORMAL     (0)  /*!< \brief Normal prediction, with transform (if needed). */
 #define C_API_PREDICT_RAW_SCORE  (1)  /*!< \brief Predict raw score. */
@@ -161,7 +162,9 @@ LIGHTGBM_C_EXPORT int LGBM_DatasetCreateFromCSR(const void* indptr,
                                                 int indptr_type,
                                                 const int32_t* indices,
                                                 const void* data,
+                                                const void* label,
                                                 int data_type,
+                                                int label_type,
                                                 int64_t nindptr,
                                                 int64_t nelem,
                                                 int64_t num_col,
@@ -206,7 +209,9 @@ LIGHTGBM_C_EXPORT int LGBM_DatasetCreateFromCSC(const void* col_ptr,
                                                 int col_ptr_type,
                                                 const int32_t* indices,
                                                 const void* data,
+                                                const void* label,
                                                 int data_type,
+                                                int label_type,
                                                 int64_t ncol_ptr,
                                                 int64_t nelem,
                                                 int64_t num_row,
@@ -227,7 +232,9 @@ LIGHTGBM_C_EXPORT int LGBM_DatasetCreateFromCSC(const void* col_ptr,
  * \return 0 when succeed, -1 when failure happens
  */
 LIGHTGBM_C_EXPORT int LGBM_DatasetCreateFromMat(const void* data,
+                                                const void* label,
                                                 int data_type,
+                                                int label_type,
                                                 int32_t nrow,
                                                 int32_t ncol,
                                                 int is_row_major,
@@ -250,7 +257,9 @@ LIGHTGBM_C_EXPORT int LGBM_DatasetCreateFromMat(const void* data,
  */
 LIGHTGBM_C_EXPORT int LGBM_DatasetCreateFromMats(int32_t nmat,
                                                  const void** data,
+                                                 const void* label,
                                                  int data_type,
+                                                 int label_type,
                                                  int32_t* nrow,
                                                  int32_t ncol,
                                                  int is_row_major,
@@ -389,6 +398,15 @@ LIGHTGBM_C_EXPORT int LGBM_DatasetGetNumData(DatasetHandle handle,
 LIGHTGBM_C_EXPORT int LGBM_DatasetGetNumFeature(DatasetHandle handle,
                                                 int* out);
 
+/*!
+ * \brief Get number of original features.
+ * \param handle Handle of dataset
+ * \param[out] out The address to hold number of features
+ * \return 0 when succeed, -1 when failure happens
+ */
+LIGHTGBM_C_EXPORT int LGBM_DatasetGetNumOriginalFeature(DatasetHandle handle,
+                                                int* out);
+
 /*!
  * \brief Add features from ``source`` to ``target``.
  * \param target The handle of the dataset to add features to
@@ -422,6 +440,8 @@ LIGHTGBM_C_EXPORT int LGBM_BoosterCreateFromModelfile(const char* filename,
                                                       int* out_num_iterations,
                                                       BoosterHandle* out);
 
+LIGHTGBM_C_EXPORT int LGBM_PrintCTRStatus(const DatasetHandle dataset);                                                
+
 /*!
  * \brief Load an existing booster from string.
  * \param model_str Model string

include/LightGBM/config.h

@@ -970,6 +970,26 @@ struct Config {
 
   #pragma endregion
 
+  #pragma region CTR Parameters
+
+  // desc = ways to convert categorical features, currently supports:
+  // desc = ctr[:prior], where prior is a real number used to smooth the calculation of CTR values
+  // desc = count, the count of the categorical feature value in the dataset
+  // desc = for example "ctr:0.5,ctr:0.0:count will convert each categorical feature into 3 numerical features, with the 3 different ways separated by ','.
+  std::string cat_converters = "";
+
+  // desc = whether to keep the original feature values after the dataset is constructed
+  // desc = if set false, then once dataset is constructed, the cat_converters cannot be changed through parameters in train method
+  bool keep_raw_cat_data = false;
+
+  // desc = number of folds that training data is divided into, to calculate ctr values
+  int num_ctr_folds = 4;
+
+  // desc = whether to use the old categorical handling
+  bool keep_old_cat_method = false;
+
+  #pragma endregion
+
   #pragma endregion
 
   size_t file_load_progress_interval_bytes = size_t(10) * 1024 * 1024 * 1024;