Init

include/LightGBM/c_api.h

@@ -31,6 +31,7 @@ typedef void* DatasetHandle;  /*!< \brief Handle of dataset. */
 typedef void* BoosterHandle;  /*!< \brief Handle of booster. */
 typedef void* FastConfigHandle; /*!< \brief Handle of FastConfig. */
 typedef void* ByteBufferHandle; /*!< \brief Handle of ByteBuffer. */
+typedef void* ObjectiveFunctionHandle; /*!< \brief Handle of ObjectiveFunction. */
 
 #define C_API_DTYPE_FLOAT32 (0)  /*!< \brief float32 (single precision float). */
 #define C_API_DTYPE_FLOAT64 (1)  /*!< \brief float64 (double precision float). */
@@ -1563,6 +1564,22 @@ LIGHTGBM_C_EXPORT int LGBM_BoosterGetUpperBoundValue(BoosterHandle handle,
 LIGHTGBM_C_EXPORT int LGBM_BoosterGetLowerBoundValue(BoosterHandle handle,
                                                      double* out_results);
 
+/*!
+ */
+LIGHTGBM_C_EXPORT int LGBM_ObjectiveFunctionCreate(const char *typ,
+                                                   const char *parameter,
+                                                   ObjectiveFunctionHandle *out);
+
+/*!
+ */
+LIGHTGBM_C_EXPORT int LGBM_ObjectiveFunctionInit(ObjectiveFunctionHandle handle,
+                                                 int *num_data,
+                                                 DatasetHandle dataset);
+
+/*!
+ */
+LIGHTGBM_C_EXPORT int LGBM_ObjectiveFunctionFree(ObjectiveFunctionHandle handle);
+
 /*!
  * \brief Initialize the network.
  * \param machines List of machines in format 'ip1:port1,ip2:port2'

python-package/lightgbm/__init__.py

@@ -6,7 +6,7 @@
 
 from pathlib import Path
 
-from .basic import Booster, Dataset, Sequence, register_logger
+from .basic import Booster, Dataset, Sequence, ObjectiveFunction, register_logger
 from .callback import EarlyStopException, early_stopping, log_evaluation, record_evaluation, reset_parameter
 from .engine import CVBooster, cv, train
 
@@ -31,6 +31,7 @@
 __all__ = [
     "Dataset",
     "Booster",
+    "ObjectiveFunction",
     "CVBooster",
     "Sequence",
     "register_logger",

python-package/lightgbm/basic.py

@@ -5281,3 +5281,79 @@ def __get_eval_info(self) -> None:
                 self.__higher_better_inner_eval = [
                     name.startswith(("auc", "ndcg@", "map@", "average_precision")) for name in self.__name_inner_eval
                 ]
+
+
+class ObjectiveFunction:
+    def __init__(self, name: str, params: Dict[str, Any]):
+        self.name = name
+        self.params = params
+        self.num_data = None
+        self.num_class = params.get("num_class", 1)
+
+        self.__create()
+
+    def init(self, dataset: Dataset) -> "ObjectiveFunction":
+        return self.__init_from_dataset(dataset)
+
+    def __create(self):
+        self._handle = ctypes.c_void_p()
+        _safe_call(
+            _LIB.LGBM_ObjectiveFunctionCreate(
+                _c_str(self.name),
+                _c_str(_param_dict_to_str(self.params)),
+                ctypes.byref(self._handle),
+            )
+        )
+
+    def __init_from_dataset(self, dataset: Dataset) -> "ObjectiveFunction":
+        if dataset._handle is None:
+            raise ValueError("Cannot create ObjectiveFunction from uninitialised Dataset")
+
+        if self._handle is None:
+            raise ValueError("Dealocated ObjectiveFunction cannot be initialized")
+
+        ref_dataset = dataset._handle
+        tmp_num_data = ctypes.c_int(0)
+        _safe_call(
+            _LIB.LGBM_ObjectiveFunctionInit(
+                self._handle,
+                ctypes.byref(tmp_num_data),
+                dataset._handle,
+            )
+        )
+        self.num_data = tmp_num_data.value
+        return self
+
+    def __del__(self) -> None:
+        try:
+            self._free_handle()
+        except AttributeError:
+            pass
+
+    def _free_handle(self) -> "ObjectiveFunction":
+        if self._handle is not None:
+            _safe_call(_LIB.LGBM_ObjectiveFunctionFree(self._handle))
+            self._handle = None
+        return self
+
+    def __call__(self, y_pred: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+        if self._handle is None:
+            raise ValueError("Objective function seems uninitialized")
+
+        if self.num_data is None or self.num_class is None:
+            # TODO: Be more descriptive
+            raise ValueError("ObjectiveFunction was not created properly")
+
+        grad = np.zeros(dtype=np.float32, shape=self.num_data * self.num_class)
+        hess = np.zeros(dtype=np.float32, shape=self.num_data * self.num_class)
+
+        _safe_call(
+            _LIB.LGBM_ObjectiveFunctionEval(
+                self._handle,
+                y_pred.ctypes.data_as(ctypes.POINTER(ctypes.c_double)),
+                grad.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
+                hess.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
+            )
+        )
+
+        return (grad, hess)

src/c_api.cpp

@@ -27,6 +27,7 @@
 #include <mutex>
 #include <stdexcept>
 #include <vector>
+#include <fstream>
 
 #include "application/predictor.hpp"
 #include <LightGBM/utils/yamc/alternate_shared_mutex.hpp>
@@ -43,7 +44,8 @@ inline int LGBM_APIHandleException(const std::string& ex) {
   return -1;
 }
 
-#define API_BEGIN() try {
+#define API_BEGIN() std::ofstream outf("logs.txt", std::ios_base::app); \
+  try {
 #define API_END() } \
 catch(std::exception& ex) { return LGBM_APIHandleException(ex); } \
 catch(std::string& ex) { return LGBM_APIHandleException(ex); } \
@@ -907,6 +909,7 @@ using LightGBM::kZeroThreshold;
 using LightGBM::LGBM_APIHandleException;
 using LightGBM::Log;
 using LightGBM::Network;
+using LightGBM::ObjectiveFunction;
 using LightGBM::Random;
 using LightGBM::ReduceScatterFunction;
 using LightGBM::SingleRowPredictor;
@@ -2587,6 +2590,7 @@ int LGBM_BoosterPredictForMats(BoosterHandle handle,
                                int64_t* out_len,
                                double* out_result) {
   API_BEGIN();
+  outf << parameter << std::endl;
   auto param = Config::Str2Map(parameter);
   Config config;
   config.Set(param);
@@ -2747,6 +2751,53 @@ int LGBM_BoosterGetLowerBoundValue(BoosterHandle handle,
   API_END();
 }
 
+LIGHTGBM_C_EXPORT int LGBM_ObjectiveFunctionCreate(const char *typ,
+                                                   const char *parameter,
+                                                   ObjectiveFunctionHandle *out) {
+  API_BEGIN();
+  auto param = Config::Str2Map(parameter);
+  Config config(param);
+  *out = ObjectiveFunction::CreateObjectiveFunction(std::string(typ), config);
+  outf << parameter << std::endl;
+  API_END();
+}
+
+LIGHTGBM_C_EXPORT int LGBM_ObjectiveFunctionInit(ObjectiveFunctionHandle handle,
+                                                 int *num_data,
+                                                 DatasetHandle dataset) {
+  API_BEGIN();
+  ObjectiveFunction* ref_fobj = reinterpret_cast<ObjectiveFunction*>(handle);
+  Dataset* ref_dataset = reinterpret_cast<Dataset*>(dataset);
+  ref_fobj->Init(ref_dataset->metadata(), ref_dataset->num_data());
+  *num_data = ref_dataset->num_data();
+  API_END();
+}
+
+LIGHTGBM_C_EXPORT int LGBM_ObjectiveFunctionEval(ObjectiveFunctionHandle handle,
+                                                 const double* score,
+                                                 float* grad,
+                                                 float* hess) {
+  API_BEGIN();
+  #ifdef SCORE_T_USE_DOUBLE
+  (void) handle;       // UNUSED VARIABLE
+  (void) grad;         // UNUSED VARIABLE
+  (void) hess;         // UNUSED VARIABLE
+  Log::Fatal("Don't support evaluating objective function when SCORE_T_USE_DOUBLE is enabled");
+  #else
+  ObjectiveFunction* ref_fobj = reinterpret_cast<ObjectiveFunction*>(handle);
+  ref_fobj->GetGradients(score, grad, hess);
+  #endif
+  API_END();
+}
+
+/*!
+ */
+LIGHTGBM_C_EXPORT int LGBM_ObjectiveFunctionFree(ObjectiveFunctionHandle handle) {
+  API_BEGIN();
+  delete reinterpret_cast<ObjectiveFunction*>(handle);
+  API_END();
+}
+
 int LGBM_NetworkInit(const char* machines,
                      int local_listen_port,
                      int listen_time_out,

src/io/config_auto.cpp

@@ -326,7 +326,6 @@ const std::unordered_set<std::string>& Config::parameter_set() {
 }
 
 void Config::GetMembersFromString(const std::unordered_map<std::string, std::string>& params) {
-  std::string tmp_str = "";
   GetString(params, "data", &data);
 
   if (GetString(params, "valid", &tmp_str)) {
@@ -588,6 +587,7 @@ void Config::GetMembersFromString(const std::unordered_map<std::string, std::str
   GetInt(params, "objective_seed", &objective_seed);
 
   GetInt(params, "num_class", &num_class);
+  std::cout << "What is this: " << num_class << std::endl;
   CHECK_GT(num_class, 0);
 
   GetBool(params, "is_unbalance", &is_unbalance);

src/objective/multiclass_objective.hpp

@@ -25,6 +25,7 @@ class MulticlassSoftmax: public ObjectiveFunction {
  public:
   explicit MulticlassSoftmax(const Config& config) {
     num_class_ = config.num_class;
+    std::cout << "We have set " << num_class_ << std::endl;
     // This factor is to rescale the redundant form of K-classification, to the non-redundant form.
     // In the traditional settings of K-classification, there is one redundant class, whose output is set to 0 (like the class 0 in binary classification).
     // This is from the Friedman GBDT paper.

src/objective/regression_objective.hpp

@@ -111,6 +111,7 @@ class RegressionL2loss: public ObjectiveFunction {
   }
 
   void Init(const Metadata& metadata, data_size_t num_data) override {
+    Log::Debug("We are here");
     num_data_ = num_data;
     label_ = metadata.label();
     if (sqrt_) {

tests/python_package_test/test_engine.py

@@ -24,6 +24,8 @@
 
 from .utils import (
     SERIALIZERS,
+    assert_all_trees_valid,
+    builtin_objective,
     dummy_obj,
     load_breast_cancer,
     load_digits,
@@ -4397,3 +4399,49 @@ def test_quantized_training():
     quant_bst = lgb.train(bst_params, ds, num_boost_round=10)
     quant_rmse = np.sqrt(np.mean((quant_bst.predict(X) - y) ** 2))
     assert quant_rmse < rmse + 6.0
+
+@pytest.mark.parametrize("use_weight", [False, True])
+def test_objective_function_regression(use_weight):
+    X, y = make_synthetic_regression()
+    weight = np.random.choice([1, 2], len(X)) if use_weight else None
+    lgb_train = lgb.Dataset(X, y, weight=weight, init_score=np.zeros(len(X)))
+
+    params = {"verbose": -1, "objective": "regression"}
+    builtin_loss = builtin_objective("multiclass", copy.deepcopy(params))
+
+    booster = lgb.train(params, lgb_train, num_boost_round=20)
+    params["objective"] = mse_obj
+    booster_custom = lgb.train(params, lgb_train, num_boost_round=20)
+    params["objective"] = builtin_loss
+    booster_exposed = lgb.train(params, lgb_train, num_boost_round=20)
+    np.testing.assert_allclose(booster_exposed.predict(X), booster.predict(X))
+    np.testing.assert_allclose(booster_exposed.predict(X), booster_custom.predict(X))
+
+    y_pred = booster.predict(X)
+    np.testing.assert_allclose(builtin_loss(y_pred, lgb_train), mse_obj(y_pred, lgb_train))
+
+@pytest.mark.parametrize("use_weight", [False, True])
+def test_objective_function_multiclass(use_weight):
+    def custom_obj(y_pred, ds):
+        y_true = ds.get_label()
+        weight = ds.get_weight()
+        grad, hess = sklearn_multiclass_custom_objective(y_true, y_pred, weight)
+        return grad, hess
+
+    X, y = make_blobs(n_samples=1_000, centers=[[-4, -4], [4, 4], [-4, 4]], random_state=42)
+    weight = np.random.choice([1, 2], y.shape) if use_weight else None
+    lgb_train = lgb.Dataset(X, y, weight=weight, init_score=np.zeros((len(y), 3)))
+
+    params = {"verbose": -1, "objective": "multiclass", "num_class": 3}
+    builtin_loss = builtin_objective("multiclass", copy.deepcopy(params))
+    booster = lgb.train(params, lgb_train, num_boost_round=20)
+    params["objective"] = custom_obj
+    booster_custom = lgb.train(params, lgb_train, num_boost_round=20)
+    params["objective"] = builtin_loss
+    booster_exposed = lgb.train(params, lgb_train, num_boost_round=20)
+
+    np.testing.assert_allclose(booster_exposed.predict(X), booster.predict(X, raw_score=True))
+    np.testing.assert_allclose(booster_exposed.predict(X), booster_custom.predict(X))
+
+    y_pred = booster.predict(X, raw_score=True)
+    np.testing.assert_allclose(builtin_loss(y_pred, lgb_train), mse_obj(y_pred, lgb_train))

tests/python_package_test/utils.py

@@ -130,6 +130,9 @@ def mse_obj(y_pred, dtrain):
     y_true = dtrain.get_label()
     grad = y_pred - y_true
     hess = np.ones(len(grad))
+    if dtrain.get_weight() is not None:
+        grad *= dtrain.get_weight()
+        hess *= dtrain.get_weight()
     return grad, hess
 
 
@@ -191,6 +194,14 @@ def pickle_and_unpickle_object(obj, serializer):
     return obj_from_disk  # noqa: RET504
 
 
+def builtin_objective(name, params):
+    def wrapper(y_pred, dtrain):
+        fobj = lgb.ObjectiveFunction(name, params)
+        fobj.init(dtrain)
+        return fobj(y_pred)
+    return wrapper
+
+
 # doing this here, at import time, to ensure it only runs once_per import
 # instead of once per assertion
 _numpy_testing_supports_strict_kwarg = "strict" in getfullargspec(np.testing.assert_array_equal).kwonlyargs