-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add prediction early stopping (#550)
* Add early stopping for prediction * Fix GBDT if-else prediction with early stopping * Small C++ embelishments to early stopping API and functions * Fix early stopping efficiency issue by creating a singleton for no early stopping * Python improvements to early stopping API * Add assertion check for binary and multiclass prediction score length * Update vcxproj and vcxproj.filters with new early stopping files * Remove inline from PredictRaw(), the linker was not able to find it otherwise
- Loading branch information
Showing
16 changed files
with
375 additions
and
76 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
#ifndef LIGHTGBM_PREDICTION_EARLY_STOP_H_ | ||
#define LIGHTGBM_PREDICTION_EARLY_STOP_H_ | ||
|
||
#include <functional> | ||
#include <string> | ||
|
||
#include <LightGBM/export.h> | ||
|
||
namespace LightGBM | ||
{ | ||
struct PredictionEarlyStopInstance | ||
{ | ||
/// Callback function type for early stopping. | ||
/// Takes current prediction and number of elements in prediction | ||
/// @returns true if prediction should stop according to criterion | ||
using FunctionType = std::function<bool(const double*, int)>; | ||
|
||
FunctionType callbackFunction; // callback function itself | ||
int roundPeriod; // call callbackFunction every `runPeriod` iterations | ||
}; | ||
|
||
struct PredictionEarlyStopConfig | ||
{ | ||
int roundPeriod; | ||
double marginThreshold; | ||
}; | ||
|
||
/// Create an early stopping algorithm of type `type`, with given roundPeriod and margin threshold | ||
LIGHTGBM_EXPORT PredictionEarlyStopInstance createPredictionEarlyStopInstance(const std::string& type, | ||
const PredictionEarlyStopConfig& config); | ||
|
||
} // namespace LightGBM | ||
|
||
#endif // LIGHTGBM_PREDICTION_EARLY_STOP_H_ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -296,7 +296,7 @@ class _InnerPredictor(object): | |
Only used for prediction, usually used for continued-train | ||
Note: Can convert from Booster, but cannot convert to Booster | ||
""" | ||
def __init__(self, model_file=None, booster_handle=None): | ||
def __init__(self, model_file=None, booster_handle=None, early_stop_instance=None): | ||
"""Initialize the _InnerPredictor. Not expose to user | ||
Parameters | ||
|
@@ -305,6 +305,8 @@ def __init__(self, model_file=None, booster_handle=None): | |
Path to the model file. | ||
booster_handle : Handle of Booster | ||
use handle to init | ||
early_stop_instance: object of type PredictionEarlyStopInstance | ||
If None, no early stopping is applied | ||
""" | ||
self.handle = ctypes.c_void_p() | ||
self.__is_manage_handle = True | ||
|
@@ -339,6 +341,11 @@ def __init__(self, model_file=None, booster_handle=None): | |
else: | ||
raise TypeError('Need Model file or Booster handle to create a predictor') | ||
|
||
if early_stop_instance is None: | ||
self.early_stop_instance = PredictionEarlyStopInstance("none") | ||
else: | ||
self.early_stop_instance = early_stop_instance | ||
|
||
def __del__(self): | ||
if self.__is_manage_handle: | ||
_safe_call(_LIB.LGBM_BoosterFree(self.handle)) | ||
|
@@ -385,6 +392,7 @@ def predict(self, data, num_iteration=-1, | |
int_data_has_header = 1 if data_has_header else 0 | ||
if num_iteration > self.num_total_iteration: | ||
num_iteration = self.num_total_iteration | ||
|
||
if isinstance(data, string_type): | ||
with _temp_file() as f: | ||
_safe_call(_LIB.LGBM_BoosterPredictForFile( | ||
|
@@ -393,6 +401,7 @@ def predict(self, data, num_iteration=-1, | |
ctypes.c_int(int_data_has_header), | ||
ctypes.c_int(predict_type), | ||
ctypes.c_int(num_iteration), | ||
self.early_stop_instance.handle, | ||
c_str(f.name))) | ||
lines = f.readlines() | ||
nrow = len(lines) | ||
|
@@ -409,7 +418,7 @@ def predict(self, data, num_iteration=-1, | |
predict_type) | ||
elif isinstance(data, DataFrame): | ||
preds, nrow = self.__pred_for_np2d(data.values, num_iteration, | ||
predict_type) | ||
predict_type, early_stop_instance_handle) | ||
This comment has been minimized.
Sorry, something went wrong.
This comment has been minimized.
Sorry, something went wrong.
cbecker
Author
Contributor
|
||
else: | ||
try: | ||
csr = scipy.sparse.csr_matrix(data) | ||
|
@@ -466,6 +475,7 @@ def __pred_for_np2d(self, mat, num_iteration, predict_type): | |
ctypes.c_int(C_API_IS_ROW_MAJOR), | ||
ctypes.c_int(predict_type), | ||
ctypes.c_int(num_iteration), | ||
self.early_stop_instance.handle, | ||
ctypes.byref(out_num_preds), | ||
preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double)))) | ||
if n_preds != out_num_preds.value: | ||
|
@@ -496,6 +506,7 @@ def __pred_for_csr(self, csr, num_iteration, predict_type): | |
ctypes.c_int64(csr.shape[1]), | ||
ctypes.c_int(predict_type), | ||
ctypes.c_int(num_iteration), | ||
self.early_stop_instance.handle, | ||
ctypes.byref(out_num_preds), | ||
preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double)))) | ||
if n_preds != out_num_preds.value: | ||
|
@@ -526,6 +537,7 @@ def __pred_for_csc(self, csc, num_iteration, predict_type): | |
ctypes.c_int64(csc.shape[0]), | ||
ctypes.c_int(predict_type), | ||
ctypes.c_int(num_iteration), | ||
self.early_stop_instance.handle, | ||
ctypes.byref(out_num_preds), | ||
preds.ctypes.data_as(ctypes.POINTER(ctypes.c_double)))) | ||
if n_preds != out_num_preds.value: | ||
|
@@ -1568,7 +1580,8 @@ def dump_model(self, num_iteration=-1): | |
ptr_string_buffer)) | ||
return json.loads(string_buffer.value.decode()) | ||
|
||
def predict(self, data, num_iteration=-1, raw_score=False, pred_leaf=False, data_has_header=False, is_reshape=True): | ||
def predict(self, data, num_iteration=-1, raw_score=False, pred_leaf=False, data_has_header=False, is_reshape=True, | ||
early_stop_instance=None): | ||
""" | ||
Predict logic | ||
|
@@ -1587,19 +1600,21 @@ def predict(self, data, num_iteration=-1, raw_score=False, pred_leaf=False, data | |
Used for txt data | ||
is_reshape : bool | ||
Reshape to (nrow, ncol) if true | ||
early_stop_instance: object of type PredictionEarlyStopInstance. | ||
If None, no early stopping is applied | ||
Returns | ||
------- | ||
Prediction result | ||
""" | ||
predictor = self._to_predictor() | ||
predictor = self._to_predictor(early_stop_instance) | ||
if num_iteration <= 0: | ||
num_iteration = self.best_iteration | ||
return predictor.predict(data, num_iteration, raw_score, pred_leaf, data_has_header, is_reshape) | ||
|
||
def _to_predictor(self): | ||
def _to_predictor(self, early_stop_instance=None): | ||
"""Convert to predictor""" | ||
predictor = _InnerPredictor(booster_handle=self.handle) | ||
predictor = _InnerPredictor(booster_handle=self.handle, early_stop_instance=early_stop_instance) | ||
predictor.pandas_categorical = self.pandas_categorical | ||
return predictor | ||
|
||
|
@@ -1785,3 +1800,35 @@ def set_attr(self, **kwargs): | |
self.__attr[key] = value | ||
else: | ||
self.__attr.pop(key, None) | ||
|
||
|
||
class PredictionEarlyStopInstance(object): | ||
""""PredictionEarlyStopInstance in LightGBM.""" | ||
def __init__(self, early_stop_type="none", round_period=20, margin_threshold=1.5): | ||
""" | ||
Create an early stopping object | ||
Parameters | ||
---------- | ||
early_stop_type: string | ||
None, "none", "binary" or "multiclass". Regression is not supported. | ||
round_period : int | ||
The score will be checked every round_period to check if the early stopping criteria is met | ||
margin_threshold : double | ||
Early stopping will kick in when the margin is greater than margin_threshold | ||
""" | ||
self.handle = ctypes.c_void_p(0) | ||
self.__attr = {} | ||
|
||
if early_stop_type is None: | ||
early_stop_type = "none" | ||
|
||
_safe_call(_LIB.LGBM_PredictionEarlyStopInstanceCreate( | ||
c_str(early_stop_type), | ||
ctypes.c_int(round_period), | ||
ctypes.c_double(margin_threshold), | ||
ctypes.byref(self.handle))) | ||
|
||
def __del__(self): | ||
if self.handle is not None: | ||
_safe_call(_LIB.LGBM_PredictionEarlyStopInstanceFree(self.handle)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
@cbecker just notice this: why is a
early_stop_instance_handle
here? look like a typo