[python-package] change default best_iteration to 0 (#495)

* make test fail

* change default best_iteration to 0

* fix test

* change data_splitter to folds in cv

* update docs

docs/Python-API.md

@@ -8,7 +8,7 @@
 
 * [Training API](Python-API.md#training-api)
     - [train](Python-API.md#trainparams-train_set-num_boost_round100-valid_setsnone-valid_namesnone-fobjnone-fevalnone-init_modelnone-feature_nameauto-categorical_featureauto-early_stopping_roundsnone-evals_resultnone-verbose_evaltrue-learning_ratesnone-callbacksnone)
-    - [cv](Python-API.md#cvparams-train_set-num_boost_round10-data_splitternone-nfold5-stratifiedfalse-shuffletrue-metricsnone-fobjnone-fevalnone-init_modelnone-feature_nameauto-categorical_featureauto-early_stopping_roundsnone-fpreprocnone-verbose_evalnone-show_stdvtrue-seed0-callbacksnone)
+    - [cv](Python-API.md#cvparams-train_set-num_boost_round10-foldsnone-nfold5-stratifiedfalse-shuffletrue-metricsnone-fobjnone-fevalnone-init_modelnone-feature_nameauto-categorical_featureauto-early_stopping_roundsnone-fpreprocnone-verbose_evalnone-show_stdvtrue-seed0-callbacksnone)
 
 * [Scikit-learn API](Python-API.md#scikit-learn-api)
     - [Common Methods](Python-API.md#common-methods)
@@ -538,7 +538,7 @@ The methods of each Class is in alphabetical order.
     booster : a trained booster model
 
 
-#### cv(params, train_set, num_boost_round=10, data_splitter=None, nfold=5, stratified=False, shuffle=True, metrics=None, fobj=None, feval=None, init_model=None, feature_name='auto', categorical_feature='auto', early_stopping_rounds=None, fpreproc=None, verbose_eval=None, show_stdv=True, seed=0, callbacks=None)
+#### cv(params, train_set, num_boost_round=10, folds=None, nfold=5, stratified=False, shuffle=True, metrics=None, fobj=None, feval=None, init_model=None, feature_name='auto', categorical_feature='auto', early_stopping_rounds=None, fpreproc=None, verbose_eval=None, show_stdv=True, seed=0, callbacks=None)
 
     Cross-validation with given paramaters.
 
@@ -550,8 +550,9 @@ The methods of each Class is in alphabetical order.
         Data to be trained.
     num_boost_round : int
         Number of boosting iterations.
-    data_splitter : an instance with split(X) method
-        Instance with split(X) method.
+    folds : a generator or iterator of (train_idx, test_idx) tuples
+        The train indices and test indices for each folds.
+        This argument has highest priority over other data split arguments.
     nfold : int
         Number of folds in CV.
     stratified : bool

python-package/lightgbm/engine.py

@@ -165,7 +165,7 @@ def train(params, train_set, num_boost_round=100,
         booster.set_train_data_name(train_data_name)
     for valid_set, name_valid_set in zip(reduced_valid_sets, name_valid_sets):
         booster.add_valid(valid_set, name_valid_set)
-    booster.best_iteration = -1
+    booster.best_iteration = 0
 
     """start training"""
     for i in range_(init_iteration, init_iteration + num_boost_round):
@@ -224,16 +224,15 @@ def handlerFunction(*args, **kwargs):
         return handlerFunction
 
 
-def _make_n_folds(full_data, data_splitter, nfold, params, seed, fpreproc=None, stratified=False, shuffle=True):
+def _make_n_folds(full_data, folds, nfold, params, seed, fpreproc=None, stratified=False, shuffle=True):
     """
     Make an n-fold list of Booster from random indices.
     """
     full_data = full_data.construct()
     num_data = full_data.num_data()
-    if data_splitter is not None:
-        if not hasattr(data_splitter, 'split'):
-            raise AttributeError("data_splitter has no method 'split'")
-        folds = data_splitter.split(np.arange(num_data))
+    if folds is not None:
+        if not hasattr(folds, '__iter__'):
+            raise AttributeError("folds should be a generator or iterator of (train_idx, test_idx)")
     else:
         if 'objective' in params and params['objective'] == 'lambdarank':
             if not SKLEARN_INSTALLED:
@@ -287,7 +286,7 @@ def _agg_cv_result(raw_results):
 
 
 def cv(params, train_set, num_boost_round=10,
-       data_splitter=None, nfold=5, stratified=False, shuffle=True,
+       folds=None, nfold=5, stratified=False, shuffle=True,
        metrics=None, fobj=None, feval=None, init_model=None,
        feature_name='auto', categorical_feature='auto',
        early_stopping_rounds=None, fpreproc=None,
@@ -304,8 +303,9 @@ def cv(params, train_set, num_boost_round=10,
         Data to be trained.
     num_boost_round : int
         Number of boosting iterations.
-    data_splitter : an instance with split(X) method
-        Instance with split(X) method.
+    folds : a generator or iterator of (train_idx, test_idx) tuples
+        The train indices and test indices for each folds.
+        This argument has highest priority over other data split arguments.
     nfold : int
         Number of folds in CV.
     stratified : bool
@@ -373,10 +373,9 @@ def cv(params, train_set, num_boost_round=10,
         params['metric'] = metrics
 
     results = collections.defaultdict(list)
-    cvfolds = _make_n_folds(train_set, data_splitter=data_splitter,
-                            nfold=nfold, params=params, seed=seed,
-                            fpreproc=fpreproc, stratified=stratified,
-                            shuffle=shuffle)
+    cvfolds = _make_n_folds(train_set, folds=folds, nfold=nfold,
+                            params=params, seed=seed, fpreproc=fpreproc,
+                            stratified=stratified, shuffle=shuffle)
 
     # setup callbacks
     if callbacks is None:

tests/python_package_test/test_engine.py

@@ -108,7 +108,7 @@ def test_early_stopping(self):
                         valid_names=valid_set_name,
                         verbose_eval=False,
                         early_stopping_rounds=5)
-        self.assertEqual(gbm.best_iteration, -1)
+        self.assertEqual(gbm.best_iteration, 0)
         self.assertIn(valid_set_name, gbm.best_score)
         self.assertIn('binary_logloss', gbm.best_score[valid_set_name])
         # early stopping occurs
@@ -189,10 +189,10 @@ def test_cv(self):
         lgb.cv(params, lgb_train, num_boost_round=10, nfold=3, shuffle=True,
                metrics='l1', verbose_eval=False,
                callbacks=[lgb.reset_parameter(learning_rate=lambda i: 0.1 - 0.001 * i)])
-        # self defined data_splitter
+        # self defined folds
         tss = TimeSeriesSplit(3)
-        lgb.cv(params, lgb_train, num_boost_round=10, data_splitter=tss, nfold=5,  # test if wrong nfold is ignored
-               metrics='l2', verbose_eval=False)
+        folds = tss.split(X_train)
+        lgb.cv(params_with_metric, lgb_train, num_boost_round=10, folds=folds, verbose_eval=False)
         # lambdarank
         X_train, y_train = load_svmlight_file('../../examples/lambdarank/rank.train')
         q_train = np.loadtxt('../../examples/lambdarank/rank.train.query')

tests/python_package_test/test_sklearn.py

@@ -27,29 +27,29 @@ def test_binary(self):
         X, y = load_breast_cancer(True)
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         gbm = lgb.LGBMClassifier(n_estimators=50, silent=True)
-        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
         ret = log_loss(y_test, gbm.predict_proba(X_test))
         self.assertLess(ret, 0.15)
-        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['binary_logloss'][-1], places=5)
+        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['binary_logloss'][gbm.best_iteration - 1], places=5)
 
     def test_regreesion(self):
         X, y = load_boston(True)
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         gbm = lgb.LGBMRegressor(n_estimators=50, silent=True)
-        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
         ret = mean_squared_error(y_test, gbm.predict(X_test))
         self.assertLess(ret, 16)
-        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['l2'][-1], places=5)
+        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['l2'][gbm.best_iteration - 1], places=5)
 
     def test_multiclass(self):
         X, y = load_digits(10, True)
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         gbm = lgb.LGBMClassifier(n_estimators=50, silent=True)
-        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
         ret = multi_error(y_test, gbm.predict(X_test))
         self.assertLess(ret, 0.2)
         ret = multi_logloss(y_test, gbm.predict_proba(X_test))
-        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['multi_logloss'][-1], places=5)
+        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['multi_logloss'][gbm.best_iteration - 1], places=5)
 
     def test_lambdarank(self):
         X_train, y_train = load_svmlight_file('../../examples/lambdarank/rank.train')
@@ -58,7 +58,7 @@ def test_lambdarank(self):
         q_test = np.loadtxt('../../examples/lambdarank/rank.test.query')
         gbm = lgb.LGBMRanker()
         gbm.fit(X_train, y_train, group=q_train, eval_set=[(X_test, y_test)],
-                eval_group=[q_test], eval_at=[1, 3], verbose=False,
+                eval_group=[q_test], eval_at=[1, 3], early_stopping_rounds=5, verbose=False,
                 callbacks=[lgb.reset_parameter(learning_rate=lambda x: 0.95 ** x * 0.1)])
 
     def test_regression_with_custom_objective(self):
@@ -69,10 +69,10 @@ def objective_ls(y_true, y_pred):
         X, y = load_boston(True)
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         gbm = lgb.LGBMRegressor(n_estimators=50, silent=True, objective=objective_ls)
-        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
         ret = mean_squared_error(y_test, gbm.predict(X_test))
         self.assertLess(ret, 100)
-        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['l2'][-1], places=5)
+        self.assertAlmostEqual(ret, gbm.evals_result['valid_0']['l2'][gbm.best_iteration - 1], places=5)
 
     def test_binary_classification_with_custom_objective(self):
         def logregobj(y_true, y_pred):
@@ -86,7 +86,7 @@ def binary_error(y_test, y_pred):
             return np.mean([int(p > 0.5) != y for y, p in zip(y_test, y_pred)])
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
         gbm = lgb.LGBMClassifier(n_estimators=50, silent=True, objective=logregobj)
-        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
+        gbm.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=5, verbose=False)
         ret = binary_error(y_test, gbm.predict(X_test))
         self.assertLess(ret, 0.1)