Save training metrics into the model metadata (best_score_, evals_res…

…ult_, best_iteration_ model attributes now work after model save/load).. Fix #1166

catboost/cuda/train_lib/train.cpp

@@ -472,7 +472,8 @@ namespace NCatboostCuda {
                 .WithCoreModelFrom(
                     modelPtr)
                 .WithObjectsDataFrom(trainingData.Learn->ObjectsData)
-                .WithFeatureEstimators(trainingData.FeatureEstimators);
+                .WithFeatureEstimators(trainingData.FeatureEstimators)
+                .WithMetrics(*metricsAndTimeHistory);
 
             if (dstModel) {
                 coreModelToFullModelConverter.Do(true, dstModel, localExecutor, &targetClassifiers);

catboost/libs/loggers/CMakeLists.darwin-arm64.txt

@@ -12,6 +12,7 @@ add_library(catboost-libs-loggers)
 target_link_libraries(catboost-libs-loggers PUBLIC
   contrib-libs-cxxsupp
   yutil
+  catboost-libs-helpers
   catboost-libs-logging
   private-libs-options
   catboost-libs-metrics

catboost/libs/loggers/CMakeLists.darwin-x86_64.txt

@@ -12,6 +12,7 @@ add_library(catboost-libs-loggers)
 target_link_libraries(catboost-libs-loggers PUBLIC
   contrib-libs-cxxsupp
   yutil
+  catboost-libs-helpers
   catboost-libs-logging
   private-libs-options
   catboost-libs-metrics

catboost/libs/loggers/CMakeLists.linux-aarch64-cuda.txt

@@ -13,6 +13,7 @@ target_link_libraries(catboost-libs-loggers PUBLIC
   contrib-libs-linux-headers
   contrib-libs-cxxsupp
   yutil
+  catboost-libs-helpers
   catboost-libs-logging
   private-libs-options
   catboost-libs-metrics

catboost/libs/loggers/CMakeLists.linux-aarch64.txt

@@ -13,6 +13,7 @@ target_link_libraries(catboost-libs-loggers PUBLIC
   contrib-libs-linux-headers
   contrib-libs-cxxsupp
   yutil
+  catboost-libs-helpers
   catboost-libs-logging
   private-libs-options
   catboost-libs-metrics

catboost/libs/loggers/CMakeLists.linux-ppc64le-cuda.txt

@@ -13,6 +13,7 @@ target_link_libraries(catboost-libs-loggers PUBLIC
   contrib-libs-linux-headers
   contrib-libs-cxxsupp
   yutil
+  catboost-libs-helpers
   catboost-libs-logging
   private-libs-options
   catboost-libs-metrics

catboost/libs/loggers/CMakeLists.linux-ppc64le.txt

@@ -13,6 +13,7 @@ target_link_libraries(catboost-libs-loggers PUBLIC
   contrib-libs-linux-headers
   contrib-libs-cxxsupp
   yutil
+  catboost-libs-helpers
   catboost-libs-logging
   private-libs-options
   catboost-libs-metrics

catboost/libs/loggers/CMakeLists.linux-x86_64-cuda.txt

@@ -13,6 +13,7 @@ target_link_libraries(catboost-libs-loggers PUBLIC
   contrib-libs-linux-headers
   contrib-libs-cxxsupp
   yutil
+  catboost-libs-helpers
   catboost-libs-logging
   private-libs-options
   catboost-libs-metrics

catboost/libs/loggers/CMakeLists.linux-x86_64.txt

@@ -13,6 +13,7 @@ target_link_libraries(catboost-libs-loggers PUBLIC
   contrib-libs-linux-headers
   contrib-libs-cxxsupp
   yutil
+  catboost-libs-helpers
   catboost-libs-logging
   private-libs-options
   catboost-libs-metrics

catboost/libs/loggers/CMakeLists.windows-x86_64-cuda.txt

@@ -12,6 +12,7 @@ add_library(catboost-libs-loggers)
 target_link_libraries(catboost-libs-loggers PUBLIC
   contrib-libs-cxxsupp
   yutil
+  catboost-libs-helpers
   catboost-libs-logging
   private-libs-options
   catboost-libs-metrics

catboost/libs/loggers/CMakeLists.windows-x86_64.txt

@@ -12,6 +12,7 @@ add_library(catboost-libs-loggers)
 target_link_libraries(catboost-libs-loggers PUBLIC
   contrib-libs-cxxsupp
   yutil
+  catboost-libs-helpers
   catboost-libs-logging
   private-libs-options
   catboost-libs-metrics

catboost/libs/loggers/catboost_logger_helpers.cpp

@@ -1,13 +1,60 @@
 #include "catboost_logger_helpers.h"
 #include "logger.h"
 
+#include <catboost/libs/helpers/json_helpers.h>
+
+#include <type_traits>
+
+
 TTimeInfo::TTimeInfo(const TProfileResults& profileResults)
     : IterationTime(profileResults.CurrentTime)
     , PassedTime(profileResults.PassedTime)
     , RemainingTime(profileResults.RemainingTime)
 {
 }
 
+NJson::TJsonValue TMetricsAndTimeLeftHistory::SaveMetrics() const {
+    auto saveToJson = [&](const auto& field) {
+        NJson::TJsonValue dst;
+        TJsonFieldHelper<std::remove_const_t<std::remove_reference_t<decltype(field)>>>::Write(field, &dst);
+        return dst;
+    };
+
+    NJson::TJsonValue result(NJson::JSON_MAP);
+    result["learn_metrics_history"] = saveToJson(LearnMetricsHistory);
+    result["test_metrics_history"] = saveToJson(TestMetricsHistory);
+    if (BestIteration) {
+        result["best_iteration"] = saveToJson(*BestIteration);
+    }
+    result["learn_best_error"] = saveToJson(LearnBestError);
+    result["test_best_error"] = saveToJson(TestBestError);
+
+    return result;
+}
+
+TMetricsAndTimeLeftHistory TMetricsAndTimeLeftHistory::LoadMetrics(const NJson::TJsonValue& rhs) {
+    const auto& rhsMap = rhs.GetMap();
+
+    auto loadFromJson = [&] (TStringBuf name, auto* field) {
+        TJsonFieldHelper<std::remove_reference_t<decltype(*field)>>::Read(
+            rhsMap.at(name),
+            field
+        );
+    };
+
+    TMetricsAndTimeLeftHistory result;
+    loadFromJson("learn_metrics_history",  &result.LearnMetricsHistory);
+    loadFromJson("test_metrics_history",  &result.TestMetricsHistory);
+    if (rhsMap.contains("best_iteration")) {
+        result.BestIteration = rhsMap.at("best_iteration").GetUIntegerSafe();
+    }
+    loadFromJson("learn_best_error", &result.LearnBestError);
+    loadFromJson("test_best_error", &result.TestBestError);
+
+    return result;
+}
+
+
 void TMetricsAndTimeLeftHistory::TryUpdateBestError(const IMetric& metric, double error, THashMap<TString, double>& bestError, bool updateBestIteration) {
     TString metricDescription = metric.GetDescription();
     bool shouldUpdate = false;

catboost/libs/loggers/catboost_logger_helpers.h

@@ -5,6 +5,8 @@
 #include <catboost/private/libs/options/enums.h>
 #include <catboost/libs/metrics/metric.h>
 
+#include <library/cpp/json/json_value.h>
+
 #include <util/generic/maybe.h>
 
 class TLogger;
@@ -32,6 +34,10 @@ struct TMetricsAndTimeLeftHistory {
 
     Y_SAVELOAD_DEFINE(LearnMetricsHistory, TestMetricsHistory, TimeHistory, BestIteration, LearnBestError, TestBestError);
 
+    // Serialization for model metadata without TimeHistory
+    NJson::TJsonValue SaveMetrics() const;
+    static TMetricsAndTimeLeftHistory LoadMetrics(const NJson::TJsonValue& rhs);
+
     void AddLearnError(const IMetric& metric, double error);
     void AddTestError(size_t testIdx, const IMetric& metric, double error, bool updateBestIteration);
 

catboost/libs/train_lib/train_model.cpp

@@ -717,6 +717,8 @@ static void SaveModel(
             trainingDataForCpu.Learn->ObjectsData
         ).WithFeatureEstimators(
             trainingDataForCpu.FeatureEstimators
+        ).WithMetrics(
+            ctx.LearnProgress->MetricsAndTimeHistory
         );
 
         const TVector<TTargetClassifier>* targetClassifiers = &ctx.CtrsHelper.GetTargetClassifiers();

catboost/private/libs/algo/full_model_saver.cpp

@@ -494,6 +494,13 @@ namespace NCB {
         return *this;
     }
 
+    TCoreModelToFullModelConverter& TCoreModelToFullModelConverter::WithMetrics(
+        const TMetricsAndTimeLeftHistory& metrics
+    ) {
+        MetricsAndTimeHistory = &metrics;
+        return *this;
+    }
+
     void TCoreModelToFullModelConverter::Do(
         bool requiresStaticCtrProvider,
         TFullModel* dstModel,
@@ -561,6 +568,15 @@ namespace NCB {
             dstModel->ModelInfo["class_params"] = ClassificationTargetHelper.Serialize();
         }
 
+        {
+            NJson::TJsonValue trainingJson(NJson::EJsonValueType::JSON_MAP);
+            if (MetricsAndTimeHistory) {
+                trainingJson["metrics"] = MetricsAndTimeHistory->SaveMetrics();
+            }
+            dstModel->ModelInfo["training"] = WriteTJsonValue(trainingJson);
+        }
+
+
         if (
             FinalFeatureCalcerComputationMode == EFinalFeatureCalcersComputationMode::Default &&
             !dstModel->ModelTrees->GetEstimatedFeatures().empty()

catboost/private/libs/algo/full_model_saver.h

@@ -4,6 +4,7 @@
 #include "projection.h"
 
 #include <catboost/libs/data/data_provider.h>
+#include <catboost/libs/loggers/catboost_logger_helpers.h>
 #include <catboost/libs/model/fwd.h>
 #include <catboost/libs/model/ctr_data.h>
 #include <catboost/libs/model/online_ctr.h>
@@ -24,6 +25,7 @@
 
 
 struct TDatasetDataForFinalCtrs;
+struct TMetricsAndTimeLeftHistory;
 
 namespace NCatboostOptions {
     class TCatBoostOptions;
@@ -79,6 +81,8 @@ namespace NCB {
             TFeatureEstimatorsPtr featureEstimators
         );
 
+        TCoreModelToFullModelConverter& WithMetrics(const TMetricsAndTimeLeftHistory& metrics);
+
         void Do(
             bool requiresStaticCtrProvider,
             TFullModel* dstModel,
@@ -127,6 +131,7 @@ namespace NCB {
 
         TFullModel* CoreModel = nullptr;
         const NCB::TPerfectHashedToHashedCatValuesMap* PerfectHashedToHashedCatValuesMap = nullptr;
+        const TMetricsAndTimeLeftHistory* MetricsAndTimeHistory = nullptr;
         TFeatureEstimatorsPtr FeatureEstimators = nullptr;
 
         TGetBinarizedDataFunc GetBinarizedDataFunc;

catboost/pytest/lib/common_helpers.py

@@ -29,6 +29,7 @@
     'permute_dataset_columns',
     'remove_time_from_json',
     'test_output_path',
+    'compare_with_limited_precision'
 ]
 
 try:
@@ -425,3 +426,34 @@ def get_limited_precision_numpy_diff_tool(rtol=None, atol=None):
     if atol is not None:
         diff_tool += ['--atol', str(atol)]
     return diff_tool
+
+
+# arguments can be JSON-like simple data structures
+def compare_with_limited_precision(lhs, rhs, rtol=1e-6, atol=1e-8):
+    if isinstance(lhs, dict):
+        if not isinstance(rhs, dict):
+            return False
+        if len(lhs) != len(rhs):
+            return False
+        for k in lhs.keys():
+            if k not in rhs:
+                return False
+            if not compare_with_limited_precision(lhs[k], rhs[k], rtol, atol):
+                return False
+        return True
+    elif isinstance(lhs, list):
+        if not isinstance(rhs, list):
+            return False
+        if len(lhs) != len(rhs):
+            return False
+        return all((compare_with_limited_precision(lhs[i], rhs[i], rtol, atol) for i in range(len(lhs))))
+    elif isinstance(lhs, np.ndarray):
+        if not isinstance(rhs, np.ndarray):
+            return False
+        return np.allclose(lhs, rhs, rtol=rtol, atol=atol, equal_nan=True)
+    elif isinstance(lhs, (float, np.floating)):
+        if not isinstance(rhs, (float, np.floating)):
+            return False
+        return abs(lhs - rhs) <= atol + rtol * abs(rhs)
+    else:
+        return lhs == rhs

catboost/python-package/catboost/_catboost.pyx

@@ -1032,6 +1032,7 @@ cdef extern from "catboost/python-package/catboost/helpers.h":
         const TVector[TString]& sampleIdsVector,
         TVector[TArrayRef[float]]* numFeaturesColumns
     ) except +ProcessException
+    TMetricsAndTimeLeftHistory GetTrainingMetrics(const TFullModel& model) except +ProcessException
 
 
 cdef extern from "catboost/python-package/catboost/helpers.h":
@@ -5198,13 +5199,15 @@ cdef class _CatBoost:
         tmp_model.Load(wrapper.Get())
         self.model_blob = None
         self.__model.Swap(tmp_model)
+        self.__metrics_history = GetTrainingMetrics(self.__model[0])
 
     cpdef _load_model(self, model_file, format):
         cdef TFullModel tmp_model
         cdef EModelType modelType = string_to_model_type(format)
         tmp_model = ReadModel(to_arcadia_string(fspath(model_file)), modelType)
         self.model_blob = None
         self.__model.Swap(tmp_model)
+        self.__metrics_history = GetTrainingMetrics(self.__model[0])
 
     cpdef _save_model(self, output_file, format, export_parameters, _PoolBase pool):
         cdef EModelType modelType = string_to_model_type(format)
@@ -5238,6 +5241,7 @@ cdef class _CatBoost:
         self.model_blob = serialized_model_str
         cdef TFullModel tmp_model = ReadZeroCopyModel(<char*>serialized_model_str, len(serialized_model_str))
         self.__model.Swap(tmp_model)
+        self.__metrics_history = GetTrainingMetrics(self.__model[0])
 
     cpdef _get_params(self):
         try:

catboost/python-package/catboost/helpers.cpp

@@ -14,6 +14,8 @@
 #include <catboost/private/libs/options/split_params.h>
 #include <catboost/private/libs/target/data_providers.h>
 
+#include <library/cpp/json/json_reader.h>
+
 #include <util/system/guard.h>
 #include <util/system/info.h>
 #include <util/system/mutex.h>
@@ -377,3 +379,16 @@ void GetNumFeatureValuesSample(
         Copy(values.begin(), values.end(), dst.begin());
     }
 }
+
+TMetricsAndTimeLeftHistory GetTrainingMetrics(const TFullModel& model) {
+    if (model.ModelInfo.contains("training"sv)) {
+        NJson::TJsonValue trainingJson;
+        ReadJsonTree(model.ModelInfo.at("training"sv), &trainingJson, /*throwOnError*/ true);
+        const auto& trainingMap = trainingJson.GetMap();
+        if (trainingMap.contains("metrics"sv)) {
+            return TMetricsAndTimeLeftHistory::LoadMetrics(trainingMap.at("metrics"sv));
+        }
+    }
+
+    return TMetricsAndTimeLeftHistory();
+}

catboost/python-package/catboost/helpers.h

@@ -13,6 +13,7 @@
 #include <catboost/private/libs/options/loss_description.h>
 #include <catboost/private/libs/options/plain_options_helper.h>
 #include <catboost/private/libs/target/data_providers.h>
+#include <catboost/libs/loggers/catboost_logger_helpers.h>
 #include <catboost/libs/train_lib/options_helper.h>
 
 #include <library/cpp/json/json_value.h>
@@ -377,3 +378,6 @@ void GetNumFeatureValuesSample(
     const TVector<TString>& sampleIdsVector,
     TVector<TArrayRef<float>>* numFeaturesColumns
 );
+
+
+TMetricsAndTimeLeftHistory GetTrainingMetrics(const TFullModel& model);