PUBDEV-7831: Add new learning curve plotting function/method

h2o-algos/src/main/java/hex/ensemble/Metalearners.java

@@ -209,6 +209,9 @@ protected void setCustomParams(GLMParameters parms) {
             //add GLM custom params
             super.setCustomParams(parms);
 
+            parms._generate_scoring_history = true;
+            parms._score_iteration_interval = (parms._valid == null) ? 5 : -1;
+
             //specific to AUTO mode
             parms._non_negative = true;
             //parms._alpha = new double[] {0.0, 0.25, 0.5, 0.75, 1.0};

h2o-bindings/bin/custom/R/gen_gam.py

@@ -61,7 +61,15 @@
       if(!missing(missing_values_handling))
         parms$missing_values_handling <- missing_values_handling
     """,
-
+    module="""
+    .h2o.fill_gam <- function(model, parameters, allparams) {
+        if (is.null(model$scoring_history))
+            model$scoring_history <- model$glm_scoring_history
+        if (is.null(model$model_summary))
+            model$model_summary <- model$glm_model_summary
+        return(model)
+    }
+"""
 )
 
 doc = dict(

h2o-bindings/bin/custom/python/gen_gam.py

@@ -15,6 +15,31 @@ def Lambda(self):
     def Lambda(self, value):
         self._parms["lambda"] = value
 
+    def _additional_used_columns(self, parms):
+        """
+        :return: Gam columns if specified.
+        """
+        return parms["gam_columns"]
+
+    def summary(self):
+        """Print a detailed summary of the model."""
+        model = self._model_json["output"]
+        if "glm_model_summary" in model and model["glm_model_summary"] is not None:
+            return model["glm_model_summary"]
+        print("No model summary for this model")
+
+    def scoring_history(self):
+        """
+        Retrieve Model Score History.
+
+        :returns: The score history as an H2OTwoDimTable or a Pandas DataFrame.
+        """
+        model = self._model_json["output"]
+        if "glm_scoring_history" in model and model["glm_scoring_history"] is not None:
+            return model["glm_scoring_history"].as_data_frame()
+        print("No score history for this model")
+
+
 extensions = dict(
     __imports__="""
 import h2o

h2o-core/src/main/java/hex/Model.java

@@ -928,6 +928,7 @@ public String[] features() {
      * User-facing model scoring history - 2D table with modeling accuracy as a function of time/trees/epochs/iterations, etc.
      */
     public TwoDimTable _scoring_history;
+    public TwoDimTable[] _cv_scoring_history;
 
     public double[] _distribution;
     public double[] _modelClassDist;

h2o-core/src/main/java/hex/ModelBuilder.java

@@ -936,6 +936,32 @@ public void cv_mainModelScores(int N, ModelMetrics.MetricBuilder mbs[], ModelBui
     Log.info(mainModel._output._cross_validation_metrics.toString());
     mainModel._output._cross_validation_metrics_summary = makeCrossValidationSummaryTable(cvModKeys);
 
+    // Put cross-validation scoring history to the main model
+    if (mainModel._output._scoring_history != null) { // check if scoring history is supported (e.g., NaiveBayes doesn't)
+      mainModel._output._cv_scoring_history = new TwoDimTable[cvModKeys.length];
+      for (int i = 0; i < cvModKeys.length; i++) {
+        TwoDimTable sh = cvModKeys[i].get()._output._scoring_history;
+        String[] rowHeaders = sh.getRowHeaders();
+        String[] colTypes = sh.getColTypes();
+        int tableSize = rowHeaders.length;
+        int colSize = colTypes.length;
+        TwoDimTable copiedScoringHistory = new TwoDimTable(
+                sh.getTableHeader(),
+                sh.getTableDescription(),
+                sh.getRowHeaders(),
+                sh.getColHeaders(),
+                sh.getColTypes(),
+                sh.getColFormats(),
+                sh.getColHeaderForRowHeaders());
+        for (int rowIndex = 0; rowIndex < tableSize; rowIndex++)  {
+          for (int colIndex = 0; colIndex < colSize; colIndex++) {
+            copiedScoringHistory.set(rowIndex, colIndex,sh.get(rowIndex, colIndex));
+          }
+        }
+        mainModel._output._cv_scoring_history[i] = copiedScoringHistory;
+      }
+    }
+
     if (!_parms._keep_cross_validation_models) {
       int count = Model.deleteAll(cvModKeys);
       Log.info(count+" CV models were removed");

h2o-core/src/main/java/water/api/schemas3/ModelOutputSchemaV3.java

@@ -50,6 +50,9 @@ public class ModelOutputSchemaV3<O extends Model.Output, S extends ModelOutputSc
   @API(help="Scoring history", direction=API.Direction.OUTPUT, level=API.Level.secondary)
   public TwoDimTableV3 scoring_history;
 
+  @API(help="Cross-Validation scoring history", direction=API.Direction.OUTPUT, level=API.Level.secondary)
+  public TwoDimTableV3 cv_scoring_history[];
+
   @API(help="Model reproducibility information", direction=API.Direction.OUTPUT, level=API.Level.secondary)
   public TwoDimTableV3[] reproducibility_information_table;
 

h2o-py/h2o/estimators/gam.py

@@ -1078,3 +1078,27 @@ def Lambda(self):
     @Lambda.setter
     def Lambda(self, value):
         self._parms["lambda"] = value
+
+    def _additional_used_columns(self, parms):
+        """
+        :return: Gam columns if specified.
+        """
+        return parms["gam_columns"]
+
+    def summary(self):
+        """Print a detailed summary of the model."""
+        model = self._model_json["output"]
+        if "glm_model_summary" in model and model["glm_model_summary"] is not None:
+            return model["glm_model_summary"]
+        print("No model summary for this model")
+
+    def scoring_history(self):
+        """
+        Retrieve Model Score History.
+
+        :returns: The score history as an H2OTwoDimTable or a Pandas DataFrame.
+        """
+        model = self._model_json["output"]
+        if "glm_scoring_history" in model and model["glm_scoring_history"] is not None:
+            return model["glm_scoring_history"].as_data_frame()
+        print("No score history for this model")

h2o-py/h2o/explanation/__init__.py

@@ -15,6 +15,7 @@ def _register_dummy_methods():
     h2o.model.ModelBase.explain_row = _complain_about_matplotlib
     h2o.model.ModelBase.pd_plot = _complain_about_matplotlib
     h2o.model.ModelBase.ice_plot = _complain_about_matplotlib
+    h2o.model.ModelBase.learning_curve_plot = _complain_about_matplotlib
 
     h2o.automl._base.H2OAutoMLBaseMixin.pd_multi_plot = _complain_about_matplotlib
     h2o.automl._base.H2OAutoMLBaseMixin.varimp_heatmap = _complain_about_matplotlib
@@ -27,7 +28,7 @@ def _register_dummy_methods():
     import numpy
     import matplotlib
     from ._explain import varimp_heatmap, model_correlation_heatmap, shap_explain_row_plot, shap_summary_plot,\
-        explain, explain_row, pd_plot, pd_multi_plot, ice_plot, residual_analysis_plot
+        explain, explain_row, pd_plot, pd_multi_plot, ice_plot, residual_analysis_plot, learning_curve_plot
 
     __all__ = [
         "explain",
@@ -52,6 +53,7 @@ def register_explain_methods():
     h2o.model.ModelBase.explain_row = explain_row
     h2o.model.ModelBase.pd_plot = pd_plot
     h2o.model.ModelBase.ice_plot = ice_plot
+    h2o.model.ModelBase.learning_curve_plot = learning_curve_plot
 
     h2o.automl._base.H2OAutoMLBaseMixin.pd_multi_plot = pd_multi_plot
     h2o.automl._base.H2OAutoMLBaseMixin.varimp_heatmap = varimp_heatmap