Expose Scoring History and Variable Importances on H2OMOJOModel

[exalate-issue-sync]

Current workaround:

{code:scala} val model = algo.fit(dataset)

import com.google.gson._
import hex.genmodel.attributes.ModelJsonReader
import hex.genmodel.attributes.Table
import hex.genmodel.attributes.Table.ColumnType
import org.apache.spark.sql.types._
import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
import collection.JavaConverters._
import org.apache.spark.sql.{DataFrame, Row}

def tableToDataFrame(table: Table): DataFrame = {
  val columnTypes = table.getColTypes.map {
    case ColumnType.LONG => LongType
    case ColumnType.INT => IntegerType
    case ColumnType.DOUBLE => DoubleType
    case ColumnType.FLOAT => FloatType
    case ColumnType.STRING => StringType
  }
  val columns = table.getColHeaders.zip(columnTypes).map {
    case (columnName, columnType) => StructField(columnName, columnType, nullable = true)
  }
  val schema = StructType(columns)
  val rows = (0 until table.rows()).map { rowId =>
    val rowData = (0 until table.columns()).map(colId => table.getCell(colId, rowId)).toArray[Any]
    val row: Row = new GenericRowWithSchema(rowData, schema)
    row
  }.asJava
  spark.createDataFrame(rows, schema)
}

// Get JSON string describing the model
val json = model.getModelDetails()
val modelDetails = new GsonBuilder().create().fromJson(json, classOf[JsonObject])

// Extract scoring history
println("Scoring history:")
val scoringHistoryTable = ModelJsonReader.readTable(modelDetails, "scoring_history")
val scoringHistoryDF = tableToDataFrame(scoringHistoryTable)
scoringHistoryDF.show(numRows = 20, truncate = false) // Increase numRows to see the full result

// Extract variable importances
println("Variable importances:")
val variableImportancesTable = ModelJsonReader.readTable(modelDetails, "variable_importances")
val variableImportancesDF = tableToDataFrame(variableImportancesTable)
variableImportancesDF.show(truncate = false)

{code}

[DinukaH2O]

JIRA Issue Migration Info

Jira Issue: SW-2559
Assignee: Marek Novotny
Reporter: Marek Novotny
State: Resolved
Fix Version: 3.32.1.3-1
Attachments: N/A
Development PRs: Available

Linked PRs from JIRA

#2525

[hasithjp]

JIRA Issue Migration Info Cont'd

Jira Issue Created Date: 2021-05-04T08:48:14.346-0700