From edbd884f1d68c0418aca6073147aac90b4f1bc56 Mon Sep 17 00:00:00 2001 From: krasinski <8573352+krasinski@users.noreply.github.com> Date: Mon, 8 Apr 2024 12:28:07 +0200 Subject: [PATCH] [GH-5720] AIC/Loglikelihood test fixes (#5723) * fix metrics generation (#5718) (cherry picked from commit 3af804b230bd488ec827dde89b54c044cddfc4a7) * [GH-5720] Add AIC/Loglikelihood metrics to MetricsTestSuite (#5721) * fix metrics generation (#5718) (cherry picked from commit 3af804b230bd488ec827dde89b54c044cddfc4a7) * [GH-5720] Add AIC/Loglikelihood metrics to MetricsTestSuite (cherry picked from commit b3bef8b46c131470cedb99baed8e46abbdec59a6) * [GH-5720] fix H2oGridSearchTestSuite NaN != NaN * [GH-5720] fix H2oGridSearchTestSuite NaN != NaN * aic/loglikelihood optional to be backwards compatible * fix metrics in test_gridsearch.py * fix test_mojo.py nan == nan --- .../api/generation/common/MetricFieldExceptions.scala | 3 ++- .../h2o/sparkling/ml/algos/H2OGridSearchTestSuite.scala | 8 +++++--- py/tests/unit/with_runtime_sparkling/test_gridsearch.py | 2 +- py/tests/unit/with_runtime_sparkling/test_mojo.py | 6 ++++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/api-generation/src/main/scala/ai/h2o/sparkling/api/generation/common/MetricFieldExceptions.scala b/api-generation/src/main/scala/ai/h2o/sparkling/api/generation/common/MetricFieldExceptions.scala index 92f15896a2..695a50f84a 100644 --- a/api-generation/src/main/scala/ai/h2o/sparkling/api/generation/common/MetricFieldExceptions.scala +++ b/api-generation/src/main/scala/ai/h2o/sparkling/api/generation/common/MetricFieldExceptions.scala @@ -21,5 +21,6 @@ object MetricFieldExceptions { def ignored(): Set[String] = Set("__meta", "domain", "model", "model_checksum", "frame", "frame_checksum", "model_category", "predictions") - def optional(): Set[String] = Set("custom_metric_name", "custom_metric_value", "mean_score", "mean_normalized_score") + def optional(): Set[String] = + Set("custom_metric_name", "custom_metric_value", "mean_score", "mean_normalized_score", "AIC", "loglikelihood") } diff --git a/ml/src/test/scala/ai/h2o/sparkling/ml/algos/H2OGridSearchTestSuite.scala b/ml/src/test/scala/ai/h2o/sparkling/ml/algos/H2OGridSearchTestSuite.scala index a169eddb5e..f34bfd0c8a 100644 --- a/ml/src/test/scala/ai/h2o/sparkling/ml/algos/H2OGridSearchTestSuite.scala +++ b/ml/src/test/scala/ai/h2o/sparkling/ml/algos/H2OGridSearchTestSuite.scala @@ -23,10 +23,9 @@ import ai.h2o.sparkling.{SharedH2OTestContext, TestUtils} import hex.Model import hex.tree.gbm.GBMModel.GBMParameters import org.apache.spark.sql.functions._ -import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.{DataFrame, Row, SparkSession} import org.apache.spark.ml.param.{ParamMap, Params} import org.apache.spark.ml.{Pipeline, PipelineModel} -import org.apache.spark.sql.SparkSession import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner import org.scalatest.{FunSuite, Matchers} @@ -268,6 +267,8 @@ class H2OGridSearchTestSuite extends FunSuite with Matchers with SharedH2OTestCo } test("The first row returned by getGridModelsMetrics() method is the same as current metrics of the best model") { + import spark.implicits._ + val drf = new H2ODRF() .setFeaturesCols(Array("AGE", "RACE", "DPROS", "DCAPS", "PSA", "VOL", "GLEASON")) .setLabelCol("CAPSULE") @@ -288,7 +289,8 @@ class H2OGridSearchTestSuite extends FunSuite with Matchers with SharedH2OTestCo val gridModelMetrics = search.getGridModelsMetrics().drop("MOJO Model ID") val modelMetricsFromGrid = gridModelMetrics.columns.zip(gridModelMetrics.head().toSeq).toMap - modelMetricsFromGrid shouldEqual expectedMetrics + modelMetricsFromGrid.filter(!_._2.asInstanceOf[Double].isNaN) should contain theSameElementsAs expectedMetrics + .filter(!_._2.isNaN) } test("The first row returned by getGridModelsParams() method is the same as training params of the best model") { diff --git a/py/tests/unit/with_runtime_sparkling/test_gridsearch.py b/py/tests/unit/with_runtime_sparkling/test_gridsearch.py index 9ce2bcb1f8..62c6ce79ad 100644 --- a/py/tests/unit/with_runtime_sparkling/test_gridsearch.py +++ b/py/tests/unit/with_runtime_sparkling/test_gridsearch.py @@ -109,7 +109,7 @@ def testGetGridModelsMetrics(prostateDataset): grid.fit(prostateDataset) metrics = grid.getGridModelsMetrics() assert metrics.count() == 3 - expectedCols = ['MOJO Model ID', 'RMSLE', 'Nobs', 'RMSE', 'MAE', 'MeanResidualDeviance', 'ScoringTime', 'MSE', 'R2'] + expectedCols = ['MOJO Model ID', 'RMSLE', 'Nobs', 'RMSE', 'MAE', 'MeanResidualDeviance', 'ScoringTime', "Loglikelihood", 'MSE', 'R2', 'AIC'] assert metrics.columns == expectedCols metrics.collect() # try materializing diff --git a/py/tests/unit/with_runtime_sparkling/test_mojo.py b/py/tests/unit/with_runtime_sparkling/test_mojo.py index 78c328cb39..fe22a04bb7 100644 --- a/py/tests/unit/with_runtime_sparkling/test_mojo.py +++ b/py/tests/unit/with_runtime_sparkling/test_mojo.py @@ -19,6 +19,7 @@ import shutil import unit_test_utils import os +import math from pyspark.mllib.linalg import * from pyspark.sql.types import * @@ -63,7 +64,7 @@ def testModelCategory(gbmModel): def testTrainingMetrics(gbmModel): metrics = gbmModel.getTrainingMetrics() assert metrics is not None - assert len(metrics) is 10 + assert len(metrics) is 12 def testFeatureTypes(gbmModel): @@ -245,7 +246,8 @@ def compareMetricValues(metricsObject, metricsMap): for metric in metricsMap: metricValue = metricsMap[metric] objectValue = getattr(metricsObject, "get" + metric)() - assert(metricValue == objectValue) + if not math.isnan(metricValue) and not math.isnan(objectValue): + assert(metricValue == objectValue) assert metricsObject.getConfusionMatrix().count() > 0 assert len(metricsObject.getConfusionMatrix().columns) > 0 assert metricsObject.getGainsLiftTable().count() > 0