fixing issues in multinode + client mode (still issues with automl re…

…moval)
h2oai · Oct 3, 2019 · 8a9a9f4 · 8a9a9f4
1 parent 22d3078
commit 8a9a9f4
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 6 deletions.
diff --git a/h2o-automl/src/main/java/ai/h2o/automl/AutoML.java b/h2o-automl/src/main/java/ai/h2o/automl/AutoML.java
@@ -74,6 +74,12 @@ public static AutoML startAutoML(AutoMLBuildSpec buildSpec) {
       lastStartTime = startTime;
     }
 
+    // TODO: problems:
+    //   if same project_name but change validation_frame or leaderboard_frame, then the existing models from previous run are scored with a different metric.
+    //   - for change of leaderboard_frame, this is currently handled in Leaderboard.getOrMake
+    //   - where to handle change of validation_frame
+    //   - unfortunately, this can't be handled on client side
+
     // if user offers a different response column,
     //   the new models will be added to a new Leaderboard, without removing the previous one.
     // otherwise, the new models will be added to the existing leaderboard.

diff --git a/h2o-automl/src/main/java/ai/h2o/automl/Leaderboard.java b/h2o-automl/src/main/java/ai/h2o/automl/Leaderboard.java
@@ -43,7 +43,7 @@ public class Leaderboard extends Lockable<Leaderboard> {
    * <p>
    * Updated inside addModels().
    */
-  private final transient IcedHashMap<Key<ModelMetrics>, ModelMetrics> _leaderboard_model_metrics_cache = new IcedHashMap<>();
+  private final IcedHashMap<Key<ModelMetrics>, ModelMetrics> _leaderboard_model_metrics_cache = new IcedHashMap<>();
 
   /**
    * Map providing for a given metric name, the list of metric values in the same order as the models

diff --git a/h2o-r/tests/testdir_algos/automl/runit_automl_memory_cleanup.R b/h2o-r/tests/testdir_algos/automl/runit_automl_memory_cleanup.R
@@ -24,11 +24,11 @@ automl.cleanup.suite <- function() {
   test_remove_automl_instance <- function() {
     ds <- import_dataset()
     aml <- h2o.automl(project_name="test_remove_R_automl_instance",
-    x=ds$x, y=ds$y,
-    training_frame=ds$train,
-    validation_frame=ds$valid,
-    leaderboard_frame=ds$test,
-    max_models=max_models)
+                      x=ds$x, y=ds$y,
+                      training_frame=ds$train,
+                      validation_frame=ds$valid,
+                      leaderboard_frame=ds$test,
+                      max_models=max_models)
 
     keys <- h2o.ls()$key
     expect_gt(length(grep("_AutoML_", keys)), nrow(aml@leaderboard))