PUBDEV-5975: Proposal for a consistent behaviour of AutoML reruns

h2o-automl/src/main/java/ai/h2o/automl/AutoML.java

@@ -32,6 +32,9 @@
  */
 public final class AutoML extends Lockable<AutoML> implements TimedH2ORunnable {
 
+  public static final Comparator<AutoML> byStartTime = Comparator.comparing(a -> a._startTime);
+  public static final String keySeparator = "@@";
+
   private final static boolean verifyImmutability = true; // check that trainingFrame hasn't been messed with
   private final static SimpleDateFormat timestampFormatForKeys = new SimpleDateFormat("yyyyMMdd_HHmmss");
 
@@ -71,7 +74,11 @@ public static AutoML startAutoML(AutoMLBuildSpec buildSpec) {
       lastStartTime = startTime;
     }
 
-    AutoML aml = makeAutoML(Key.make(buildSpec.project()), startTime, buildSpec);
+    // if user offers a different response column,
+    //   the new models will be added to a new Leaderboard, without removing the previous one.
+    // otherwise, the new models will be added to the existing leaderboard.
+    Key<AutoML> key = Key.make(buildSpec.project()+keySeparator+buildSpec.input_spec.response_column);
+    AutoML aml = new AutoML(key, startTime, buildSpec);
     startAutoML(aml);
     return aml;
   }
@@ -86,6 +93,8 @@ public static void startAutoML(AutoML aml) {
     if (aml._job == null || !aml._job.isRunning()) {
       H2OJob<AutoML> j = new H2OJob<>(aml, aml._key, aml._runCountdown.remainingTime());
       aml._job = j._job;
+      aml.eventLog().info(Stage.Workflow, "AutoML job created: " + EventLogEntry.dateTimeFormat.format(aml._startTime))
+              .setNamedValue("creation_epoch", aml._startTime, EventLogEntry.epochFormat);
       j.start(aml._workAllocations.remainingWork());
       DKV.put(aml);
     }
@@ -120,10 +129,10 @@ public Class<AutoMLV99.AutoMLKeyV3> makeSchema() {
 
   public StepDefinition[] getActualModelingSteps() { return _actualModelingSteps; }
 
-  Frame _trainingFrame;    // required training frame: can add and remove Vecs, but not mutate Vec data in place
-  Frame _validationFrame;  // optional validation frame; the training_frame is split automagically if it's not specified
-  Frame _blendingFrame;
-  Frame _leaderboardFrame; // optional test frame used for leaderboard scoring; if not specified, leaderboard will use xval metrics
+  Frame _trainingFrame;    // required training frame: can add and remove Vecs, but not mutate Vec data in place.
+  Frame _validationFrame;  // optional validation frame; the training_frame is split automatically if it's not specified.
+  Frame _blendingFrame;    // optional blending frame for SE (usually if xval is disabled).
+  Frame _leaderboardFrame; // optional test frame used for leaderboard scoring; if not specified, leaderboard will use xval metrics.
 
   Vec _responseColumn;
   Vec _foldColumn;
@@ -153,18 +162,16 @@ public AutoML() {
   }
 
   public AutoML(Key<AutoML> key, Date startTime, AutoMLBuildSpec buildSpec) {
-    super(key);
+    super(key == null ? Key.make(buildSpec.project()) : key);
     _startTime = startTime;
     _buildSpec = buildSpec;
     _runCountdown = Countdown.fromSeconds(buildSpec.build_control.stopping_criteria.max_runtime_secs());
     _modelingPlan = buildSpec.build_models.modeling_plan == null ? defaultModelingPlan : buildSpec.build_models.modeling_plan;
     _modelingStepsRegistry = new ModelingStepsRegistry();
 
     try {
-      _eventLog = EventLog.make(_key);
+      _eventLog = EventLog.getOrMake(_key);
       eventLog().info(Stage.Workflow, "Project: " + projectName());
-      eventLog().info(Stage.Workflow, "AutoML job created: " + EventLogEntry.dateTimeFormat.format(_startTime))
-              .setNamedValue("creation_epoch", _startTime, EventLogEntry.epochFormat);
 
       handleCVParameters(buildSpec);
 
@@ -187,7 +194,7 @@ public AutoML(Key<AutoML> key, Date startTime, AutoMLBuildSpec buildSpec) {
 
   private void initLeaderboard(AutoMLBuildSpec buildSpec) {
     String sort_metric = buildSpec.input_spec.sort_metric == null ? null : buildSpec.input_spec.sort_metric.toLowerCase();
-    _leaderboard = Leaderboard.getOrMake(projectName(), _eventLog, _leaderboardFrame, sort_metric);
+    _leaderboard = Leaderboard.getOrMake(_key.toString(), _eventLog, _leaderboardFrame, sort_metric);
   }
 
   private void handleReproducibilityParameters(AutoMLBuildSpec buildSpec) {

h2o-automl/src/main/java/ai/h2o/automl/AutoMLBuildSpec.java

@@ -4,13 +4,20 @@
 import hex.grid.HyperSpaceSearchCriteria.RandomDiscreteValueSearchCriteria;
 import water.Iced;
 import water.Key;
+import water.api.schemas3.JobV3;
 import water.fvec.Frame;
 
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
 /**
  * Parameters which specify the build (or extension) of an AutoML build job.
  */
 public class AutoMLBuildSpec extends Iced {
 
+  private static final DateFormat projectTimeStampFormat = new SimpleDateFormat("yyyyMMdd_HmmssSSS");
+
   /**
    * Default constructor provides the default behavior.
    */
@@ -173,17 +180,13 @@ public static final class AutoMLBuildModels extends Iced {
   public AutoMLInput input_spec;
   public AutoMLBuildSpec.AutoMLBuildModels build_models;
 
-  private transient String project_cached = null;
-  public String project() {
-    if (null != project_cached)
-      return project_cached;
+  // output
+  public JobV3 job;
 
-    // allow the user to override:
-    if (null != build_control.project_name) {
-      project_cached = build_control.project_name;
-      return project_cached;
+  public String project() {
+    if (build_control.project_name == null) {
+      build_control.project_name = "AutoML_"+ projectTimeStampFormat.format(new Date());
     }
-    project_cached = "automl_"+input_spec.training_frame+"_"+input_spec.response_column;
-    return project_cached;
+    return build_control.project_name;
   }
 }

h2o-automl/src/main/java/ai/h2o/automl/EventLog.java

@@ -18,11 +18,11 @@
  */
 public class EventLog extends Keyed<EventLog> {
 
-  public Key<AutoML> _automlKey;
+  public final Key<AutoML> _automl_id;
   public EventLogEntry[] _events;
 
   public EventLog(Key<AutoML> automlKey) {
-    _automlKey = automlKey;
+    _automl_id = automlKey;
     _key = Key.make(idForRun(automlKey));
     _events = new EventLogEntry[0];
   }
@@ -36,12 +36,6 @@ static EventLog getOrMake(Key<AutoML> runKey) {
     return eventLog;
   }
 
-  static EventLog make(Key<AutoML> runKey) {
-    EventLog eventLog = new EventLog(runKey);
-    DKV.put(eventLog);
-    return eventLog;
-  }
-
   private static String idForRun(Key<AutoML> runKey) {
     if (null == runKey)
       return "AutoML_Events_dummy";
@@ -68,7 +62,7 @@ public <V extends Serializable> EventLogEntry<V> warn(Stage stage, String messag
 
   /** Add a EventLogEntry, but don't log. */
   public <V extends Serializable> EventLogEntry<V> addEvent(Level level, Stage stage, String message) {
-    EventLogEntry<V> entry = new EventLogEntry<>(_automlKey, level, stage, message);
+    EventLogEntry<V> entry = new EventLogEntry<>(_automl_id, level, stage, message);
     addEvent(entry);
     return entry;
   }
@@ -91,7 +85,7 @@ protected Futures remove_impl(Futures fs, boolean cascade) {
   }
 
   public TwoDimTable toTwoDimTable() {
-    String name = _automlKey == null ? "(new)" : _automlKey.toString();
+    String name = _automl_id == null ? "(new)" : _automl_id.toString();
     return toTwoDimTable("Event Log for AutoML:" + name);
   }