diff --git a/h2o-automl/src/main/java/ai/h2o/automl/leaderboard/Leaderboard.java b/h2o-automl/src/main/java/ai/h2o/automl/leaderboard/Leaderboard.java
index 0336f21d8a72..724d7254ed30 100644
--- a/h2o-automl/src/main/java/ai/h2o/automl/leaderboard/Leaderboard.java
+++ b/h2o-automl/src/main/java/ai/h2o/automl/leaderboard/Leaderboard.java
@@ -356,18 +356,26 @@ public void addModels(final Key<Model>[] modelKeys) {
 
     final List<ModelMetrics> modelMetrics = new ArrayList<>();
     final Map<Key<Model>, LeaderboardCell[]> extensions = new HashMap<>();
-
+    final List<Key<Model>> badKeys = new ArrayList<>();
     for (Key<Model> modelKey : allModelKeys) {  // fully rebuilding modelMetrics, so we loop through all keys, not only new ones
       Model model = modelKey.get();
       if (model == null) {
-        eventLog().warn(Stage.ModelTraining, "Model in the leaderboard has unexpectedly been deleted from H2O: " + modelKey);
+        badKeys.add(modelKey);
+        eventLog().warn(Stage.ModelTraining, "Model `"+modelKey+"` has unexpectedly been deleted from H2O: ignoring the model and/or removing it from the leaderboard.");
         continue;
       }
 
       if (_extensionsProvider != null) {
         extensions.put(modelKey, _extensionsProvider.createExtensions(model));
       }
-      modelMetrics.add(getOrCreateModelMetrics(modelKey, extensions));
+      ModelMetrics mm = getOrCreateModelMetrics(modelKey, extensions);
+      assert mm != null: "Missing metrics for model "+modelKey;
+      if (mm == null) {
+        badKeys.add(modelKey);
+        eventLog().warn(Stage.ModelTraining, "Metrics for model `"+modelKey+"` are missing: ignoring the model and/or removing it from the leaderboard.");
+        continue;
+      }
+      modelMetrics.add(mm);
     }
 
     if (_metrics == null) {
@@ -375,6 +383,12 @@ public void addModels(final Key<Model>[] modelKeys) {
       setDefaultMetrics(modelKeys[0].get());
     }
 
+    for (Key<Model> key : badKeys) {
+      // keep everything clean for the update
+      allModelKeys.remove(key);
+      extensions.remove(key);
+    }
+
     atomicUpdate(() -> {
       _leaderboard_model_metrics.clear();
       modelMetrics.forEach(this::addModelMetrics);