debug

Signed-off-by: stevehuang52 <heh@nvidia.com>
NVIDIA · Dec 22, 2023 · f97c901 · f97c901
1 parent 4f947ce
commit f97c901
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/nemo/collections/multimodal/speechllm/models/speechllm_models.py b/nemo/collections/multimodal/speechllm/models/speechllm_models.py
@@ -1092,6 +1092,9 @@ def inference_epoch_end(self, outputs, mode, data_cfg):
         averaged_metric = []
         # Log metrics for each provided validation/test dataset.
         for dataloader_idx, output in enumerate(outputs):
+            if len(output) == 0:
+                logging.warning(f"Empty output for dataloader_idx: {dataloader_idx}")
+                continue
             # Expand on_validation_epoch_end from parent class MegatronGPTModel as on_validation_epoch_end doesnt take outputs arg
             loss_vals = [x['loss'] for x in output]
             if parallel_state.is_pipeline_last_stage():