add comment to atomic checkpoint copying process

ludwig-ai · Mar 25, 2021 · c924b9f · c924b9f
1 parent 7c244df
commit c924b9f
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/ludwig/hyperopt/execution.py b/ludwig/hyperopt/execution.py
@@ -723,6 +723,9 @@ def on_epoch_end(self, trainer, progress_tracker, save_path):
                 if trainer.is_coordinator():
                     with tune.checkpoint_dir(step=progress_tracker.epoch) as checkpoint_dir:
                         checkpoint_model = os.path.join(checkpoint_dir, 'model')
+                        # shutil.copytree(save_path, checkpoint_model)
+                        # Note: A previous implementation used shutil.copytree() 
+                        # however, this copying method is non atomic
                         if not os.path.isdir(checkpoint_model):
                             copy_id = uuid.uuid4()
                             tmp_dst = "%s.%s.tmp" % (checkpoint_model, copy_id)