Mark end of process in sessions and exit benignly

digideep/pipeline/runner.py

@@ -269,6 +269,7 @@ def train(self):
             logger.fatal('Operation stopped by the user ...')
         finally:
             logger.fatal('End of operation ...')
+            self.finalize()
 
     def test(self):
         # Make the states of the two explorers train/test exactly the same, for the states of the environments.
@@ -311,6 +312,15 @@ def enjoy(self): #i.e. eval
         finally:
             logger.fatal('End of operation ...')
 
+    def finalize(self):
+        # Mark session as done if we have went through all epochs.
+        if self.state["i_epoch"] == self.params["runner"]["n_epochs"]:
+            self.session.mark_as_done()
+
+        # Close all explorers benignly:
+        for key in self.explorer:
+            self.explorer[key].close()
+
     #####################
     ## Logging Summary ##
     #####################

digideep/pipeline/session.py

@@ -278,6 +278,10 @@ def set_device(self):
         # self.device = torch.device("cpu")
     def get_device(self):
         return self.device
+
+    def mark_as_done(self):
+        with open(os.path.join(self.state['path_session'], 'done.lock'), 'w') as f:
+            print("", file=f)
 
     #################################
     # Apparatus for model save/load #