Merge pull request #759 from ufal/final-saving

Saving of variables upon finish
ufal · Sep 19, 2018 · 98d9db9 · 98d9db9
2 parents aabfc26 + 1baba63
commit 98d9db9
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 2 deletions.
diff --git a/neuralmonkey/experiment.py b/neuralmonkey/experiment.py
@@ -185,7 +185,8 @@ def train(self) -> None:
                 postprocess=self.model.postprocess,
                 train_start_offset=self.model.train_start_offset,
                 runners_batch_size=self.model.runners_batch_size,
-                initial_variables=self.model.initial_variables)
+                initial_variables=self.model.initial_variables,
+                final_variables=self.get_path("variables.data.final"))
 
             self._vars_loaded = True
 

diff --git a/neuralmonkey/learning_utils.py b/neuralmonkey/learning_utils.py
@@ -39,6 +39,7 @@ def training_loop(tf_manager: TensorFlowManager,
                   log_directory: str,
                   evaluators: EvalConfiguration,
                   runners: List[BaseRunner],
+                  final_variables: str,
                   train_dataset: Dataset,
                   val_dataset: Union[Dataset, List[Dataset]],
                   test_datasets: Optional[List[Dataset]] = None,
@@ -286,6 +287,9 @@ def training_loop(tf_manager: TensorFlowManager,
         .format(main_metric, tf_manager.best_score,
                 tf_manager.best_score_epoch))
 
+    log("Saving final variables in {}".format(final_variables))
+    tf_manager.save(final_variables)
+
     if test_datasets:
         tf_manager.restore_best_vars()
 

diff --git a/neuralmonkey/tf_manager.py b/neuralmonkey/tf_manager.py
@@ -88,7 +88,7 @@ def __init__(self,
         init_op = tf.global_variables_initializer()
         for sess in self.sessions:
             sess.run(init_op)
-        self.saver = tf.train.Saver(max_to_keep=self.saver_max_to_keep,
+        self.saver = tf.train.Saver(max_to_keep=None,
                                     var_list=[g for g in tf.global_variables()
                                               if "reward_" not in g.name])