Various fixes & improvements (#408)

* started adding unit tests

* mark module-private parts of persistence module

* finish unit tests for TestPreloader

* fix bug regarding duplicate objects returned by yaml.load()

* check for malformed LoadSerialized

* support placeholder strings in LoadSerialized

* log param count

* mention xnmt_decode in doc

* mention that Serializable must be explicitly specified even if it’s inherited.

* clean up test output

docs/getting_started.rst

@@ -19,12 +19,22 @@ development.
 Command line tools
 ------------------
 
-*xnmt* currently ships with two command line interfaces:
+*xnmt* comes with the following command line interfaces:
 
 * ``xnmt`` runs experiments given a configuration file that can specify preprocessing, model training, and evaluation.
-  The corresponding Python file is xnmt/xnmt_run_experiments.py.
+  The corresponding Python file is ``xnmt/xnmt_run_experiments.py``. Typical example call::
+
+    xnmt --dynet-gpu my-training.yaml
+
+* ``xnmt_decode`` decodes a hypothesis using a specified model. The corresponding Python file is
+  ``xnmt/xnmt_decode.py``. Typical example call::
+
+    xnmt_decode --src src.txt --hyp out.txt --mod saved-model.mod
+
 * ``xnmt_evaluate`` computes an evaluation metric given hypothesis and reference files. The corresponding Python file
-  is xnmt/xnmt_evaluate.py.
+  is ``xnmt/xnmt_evaluate.py``. Typical example call::
+
+    xnmt_evaluate --hyp out.txt --ref ref.txt --metric bleu
 
 
 Running the examples

docs/writing_xnmt_classes.rst

@@ -12,9 +12,10 @@ sharing of DyNet parameters, etc., one must adhere to the Serializable interface
 Marking classes as serializable
 """""""""""""""""""""""""""""""
 
-Classes are marked as serializable by specifying :class:`xnmt.persistence.Serializable` as super class.
-They must specify a unique yaml_tag class attribute, set to ``!ClassName`` with ClassName replaced by the class
-name. It follows that class names must be unique, even across different XNMT modules.
+Classes are marked as serializable by specifying :class:`xnmt.persistence.Serializable` as super class. They must
+specify a unique yaml_tag class attribute, set to ``!ClassName`` with ClassName replaced by the class name. It follows
+that class names must be unique, even across different XNMT modules.
+(Note: Serializable should be explicitly specified even if another super class already does the same)
 
 Specifying init arguments
 """""""""""""""""""""""""

test/config/assemble.yaml

@@ -1,7 +1,4 @@
 exp1-pretrain1: !Experiment
-  exp_global: !ExpGlobal
-    model_file: examples/output/{EXP}.mod
-    log_file: examples/output/{EXP}.log
   model: !DefaultTranslator
     src_reader: !PlainTextReader
       vocab: !Vocab {vocab_file: examples/data/head.ja.vocab}
@@ -32,9 +29,6 @@ exp1-pretrain1: !Experiment
     trg_file: examples/data/head.en
 
 exp2-pretrain2: !Experiment
-  exp_global: !ExpGlobal
-    model_file: examples/output/{EXP}.mod
-    log_file: examples/output/{EXP}.log
   model: !DefaultTranslator
     src_reader: !PlainTextReader
       vocab: !Vocab {vocab_file: examples/data/head.ja.vocab}
@@ -66,16 +60,13 @@ exp2-pretrain2: !Experiment
 
 
 exp3-assemble: !Experiment
-  exp_global: !ExpGlobal
-    model_file: examples/output/{EXP}.mod
-    log_file: examples/output/{EXP}.log
   model: !DefaultTranslator
     src_reader: !PlainTextReader
       vocab: !Vocab {vocab_file: examples/data/head.en.vocab}
     trg_reader: !PlainTextReader
       vocab: !Vocab {vocab_file: examples/data/head.ja.vocab}
     src_embedder: !LoadSerialized
-      filename: examples/output/exp1-pretrain1.mod
+      filename: test/tmp/exp1-pretrain1.mod
       path: 'model.trg_embedder'
     encoder: !BiLSTMSeqTransducer
       layers: 2
@@ -85,7 +76,7 @@ exp3-assemble: !Experiment
       hidden_dim: 64
       input_dim: 64
     trg_embedder: !LoadSerialized
-      filename: examples/output/exp2-pretrain2.mod
+      filename: test/tmp/exp2-pretrain2.mod
       path: 'model.src_embedder'
     decoder: !MlpSoftmaxDecoder
       rnn_layer: !UniLSTMSeqTransducer
@@ -103,9 +94,6 @@ exp3-assemble: !Experiment
 
 
 exp4-assemble: !Experiment
-  exp_global: !ExpGlobal
-    model_file: examples/output/{EXP}.mod
-    log_file: examples/output/{EXP}.log
   model: !DefaultTranslator
     src_reader: !PlainTextReader
       vocab: !Vocab {vocab_file: examples/data/head.en.vocab}
@@ -116,10 +104,10 @@ exp4-assemble: !Experiment
     encoder: !ModularSeqTransducer
       modules:
       - !LoadSerialized
-        filename: examples/output/exp1-pretrain1.mod
+        filename: test/tmp/exp1-pretrain1.mod
         path: model.encoder
       - !LoadSerialized
-        filename: examples/output/exp2-pretrain2.mod
+        filename: test/tmp/exp2-pretrain2.mod
         path: model.encoder
     attender: !MlpAttender
       state_dim: 64

test/config/component_sharing.yaml

@@ -11,8 +11,6 @@
 
 exp1.pretrain: !Experiment
   exp_global: !ExpGlobal
-    model_file: examples/output/{EXP}.mod
-    log_file: examples/output/{EXP}.log
     default_layer_dim: 32
   model: !DefaultTranslator
     src_reader: !PlainTextReader
@@ -51,8 +49,8 @@ exp1.pretrain: !Experiment
       eval_metrics: bleu,wer
       src_file: *dev_src # Copy over the file path from the dev tasks using YAML anchors.
       ref_file: *dev_trg # The same could also be done for more complex objects.
-      hyp_file: examples/output/{EXP}.test_hyp
+      hyp_file: test/tmp/{EXP}.test_hyp
 
 exp2.load: !LoadSerialized
-  filename: examples/output/exp1.pretrain.mod  
+  filename: test/tmp/exp1.pretrain.mod
 

test/config/dev_combinator.yaml

@@ -1,8 +1,8 @@
 # A standard training run, should almost never break
 dev_combinator: !Experiment
   exp_global: !ExpGlobal
-    model_file: examples/output/{EXP}.mod
-    log_file: examples/output/{EXP}.log
+    model_file: test/tmp/{EXP}.mod
+    log_file: test/tmp/{EXP}.log
     default_layer_dim: 64
   model: !DefaultTranslator
     src_reader: !PlainTextReader
@@ -20,7 +20,7 @@ dev_combinator: !Experiment
         eval_metrics: bleu
         src_file: examples/data/head.ja
         ref_file: examples/data/head.en
-        hyp_file: examples/output/{EXP}.test_hyp
+        hyp_file: test/tmp/{EXP}.test_hyp
       - !LossEvalTask
         src_file: examples/data/head.ja
         ref_file: examples/data/head.en
@@ -30,6 +30,6 @@ dev_combinator: !Experiment
       eval_metrics: bleu
       src_file: examples/data/head.ja
       ref_file: examples/data/head.en
-      hyp_file: examples/output/{EXP}.test_hyp
+      hyp_file: test/tmp/{EXP}.test_hyp
 
 

test/config/encoders.yaml

@@ -1,8 +1,6 @@
 defaults: !Experiment
   kwargs: &defaults
     exp_global: !ExpGlobal
-      model_file: examples/output/{EXP}.mod
-      log_file: examples/output/{EXP}.log
       default_layer_dim: 64
       dropout: 0.5
       weight_noise: 0.1
@@ -19,7 +17,7 @@ defaults: !Experiment
         eval_metrics: bleu,wer
         src_file: examples/data/head.ja
         ref_file: examples/data/head.en
-        hyp_file: examples/output/{EXP}.test_hyp
+        hyp_file: test/tmp/{EXP}.test_hyp
 
 
 exp1-lstm-encoder: !Experiment

test/config/ensembling.yaml

@@ -3,8 +3,6 @@
 
 ensemble1: !Experiment
   exp_global: &globals !ExpGlobal
-    model_file: examples/output/{EXP}.mod
-    log_file: examples/output/{EXP}.log
     default_layer_dim: 32
   # Just use default model settings here
   model: !DefaultTranslator
@@ -50,14 +48,14 @@ ensemble3-load: !Experiment
     trg_reader: !Ref {path: model.models.0.trg_reader}
     models:
       - !LoadSerialized
-        filename: 'examples/output/ensemble1.mod'
+        filename: 'test/tmp/ensemble1.mod'
         path: model
       - !LoadSerialized
-        filename: 'examples/output/ensemble2.mod'
+        filename: 'test/tmp/ensemble2.mod'
         path: model
   evaluate:
     - !AccuracyEvalTask
       eval_metrics: bleu,wer
       src_file: examples/data/head.ja
       ref_file: examples/data/head.en
-      hyp_file: examples/output/{EXP}.test_hyp
+      hyp_file: test/tmp/{EXP}.test_hyp

test/config/forced.yaml

@@ -1,7 +1,5 @@
 forced: !Experiment
   exp_global: !ExpGlobal
-    model_file: examples/output/{EXP}.mod
-    log_file: examples/output/{EXP}.log
     default_layer_dim: 32
   model: !DefaultTranslator
     src_reader: !PlainTextReader
@@ -38,7 +36,7 @@ forced: !Experiment
       inference: !SimpleInference {}
       src_file: examples/data/head.ja
       ref_file: examples/data/head.en
-      hyp_file: examples/output/{EXP}.test_hyp
+      hyp_file: test/tmp/{EXP}.test_hyp
     - !AccuracyEvalTask # TODO: can this inherit settings from acc_task?
       eval_metrics: bleu,wer
       inference: !SimpleInference
@@ -47,7 +45,7 @@ forced: !Experiment
         batcher: !InOrderBatcher { batch_size: 32 }
       src_file: examples/data/head.ja
       ref_file: examples/data/head.en
-      hyp_file: examples/output/{EXP}.forced_hyp
+      hyp_file: test/tmp/{EXP}.forced_hyp
     - !AccuracyEvalTask
       eval_metrics: bleu,wer
       inference: !SimpleInference
@@ -56,4 +54,4 @@ forced: !Experiment
         batcher: !InOrderBatcher { batch_size: 32 }
       src_file: examples/data/head.ja
       ref_file: examples/data/head.en
-      hyp_file: examples/output/{EXP}.forceddebug_hyp
+      hyp_file: test/tmp/{EXP}.forceddebug_hyp

test/config/load_model.yaml

@@ -2,8 +2,6 @@ exp1-pretrain-model: !Experiment
   exp_global: !ExpGlobal
     # The model file contain the whole contents of this experiment in YAML
     # format. Note that {EXP} expressions are left intact when saving.
-    model_file: examples/output/{EXP}.mod
-    log_file: examples/output/{EXP}.log
     default_layer_dim: 64
     dropout: 0.5
     weight_noise: 0.1
@@ -40,13 +38,13 @@ exp1-pretrain-model: !Experiment
       eval_metrics: bleu
       src_file: examples/data/head.ja
       ref_file: examples/data/head.en
-      hyp_file: examples/output/{EXP}.dev_hyp
+      hyp_file: test/tmp/{EXP}.dev_hyp
   evaluate:
     - !AccuracyEvalTask
       eval_metrics: bleu
       src_file: examples/data/head.ja
       ref_file: examples/data/head.en
-      hyp_file: examples/output/{EXP}.test_hyp
+      hyp_file: test/tmp/{EXP}.test_hyp
 
 exp2-finetune-model: !LoadSerialized
   # This will load the contents of the above experiments that were saved to the
@@ -56,7 +54,7 @@ exp2-finetune-model: !LoadSerialized
   # and except for the things explicitly overwritten in the overwrite: section.
   # It's possible to change any settings as long as these don't change the number
   # or nature of DyNet parameters allocated for the component.
-  filename: examples/output/exp1-pretrain-model.mod
+  filename: test/tmp/exp1-pretrain-model.mod
   path: ''
   overwrite: # list of [path, value] pairs. Value can be scalar or an arbitrary object
   - path: train.trainer

test/config/multi_task.yaml

@@ -2,8 +2,6 @@
 # some shared parameters.
 exp1-multi_task_exp: !Experiment
   exp_global: !ExpGlobal
-    model_file: examples/output/{EXP}.mod
-    log_file: examples/output/{EXP}.log
     default_layer_dim: 64
   train: !SameBatchMultiTaskTrainingRegimen
     trainer: !AdamTrainer {}
@@ -50,7 +48,7 @@ exp1-multi_task_exp: !Experiment
           model: !Ref { name: first_task_model }
           src_file: &first_task_dev_src examples/data/head.ja  # value-sharing between first task dev and final eval
           ref_file: &first_task_dev_trg examples/data/head.en  # value-sharing between first task dev and final eval
-          hyp_file: examples/output/{EXP}.first_dev_hyp
+          hyp_file: test/tmp/{EXP}.first_dev_hyp
           eval_metrics: bleu # tasks can specify different dev_metrics
     - !SimpleTrainingTask
       name: second_task
@@ -87,18 +85,18 @@ exp1-multi_task_exp: !Experiment
           model: !Ref { name: second_task_model }
           src_file: examples/data/head.ja
           ref_file: examples/data/head.en
-          hyp_file: examples/output/{EXP}.second_dev_hyp
+          hyp_file: test/tmp/{EXP}.second_dev_hyp
           eval_metrics: gleu # tasks can specify different dev_metrics
   evaluate:
     - !AccuracyEvalTask
       model: !Ref { name: first_task_model }
       eval_metrics: bleu,wer
       src_file: *first_task_dev_src
       ref_file: *first_task_dev_trg
-      hyp_file: examples/output/{EXP}.test_hyp
+      hyp_file: test/tmp/{EXP}.test_hyp
 
 exp2-finetune-model: !LoadSerialized
-  filename: examples/output/exp1-multi_task_exp.mod
+  filename: test/tmp/exp1-multi_task_exp.mod
 
 exp3-multi_task_exp-altbatch: !Experiment
   exp_global: !ExpGlobal

test/config/multi_task_speech.yaml

@@ -2,8 +2,6 @@
 # some shared parameters.
 exp1-multi_task_exp: !Experiment
   exp_global: !ExpGlobal
-    model_file: examples/output/{EXP}.mod
-    log_file: examples/output/{EXP}.log
     default_layer_dim: 64
   train: !SameBatchMultiTaskTrainingRegimen
     trainer: !AdamTrainer {}
@@ -55,7 +53,7 @@ exp1-multi_task_exp: !Experiment
           model: !Ref { name: first_task_model }
           src_file: &first_task_dev_src examples/data/LDC94S13A.h5    # value-sharing between train.training_corpus.dev_src and inference.src_file
           ref_file: &first_task_dev_trg examples/data/LDC94S13A.char  # value-sharing between train.training_corpus.dev_trg and evaluate.ref_file
-          hyp_file: examples/output/{EXP}.first_dev_hyp
+          hyp_file: test/tmp/{EXP}.first_dev_hyp
           eval_metrics: bleu # tasks can specify different dev_metrics
     - !SimpleTrainingTask
       name: second_task
@@ -94,15 +92,15 @@ exp1-multi_task_exp: !Experiment
           model: !Ref { name: second_task_model }
           src_file: examples/data/LDC94S13A.h5
           ref_file: examples/data/LDC94S13A.char  # value-sharing between train.training_corpus.dev_trg and evaluate.ref_file
-          hyp_file: examples/output/{EXP}.second_dev_hyp
+          hyp_file: test/tmp/{EXP}.second_dev_hyp
           eval_metrics: gleu # tasks can specify different dev_metrics
   evaluate:
     - !AccuracyEvalTask
       model: !Ref { name: first_task_model }
       eval_metrics: bleu,wer
       src_file: *first_task_dev_src
       ref_file: *first_task_dev_trg
-      hyp_file: examples/output/{EXP}.test_hyp
+      hyp_file: test/tmp/{EXP}.test_hyp
 
 exp2-finetune-model: !LoadSerialized
-  filename: examples/output/exp1-multi_task_exp.mod
+  filename: test/tmp/exp1-multi_task_exp.mod