Improved import conventions Pt 1 (#484)

* renames + minor cleanup * more renaming / cleanup * more renaming / cleanup * clean up tests * fix unit tests * attempt more consistent module naming * rename lstm.py and loss_calc.py * rename recurrent to recurrent_transducers
neulab · Jul 30, 2018 · 14d8d2d · 14d8d2d
1 parent 92a5fbf
commit 14d8d2d
Show file tree

Hide file tree

Showing 76 changed files with 953 additions and 1,004 deletions.
diff --git a/examples/09_programmatic.py b/examples/09_programmatic.py
@@ -18,25 +18,25 @@
 
 import numpy as np
 
-from xnmt.attender import MlpAttender
-from xnmt.batcher import SrcBatcher, InOrderBatcher
-from xnmt.bridge import CopyBridge
-from xnmt.decoder import AutoRegressiveDecoder
-from xnmt.embedder import SimpleWordEmbedder
-from xnmt.eval_task import LossEvalTask, AccuracyEvalTask
-from xnmt.experiment import Experiment
-from xnmt.inference import AutoRegressiveInference
-from xnmt.input_reader import PlainTextReader
-from xnmt.lstm import BiLSTMSeqTransducer, UniLSTMSeqTransducer
-from xnmt.transform import AuxNonLinear
-from xnmt.scorer import Softmax
-from xnmt.optimizer import AdamTrainer
-from xnmt.param_collection import ParamManager
+from xnmt.attenders import MlpAttender
+from xnmt.batchers import SrcBatcher, InOrderBatcher
+from xnmt.bridges import CopyBridge
+from xnmt.decoders import AutoRegressiveDecoder
+from xnmt.embedders import SimpleWordEmbedder
+from xnmt.eval_tasks import LossEvalTask, AccuracyEvalTask
+from xnmt.experiments import Experiment
+from xnmt.infererences import AutoRegressiveInference
+from xnmt.input_readers import PlainTextReader
+from xnmt.recurrent_transducers import BiLSTMSeqTransducer, UniLSTMSeqTransducer
+from xnmt.transforms import AuxNonLinear
+from xnmt.scorers import Softmax
+from xnmt.optimizers import AdamTrainer
+from xnmt.param_collections import ParamManager
 from xnmt.persistence import save_to_file
 import xnmt.tee
-from xnmt.training_regimen import SimpleTrainingRegimen
-from xnmt.translator import DefaultTranslator
-from xnmt.vocab import Vocab
+from xnmt.training_regimens import SimpleTrainingRegimen
+from xnmt.translators import DefaultTranslator
+from xnmt.vocabs import Vocab
 
 seed=13
 random.seed(seed)

diff --git a/examples/10_programmatic_load.py b/examples/10_programmatic_load.py
@@ -4,7 +4,7 @@
 import os
 
 import xnmt.tee
-from xnmt.param_collection import ParamManager
+from xnmt.param_collections import ParamManager
 from xnmt.persistence import initialize_if_needed, YamlPreloader, LoadSerialized, save_to_file
 
 EXP_DIR = os.path.dirname(__file__)

diff --git a/examples/12_multi_task.yaml b/examples/12_multi_task.yaml
@@ -2,7 +2,7 @@
 # models and using the !Ref mechanism for weight sharing, as demonstrated
 # in this config file.
 # The possible multi-task training strategies can be looked up in
-# xnmt/training_regimen.py and include same-batch, alternating-batch, and serial
+# xnmt/training_regimens.py and include same-batch, alternating-batch, and serial
 # strategies.
 exp1-multi_task: !Experiment
   exp_global: !ExpGlobal

diff --git a/test/test_batcher.py → test/test_batchers.py b/test/test_batcher.py → test/test_batchers.py
@@ -1,20 +1,19 @@
 import unittest
 
-import xnmt.batcher
+from xnmt import batchers, events
 import xnmt.input
-import xnmt.events
-from xnmt.param_collection import ParamManager
+from xnmt.param_collections import ParamManager
 
 class TestBatcher(unittest.TestCase):
 
   def setUp(self):
-    xnmt.events.clear()
+    events.clear()
     ParamManager.init_param_col()
 
   def test_batch_src(self):
     src_sents = [xnmt.input.SimpleSentenceInput([0] * i) for i in range(1,7)]
     trg_sents = [xnmt.input.SimpleSentenceInput([0] * ((i+3)%6 + 1)) for i in range(1,7)]
-    my_batcher = xnmt.batcher.SrcBatcher(batch_size=3, src_pad_token=1, trg_pad_token=2)
+    my_batcher = batchers.SrcBatcher(batch_size=3, src_pad_token=1, trg_pad_token=2)
     src, trg = my_batcher.pack(src_sents, trg_sents)
     self.assertEqual([[0, 0, 1], [0, 1, 1], [0, 0, 0]], [x.words for x in src[0]])
     self.assertEqual([[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 2], [0, 2, 2, 2, 2, 2]], [x.words for x in trg[0]])
@@ -24,7 +23,7 @@ def test_batch_src(self):
   def test_batch_word_src(self):
     src_sents = [xnmt.input.SimpleSentenceInput([0] * i) for i in range(1,7)]
     trg_sents = [xnmt.input.SimpleSentenceInput([0] * ((i+3)%6 + 1)) for i in range(1,7)]
-    my_batcher = xnmt.batcher.WordSrcBatcher(words_per_batch=12, src_pad_token=1, trg_pad_token=2)
+    my_batcher = batchers.WordSrcBatcher(words_per_batch=12, src_pad_token=1, trg_pad_token=2)
     src, trg = my_batcher.pack(src_sents, trg_sents)
     self.assertEqual([[0]], [x.words for x in src[0]])
     self.assertEqual([[0, 0, 0, 0, 0]], [x.words for x in trg[0]])
@@ -40,7 +39,7 @@ def test_batch_word_src(self):
   def test_batch_random_no_ties(self):
     src_sents = [xnmt.input.SimpleSentenceInput([0] * i) for i in range(1,7)]
     trg_sents = [xnmt.input.SimpleSentenceInput([0] * ((i+3)%6 + 1)) for i in range(1,7)]
-    my_batcher = xnmt.batcher.SrcBatcher(batch_size=3, src_pad_token=1, trg_pad_token=2)
+    my_batcher = batchers.SrcBatcher(batch_size=3, src_pad_token=1, trg_pad_token=2)
     _, trg = my_batcher.pack(src_sents, trg_sents)
     l0 = trg[0].sent_len()
     for _ in range(10):
@@ -51,7 +50,7 @@ def test_batch_random_no_ties(self):
   def test_batch_random_ties(self):
     src_sents = [xnmt.input.SimpleSentenceInput([0] * 5) for _ in range(1,7)]
     trg_sents = [xnmt.input.SimpleSentenceInput([0] * ((i+3)%6 + 1)) for i in range(1,7)]
-    my_batcher = xnmt.batcher.SrcBatcher(batch_size=3, src_pad_token=1, trg_pad_token=2)
+    my_batcher = batchers.SrcBatcher(batch_size=3, src_pad_token=1, trg_pad_token=2)
     _, trg = my_batcher.pack(src_sents, trg_sents)
     l0 = trg[0].sent_len()
     for _ in range(10):

diff --git a/test/test_beam_search.py b/test/test_beam_search.py
@@ -3,21 +3,20 @@
 import dynet_config
 import dynet as dy
 
-from xnmt.attender import MlpAttender
-import xnmt.batcher
-from xnmt.bridge import CopyBridge
-from xnmt.decoder import AutoRegressiveDecoder
-from xnmt.embedder import SimpleWordEmbedder
-import xnmt.events
-from xnmt.input_reader import PlainTextReader
-from xnmt.lstm import UniLSTMSeqTransducer, BiLSTMSeqTransducer
-from xnmt.loss_calculator import AutoRegressiveMLELoss
-from xnmt.param_collection import ParamManager
-from xnmt.transform import NonLinear
-from xnmt.scorer import Softmax
-from xnmt.translator import DefaultTranslator
-from xnmt.search_strategy import BeamSearch, GreedySearch
-from xnmt.vocab import Vocab
+from xnmt.attenders import MlpAttender
+from xnmt import batchers, events
+from xnmt.bridges import CopyBridge
+from xnmt.decoders import AutoRegressiveDecoder
+from xnmt.embedders import SimpleWordEmbedder
+from xnmt.input_readers import PlainTextReader
+from xnmt.recurrent_transducers import UniLSTMSeqTransducer, BiLSTMSeqTransducer
+from xnmt.loss_calculators import AutoRegressiveMLELoss
+from xnmt.param_collections import ParamManager
+from xnmt.transforms import NonLinear
+from xnmt.scorers import Softmax
+from xnmt.translators import DefaultTranslator
+from xnmt.search_strategies import BeamSearch, GreedySearch
+from xnmt.vocabs import Vocab
 
 class TestForcedDecodingOutputs(unittest.TestCase):
 
@@ -28,7 +27,7 @@ def assertItemsEqual(self, l1, l2):
 
   def setUp(self):
     layer_dim = 512
-    xnmt.events.clear()
+    events.clear()
     ParamManager.init_param_col()
     src_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
     trg_vocab = Vocab(vocab_file="examples/data/head.en.vocab")
@@ -53,8 +52,8 @@ def setUp(self):
 
   def assert_forced_decoding(self, sent_id):
     dy.renew_cg()
-    outputs = self.model.generate(xnmt.batcher.mark_as_batch([self.src_data[sent_id]]), [sent_id], BeamSearch(),
-                                  forced_trg_ids=xnmt.batcher.mark_as_batch([self.trg_data[sent_id]]))
+    outputs = self.model.generate(batchers.mark_as_batch([self.src_data[sent_id]]), [sent_id], BeamSearch(),
+                                  forced_trg_ids=batchers.mark_as_batch([self.trg_data[sent_id]]))
     self.assertItemsEqual(self.trg_data[sent_id].words, outputs[0].actions)
 
   def test_forced_decoding(self):
@@ -65,7 +64,7 @@ class TestForcedDecodingLoss(unittest.TestCase):
 
   def setUp(self):
     layer_dim = 512
-    xnmt.events.clear()
+    events.clear()
     ParamManager.init_param_col()
     src_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
     trg_vocab = Vocab(vocab_file="examples/data/head.en.vocab")
@@ -94,15 +93,15 @@ def test_single(self):
                                       trg=self.trg_data[0],
                                       loss_calculator=AutoRegressiveMLELoss()).value()
     dy.renew_cg()
-    outputs = self.model.generate(xnmt.batcher.mark_as_batch([self.src_data[0]]), [0], BeamSearch(beam_size=1),
-                                  forced_trg_ids=xnmt.batcher.mark_as_batch([self.trg_data[0]]))
+    outputs = self.model.generate(batchers.mark_as_batch([self.src_data[0]]), [0], BeamSearch(beam_size=1),
+                                  forced_trg_ids=batchers.mark_as_batch([self.trg_data[0]]))
     self.assertAlmostEqual(-outputs[0].score, train_loss, places=4)
 
 class TestFreeDecodingLoss(unittest.TestCase):
 
   def setUp(self):
     layer_dim = 512
-    xnmt.events.clear()
+    events.clear()
     ParamManager.init_param_col()
     src_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
     trg_vocab = Vocab(vocab_file="examples/data/head.en.vocab")
@@ -127,8 +126,8 @@ def setUp(self):
 
   def test_single(self):
     dy.renew_cg()
-    outputs = self.model.generate(xnmt.batcher.mark_as_batch([self.src_data[0]]), [0], BeamSearch(),
-                                         forced_trg_ids=xnmt.batcher.mark_as_batch([self.trg_data[0]]))
+    outputs = self.model.generate(batchers.mark_as_batch([self.src_data[0]]), [0], BeamSearch(),
+                                  forced_trg_ids=batchers.mark_as_batch([self.trg_data[0]]))
     dy.renew_cg()
     train_loss = self.model.calc_loss(src=self.src_data[0],
                                       trg=outputs[0],
@@ -142,7 +141,7 @@ class TestGreedyVsBeam(unittest.TestCase):
   """
   def setUp(self):
     layer_dim = 512
-    xnmt.events.clear()
+    events.clear()
     ParamManager.init_param_col()
     src_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
     trg_vocab = Vocab(vocab_file="examples/data/head.en.vocab")
@@ -167,13 +166,13 @@ def setUp(self):
 
   def test_greedy_vs_beam(self):
     dy.renew_cg()
-    outputs = self.model.generate(xnmt.batcher.mark_as_batch([self.src_data[0]]), [0], BeamSearch(beam_size=1),
-                                         forced_trg_ids=xnmt.batcher.mark_as_batch([self.trg_data[0]]))
+    outputs = self.model.generate(batchers.mark_as_batch([self.src_data[0]]), [0], BeamSearch(beam_size=1),
+                                  forced_trg_ids=batchers.mark_as_batch([self.trg_data[0]]))
     output_score1 = outputs[0].score
 
     dy.renew_cg()
-    outputs = self.model.generate(xnmt.batcher.mark_as_batch([self.src_data[0]]), [0], GreedySearch(),
-                                  forced_trg_ids=xnmt.batcher.mark_as_batch([self.trg_data[0]]))
+    outputs = self.model.generate(batchers.mark_as_batch([self.src_data[0]]), [0], GreedySearch(),
+                                  forced_trg_ids=batchers.mark_as_batch([self.trg_data[0]]))
     output_score2 = outputs[0].score
 
     self.assertAlmostEqual(output_score1, output_score2)

diff --git a/test/test_decoding.py → test/test_decoders.py b/test/test_decoding.py → test/test_decoders.py
@@ -3,21 +3,20 @@
 import dynet_config
 import dynet as dy
 
-from xnmt.attender import MlpAttender
-import xnmt.batcher
-from xnmt.bridge import CopyBridge
-from xnmt.decoder import AutoRegressiveDecoder
-from xnmt.embedder import SimpleWordEmbedder
-import xnmt.events
-from xnmt.input_reader import PlainTextReader
-from xnmt.loss_calculator import AutoRegressiveMLELoss
-from xnmt.lstm import UniLSTMSeqTransducer, BiLSTMSeqTransducer
-from xnmt.param_collection import ParamManager
-from xnmt.transform import NonLinear
-from xnmt.translator import DefaultTranslator
-from xnmt.scorer import Softmax
-from xnmt.search_strategy import GreedySearch
-from xnmt.vocab import Vocab
+from xnmt.attenders import MlpAttender
+from xnmt import batchers, events
+from xnmt.bridges import CopyBridge
+from xnmt.decoders import AutoRegressiveDecoder
+from xnmt.embedders import SimpleWordEmbedder
+from xnmt.input_readers import PlainTextReader
+from xnmt.loss_calculators import AutoRegressiveMLELoss
+from xnmt.recurrent_transducers import UniLSTMSeqTransducer, BiLSTMSeqTransducer
+from xnmt.param_collections import ParamManager
+from xnmt.transforms import NonLinear
+from xnmt.translators import DefaultTranslator
+from xnmt.scorers import Softmax
+from xnmt.search_strategies import GreedySearch
+from xnmt.vocabs import Vocab
 
 class TestForcedDecodingOutputs(unittest.TestCase):
 
@@ -28,7 +27,7 @@ def assertItemsEqual(self, l1, l2):
 
   def setUp(self):
     layer_dim = 512
-    xnmt.events.clear()
+    events.clear()
     ParamManager.init_param_col()
     src_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
     trg_vocab = Vocab(vocab_file="examples/data/head.en.vocab")
@@ -55,8 +54,8 @@ def setUp(self):
 
   def assert_forced_decoding(self, sent_id):
     dy.renew_cg()
-    outputs = self.model.generate(xnmt.batcher.mark_as_batch([self.src_data[sent_id]]), [sent_id], self.search,
-                                  forced_trg_ids=xnmt.batcher.mark_as_batch([self.trg_data[sent_id]]))
+    outputs = self.model.generate(batchers.mark_as_batch([self.src_data[sent_id]]), [sent_id], self.search,
+                                  forced_trg_ids=batchers.mark_as_batch([self.trg_data[sent_id]]))
     self.assertItemsEqual(self.trg_data[sent_id].words, outputs[0].actions)
 
   def test_forced_decoding(self):
@@ -67,7 +66,7 @@ class TestForcedDecodingLoss(unittest.TestCase):
 
   def setUp(self):
     layer_dim = 512
-    xnmt.events.clear()
+    events.clear()
     ParamManager.init_param_col()
     src_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
     trg_vocab = Vocab(vocab_file="examples/data/head.en.vocab")
@@ -96,16 +95,16 @@ def test_single(self):
                                       trg=self.trg_data[0],
                                       loss_calculator=AutoRegressiveMLELoss()).value()
     dy.renew_cg()
-    outputs = self.model.generate(xnmt.batcher.mark_as_batch([self.src_data[0]]), [0], GreedySearch(),
-                                  forced_trg_ids=xnmt.batcher.mark_as_batch([self.trg_data[0]]))
+    outputs = self.model.generate(batchers.mark_as_batch([self.src_data[0]]), [0], GreedySearch(),
+                                  forced_trg_ids=batchers.mark_as_batch([self.trg_data[0]]))
     output_score = outputs[0].score
     self.assertAlmostEqual(-output_score, train_loss, places=5)
 
 class TestFreeDecodingLoss(unittest.TestCase):
 
   def setUp(self):
     layer_dim = 512
-    xnmt.events.clear()
+    events.clear()
     ParamManager.init_param_col()
     src_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
     trg_vocab = Vocab(vocab_file="examples/data/head.en.vocab")
@@ -130,8 +129,8 @@ def setUp(self):
 
   def test_single(self):
     dy.renew_cg()
-    outputs = self.model.generate(xnmt.batcher.mark_as_batch([self.src_data[0]]), [0], GreedySearch(),
-                                  forced_trg_ids=xnmt.batcher.mark_as_batch([self.trg_data[0]]))
+    outputs = self.model.generate(batchers.mark_as_batch([self.src_data[0]]), [0], GreedySearch(),
+                                  forced_trg_ids=batchers.mark_as_batch([self.trg_data[0]]))
     output_score = outputs[0].score
 
     dy.renew_cg()

diff --git a/test/test_embedder.py → test/test_embedders.py b/test/test_embedder.py → test/test_embedders.py
@@ -3,15 +3,15 @@
 import numpy as np
 from itertools import islice
 
-from xnmt.input_reader import PlainTextReader
-from xnmt.embedder import PretrainedSimpleWordEmbedder
-from xnmt.param_collection import ParamManager
-import xnmt.events
-from xnmt.vocab import Vocab
+from xnmt.input_readers import PlainTextReader
+from xnmt.embedders import PretrainedSimpleWordEmbedder
+from xnmt.param_collections import ParamManager
+from xnmt import events
+from xnmt.vocabs import Vocab
 
 class PretrainedSimpleWordEmbedderSanityTest(unittest.TestCase):
   def setUp(self):
-    xnmt.events.clear()
+    events.clear()
     self.input_reader = PlainTextReader(vocab=Vocab(vocab_file="examples/data/head.ja.vocab"))
     list(self.input_reader.read_sents('examples/data/head.ja'))
     ParamManager.init_param_col()