Fix collisions between oov words and in-vocab words (#447)

test/test_vocab.py

@@ -49,6 +49,38 @@ def test_vocab_specials_first(self):
         self.assertEqual(v.itos, expected_itos)
         self.assertEqual(dict(v.stoi), expected_stoi)
 
+    def test_vocab_without_unk(self):
+        c = Counter({'hello': 4, 'world': 3, 'ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T': 5, 'freq_too_low': 2})
+        oov_word = 'OOVWORD'
+        self.assertNotIn(oov_word, c)
+
+        # tests for specials_first=True
+        v_first = vocab.Vocab(c, min_freq=3, specials=['<pad>'], specials_first=True)
+        expected_itos_first = ['<pad>', 'ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T', 'hello', 'world']
+        expected_stoi_first = {x: index for index, x in enumerate(expected_itos_first)}
+        self.assertEqual(v_first.itos, expected_itos_first)
+        self.assertEqual(dict(v_first.stoi), expected_stoi_first)
+        self.assertNotIn(oov_word, v_first.itos)
+        self.assertNotIn(oov_word, v_first.stoi)
+
+        # tests for specials_first=False
+        v_last = vocab.Vocab(c, min_freq=3, specials=['<pad>'], specials_first=False)
+        expected_itos_last = ['ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T', 'hello', 'world', '<pad>']
+        expected_stoi_last = {x: index for index, x in enumerate(expected_itos_last)}
+        self.assertEqual(v_last.itos, expected_itos_last)
+        self.assertEqual(dict(v_last.stoi), expected_stoi_last)
+        self.assertNotIn(oov_word, v_last.itos)
+        self.assertNotIn(oov_word, v_last.stoi)
+
+        # check if pad is mapped to the first index
+        self.assertEqual(v_first.stoi['<pad>'], 0)
+        # check if pad is mapped to the last index
+        self.assertEqual(v_last.stoi['<pad>'], max(v_last.stoi.values()))
+
+        # check if an oovword is not in vocab and a default unk_id is not assigned to it
+        self.assertRaises(KeyError, v_first.stoi.__getitem__, oov_word)
+        self.assertRaises(KeyError, v_last.stoi.__getitem__, oov_word)
+
     def test_vocab_set_vectors(self):
         c = Counter({'hello': 4, 'world': 3, 'ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T': 5,
                      'ｔｅｓｔ': 4, 'freq_too_low': 2})

torchtext/vocab.py

@@ -79,7 +79,11 @@ def __init__(self, counter, max_size=None, min_freq=1, specials=['<pad>'],
         if not specials_first:
             self.itos.extend(list(specials))
 
-        self.stoi = defaultdict(_default_unk_index)
+        if '<unk>' in specials:  # hard-coded for now
+            self.stoi = defaultdict(_default_unk_index)
+        else:
+            self.stoi = defaultdict()
+
         # stoi is simply a reverse dict for itos
         self.stoi.update({tok: i for i, tok in enumerate(self.itos)})