fix: many minor fixes

* fix: fix mark_done data_path

* refactor: rename ranking_dataset to ranking_iterator.py and move it to the dataset_iterators folder

* fix: fix embedding matrix construction, change epochs num
default parameter value

* refactor: rename registered name and name of the class

* refactor: rename files and classes

* refactor: change dataset downlaod

* feat: add insurance embeddings and datasets in urls.py

deeppavlov/__init__.py

@@ -32,8 +32,8 @@
 import deeppavlov.skills.go_bot.network
 import deeppavlov.skills.go_bot.tracker
 import deeppavlov.vocabs.typos
-import deeppavlov.dataset_readers.ranking_dataset_reader
-import deeppavlov.datasets.ranking_dataset
+import deeppavlov.dataset_readers.insurance_reader
+import deeppavlov.dataset_iterators.ranking_iterator
 import deeppavlov.models.ranking.ranking_model
 import deeppavlov.models.ranking.metrics
 

deeppavlov/configs/ranking/insurance_config.json

@@ -1,10 +1,10 @@
 {
   "dataset_reader": {
-    "name": "insurance_dataset_reader",
-    "data_path": "insuranceQA-master/V1"
+    "name": "insurance_reader",
+    "data_path": "./"
   },
   "dataset_iterator": {
-    "name": "insurance_dataset",
+    "name": "ranking_iterator",
     "seed": 243,
     "sample_candidates": "global",
     "sample_candidates_valid": "pool",
@@ -44,7 +44,7 @@
     "out": ["y_predicted"]
   },
   "train": {
-    "epochs": 100,
+    "epochs": 150,
     "batch_size": 256,
     "metrics": ["r@1", "rank_response"],
     "validation_patience": 5,

deeppavlov/core/data/urls.py

@@ -21,6 +21,7 @@
     'http://lnsigo.mipt.ru/export/deeppavlov_data/ner_conll2003.tar.gz',
     'http://lnsigo.mipt.ru/export/deeppavlov_data/error_model.tar.gz',
     'http://lnsigo.mipt.ru/export/deeppavlov_data/ranking.tar.gz',
+    'http://lnsigo.mipt.ru/export/embeddings/insurance_v1_word2vec',
     'http://lnsigo.mipt.ru/export/deeppavlov_data/vocabs.tar.gz',
     'http://lnsigo.mipt.ru/export/deeppavlov_data/slots.tar.gz',
     'http://lnsigo.mipt.ru/export/deeppavlov_data/embeddings/dstc2_fastText_model.bin',
@@ -35,9 +36,11 @@
 
 EMBEDDING_URLS = {
     'http://lnsigo.mipt.ru/export/deeppavlov_data/embeddings/wiki.en.bin',
-    'http://lnsigo.mipt.ru/export/deeppavlov_data/embeddings/dstc2_fastText_model.bin'
+    'http://lnsigo.mipt.ru/export/deeppavlov_data/embeddings/dstc2_fastText_model.bin',
+    'http://lnsigo.mipt.ru/export/embeddings/insurance_v1_word2vec'
 }
 
 DATA_URLS = {
-    'http://lnsigo.mipt.ru/export/datasets/dstc2.tar.gz'
+    'http://lnsigo.mipt.ru/export/datasets/dstc2.tar.gz',
+    'http://lnsigo.mipt.ru/export/datasets/insuranceQA-master.zip'
 }

deeppavlov/datasets/ranking_dataset.py → deeppavlov/dataset_iterators/ranking_iterator.py

@@ -3,8 +3,8 @@
 import numpy as np
 
 
-@register('insurance_dataset')
-class InsuranceDataset:
+@register('ranking_iterator')
+class RankingIterator:
 
     def __init__(self, data,
                  sample_candidates, sample_candidates_valid, sample_candidates_test,

deeppavlov/dataset_readers/ranking_dataset_reader.py → deeppavlov/dataset_readers/insurance_reader.py

@@ -2,26 +2,28 @@
 from pathlib import Path
 from deeppavlov.core.common.registry import register
 from deeppavlov.core.data.utils import download_decompress, mark_done, is_done
-from deeppavlov.core.commands.utils import get_deeppavlov_root
+from deeppavlov.core.commands.utils import get_deeppavlov_root, expand_path
 
-@register('insurance_dataset_reader')
-class InsuranceDatasetReader(DatasetReader):
+
+@register('insurance_reader')
+class InsuranceReader(DatasetReader):
 
     def read(self, data_path):
+        data_path = expand_path(data_path)
         self.download_data(data_path)
         dataset = {'train': None, 'valid': None, 'test': None}
-        train_fname = Path(data_path) / 'question.train.token_idx.label'
+        train_fname = Path(data_path) / 'insuranceQA-master/V1/question.train.token_idx.label'
         dataset["train"] = self.preprocess_data_train(train_fname)
-        valid_fname = Path(data_path) / 'question.dev.label.token_idx.pool'
+        valid_fname = Path(data_path) / 'insuranceQA-master/V1/question.dev.label.token_idx.pool'
         dataset["valid"] = self.preprocess_data_valid_test(valid_fname)
-        test_fname = Path(data_path) / 'question.test1.label.token_idx.pool'
+        test_fname = Path(data_path) / 'insuranceQA-master/V1/question.test1.label.token_idx.pool'
         dataset["test"] = self.preprocess_data_valid_test(test_fname)
         return dataset
 
     def download_data(self, data_path):
         if not is_done(Path(data_path)):
-            download_decompress(url="https://github.com/shuzi/insuranceQA/archive/master.zip",
-                                download_path=get_deeppavlov_root())
+            download_decompress(url="http://lnsigo.mipt.ru/export/datasets/insuranceQA-master.zip",
+                                download_path=data_path)
             mark_done(data_path)
 
     def preprocess_data_train(self, fname):

deeppavlov/models/ranking/emb_dict.py

@@ -35,7 +35,10 @@ def create_emb_matrix(self, tok2int_vocab):
         dummy_emb = list(np.zeros(self.embedding_dim))
         self.emb_matrix = np.zeros((len(tok2int_vocab), self.embedding_dim))
         for tok, i in tok2int_vocab.items():
-            try:
-                self.emb_matrix[i] = self.embeddings_model[tok]
-            except:
+            if tok == '<UNK>':
                 self.emb_matrix[i] = dummy_emb
+            else:
+                try:
+                    self.emb_matrix[i] = self.embeddings_model[tok]
+                except:
+                    self.emb_matrix[i] = dummy_emb