feat: cudnn layers for cpu

* chore: add reminder to update model links * feat&refactor: add CudnnCompatible cells support to run on CPU * feat: upd some ner configs with cpu compatible models * fix: save/load all saveable instead of only global_variables * feat: upd squad configs with cpu compatible models * fix: upd squad configs * fix: remove link to model in conll2003_pos config * fix: typo in docstring * refactor: some codestyle changes
deeppavlov · Aug 6, 2018 · ba675a4 · ba675a4
1 parent d44e66f
commit ba675a4
Show file tree

Hide file tree

Showing 12 changed files with 309 additions and 74 deletions.
diff --git a/deeppavlov/configs/ner/ner_conll2003.json b/deeppavlov/configs/ner/ner_conll2003.json
@@ -152,7 +152,7 @@
       "server_utils": "NER"
     },
     "download": [
-      "http://lnsigo.mipt.ru/export/deeppavlov_data/ner_conll2003_v4.tar.gz",
+      "http://lnsigo.mipt.ru/export/deeppavlov_data/ner_conll2003_v4_cpu_compatible.tar.gz",
       {
         "url": "http://lnsigo.mipt.ru/export/embeddings/glove.6B.100d.txt",
         "subdir": "embeddings"

diff --git a/deeppavlov/configs/ner/ner_conll2003_pos.json b/deeppavlov/configs/ner/ner_conll2003_pos.json
@@ -171,7 +171,6 @@
       "server_utils": "NER"
     },
     "download": [
-      "http://lnsigo.mipt.ru/export/deeppavlov_data/ner_conll2003_v2.tar.gz",
       {
         "url": "http://lnsigo.mipt.ru/export/embeddings/glove.6B.100d.txt",
         "subdir": "embeddings"

diff --git a/deeppavlov/configs/ner/ner_ontonotes.json b/deeppavlov/configs/ner/ner_ontonotes.json
@@ -137,7 +137,7 @@
       "server_utils": "NER"
     },
     "download": [
-      "http://lnsigo.mipt.ru/export/deeppavlov_data/ner_ontonotes_v3.tar.gz",
+      "http://lnsigo.mipt.ru/export/deeppavlov_data/ner_ontonotes_v3_cpu_compatible.tar.gz",
       {
         "url": "http://lnsigo.mipt.ru/export/embeddings/glove.6B.100d.txt",
         "subdir": "embeddings"

diff --git a/deeppavlov/configs/ner/ner_rus.json b/deeppavlov/configs/ner/ner_rus.json
@@ -151,7 +151,7 @@
       "server_utils": "NER"
     },
     "download": [
-      "http://lnsigo.mipt.ru/export/deeppavlov_data/ner_rus_v2.tar.gz",
+      "http://lnsigo.mipt.ru/export/deeppavlov_data/ner_rus_v2_cpu_compatible.tar.gz",
       {
         "url": "http://lnsigo.mipt.ru/export/embeddings/lenta_lower_100.bin",
         "subdir": "embeddings"

diff --git a/deeppavlov/configs/squad/squad.json b/deeppavlov/configs/squad/squad.json
@@ -39,7 +39,7 @@
         "load_path": "squad_model/emb/vocab_embedder.pckl",
         "context_limit": "#squad_prepr.context_limit",
         "question_limit": "#squad_prepr.question_limit",
-        "char_limit": "#squad_prepr.context_limit",
+        "char_limit": "#squad_prepr.char_limit",
         "fit_on": ["context_tokens", "question_tokens"],
         "in": ["context_tokens", "question_tokens"],
         "out": ["context_tokens_idxs", "question_tokens_idxs"]
@@ -112,7 +112,7 @@
       "server_utils": "SquadModel"
     },
     "download": [
-      "http://lnsigo.mipt.ru/export/deeppavlov_data/squad_model_1.2.tar.gz",
+      "http://lnsigo.mipt.ru/export/deeppavlov_data/squad_model_1.3_cpu_compatible.tar.gz",
       {
         "url": "http://lnsigo.mipt.ru/export/embeddings/wiki-news-300d-1M.vec",
         "subdir": "embeddings"

diff --git a/deeppavlov/configs/squad/squad_ru.json b/deeppavlov/configs/squad/squad_ru.json
@@ -40,7 +40,7 @@
         "load_path": "squad_model_ru/emb/vocab_embedder.pckl",
         "context_limit": "#squad_prepr.context_limit",
         "question_limit": "#squad_prepr.question_limit",
-        "char_limit": "#squad_prepr.context_limit",
+        "char_limit": "#squad_prepr.char_limit",
         "fit_on": ["context_tokens", "question_tokens"],
         "in": ["context_tokens", "question_tokens"],
         "out": ["context_tokens_idxs", "question_tokens_idxs"]
@@ -113,7 +113,7 @@
       "server_utils": "SquadModel"
     },
     "download": [
-      "http://lnsigo.mipt.ru/export/deeppavlov_data/squad_model_ru_1.2.tar.gz",
+      "http://lnsigo.mipt.ru/export/deeppavlov_data/squad_model_ru_1.3_cpu_compatible.tar.gz",
       {
         "url": "http://lnsigo.mipt.ru/export/embeddings/ft_native_300_ru_wiki_lenta_nltk_word_tokenize/ft_native_300_ru_wiki_lenta_nltk_word_tokenize.vec",
         "subdir": "embeddings"

diff --git a/deeppavlov/core/common/check_gpu.py b/deeppavlov/core/common/check_gpu.py
@@ -17,10 +17,13 @@
 from tensorflow.python.client import device_lib
 
 
-def check_gpu_existence():
+def _check_gpu_existence():
     r"""Return True if at least one GPU available"""
     sess_config = tf.ConfigProto()
     sess_config.gpu_options.allow_growth = True
     with tf.Session(config=sess_config):
         device_list = device_lib.list_local_devices()
         return any(device.device_type == 'GPU' for device in device_list)
+
+
+GPU_AVAILABLE = _check_gpu_existence()