Release 0.1.5.1 (#678)

Release 0.1.5.1

deeppavlov/__init__.py

@@ -35,7 +35,7 @@ def evaluate_model(config: [str, Path, dict], download: bool = False, recursive:
 except ImportError:
     'Assuming that requirements are not yet installed'
 
-__version__ = '0.1.5'
+__version__ = '0.1.5.1'
 __author__ = 'Neural Networks and Deep Learning lab, MIPT'
 __description__ = 'An open source library for building end-to-end dialog systems and training chatbots.'
 __keywords__ = ['NLP', 'NER', 'SQUAD', 'Intents', 'Chatbot']

deeppavlov/configs/go_bot/gobot_dstc2.json

@@ -32,7 +32,7 @@
         "save_path": "{DOWNLOADS_PATH}/dstc2/resto.sqlite"
       },
       {
-        "class_name": "go_bot_rnn",
+        "class_name": "go_bot",
         "load_path": "{MODELS_PATH}/gobot_dstc2/model",
         "save_path": "{MODELS_PATH}/gobot_dstc2/model",
         "in": ["x"],

deeppavlov/configs/go_bot/gobot_dstc2_best.json

@@ -32,7 +32,7 @@
         "save_path": "{DOWNLOADS_PATH}/dstc2/resto.sqlite"
       },
       {
-        "class_name": "go_bot_rnn",
+        "class_name": "go_bot",
         "load_path": "{MODELS_PATH}/gobot_dstc2_best/model",
         "save_path": "{MODELS_PATH}/gobot_dstc2_best/model",
         "in": ["x"],

deeppavlov/configs/go_bot/gobot_dstc2_minimal.json

@@ -32,7 +32,7 @@
         "save_path": "{DOWNLOADS_PATH}/dstc2/resto.sqlite"
       },
       {
-        "class_name": "go_bot_rnn",
+        "class_name": "go_bot",
         "load_path": "{MODELS_PATH}/gobot_dstc2_minimal/model",
         "save_path": "{MODELS_PATH}/gobot_dstc2_minimal/model",
         "in": ["x"],

deeppavlov/core/commands/utils.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 from pathlib import Path
 from typing import Union, Dict, TypeVar
 
@@ -38,9 +39,12 @@ def parse_config(config: Union[str, Path, dict]) -> dict:
         config = read_json(find_config(config))
 
     variables = {
-        'DEEPPAVLOV_PATH': Path(__file__).parent.parent.parent
+        'DEEPPAVLOV_PATH': os.getenv(f'DP_DEEPPAVLOV_PATH', Path(__file__).parent.parent.parent)
     }
     for name, value in config.get('metadata', {}).get('variables', {}).items():
+        env_name = f'DP_{name}'
+        if env_name in os.environ:
+            value = os.getenv(env_name)
         variables[name] = value.format(**variables)
 
     return _parse_config_property(config, variables)

deeppavlov/core/common/check_gpu.py

@@ -20,17 +20,20 @@
 log = get_logger(__name__)
 
 
-def _check_gpu_existence():
-    r"""Return True if at least one GPU available"""
-    sess_config = tf.ConfigProto()
-    sess_config.gpu_options.allow_growth = True
-    try:
-        with tf.Session(config=sess_config):
-            device_list = device_lib.list_local_devices()
-            return any(device.device_type == 'GPU' for device in device_list)
-    except AttributeError as e:
-        log.warning(f'Got an AttributeError `{e}`, assuming documentation building')
-        return False
+_gpu_available = None
 
 
-GPU_AVAILABLE = _check_gpu_existence()
+def check_gpu_existence():
+    r"""Return True if at least one GPU is available"""
+    global _gpu_available
+    if _gpu_available is None:
+        sess_config = tf.ConfigProto()
+        sess_config.gpu_options.allow_growth = True
+        try:
+            with tf.Session(config=sess_config):
+                device_list = device_lib.list_local_devices()
+                _gpu_available = any(device.device_type == 'GPU' for device in device_list)
+        except AttributeError as e:
+            log.warning(f'Got an AttributeError `{e}`, assuming documentation building')
+            _gpu_available = False
+    return _gpu_available

deeppavlov/core/common/registry.json

@@ -37,7 +37,7 @@
   "file_paths_iterator": "deeppavlov.dataset_iterators.file_paths_iterator:FilePathsIterator",
   "file_paths_reader": "deeppavlov.dataset_readers.file_paths_reader:FilePathsReader",
   "glove": "deeppavlov.models.embedders.glove_embedder:GloVeEmbedder",
-  "go_bot_rnn": "deeppavlov.models.go_bot.network:GoalOrientedBotNetwork",
+  "go_bot": "deeppavlov.models.go_bot.network:GoalOrientedBot",
   "hashing_tfidf_vectorizer": "deeppavlov.models.vectorizers.hashing_tfidf_vectorizer:HashingTfIdfVectorizer",
   "insurance_reader": "deeppavlov.dataset_readers.insurance_reader:InsuranceReader",
   "kenlm_elector": "deeppavlov.models.spelling_correction.electors.kenlm_elector:KenlmElector",

deeppavlov/core/layers/tf_layers.py

@@ -13,11 +13,10 @@
 # limitations under the License.
 
 import tensorflow as tf
-from tensorflow.contrib.layers import xavier_initializer
 import numpy as np
 from typing import List
 
-from deeppavlov.core.common.check_gpu import GPU_AVAILABLE
+from deeppavlov.core.common.check_gpu import check_gpu_existence
 from deeppavlov.core.common.log import get_logger
 
 
@@ -608,7 +607,7 @@ def single_cell(): return tf.contrib.cudnn_rnn.CudnnCompatibleGRUCell(n_hidden)
 def cudnn_gru_wrapper(units, n_hidden, n_layers=1, trainable_initial_states=False,
                       seq_lengths=None, input_initial_h=None, name='cudnn_gru', reuse=False):
 
-    if GPU_AVAILABLE:
+    if check_gpu_existence():
         return cudnn_gru(units, n_hidden, n_layers, trainable_initial_states,
                          seq_lengths, input_initial_h, name, reuse)
 
@@ -750,7 +749,7 @@ def single_cell(): return tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell(n_hidden)
 def cudnn_lstm_wrapper(units, n_hidden, n_layers=1, trainable_initial_states=None, seq_lengths=None, initial_h=None,
                        initial_c=None, name='cudnn_lstm', reuse=False):
 
-    if GPU_AVAILABLE:
+    if check_gpu_existence():
         return cudnn_lstm(units, n_hidden, n_layers, trainable_initial_states,
                           seq_lengths, initial_h, initial_c, name, reuse)
 

deeppavlov/models/doc_retrieval/pop_ranker.py

@@ -19,13 +19,15 @@
 from sklearn.externals import joblib
 
 from deeppavlov.core.common.log import get_logger
+from deeppavlov.core.common.registry import register
 from deeppavlov.core.models.estimator import Component
 from deeppavlov.core.commands.utils import expand_path
 from deeppavlov.core.common.file import read_json
 
 logger = get_logger(__name__)
 
 
+@register('pop_ranker')
 class PopRanker(Component):
     """Rank documents according to their tfidf scores and popularities. It is not a standalone ranker,
     it should be used for re-ranking the results of TF-IDF Ranker.

deeppavlov/models/go_bot/network.py

@@ -14,6 +14,7 @@
 
 import re
 import collections
+from typing import Dict, Any
 import json
 import numpy as np
 import tensorflow as tf
@@ -34,8 +35,8 @@
 log = get_logger(__name__)
 
 
-@register("go_bot_rnn")
-class GoalOrientedBotNetwork(LRScheduledTFModel):
+@register("go_bot")
+class GoalOrientedBot(LRScheduledTFModel):
     """
     The dialogue bot is based on  https://arxiv.org/abs/1702.03274, which introduces
     Hybrid Code Networks that combine an RNN with domain-specific knowledge
@@ -47,7 +48,7 @@ class GoalOrientedBotNetwork(LRScheduledTFModel):
 
     An LSTM with a dense layer for input features and a dense layer for it's output.
     Softmax is used as an output activation function.
-        
+
     Todo:
         add docstring for trackers.
 
@@ -76,6 +77,7 @@ class GoalOrientedBotNetwork(LRScheduledTFModel):
               to attention.
             * **intent_as_key** – use utterance intents as attention key or not.
             * **projected_align** – whether to use output projection.
+        network_parameters: dictionary with network parameters (for compatibility with release 0.1.1, deprecated in the future)
 
         template_path: file with mapping between actions and text templates
             for response generation.
@@ -107,19 +109,21 @@ class GoalOrientedBotNetwork(LRScheduledTFModel):
 
     GRAPH_PARAMS = ["hidden_size", "action_size", "dense_size", "obs_size",
                     "attention_mechanism"]
+    DEPRECATED = ["end_learning_rate", "decay_steps", "decay_power"]
 
     def __init__(self,
                  tokenizer: Component,
                  tracker: Tracker,
                  template_path: str,
                  save_path: str,
-                 hidden_size: int,
+                 hidden_size: int = 128,
                  obs_size: int = None,
                  action_size: int = None,
                  dropout_rate: float = 0.,
                  l2_reg_coef: float = 0.,
                  dense_size: int = None,
                  attention_mechanism: dict = None,
+                 network_parameters: Dict[str, Any] = {},
                  load_path: str = None,
                  template_type: str = "DefaultTemplate",
                  word_vocab: Component = None,
@@ -132,6 +136,13 @@ def __init__(self,
                  use_action_mask: bool = False,
                  debug: bool = False,
                  **kwargs):
+        if any(p in network_parameters for p in self.DEPRECATED):
+            log.warning(f"parameters {self.DEPRECATED} are deprecated,"
+                              " for learning rate schedule documentation see"
+                              " deeppavlov.core.models.lr_scheduled_tf_model"
+                              " or read gitub tutorial on super convergence.")
+        if 'learning_rate' in network_parameters:
+            kwargs['learning_rate'] = network_parameters.pop('learning_rate')
         super().__init__(load_path=load_path, save_path=save_path, **kwargs)
 
         self.tokenizer = tokenizer
@@ -160,13 +171,19 @@ def __init__(self,
         if callable(self.intent_classifier):
             self.intents = self.intent_classifier.get_main_component().classes
 
-        self._init_network(hidden_size=hidden_size,
-                           action_size=action_size,
-                           obs_size=obs_size,
-                           dropout_rate=dropout_rate,
-                           l2_reg_coef=l2_reg_coef,
-                           dense_size=dense_size,
-                           attn=attention_mechanism)
+        new_network_parameters = {
+            'hidden_size': hidden_size,
+            'action_size': action_size,
+            'obs_size': obs_size,
+            'dropout_rate': dropout_rate,
+            'l2_reg_coef': l2_reg_coef,
+            'dense_size': dense_size,
+            'attn': attention_mechanism
+        }
+        if 'attention_mechanism' in network_parameters:
+            network_parameters['attn'] = network_parameters.pop('attention_mechanism')
+        new_network_parameters.update(network_parameters)
+        self._init_network(**new_network_parameters)
 
         self.reset()
 

deeppavlov/models/squad/squad.py

@@ -20,7 +20,7 @@
 from deeppavlov.core.common.registry import register
 from deeppavlov.core.models.lr_scheduled_tf_model import LRScheduledTFModel
 from deeppavlov.models.squad.utils import dot_attention, simple_attention, PtrNet, CudnnGRU, CudnnCompatibleGRU
-from deeppavlov.core.common.check_gpu import GPU_AVAILABLE
+from deeppavlov.core.common.check_gpu import check_gpu_existence
 from deeppavlov.core.layers.tf_layers import cudnn_bi_gru, variational_dropout
 from deeppavlov.core.common.log import get_logger
 
@@ -76,7 +76,7 @@ def __init__(self, word_emb: np.ndarray, char_emb: np.ndarray, context_limit: in
         self.last_impatience = 0
         self.lr_impatience = 0
 
-        if GPU_AVAILABLE:
+        if check_gpu_existence():
             self.GRU = CudnnGRU
         else:
             self.GRU = CudnnCompatibleGRU

docs/apiref/models/go_bot.rst

@@ -3,7 +3,7 @@ deeppavlov.models.go_bot
 
 .. automodule:: deeppavlov.models.go_bot
 
-.. autoclass:: deeppavlov.models.go_bot.network.GoalOrientedBotNetwork
+.. autoclass:: deeppavlov.models.go_bot.network.GoalOrientedBot
    :members:
 
 .. autoclass:: deeppavlov.models.go_bot.tracker.Tracker

docs/intro/config_description.rst

@@ -96,6 +96,10 @@ from ``metadata.variables`` element:
 
 Variable ``DEEPPAVLOV_PATH`` is always preset to be a path to the ``deeppavlov`` python module.
 
+One can override configuration variables using environment variables with prefix ``DP_``. So environment variable
+``DP_VARIABLE_NAME`` will override ``VARIABLE_NAME`` inside a configuration file.
+
+For example, adding ``DP_ROOT_PATH=/my_path/to/large_hard_drive`` will make most configs use this path for downloading and reading  embeddings/models/datasets.
 
 Training
 --------

tests/test_quick_start.py

@@ -6,6 +6,7 @@
 import signal
 import shutil
 import sys
+from concurrent.futures import ProcessPoolExecutor
 from pathlib import Path
 from typing import Union
 
@@ -247,10 +248,6 @@ def download_config(config_path):
     config.setdefault('train', {}).setdefault('pytest_epochs', 1)
     _override_with_test_values(config)
 
-    config_vars = config.setdefault('metadata', {}).setdefault('variables', {})
-    config_vars['ROOT_PATH'] = str(download_path)
-    config_vars['CONFIGS_PATH'] = str(test_configs_path)
-
     config_path = test_configs_path / config_path
     config_path.parent.mkdir(exist_ok=True, parents=True)
     with config_path.open("w", encoding='utf8') as fout:
@@ -277,6 +274,9 @@ def setup_module():
         for (config_path, _, _), _ in conf_dict.items():
             download_config(config_path)
 
+    os.environ['DP_ROOT_PATH'] = str(download_path)
+    os.environ['DP_CONFIGS_PATH'] = str(test_configs_path)
+
     if cache_dir:
         cache_dir.mkdir(parents=True, exist_ok=True)
         os.environ['DP_CACHE_DIR'] = str(cache_dir.resolve())
@@ -290,6 +290,23 @@ def teardown_module():
         shutil.rmtree(str(cache_dir), ignore_errors=True)
 
 
+def _serialize(config):
+    chainer = build_model(config, download=True)
+    return chainer.serialize()
+
+
+def _deserialize(config, raw_bytes, examples):
+    chainer = build_model(config, serialized=raw_bytes)
+    for *query, expected_response in examples:
+        query = [[q] for q in query]
+        actual_response = chainer(*query)
+        if expected_response is not None:
+            if actual_response is not None and len(actual_response) > 0:
+                actual_response = actual_response[0]
+            assert expected_response == str(actual_response), \
+                f"Error in interacting with {model_dir} ({conf_file}): {query}"
+
+
 @pytest.mark.parametrize("model,conf_file,model_dir,mode", TEST_GRID, scope='class')
 class TestQuickStart(object):
     @staticmethod
@@ -379,29 +396,24 @@ def test_serialization(self, model, conf_file, model_dir, mode):
         if 'IP' not in mode:
             return pytest.skip("Unsupported mode: {}".format(mode))
 
-        download_config(conf_file)
         config_file_path = test_configs_path / conf_file
-        install_config(config_file_path)
-        chainer = build_model(config_file_path, download=True)
-        raw_bytes = chainer.serialize()
-        chainer.destroy()
 
-        serialized: list = pickle.loads(raw_bytes)
-        if any(serialized):
-            serialized.clear()
+        with ProcessPoolExecutor(max_workers=1) as executor:
+            f = executor.submit(_serialize, config_file_path)
+        raw_bytes = f.result()
 
-            chainer = build_model(config_file_path, serialized=raw_bytes)
-            for *query, expected_response in PARAMS[model][(conf_file, model_dir, mode)]:
-                query = [[q] for q in query]
-                actual_response = chainer(*query)
-                if expected_response is not None:
-                    if actual_response is not None and len(actual_response) > 0:
-                        actual_response = actual_response[0]
-                    assert expected_response == str(actual_response), \
-                        f"Error in interacting with {model_dir} ({conf_file}): {query}"
-            chainer.destroy()
-        else:
+        serialized: list = pickle.loads(raw_bytes)
+        if not any(serialized):
             pytest.skip("Serialization not supported: {}".format(conf_file))
+            return
+        serialized.clear()
+
+        with ProcessPoolExecutor(max_workers=1) as executor:
+            f = executor.submit(_deserialize, config_file_path, raw_bytes, PARAMS[model][(conf_file, model_dir, mode)])
+
+        exc = f.exception()
+        if exc is not None:
+            raise exc
 
     def test_consecutive_training_and_interacting(self, model, conf_file, model_dir, mode):
         if 'TI' in mode: