Release 0.10.0 (#1233)

deeppavlov · May 27, 2020 · 830940e · 830940e
2 parents 5f862b0 + ae9e5b6
commit 830940e
Show file tree

Hide file tree

Showing 107 changed files with 6,012 additions and 1,825 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Report on a bug you encountered
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Want to contribute to DeepPavlov? Please read the [contributing guideline](http://docs.deeppavlov.ai/en/master/devguides/contribution_guide.html) first.
+
+Please enter all the information below, otherwise your issue may be closed without a warning. 
+
+
+**DeepPavlov version** (you can look it up by running `pip show deeppavlov`):
+
+**Python version**:
+
+**Operating system** (ubuntu linux, windows, ...):
+
+**Issue**:
+
+
+**Content or a name of a configuration file**:
+```
+
+```
+
+
+**Command that led to error**:
+```
+
+```
+
+**Error (including full traceback)**:
+```
+
+```
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a question
+    url: https://forum.deeppavlov.ai/
+    about: If you have a different question, please ask it in the forum https://forum.deeppavlov.ai
diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md
@@ -0,0 +1,26 @@
+---
+name: Feature request
+about: Suggest a feature to improve the DeepPavlov library
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Want to contribute to DeepPavlov? Please read the [contributing guideline](http://docs.deeppavlov.ai/en/master/devguides/contribution_guide.html) first.
+
+
+**What problem are we trying to solve?**:
+```
+
+```
+
+**How can we solve it?**:
+```
+
+```
+
+**Are there other issues that block this solution?**:
+```
+
+```
diff --git a/README.md b/README.md
@@ -30,11 +30,15 @@ Please leave us [your feedback](https://forms.gle/i64fowQmiVhMMC7f9) on how we c
 
 [Intent/Sentence Classification](http://docs.deeppavlov.ai/en/master/features/models/classifiers.html) |  [Question Answering over Text (SQuAD)](http://docs.deeppavlov.ai/en/master/features/models/squad.html) 
 
+[Knowledge Base Question Answering](http://docs.deeppavlov.ai/en/master/features/models/kbqa.html)
+
 [Sentence Similarity/Ranking](http://docs.deeppavlov.ai/en/master/features/models/neural_ranking.html) | [TF-IDF Ranking](http://docs.deeppavlov.ai/en/master/features/models/tfidf_ranking.html) 
 
-[Morphological tagging](http://docs.deeppavlov.ai/en/master/features/models/morphotagger.html) | [Automatic Spelling Correction](http://docs.deeppavlov.ai/en/master/features/models/spelling_correction.html)
+[Morphological tagging](http://docs.deeppavlov.ai/en/master/features/models/morphotagger.html) | [Syntactic parsing](http://docs.deeppavlov.ai/en/master/features/models/syntaxparser.html)
+
+[Automatic Spelling Correction](http://docs.deeppavlov.ai/en/master/features/models/spelling_correction.html) | [ELMo training and fine-tuning](http://docs.deeppavlov.ai/en/master/apiref/models/elmo.html)
 
-[ELMo training and fine-tuning](http://docs.deeppavlov.ai/en/master/apiref/models/elmo.html)
+[Speech recognition and synthesis (ASR and TTS)](http://docs.deeppavlov.ai/en/master/features/models/nemo.html) based on [NVIDIA NeMo](https://nvidia.github.io/NeMo/index.html)
 
 **Skills**
 

diff --git a/deeppavlov/__init__.py b/deeppavlov/__init__.py
@@ -15,36 +15,24 @@
 import sys
 from pathlib import Path
 
+from ._meta import __author__, __description__, __email__, __keywords__, __license__, __version__
+from .configs import configs
+from .core.commands.infer import build_model
+from .core.commands.train import train_evaluate_model_from_config
+from .core.common.chainer import Chainer
 from .core.common.log import init_logger
+from .download import deep_download
 
-try:
-    from .configs import configs
-    # noinspection PyUnresolvedReferences
-    from .core.commands.infer import build_model
-    # noinspection PyUnresolvedReferences
-    from .core.commands.train import train_evaluate_model_from_config
-    from .download import deep_download
-    from .core.common.chainer import Chainer
 
+# TODO: make better
+def train_model(config: [str, Path, dict], download: bool = False, recursive: bool = False) -> Chainer:
+    train_evaluate_model_from_config(config, download=download, recursive=recursive)
+    return build_model(config, load_trained=True)
 
-    # TODO: make better
-    def train_model(config: [str, Path, dict], download: bool = False, recursive: bool = False) -> Chainer:
-        train_evaluate_model_from_config(config, download=download, recursive=recursive)
-        return build_model(config, load_trained=True)
 
+def evaluate_model(config: [str, Path, dict], download: bool = False, recursive: bool = False) -> dict:
+    return train_evaluate_model_from_config(config, to_train=False, download=download, recursive=recursive)
 
-    def evaluate_model(config: [str, Path, dict], download: bool = False, recursive: bool = False) -> dict:
-        return train_evaluate_model_from_config(config, to_train=False, download=download, recursive=recursive)
-
-except ImportError:
-    'Assuming that requirements are not yet installed'
-
-__version__ = '0.9.1'
-__author__ = 'Neural Networks and Deep Learning lab, MIPT'
-__description__ = 'An open source library for building end-to-end dialog systems and training chatbots.'
-__keywords__ = ['NLP', 'NER', 'SQUAD', 'Intents', 'Chatbot']
-__license__ = 'Apache License, Version 2.0'
-__email__ = 'info@deeppavlov.ai'
 
 # check version
 assert sys.hexversion >= 0x3060000, 'Does not work in python3.5 or lower'

diff --git a/deeppavlov/_meta.py b/deeppavlov/_meta.py
@@ -0,0 +1,6 @@
+__version__ = '0.10.0'
+__author__ = 'Neural Networks and Deep Learning lab, MIPT'
+__description__ = 'An open source library for building end-to-end dialog systems and training chatbots.'
+__keywords__ = ['NLP', 'NER', 'SQUAD', 'Intents', 'Chatbot']
+__license__ = 'Apache License, Version 2.0'
+__email__ = 'info@deeppavlov.ai'
diff --git a/deeppavlov/configs/classifiers/query_pr.json b/deeppavlov/configs/classifiers/query_pr.json
@@ -0,0 +1,120 @@
+{
+  "dataset_reader": {
+    "class_name": "basic_classification_reader",
+    "x": "Question",
+    "y": "Class",
+    "data_path": "{DOWNLOADS_PATH}/query_prediction"
+  },
+  "dataset_iterator": {
+    "class_name": "basic_classification_iterator",
+    "seed": 42
+  },
+  "chainer": {
+    "in": ["x"],
+    "in_y": ["y"],
+    "pipe": [
+      {
+        "class_name": "bert_preprocessor",
+        "vocab_file": "{DOWNLOADS_PATH}/bert_models/cased_L-12_H-768_A-12/vocab.txt",
+        "do_lower_case": false,
+        "max_seq_length": 64,
+        "in": ["x"],
+        "out": ["bert_features"]
+      },
+      {
+        "id": "classes_vocab",
+        "class_name": "simple_vocab",
+        "fit_on": ["y"],
+        "save_path": "{MODEL_PATH}/classes.dict",
+        "load_path": "{MODEL_PATH}/classes.dict",
+        "in": "y",
+        "out": "y_ids"
+      },
+      {
+        "in": "y_ids",
+        "out": "y_onehot",
+        "class_name": "one_hotter",
+        "depth": "#classes_vocab.len",
+        "single_vector": true
+      },
+      {
+        "class_name": "bert_classifier",
+        "n_classes": "#classes_vocab.len",
+        "return_probas": true,
+        "one_hot_labels": true,
+        "bert_config_file": "{DOWNLOADS_PATH}/bert_models/cased_L-12_H-768_A-12/bert_config.json",
+        "pretrained_bert": "{DOWNLOADS_PATH}/bert_models/cased_L-12_H-768_A-12/bert_model.ckpt",
+        "save_path": "{MODEL_PATH}/model",
+        "load_path": "{MODEL_PATH}/model",
+        "keep_prob": 0.5,
+        "learning_rate": 1e-05,
+        "learning_rate_drop_patience": 5,
+        "learning_rate_drop_div": 2.0,
+        "in": ["bert_features"],
+        "in_y": ["y_onehot"],
+        "out": ["y_pred_probas"]
+      },
+      {
+        "in": "y_pred_probas",
+        "out": "y_pred_ids",
+        "class_name": "proba2labels",
+        "max_proba": true
+      },
+      {
+        "in": "y_pred_ids",
+        "out": "y_pred_labels",
+        "ref": "classes_vocab"
+      }
+    ],
+    "out": ["y_pred_labels"]
+  },
+  "train": {
+    "epochs": 100,
+    "batch_size": 64,
+    "metrics": [
+      "sets_accuracy",
+      "f1_macro",
+      {
+        "name": "roc_auc",
+        "inputs": ["y_onehot", "y_pred_probas"]
+      }
+    ],
+    "validation_patience": 5,
+    "val_every_n_epochs": 1,
+    "log_every_n_epochs": 1,
+    "show_examples": false,
+    "evaluation_targets": ["train", "valid", "test"],
+    "class_name": "nn_trainer",
+    "tensorboard_log_dir": "{MODEL_PATH}/"
+  },
+  "metadata": {
+    "variables": {
+      "ROOT_PATH": "~/.deeppavlov",
+      "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
+      "MODELS_PATH": "{ROOT_PATH}/models",
+      "MODEL_PATH": "{MODELS_PATH}/classifiers/query_prediction"
+    },
+    "requirements": [
+      "{DEEPPAVLOV_PATH}/requirements/tf.txt",
+      "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt"
+    ],
+    "labels": {
+      "telegram_utils": "IntentModel",
+      "server_utils": "KerasIntentModel"
+    },
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/kbqa/datasets/query_prediction.tar.gz",
+        "subdir": "{DOWNLOADS_PATH}/query_prediction"
+      },
+      {
+        "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/cased_L-12_H-768_A-12.zip",
+        "subdir": "{DOWNLOADS_PATH}/bert_models"
+      },
+      {
+        "url": "http://files.deeppavlov.ai/kbqa/models/query_prediction.tar.gz",
+        "subdir": "{MODELS_PATH}/classifiers/query_prediction"
+      }
+    ]
+  }
+}
diff --git a/deeppavlov/configs/classifiers/rel_ranking_bert.json b/deeppavlov/configs/classifiers/rel_ranking_bert.json
@@ -0,0 +1,80 @@
+{
+  "dataset_reader": {
+    "class_name": "paraphraser_reader",
+    "data_path": "{DOWNLOADS_PATH}/rel_ranking_bert",
+    "do_lower_case": false
+  },
+  "dataset_iterator": {
+    "class_name": "siamese_iterator",
+    "seed": 243,
+    "len_valid": 500
+  },
+  "chainer": {
+    "in": ["text_a", "text_b"],
+    "in_y": ["y"],
+    "pipe": [
+      {
+        "class_name": "bert_preprocessor",
+        "vocab_file": "{DOWNLOADS_PATH}/bert_models/cased_L-12_H-768_A-12/vocab.txt",
+        "do_lower_case": false,
+        "max_seq_length": 64,
+        "in": ["text_a", "text_b"],
+        "out": ["bert_features"]
+      },
+      {
+        "class_name": "bert_classifier",
+        "n_classes": 2,
+        "return_probas": true,
+        "one_hot_labels": false,
+        "bert_config_file": "{DOWNLOADS_PATH}/bert_models/cased_L-12_H-768_A-12/bert_config.json",
+        "pretrained_bert": "{DOWNLOADS_PATH}/bert_models/cased_L-12_H-768_A-12/bert_model.ckpt",
+        "save_path": "{MODEL_PATH}/model",
+        "load_path": "{MODEL_PATH}/model",
+        "keep_prob": 0.5,
+        "learning_rate": 2e-05,
+        "learning_rate_drop_patience": 2,
+        "learning_rate_drop_div": 2.0,
+        "in": ["bert_features"],
+        "in_y": ["y"],
+        "out": ["predictions"]
+      }
+    ],
+    "out": ["predictions"]
+  },
+  "train": {
+    "batch_size": 32,
+    "pytest_max_batches": 2,
+    "metrics": ["f1", "acc"],
+    "validation_patience": 10,
+    "val_every_n_batches": 100,
+    "log_every_n_batches": 100,
+    "evaluation_targets": ["train", "valid", "test"],
+    "tensorboard_log_dir": "{MODEL_PATH}/"
+  },
+  "metadata": {
+    "variables": {
+      "ROOT_PATH": "~/.deeppavlov",
+      "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
+      "MODELS_PATH": "{ROOT_PATH}/models",
+      "MODEL_PATH": "{MODELS_PATH}/rel_ranking_bert"
+    },
+    "requirements": [
+      "{DEEPPAVLOV_PATH}/requirements/tf.txt",
+      "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt"
+    ],
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/kbqa/datasets/rel_ranking_bert.tar.gz",
+        "subdir": "{DOWNLOADS_PATH}/rel_ranking_bert"
+      },
+      {
+        "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/cased_L-12_H-768_A-12.zip",
+        "subdir": "{DOWNLOADS_PATH}/bert_models"
+      },
+      {
+        "url": "http://files.deeppavlov.ai/kbqa/models/rel_ranking_bert.tar.gz",
+        "subdir": "{MODELS_PATH}/rel_ranking_bert"
+      }
+    ]
+  }
+}