Release 0.5.0 (#950)

deeppavlov · Jul 29, 2019 · 4542f08 · 4542f08
2 parents 8726173 + 4317150
commit 4542f08
Show file tree

Hide file tree

Showing 132 changed files with 2,460 additions and 1,491 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,4 +1,4 @@
-node('gpu') {
+node('cuda-module') {
     timestamps {
         try {
             stage('Clean') {
@@ -9,9 +9,9 @@ node('gpu') {
             }
             stage('Setup') {
                 env.TFHUB_CACHE_DIR="tfhub_cache"
-                env.LD_LIBRARY_PATH="/usr/local/cuda-9.0/lib64"
+                env.LD_LIBRARY_PATH="/usr/local/cuda-10.0/lib64"
                 sh """
-                    virtualenv --python=python3 '.venv-$BUILD_NUMBER'
+                    virtualenv --python=python3.7 '.venv-$BUILD_NUMBER'
                     . '.venv-$BUILD_NUMBER/bin/activate'
                     pip install .[tests,docs]
                     pip install -r deeppavlov/requirements/tf-gpu.txt
@@ -20,11 +20,17 @@ node('gpu') {
             }
             stage('Tests') {
                 sh """
+                    . /etc/profile
+                    module add cuda/10.0
                     . .venv-$BUILD_NUMBER/bin/activate
-                    pytest -v --disable-warnings
+
                     cd docs
                     make clean
                     make html
+                    cd ..
+
+                    flake8 `python -c 'import deeppavlov; print(deeppavlov.__path__[0])'` --count --select=E9,F63,F7,F82 --show-source --statistics
+                    pytest -v --disable-warnings
                 """
                 currentBuild.result = 'SUCCESS'
             }

diff --git a/README.md b/README.md
diff --git a/deeppavlov/__init__.py b/deeppavlov/__init__.py
@@ -37,7 +37,7 @@ def evaluate_model(config: [str, Path, dict], download: bool = False, recursive:
 except ImportError:
     'Assuming that requirements are not yet installed'
 
-__version__ = '0.4.0'
+__version__ = '0.5.0'
 __author__ = 'Neural Networks and Deep Learning lab, MIPT'
 __description__ = 'An open source library for building end-to-end dialog systems and training chatbots.'
 __keywords__ = ['NLP', 'NER', 'SQUAD', 'Intents', 'Chatbot']

diff --git a/deeppavlov/configs/classifiers/insults_kaggle_conv_bert.json b/deeppavlov/configs/classifiers/insults_kaggle_conv_bert.json
@@ -0,0 +1,161 @@
+{
+  "dataset_reader": {
+    "class_name": "basic_classification_reader",
+    "x": "Comment",
+    "y": "Class",
+    "data_path": "{DOWNLOADS_PATH}/insults_data"
+  },
+  "dataset_iterator": {
+    "class_name": "basic_classification_iterator",
+    "seed": 42
+  },
+  "chainer": {
+    "in": [
+      "x"
+    ],
+    "in_y": [
+      "y"
+    ],
+    "pipe": [
+      {
+        "class_name": "bert_preprocessor",
+        "vocab_file": "{DOWNLOADS_PATH}/bert_models/conversational_cased_L-12_H-768_A-12/vocab.txt",
+        "do_lower_case": false,
+        "max_seq_length": 64,
+        "in": [
+          "x"
+        ],
+        "out": [
+          "bert_features"
+        ]
+      },
+      {
+        "id": "classes_vocab",
+        "class_name": "simple_vocab",
+        "fit_on": [
+          "y"
+        ],
+        "save_path": "{MODEL_PATH}/classes.dict",
+        "load_path": "{MODEL_PATH}/classes.dict",
+        "in": [
+          "y"
+        ],
+        "out": [
+          "y_ids"
+        ]
+      },
+      {
+        "in": [
+          "y_ids"
+        ],
+        "out": [
+          "y_onehot"
+        ],
+        "class_name": "one_hotter",
+        "depth": "#classes_vocab.len",
+        "single_vector": true
+      },
+      {
+        "class_name": "bert_classifier",
+        "n_classes": "#classes_vocab.len",
+        "return_probas": true,
+        "one_hot_labels": true,
+        "bert_config_file": "{DOWNLOADS_PATH}/bert_models/conversational_cased_L-12_H-768_A-12/bert_config.json",
+        "pretrained_bert": "{DOWNLOADS_PATH}/bert_models/conversational_cased_L-12_H-768_A-12/bert_model.ckpt",
+        "save_path": "{MODEL_PATH}/model",
+        "load_path": "{MODEL_PATH}/model",
+        "keep_prob": 0.5,
+        "learning_rate": 1e-05,
+        "learning_rate_drop_patience": 5,
+        "learning_rate_drop_div": 2.0,
+        "in": [
+          "bert_features"
+        ],
+        "in_y": [
+          "y_onehot"
+        ],
+        "out": [
+          "y_pred_probas"
+        ]
+      },
+      {
+        "in": [
+          "y_pred_probas"
+        ],
+        "out": [
+          "y_pred_ids"
+        ],
+        "class_name": "proba2labels",
+        "max_proba": true
+      },
+      {
+        "in": [
+          "y_pred_ids"
+        ],
+        "out": [
+          "y_pred_labels"
+        ],
+        "ref": "classes_vocab"
+      }
+    ],
+    "out": [
+      "y_pred_labels"
+    ]
+  },
+  "train": {
+    "epochs": 100,
+    "batch_size": 64,
+    "metrics": [
+      {
+        "name": "roc_auc",
+        "inputs": [
+          "y_onehot",
+          "y_pred_probas"
+        ]
+      },
+      "sets_accuracy",
+      "f1_macro"
+    ],
+    "validation_patience": 5,
+    "val_every_n_epochs": 1,
+    "log_every_n_epochs": 1,
+    "show_examples": false,
+    "evaluation_targets": [
+      "train",
+      "valid",
+      "test"
+    ],
+    "class_name": "nn_trainer",
+    "tensorboard_log_dir": "{MODEL_PATH}/"
+  },
+  "metadata": {
+    "variables": {
+      "ROOT_PATH": "~/.deeppavlov",
+      "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
+      "MODELS_PATH": "{ROOT_PATH}/models",
+      "MODEL_PATH": "{MODELS_PATH}/classifiers/insults_kaggle_v4"
+    },
+    "requirements": [
+      "{DEEPPAVLOV_PATH}/requirements/tf.txt",
+      "{DEEPPAVLOV_PATH}/requirements/bert_dp.txt"
+    ],
+    "labels": {
+      "telegram_utils": "IntentModel",
+      "server_utils": "KerasIntentModel"
+    },
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/datasets/insults_data.tar.gz",
+        "subdir": "{DOWNLOADS_PATH}"
+      },
+      {
+        "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/conversational_cased_L-12_H-768_A-12.tar.gz",
+        "subdir": "{DOWNLOADS_PATH}/bert_models"
+      },
+      {
+        "url": "http://files.deeppavlov.ai/deeppavlov_data/classifiers/insults_kaggle_v4.tar.gz",
+        "subdir": "{MODELS_PATH}/classifiers"
+      }
+    ]
+  }
+}
diff --git a/deeppavlov/configs/classifiers/intents_sample_csv.json b/deeppavlov/configs/classifiers/intents_sample_csv.json
@@ -83,9 +83,9 @@
         "embedding_size": "#my_embedder.dim",
         "n_classes": "#classes_vocab.len",
         "kernel_sizes_cnn": [
-          3,
-          5,
-          7
+          1,
+          2,
+          3
         ],
         "filters_cnn": 256,
         "optimizer": "Adam",

diff --git a/deeppavlov/configs/classifiers/intents_sample_json.json b/deeppavlov/configs/classifiers/intents_sample_json.json
@@ -78,9 +78,9 @@
         "embedding_size": "#my_embedder.dim",
         "n_classes": "#classes_vocab.len",
         "kernel_sizes_cnn": [
-          3,
-          5,
-          7
+          1,
+          2,
+          3
         ],
         "filters_cnn": 256,
         "optimizer": "Adam",

diff --git a/deeppavlov/configs/classifiers/intents_snips.json b/deeppavlov/configs/classifiers/intents_snips.json
@@ -1,22 +1,13 @@
 {
   "dataset_reader": {
-    "class_name": "basic_classification_reader",
+    "class_name": "snips_reader",
     "x": "text",
     "y": "intents",
     "data_path": "{DOWNLOADS_PATH}/snips"
   },
   "dataset_iterator": {
-    "class_name": "basic_classification_iterator",
-    "seed": 42,
-    "field_to_split": "train",
-    "split_fields": [
-      "train",
-      "valid"
-    ],
-    "split_proportions": [
-      0.9,
-      0.1
-    ]
+    "class_name": "snips_intents_iterator",
+    "seed": 42
   },
   "chainer": {
     "in": [
@@ -145,11 +136,7 @@
       "server_utils": "KerasIntentModel"
     },
     "download": [
-      {
-        "url": "http://files.deeppavlov.ai/datasets/snips_intents/train.csv",
-        "subdir": "{DOWNLOADS_PATH}/snips"
-      },
-      {
+{
         "url": "http://files.deeppavlov.ai/deeppavlov_data/embeddings/dstc2_fastText_model.bin",
         "subdir": "{DOWNLOADS_PATH}/embeddings"
       },

diff --git a/deeppavlov/configs/classifiers/intents_snips_big.json b/deeppavlov/configs/classifiers/intents_snips_big.json
@@ -1,22 +1,13 @@
 {
   "dataset_reader": {
-    "class_name": "basic_classification_reader",
+    "class_name": "snips_reader",
     "x": "text",
     "y": "intents",
     "data_path": "{DOWNLOADS_PATH}/snips"
   },
   "dataset_iterator": {
-    "class_name": "basic_classification_iterator",
-    "seed": 42,
-    "field_to_split": "train",
-    "split_fields": [
-      "train",
-      "valid"
-    ],
-    "split_proportions": [
-      0.9,
-      0.1
-    ]
+    "class_name": "snips_intents_iterator",
+    "seed": 42
   },
   "chainer": {
     "in": [
@@ -145,11 +136,7 @@
       "server_utils": "KerasIntentModel"
     },
     "download": [
-      {
-        "url": "http://files.deeppavlov.ai/datasets/snips_intents/train.csv",
-        "subdir": "{DOWNLOADS_PATH}/snips"
-      },
-      {
+{
         "url": "http://files.deeppavlov.ai/deeppavlov_data/embeddings/wiki.en.bin",
         "subdir": "{DOWNLOADS_PATH}/embeddings"
       },