BiLSTM_POS_Tagging.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Full_code_BiLSTM_POS_Tagging.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.7"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ICI0prX0vGz"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import nltk\n",
        "import numpy as np\n",
        "import os\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "from collections import defaultdict\n",
        "from keras import backend as K\n",
        "from keras.layers import Dense, LSTM, InputLayer, Bidirectional, TimeDistributed\n",
        "from keras.layers import Embedding, Activation\n",
        "from keras.models import Sequential\n",
        "from keras.optimizers import Adam\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from nltk.corpus import brown\n",
        "from sklearn.model_selection import train_test_split, KFold"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tiher5s9Vke-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 571
        },
        "outputId": "ae906d71-a2dc-483d-ee1f-c9fec32ecd6d"
      },
      "source": [
        "#Downloading the dataset\n",
        "nltk.download('brown')\n",
        "nltk.download('universal_tagset')\n",
        "\n",
        "data = nltk.corpus.brown.tagged_sents(tagset='universal')\n",
        "\n",
        "#Downloading Glove Word Embeddings\n",
        "!wget http://nlp.stanford.edu/data/glove.6B.zip\n",
        "!unzip glove.6B.zip\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package brown to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/brown.zip.\n",
            "[nltk_data] Downloading package universal_tagset to /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/universal_tagset.zip.\n",
            "--2020-10-18 06:32:51--  http://nlp.stanford.edu/data/glove.6B.zip\n",
            "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://nlp.stanford.edu/data/glove.6B.zip [following]\n",
            "--2020-10-18 06:32:52--  https://nlp.stanford.edu/data/glove.6B.zip\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip [following]\n",
            "--2020-10-18 06:32:52--  http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip\n",
            "Resolving downloads.cs.stanford.edu (downloads.cs.stanford.edu)... 171.64.64.22\n",
            "Connecting to downloads.cs.stanford.edu (downloads.cs.stanford.edu)|171.64.64.22|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 862182613 (822M) [application/zip]\n",
            "Saving to: ‘glove.6B.zip’\n",
            "\n",
            "glove.6B.zip        100%[===================>] 822.24M  2.18MB/s    in 6m 29s  \n",
            "\n",
            "2020-10-18 06:39:22 (2.11 MB/s) - ‘glove.6B.zip’ saved [862182613/862182613]\n",
            "\n",
            "Archive:  glove.6B.zip\n",
            "  inflating: glove.6B.50d.txt        \n",
            "  inflating: glove.6B.100d.txt       \n",
            "  inflating: glove.6B.200d.txt       \n",
            "  inflating: glove.6B.300d.txt       \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L4Ai_2xCKLJO"
      },
      "source": [
        "#separating sentences and tags as separate sequences\n",
        "sentences, sentence_tags =[], [] \n",
        "for tagged_sentence in data:\n",
        "    sentence, tags = zip(*tagged_sentence)\n",
        "    sentences.append(np.array(sentence))\n",
        "    sentence_tags.append(np.array(tags))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jlP9FpEL0ZV7"
      },
      "source": [
        "#Function to ignore the 0 padding while calculating accuracy\n",
        "def ignore_class_accuracy(to_ignore=0):\n",
        "    def ignore_accuracy(y_true, y_pred):\n",
        "        y_true_class = K.argmax(y_true, axis=-1)\n",
        "        y_pred_class = K.argmax(y_pred, axis=-1)\n",
        " \n",
        "        ignore_mask = K.cast(K.not_equal(y_pred_class, to_ignore), 'int32')\n",
        "        matches = K.cast(K.equal(y_true_class, y_pred_class), 'int32') * ignore_mask\n",
        "        accuracy = K.sum(matches) / K.maximum(K.sum(ignore_mask), 1)\n",
        "        return accuracy\n",
        "    return ignore_accuracy\n",
        "\n",
        "#Function to return one code encoding of tags\n",
        "def one_hot_encoding(tag_sents, n_tags):\n",
        "    tag_one_hot_sent = []\n",
        "    for tag_sent in tag_sents:\n",
        "        tags_one_hot = []\n",
        "        for tag in tag_sent:\n",
        "            tags_one_hot.append(np.zeros(n_tags))\n",
        "            tags_one_hot[-1][tag] = 1.0\n",
        "        tag_one_hot_sent.append(tags_one_hot)\n",
        "    return np.array(tag_one_hot_sent)\n",
        "\n",
        "#Function to convert output into tags\n",
        "def logits_to_tags(tag_sentences, index):\n",
        "    tag_sequences = []\n",
        "    for tag_sentence in tag_sentences:\n",
        "        tag_sequence = []\n",
        "        for tag in tag_sentence:\n",
        "            # if index[np.argmax(tag)] == \"-PAD-\":\n",
        "            #     break\n",
        "            # else:\n",
        "                tag_sequence.append(index[np.argmax(tag)])\n",
        "        tag_sequences.append(np.array(tag_sequence))\n",
        "    return tag_sequences"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OgfevJX3Vkfl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "a50ae7c1-4729-4048-a314-38b1af484e4c"
      },
      "source": [
        "#Use the 300 dimensional GLove Word Embeddings\n",
        "glove_dir = './'\n",
        "\n",
        "embeddings_index = {} #initialize dictionary\n",
        "f = open(os.path.join(glove_dir, 'glove.6B.300d.txt'))\n",
        "for line in f:\n",
        "    values = line.split()\n",
        "    word = values[0]\n",
        "    coefs = np.asarray(values[1:], dtype='float32')\n",
        "    embeddings_index[word] = coefs\n",
        "f.close()\n",
        "\n",
        "print('Found %s word vectors.' % len(embeddings_index))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 400000 word vectors.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PJ-t17BlVkft"
      },
      "source": [
        "acc = []\n",
        "conf_matrix = []\n",
        "precision_fold = []\n",
        "accuracy_fold = []\n",
        "recall_fold = []\n",
        "f1score_fold = []\n",
        "\n",
        "tag_list=['-PAD-','CONJ','PRT','PRON','DET','VERB','ADP','ADJ','ADV','.','X','NUM','NOUN']\n",
        "\n",
        "# The integers for each tag are the same as above\n",
        "\n",
        "MAX_LENGTH = len(max(sentences, key=len)) # maximum words in a sentence\n",
        "\n",
        "conf_mat_df = pd.DataFrame(columns=tag_list, index=tag_list)\n",
        "conf_mat_df = conf_mat_df.fillna(0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mt3WSLZuyPc7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 705
        },
        "outputId": "f95ef3ee-da2e-474c-dfa0-5f892767657e"
      },
      "source": [
        "num_folds = 5\n",
        "iteration = 1\n",
        "kfold = KFold(num_folds)\n",
        "\n",
        "for train_index, test_index in kfold.split(range(len(data))):\n",
        "\n",
        "    print(\"Iteration \" + str(iteration) + \" started.\")    \n",
        "    train_sentences = np.take(sentences,train_index).tolist()\n",
        "    test_sentences = np.take(sentences,test_index).tolist()\n",
        "    train_tags = np.take(sentence_tags,train_index).tolist()\n",
        "    test_tags = np.take(sentence_tags,test_index).tolist()\n",
        "    \n",
        "    true_pos_tag = defaultdict(int)\n",
        "    false_pos_tag = defaultdict(int)\n",
        "    false_neg_tag = defaultdict(int)\n",
        "    precision_tags = defaultdict(float)\n",
        "    accuracy_tags = defaultdict(float)\n",
        "    recall_tags = defaultdict(float)\n",
        "    f1score_tags = defaultdict(float)\n",
        "\n",
        "    words, tags = set([]), set([])\n",
        "    #creating sets of words and tags \n",
        "    for sentence in train_sentences:\n",
        "        for word in sentence:\n",
        "            words.add(word.lower())\n",
        " \n",
        "    for tag_sent in train_tags:\n",
        "        for tag in tag_sent:\n",
        "            tags.add(tag)\n",
        "\n",
        "    #bulding vocabulary of words and tags \n",
        "    word2index = {word: i + 2 for i, word in enumerate(list(words))}\n",
        "    word2index['-PAD-'] = 0  # 0 is assigned for padding\n",
        "    word2index['-OOV-'] = 1  # 1 is assigned for unknown words\n",
        "    tag2index = {tag: i + 1 for i, tag in enumerate(list(tags))}\n",
        "    tag2index['-PAD-'] = 0  # 0 is assigned for padding\n",
        "\n",
        "    #Tokenising words and  by their indexes in vocabulary\n",
        "    train_sentences_X, test_sentences_X, train_tags_y, test_tags_y = [], [], [], []\n",
        " \n",
        "    for sentence in train_sentences:\n",
        "        sent_int = []\n",
        "        for word in sentence:\n",
        "            try:\n",
        "                sent_int.append(word2index[word.lower()])\n",
        "            except KeyError:\n",
        "                sent_int.append(word2index['-OOV-'])\n",
        "        train_sentences_X.append(sent_int)\n",
        " \n",
        "    for sentence in test_sentences:\n",
        "        sent_int = []\n",
        "        for word in sentence:\n",
        "            try:\n",
        "                sent_int.append(word2index[word.lower()])\n",
        "            except KeyError:\n",
        "                sent_int.append(word2index['-OOV-'])\n",
        "        test_sentences_X.append(sent_int)\n",
        " \n",
        "    for sent_tags in train_tags:\n",
        "        train_tags_y.append([tag2index[tag] for tag in sent_tags])\n",
        " \n",
        "    for sent_tags in test_tags:\n",
        "        test_tags_y.append([tag2index[tag] for tag in sent_tags])\n",
        "\n",
        "    #Add padding to sentences\n",
        "    train_sentences_X = pad_sequences(train_sentences_X, maxlen=MAX_LENGTH, padding='post')\n",
        "    test_sentences_X = pad_sequences(test_sentences_X, maxlen=MAX_LENGTH, padding='post')\n",
        "    train_tags_y = pad_sequences(train_tags_y, maxlen=MAX_LENGTH, padding='post')\n",
        "    test_tags_y = pad_sequences(test_tags_y, maxlen=MAX_LENGTH, padding='post')\n",
        "\n",
        "    #Building the Embedding Layer \n",
        "    embedding_dim = 300\n",
        "\n",
        "    embedding_matrix = np.zeros((len(word2index), embedding_dim))\n",
        "    for word, i in word2index.items():\n",
        "        embedding_vector = embeddings_index.get(word)\n",
        "        if i < len(word2index):\n",
        "            if embedding_vector is not None:\n",
        "                # Words not found in embedding index will be all-zeros.\n",
        "                embedding_matrix[i] = embedding_vector\n",
        "    \n",
        "    #Building the BiLSTM model\n",
        "    model = Sequential()\n",
        "    model.add(InputLayer(input_shape=(MAX_LENGTH, ))) \n",
        "    model.add(Embedding(len(word2index), 300, weights=[embedding_matrix],trainable=False))\n",
        "    model.add(Bidirectional(LSTM(256, return_sequences=True)))\n",
        "    model.add(TimeDistributed(Dense(len(tag2index))))\n",
        "    model.add(Activation('softmax'))\n",
        " \n",
        "    model.compile(loss='categorical_crossentropy', optimizer=Adam(0.001),\n",
        "                  metrics=['accuracy', ignore_class_accuracy(0)])\n",
        "    model.summary()\n",
        "    one_hot_train_tags_y = one_hot_encoding(train_tags_y, len(tag2index))\n",
        "    \n",
        "    #Training the model\n",
        "    model.fit(train_sentences_X, one_hot_encoding(train_tags_y, len(tag2index)), \n",
        "              batch_size=128, epochs= 9, validation_split=0.2)\n",
        "    \n",
        "    scores = model.evaluate(test_sentences_X, one_hot_encoding(test_tags_y, len(tag2index)))\n",
        "    acc.append(scores[2]*100)\n",
        "    \n",
        "    \n",
        "    predictions = model.predict(test_sentences_X)\n",
        "    pred_sequence = logits_to_tags(predictions, {i: t for t, i in tag2index.items()})\n",
        "    #y_prob_class = model.predict_classes(test_sentences_X, verbose = 1)\n",
        "    \n",
        "    for sen_num in range(len(test_tags)):\n",
        "        for i,tag in enumerate(test_tags[sen_num]):\n",
        "            conf_mat_df[tag][pred_sequence[sen_num][i]] +=1\n",
        "            if test_tags[sen_num][i] == pred_sequence[sen_num][i]:\n",
        "              true_pos_tag[tag] += 1\n",
        "            else:\n",
        "              false_neg_tag[tag] += 1\n",
        "              false_pos_tag[pred_sequence[sen_num][i]] += 1\n",
        "  \n",
        "    for tag in tag_list[1:]:\n",
        "        precision_tags[tag] = true_pos_tag[tag] / (true_pos_tag[tag] + false_pos_tag[tag])\n",
        "        recall_tags[tag] = true_pos_tag[tag] / (true_pos_tag[tag] + false_neg_tag[tag])\n",
        "        f1score_tags[tag] = 2 * precision_tags[tag] * recall_tags[tag] / (precision_tags[tag] + recall_tags[tag])\n",
        "        accuracy_tags[tag] = true_pos_tag[tag] / (true_pos_tag[tag] + false_neg_tag[tag] + false_pos_tag[tag])\n",
        "        \n",
        "    #conf_matrix.append(conf_mat_df)\n",
        "    accuracy_fold.append(accuracy_tags)\n",
        "    precision_fold.append(precision_tags)\n",
        "    recall_fold.append(recall_tags)\n",
        "    f1score_fold.append(f1score_tags)\n",
        "    if iteration == 1:\n",
        "        break\n",
        "    iteration += 1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Iteration 1 started.\n",
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, 180, 300)          12708300  \n",
            "_________________________________________________________________\n",
            "bidirectional_1 (Bidirection (None, 180, 512)          1140736   \n",
            "_________________________________________________________________\n",
            "time_distributed_1 (TimeDist (None, 180, 13)           6669      \n",
            "_________________________________________________________________\n",
            "activation_1 (Activation)    (None, 180, 13)           0         \n",
            "=================================================================\n",
            "Total params: 13,855,705\n",
            "Trainable params: 1,147,405\n",
            "Non-trainable params: 12,708,300\n",
            "_________________________________________________________________\n",
            "Epoch 1/9\n",
            "287/287 [==============================] - 54s 188ms/step - loss: 0.1168 - accuracy: 0.9758 - ignore_accuracy: 0.8090 - val_loss: 0.0277 - val_accuracy: 0.9917 - val_ignore_accuracy: 0.9075\n",
            "Epoch 2/9\n",
            "287/287 [==============================] - 53s 184ms/step - loss: 0.0256 - accuracy: 0.9922 - ignore_accuracy: 0.9330 - val_loss: 0.0169 - val_accuracy: 0.9948 - val_ignore_accuracy: 0.9426\n",
            "Epoch 3/9\n",
            "287/287 [==============================] - 53s 184ms/step - loss: 0.0163 - accuracy: 0.9951 - ignore_accuracy: 0.9583 - val_loss: 0.0129 - val_accuracy: 0.9960 - val_ignore_accuracy: 0.9557\n",
            "Epoch 4/9\n",
            "287/287 [==============================] - 53s 183ms/step - loss: 0.0123 - accuracy: 0.9963 - ignore_accuracy: 0.9685 - val_loss: 0.0111 - val_accuracy: 0.9966 - val_ignore_accuracy: 0.9617\n",
            "Epoch 5/9\n",
            "287/287 [==============================] - 53s 183ms/step - loss: 0.0100 - accuracy: 0.9970 - ignore_accuracy: 0.9745 - val_loss: 0.0103 - val_accuracy: 0.9968 - val_ignore_accuracy: 0.9645\n",
            "Epoch 6/9\n",
            "287/287 [==============================] - 53s 184ms/step - loss: 0.0085 - accuracy: 0.9975 - ignore_accuracy: 0.9781 - val_loss: 0.0094 - val_accuracy: 0.9971 - val_ignore_accuracy: 0.9681\n",
            "Epoch 7/9\n",
            "287/287 [==============================] - 53s 185ms/step - loss: 0.0074 - accuracy: 0.9978 - ignore_accuracy: 0.9811 - val_loss: 0.0092 - val_accuracy: 0.9972 - val_ignore_accuracy: 0.9686\n",
            "Epoch 8/9\n",
            "287/287 [==============================] - 52s 182ms/step - loss: 0.0065 - accuracy: 0.9981 - ignore_accuracy: 0.9834 - val_loss: 0.0087 - val_accuracy: 0.9973 - val_ignore_accuracy: 0.9700\n",
            "Epoch 9/9\n",
            "287/287 [==============================] - 53s 184ms/step - loss: 0.0057 - accuracy: 0.9983 - ignore_accuracy: 0.9854 - val_loss: 0.0086 - val_accuracy: 0.9973 - val_ignore_accuracy: 0.9701\n",
            "359/359 [==============================] - 14s 40ms/step - loss: 0.0121 - accuracy: 0.9962 - ignore_accuracy: 0.9687\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wjHAIA7EjPXD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "6493423f-1e9c-4ce1-84df-f48bf7f5fe8c"
      },
      "source": [
        "len(predictions) #11468 X 180 test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "11468"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uxc9pwjbliON",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "d3700dc6-ddc4-4a52-bf6e-865ef55f8431"
      },
      "source": [
        "train_tags[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['ADP', 'PRON', 'VERB', 'ADP', 'PRON', 'NOUN', '.', 'PRON', 'VERB',\n",
              "       'DET', 'NOUN', 'VERB', 'DET', 'NOUN', 'CONJ', 'PRON', 'VERB',\n",
              "       'PRON', 'ADV', 'ADJ', '.'], dtype='<U4')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hmojEjB4lmFl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 93
        },
        "outputId": "fd989184-09e3-4a7d-8078-b5b2434809bd"
      },
      "source": [
        "predictions[0][2]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([5.38698657e-07, 1.01106925e-04, 4.76922112e-04, 8.31685669e-04,\n",
              "       5.91829530e-10, 9.98532653e-01, 2.30038477e-08, 2.82520500e-06,\n",
              "       3.32592549e-06, 9.16709178e-06, 4.04218845e-05, 1.37006992e-07,\n",
              "       1.23911263e-06], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rc8YP5VUVkf9"
      },
      "source": [
        "tot_acc = np.mean(acc)\n",
        "tot_acc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UL4kLo4R2z-Y"
      },
      "source": [
        "conf_mat_df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MfmImiKh-eAt"
      },
      "source": [
        "import seaborn as sns\n",
        "sns.heatmap(conf_mat_df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1OeJFlid-ziJ"
      },
      "source": [
        "sns.heatmap(conf_mat_df, \n",
        "             cmap='Blues')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EkrSVi7KVkgt"
      },
      "source": [
        "avg_pre_tag = defaultdict(float)\n",
        "avg_rec_tag = defaultdict(float)\n",
        "avg_fsc_tag = defaultdict(float)\n",
        "avg_acc_tag = defaultdict(float)\n",
        "\n",
        "for tag in tag_list[1:]:\n",
        "    for i in range(5):\n",
        "        avg_pre_tag[tag] += precision_fold[i][tag]\n",
        "        avg_acc_tag[tag] += accuracy_fold[i][tag]\n",
        "        avg_rec_tag[tag] += recall_fold[i][tag]\n",
        "        avg_fsc_tag[tag] += f1score_fold[i][tag]\n",
        "    print(tag + \"\\tprecision: \" + str(avg_pre_tag[tag]/5) + \"\\tf1_score: \" + str(avg_fsc_tag[tag]/5) + \"\\taccuracy: \" + str(avg_acc_tag[tag]/5) +\"\\trecall: \" + str(avg_rec_tag[tag]/5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2nnyXv4j5KJ2"
      },
      "source": [
        "\n",
        "# #\"i am a boy and I can run.\".split()\n",
        "# test_samples = test_sentences\n",
        "# test_samples_X = []\n",
        "# for s in test_samples:\n",
        "#     s_int = []\n",
        "#     for w in s:\n",
        "#         try:\n",
        "#             s_int.append(word2index[w.lower()])\n",
        "#         except KeyError:\n",
        "#             s_int.append(word2index['-OOV-'])\n",
        "#     test_samples_X.append(s_int)\n",
        "\n",
        "# test_samples_X = pad_sequences(test_samples_X, maxlen=MAX_LENGTH, padding='post')\n",
        "\n",
        "# predictions = model.predict(test_samples_X)\n",
        "# pred_sequence = logits_to_tags(predictions, {i: t for t, i in tag2index.items()})\n",
        "# # print(pred_sequence)\n",
        "# # print(test_tags[0])\n",
        "# #pred_sequence"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}