SeqNLP_Sentiment_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xT7MKZuMRaCg"
   },
   "source": [
    "# Sentiment Classification\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Wq4RCyyPSYRp"
   },
   "source": [
    "## Loading the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3430,
     "status": "ok",
     "timestamp": 1589114541670,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "rbAddtT7fFud",
    "outputId": "e8063d75-ddb2-4bd0-87c0-44e558d6900d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 32272,
     "status": "ok",
     "timestamp": 1589114570754,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "NGCtiXUhSWss",
    "outputId": "fd76a065-fc05-403c-ab4d-e4f50add08f3"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import imdb\n",
    "\n",
    "vocab_size = 10000 #vocab size\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=vocab_size) # vocab_size is no.of words to consider from the dataset, ordering based on frequency."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fCPC_WN-eCyw"
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "vocab_size = 10000 #vocab size\n",
    "maxlen = 300  #number of word used from each review"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qMEsHYrWxdtk"
   },
   "source": [
    "## Train test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "h0g381XzeCyz"
   },
   "outputs": [],
   "source": [
    "#load dataset as a list of ints\n",
    "(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=vocab_size)\n",
    "#make all sequences of the same length\n",
    "x_train = pad_sequences(x_train, maxlen=maxlen)\n",
    "x_test =  pad_sequences(x_test, maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 57559,
     "status": "ok",
     "timestamp": 1589114599359,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "Jy6n-uM2eCy2",
    "outputId": "bf87dd25-3688-4d90-ec12-288bf1fa8a52"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25000, 300)\n",
      "(25000, 300)\n"
     ]
    }
   ],
   "source": [
    "print(x_train.shape)\n",
    "print(x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 57030,
     "status": "ok",
     "timestamp": 1589114599365,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "tZhMAgaNeCy5",
    "outputId": "301c32d2-7a76-422f-8f2f-c2744d704edc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "300"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 56366,
     "status": "ok",
     "timestamp": 1589114599370,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "PfcoN5RfgfNa",
    "outputId": "5124596b-2b0d-45c4-ed66-29fff1b9e4e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "u'the'"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the words and create a dictionary with index as key and the word as value\n",
    "words=imdb.get_word_index()\n",
    "print(words['the'])\n",
    "\n",
    "index=dict([(v,k) for (k,v) in words.items() if v<=10000])\n",
    "index[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 55852,
     "status": "ok",
     "timestamp": 1589114599371,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "qG7K_Ln_ghuT",
    "outputId": "fc091012-98eb-40b9-d0ac-74063b4c7ac5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 8,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8hVuOXHrglN3"
   },
   "outputs": [],
   "source": [
    "# Load the embeddings from Glove\n",
    "from zipfile import ZipFile\n",
    "\n",
    "with ZipFile('/content/drive/My Drive/Colab Notebooks/NLP/glove.6B.zip') as f:\n",
    "  f.extractall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 90969,
     "status": "ok",
     "timestamp": 1589114635375,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "XI8VAvdzjspN",
    "outputId": "97bafa06-964c-4cb1-dd9d-34f0ec8d999f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# load the whole embedding into memory\n",
    "embeddings_index = dict()\n",
    "f = open('./glove.6B.100d.txt')\n",
    "\n",
    "for line in f:\n",
    "\tvalues = line.split()\n",
    "\tword = values[0]\n",
    "\tcoefs = np.asarray(values[1:], dtype='float32')\n",
    "\tembeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Loaded %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 90444,
     "status": "ok",
     "timestamp": 1589114635378,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "9kKN84Lhj-4n",
    "outputId": "772c8014-d328-4cf3-8249-f48ef9b17416"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10001, 100)\n"
     ]
    }
   ],
   "source": [
    "# Map the embedding vector with the words in the index:word dictionary defined before and store in embedding_matrix\n",
    "embedding_matrix = np.zeros((vocab_size+1, 100))\n",
    "\n",
    "for i,word in index.items():\n",
    "\tembedding_vector = embeddings_index.get(word)\n",
    "\tif embedding_vector is not None:\n",
    "\t\tembedding_matrix[i] = embedding_vector\n",
    "\n",
    "print(embedding_matrix.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dybtUgUReCy8"
   },
   "source": [
    "## Build Keras Embedding Layer Model\n",
    "We can think of the Embedding layer as a dicionary that maps a index assigned to a word to a word vector. This layer is very flexible and can be used in a few ways:\n",
    "\n",
    "* The embedding layer can be used at the start of a larger deep learning model. \n",
    "* Also we could load pre-train word embeddings into the embedding layer when we create our model.\n",
    "* Use the embedding layer to train our own word2vec models.\n",
    "\n",
    "The keras embedding layer doesn't require us to onehot encode our words, instead we have to give each word a unqiue intger number as an id. For the imdb dataset we've loaded this has already been done, but if this wasn't the case we could use sklearn [LabelEncoder](http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.LabelEncoder.html)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "A5OLM4eBeCy9"
   },
   "outputs": [],
   "source": [
    "%tensorflow_version 2.x\n",
    "import tensorflow\n",
    "tensorflow.__version__\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Input, LSTM, Dense, TimeDistributed,Bidirectional\n",
    "from tensorflow.keras.layers import Embedding\n",
    "import tensorflow as tf\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "input_layer=Input(shape=(maxlen,),dtype=tensorflow.int64)\n",
    "embedding = Embedding(vocab_size+1,100,weights=[embedding_matrix],trainable=True, input_length=maxlen)(input_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TxNDNhrseCzA"
   },
   "outputs": [],
   "source": [
    "lstm = Bidirectional(LSTM(20))(embedding)\n",
    "out = (Dense(2, activation=\"softmax\"))(lstm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "L3CSVVPPeCzD"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras import backend as K\n",
    "def custom_sparse_categorical_accuracy(y_true, y_pred):\n",
    "    return K.cast(K.equal(K.max(y_true, axis=-1),\n",
    "                          K.cast(K.argmax(y_pred, axis=-1), K.floatx())),\n",
    "                  K.floatx())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 697,
     "status": "ok",
     "timestamp": 1589114652576,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "OAhU_IztldSr",
    "outputId": "7d518da3-8a6f-473f-af0c-48e8ccea9c54"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 300)]             0         \n",
      "_________________________________________________________________\n",
      "embedding (Embedding)        (None, 300, 100)          1000100   \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 40)                19360     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2)                 82        \n",
      "=================================================================\n",
      "Total params: 1,019,542\n",
      "Trainable params: 1,019,542\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Model(input_layer, out)\n",
    "model.compile(optimizer=\"adam\", loss=\"sparse_categorical_crossentropy\", metrics=[custom_sparse_categorical_accuracy])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 225
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 92613,
     "status": "ok",
     "timestamp": 1589114746087,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "susOXSy1llsz",
    "outputId": "4513c950-c180-4dae-dd0e-55f11981475d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 10000 samples\n",
      "Epoch 1/5\n",
      "10000/10000 [==============================] - 21s 2ms/sample - loss: 0.6407 - custom_sparse_categorical_accuracy: 0.6108\n",
      "Epoch 2/5\n",
      "10000/10000 [==============================] - 18s 2ms/sample - loss: 0.3529 - custom_sparse_categorical_accuracy: 0.8466\n",
      "Epoch 3/5\n",
      "10000/10000 [==============================] - 18s 2ms/sample - loss: 0.1822 - custom_sparse_categorical_accuracy: 0.9319\n",
      "Epoch 4/5\n",
      "10000/10000 [==============================] - 18s 2ms/sample - loss: 0.0855 - custom_sparse_categorical_accuracy: 0.9733\n",
      "Epoch 5/5\n",
      "10000/10000 [==============================] - 18s 2ms/sample - loss: 0.0369 - custom_sparse_categorical_accuracy: 0.9907\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f72b8208f10>"
      ]
     },
     "execution_count": 17,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train=x_train[:10000,:]\n",
    "y_train=y_train[:10000]\n",
    "\n",
    "model.fit(np.array(x_train),np.array(y_train),batch_size=20, epochs=5, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 85668,
     "status": "ok",
     "timestamp": 1589114748251,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "nsahrE5NlusR",
    "outputId": "5c88ae19-4734-4733-d1be-755d87612e85"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5000/5000 [==============================] - 2s 408us/sample\n"
     ]
    }
   ],
   "source": [
    "x_test=x_test[:5000,:]\n",
    "y_test=y_test[:5000]\n",
    "\n",
    "test_pred = model.predict(np.array(x_test), verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 84863,
     "status": "ok",
     "timestamp": 1589114748255,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "_T-j9mLImChP",
    "outputId": "0f30dd37-c3ee-4566-e2ac-75614ede3677"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 19,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mapping the greater value as 1 and the lesser probability as 0\n",
    "test_pred = [1 if j>i else 0 for i,j in test_pred]\n",
    "\n",
    "test_pred[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 84139,
     "status": "ok",
     "timestamp": 1589114748257,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "InPBPuBHmGMm",
    "outputId": "0418a800-86ba-4b12-deb7-613e43a6f78f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2082,  489],\n",
       "       [ 351, 2078]])"
      ]
     },
     "execution_count": 20,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, test_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Igq8Qm8GeCzG"
   },
   "source": [
    "## Retrive the output of each layer in keras for a given single test sample from the trained model you built"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 82702,
     "status": "ok",
     "timestamp": 1589114748257,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "0AqOnLa2eCzH",
    "outputId": "d62bcf22-ddda-41df-d5c8-3bc82a009f5f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'bidirectional_1/Identity:0' shape=(None, 40) dtype=float32>"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[2].output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-dUDSg7VeCzM"
   },
   "outputs": [],
   "source": [
    "layer_outputs=[out.output for out in model.layers][1:]\n",
    "model_input = model.input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 83008,
     "status": "ok",
     "timestamp": 1589114750114,
     "user": {
      "displayName": "Kavitha K",
      "photoUrl": "",
      "userId": "09732668797975130994"
     },
     "user_tz": -330
    },
    "id": "Tskt_1npeCzP",
    "outputId": "e66cdecb-a9d5-418b-fde8-810b02ad12f7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[array([[-0.06290968,  0.01574332, -0.08873846, ..., -0.00488403,\n",
      "         0.00516531, -0.00949626],\n",
      "       [-0.83777916,  0.3790793 , -0.24576673, ..., -0.00471083,\n",
      "         0.00494218, -0.00946333],\n",
      "       [-0.5466458 ,  0.24356303, -0.13953371, ..., -0.3159853 ,\n",
      "         0.13335492, -0.00292477],\n",
      "       ...,\n",
      "       [-0.8677911 ,  0.11779376, -0.36396813, ..., -0.00488403,\n",
      "         0.00516531, -0.00949626],\n",
      "       [-0.81795   ,  0.22889858, -0.21950084, ..., -0.00488403,\n",
      "         0.00516531, -0.00949626],\n",
      "       [-0.7861555 ,  0.05340945, -0.30148953, ..., -0.00488403,\n",
      "         0.00516531, -0.00949626]], dtype=float32)]\n",
      "[array([[1.21324725e-01, 8.78675222e-01],\n",
      "       [3.18076258e-04, 9.99681950e-01],\n",
      "       [7.54319504e-03, 9.92456794e-01],\n",
      "       ...,\n",
      "       [6.83021091e-04, 9.99316931e-01],\n",
      "       [5.02069760e-03, 9.94979322e-01],\n",
      "       [3.71700473e-04, 9.99628305e-01]], dtype=float32)]\n",
      "[array([[[-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        [-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        [-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        ...,\n",
      "        [-0.25009653,  0.1558156 ,  0.29005256, ..., -0.50640386,\n",
      "          0.49600402,  0.15009421],\n",
      "        [-0.5633234 ,  0.3514712 ,  1.0457319 , ..., -1.2479844 ,\n",
      "          0.7307707 ,  0.516163  ],\n",
      "        [-0.6560009 ,  0.3228403 ,  0.48765302, ..., -0.99230283,\n",
      "         -0.7464403 ,  0.648284  ]],\n",
      "\n",
      "       [[-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        [-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        [-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        ...,\n",
      "        [ 0.03757556,  0.6069964 ,  0.274184  , ..., -0.22671428,\n",
      "          0.5737247 ,  0.4561237 ],\n",
      "        [-0.15752225, -0.33359134,  0.9027184 , ..., -0.6381364 ,\n",
      "          0.9390895 ,  0.22076908],\n",
      "        [ 0.24915297,  1.3832814 , -0.13677393, ..., -0.8264307 ,\n",
      "          0.05414074,  0.8594739 ]],\n",
      "\n",
      "       [[ 0.42669994,  1.0812908 ,  0.44221175, ..., -0.6146979 ,\n",
      "          0.14069092, -0.03681722],\n",
      "        [ 0.4843771 , -0.6834083 , -0.11419281, ..., -0.29322302,\n",
      "         -0.70836204, -0.33378357],\n",
      "        [-0.17189434,  0.34198853,  0.6535652 , ...,  0.27249375,\n",
      "          0.52038306,  0.35993695],\n",
      "        ...,\n",
      "        [-0.40702182,  0.15501732,  0.9770148 , ..., -0.52570814,\n",
      "          0.33063468,  0.5890914 ],\n",
      "        [-0.02663568,  0.6950425 ,  0.5858706 , ...,  0.11222595,\n",
      "          0.47243753,  0.3021525 ],\n",
      "        [-0.2348269 ,  0.8151834 ,  0.47284496, ..., -0.5735323 ,\n",
      "         -0.13818745,  0.11309975]],\n",
      "\n",
      "       ...,\n",
      "\n",
      "       [[-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        [-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        [-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        ...,\n",
      "        [-0.28457344, -0.02653714,  0.26639223, ..., -0.4025916 ,\n",
      "          0.35030547, -0.048234  ],\n",
      "        [ 0.14040755, -0.15044655,  0.04572811, ..., -0.47243086,\n",
      "          0.9394365 ,  0.49673212],\n",
      "        [-0.4127605 ,  0.62918407,  0.1676075 , ...,  0.07150126,\n",
      "          0.21779795,  0.07950599]],\n",
      "\n",
      "       [[-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        [-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        [-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        ...,\n",
      "        [ 0.17723729, -0.55134434, -0.09077956, ...,  0.33258402,\n",
      "          0.66542816,  0.26600716],\n",
      "        [-0.28457344, -0.02653714,  0.26639223, ..., -0.4025916 ,\n",
      "          0.35030547, -0.048234  ],\n",
      "        [ 0.11452609,  0.15162769, -0.20229006, ..., -0.21735716,\n",
      "          1.0238339 ,  0.39330277]],\n",
      "\n",
      "       [[-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        [-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        [-0.01995359, -0.03995002, -0.0019082 , ...,  0.03327145,\n",
      "         -0.04077213, -0.01152533],\n",
      "        ...,\n",
      "        [ 0.06865751, -0.24622782,  0.17665814, ..., -0.12573294,\n",
      "          0.7438595 , -0.33437586],\n",
      "        [ 0.04744919,  0.27464044,  0.35191178, ..., -0.4974542 ,\n",
      "          0.5836645 , -0.04890705],\n",
      "        [-0.1433879 ,  0.48588526,  0.25707638, ..., -0.3869816 ,\n",
      "          0.9063313 ,  0.9141414 ]]], dtype=float32)]\n"
     ]
    }
   ],
   "source": [
    "functions = [K.function([model_input],[out]) for out in layer_outputs]\n",
    "\n",
    "outs = [func([x_test]) for func in functions]\n",
    "\n",
    "print(outs[1])    # Output of Bidirectional layer\n",
    "\n",
    "print(outs[2])    # Output of Dense softmax layer\n",
    "\n",
    "print(outs[0])    # Output of the embedding layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FRQ-lwHcmrgT"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "SeqNLP_Project1_Kavitha_Kothandaraman.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}