question_topic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_unzip(ip_file):\n",
    "    with open(ip_file,\"r\", encoding=\"latin1\") as f:\n",
    "        ip = f.readlines()\n",
    "    pr_list = [i.split(\" \", 1) for i in ip]\n",
    "    intent = []\n",
    "    sub_intent = []\n",
    "    que_text = []\n",
    "    for i in pr_list:\n",
    "        intent.append(i[0].split(':')[0])\n",
    "        sub_intent.append(i[0].split(':')[1])\n",
    "        que_text.append(i[1])    \n",
    "    return intent, sub_intent, que_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "intents, sub_intents, ques = data_unzip(\"train_5500.label\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def replace_contraction(text):\n",
    "    contraction_patterns = [ (r'won\\'t', 'will not'), (r'can\\'t', 'can not'), (r'i\\'m', 'i am'), (r'ain\\'t', 'is not'), (r'(\\w+)\\'ll', '\\g<1> will'), (r'(\\w+)n\\'t', '\\g<1> not'),\n",
    "                         (r'(\\w+)\\'ve', '\\g<1> have'), (r'(\\w+)\\'s', '\\g<1> is'), (r'(\\w+)\\'re', '\\g<1> are'), (r'(\\w+)\\'d', '\\g<1> would'), (r'&', 'and'), (r'dammit', 'damn it'), (r'dont', 'do not'), (r'wont', 'will not') ]\n",
    "    patterns = [(re.compile(regex), repl) for (regex, repl) in contraction_patterns]\n",
    "    for (pattern, repl) in patterns:\n",
    "        (text, count) = re.subn(pattern, repl, text)\n",
    "    return text\n",
    "def replace_links(text, filler=' '):\n",
    "        text = re.sub(r'((http|https)\\:\\/\\/)?[a-zA-Z0-9\\.\\/\\?\\:@\\-_=#]+\\.([a-zA-Z]){2,6}([a-zA-Z0-9\\.\\&\\/\\?\\:@\\-_=#])*',\n",
    "                      filler, text).strip()\n",
    "        return text\n",
    "def remove_numbers(text):\n",
    "    text = ''.join([i for i in text if not i.isdigit()])\n",
    "    return text\n",
    "def cleanText(text):\n",
    "    text = text.strip().replace(\"\\n\", \" \").replace(\"\\r\", \" \")\n",
    "    text = replace_contraction(text)\n",
    "    text = replace_links(text, \"link\")\n",
    "    text = remove_numbers(text)\n",
    "    text = re.sub(r'[,!@#$%^&*)(|/><\";:.?\\'\\\\}{]',\"\",text)\n",
    "    text = text.lower()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [cleanText(i) for i in ques]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'DESC': 1162,\n",
       "         'ENTY': 1250,\n",
       "         'ABBR': 86,\n",
       "         'HUM': 1223,\n",
       "         'NUM': 896,\n",
       "         'LOC': 835})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter(intents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LabelEncoder()"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import preprocessing\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(intents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = le.classes_.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"le_quetopic.pickle\",\"wb\") as f:\n",
    "    pickle.dump(le, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le.transform(intents[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DESC', 'ENTY']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "intents[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = le.transform(intents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "y_encoded = tf.keras.backend.one_hot(y, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'one_hot:0' shape=(5452, 6) dtype=float32>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    y_encoded = sess.run(y_encoded )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 1., 0., 0., 0.], dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_encoded[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow_hub as hub\n",
    "embed = hub.Module(\"../module/module_useT\")\n",
    "def UseTEmbedding(x):\n",
    "    return embed(tf.squeeze(tf.cast(x, tf.string)), signature=\"default\", as_dict=True)[\"default\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow.keras as keras\n",
    "from keras.layers import Input, Lambda, Dense\n",
    "from keras.models import Model\n",
    "import keras.backend as K\n",
    "def build_model(): \n",
    "    input_text = Input(shape=(1,), dtype=\"string\")\n",
    "    embedding = Lambda(UseTEmbedding, output_shape=(512, ))(input_text)\n",
    "    dense1 = Dense(256, kernel_regularizer=keras.regularizers.l2(0.001), \\\n",
    "                   activation=tf.nn.relu)(embedding)\n",
    "    dense2 = Dense(256, kernel_regularizer=keras.regularizers.l2(0.001), \\\n",
    "                   activation=tf.nn.relu)(dense1)\n",
    "    pred = Dense(num_classes, activation='sigmoid')(dense2)\n",
    "    model = Model(inputs=[input_text], outputs=[pred])\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
     ]
    }
   ],
   "source": [
    "model_useT = build_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 1)                 0         \n",
      "_________________________________________________________________\n",
      "lambda_1 (Lambda)            (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 6)                 1542      \n",
      "=================================================================\n",
      "Total params: 198,662\n",
      "Trainable params: 198,662\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_useT.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5452, 6)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_encoded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [\n",
    "         tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, verbose=0),\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 4906 samples, validate on 546 samples\n",
      "Epoch 1/50\n",
      "4906/4906 [==============================] - 119s 24ms/step - loss: 2.3301 - acc: 0.3349 - val_loss: 2.1732 - val_acc: 0.5110\n",
      "Epoch 2/50\n",
      "4906/4906 [==============================] - 115s 23ms/step - loss: 2.1149 - acc: 0.5151 - val_loss: 1.9730 - val_acc: 0.6630\n",
      "Epoch 3/50\n",
      "4906/4906 [==============================] - 111s 23ms/step - loss: 1.9063 - acc: 0.6786 - val_loss: 1.7419 - val_acc: 0.7418\n",
      "Epoch 4/50\n",
      "4906/4906 [==============================] - 115s 23ms/step - loss: 1.6630 - acc: 0.7623 - val_loss: 1.4864 - val_acc: 0.8077\n",
      "Epoch 5/50\n",
      "4906/4906 [==============================] - 116s 24ms/step - loss: 1.4298 - acc: 0.7797 - val_loss: 1.3123 - val_acc: 0.8114\n",
      "Epoch 6/50\n",
      "4906/4906 [==============================] - 113s 23ms/step - loss: 1.2789 - acc: 0.8025 - val_loss: 1.1850 - val_acc: 0.8004\n",
      "Epoch 7/50\n",
      "4906/4906 [==============================] - 115s 24ms/step - loss: 1.1560 - acc: 0.8161 - val_loss: 1.0869 - val_acc: 0.8590\n",
      "Epoch 8/50\n",
      "4906/4906 [==============================] - 115s 23ms/step - loss: 1.0621 - acc: 0.8371 - val_loss: 1.0305 - val_acc: 0.8370\n",
      "Epoch 9/50\n",
      "4906/4906 [==============================] - 114s 23ms/step - loss: 1.0205 - acc: 0.8320 - val_loss: 0.9826 - val_acc: 0.8608\n",
      "Epoch 10/50\n",
      "4906/4906 [==============================] - 115s 23ms/step - loss: 0.9499 - acc: 0.8647 - val_loss: 0.9133 - val_acc: 0.8883\n",
      "Epoch 11/50\n",
      "4906/4906 [==============================] - 115s 23ms/step - loss: 0.9002 - acc: 0.8714 - val_loss: 0.8894 - val_acc: 0.8810\n",
      "Epoch 12/50\n",
      "4906/4906 [==============================] - 114s 23ms/step - loss: 0.8766 - acc: 0.8708 - val_loss: 0.9082 - val_acc: 0.8297\n",
      "Epoch 13/50\n",
      "4906/4906 [==============================] - 114s 23ms/step - loss: 0.9012 - acc: 0.8408 - val_loss: 0.8403 - val_acc: 0.8901\n",
      "Epoch 14/50\n",
      "4906/4906 [==============================] - 115s 23ms/step - loss: 0.8193 - acc: 0.8806 - val_loss: 0.8084 - val_acc: 0.8901\n",
      "Epoch 15/50\n",
      "4906/4906 [==============================] - 116s 24ms/step - loss: 0.7971 - acc: 0.8842 - val_loss: 0.8101 - val_acc: 0.8846\n",
      "Epoch 16/50\n",
      "4906/4906 [==============================] - 116s 24ms/step - loss: 0.8092 - acc: 0.8734 - val_loss: 0.8038 - val_acc: 0.8791\n",
      "Epoch 17/50\n",
      "4906/4906 [==============================] - 114s 23ms/step - loss: 0.7818 - acc: 0.8759 - val_loss: 0.7817 - val_acc: 0.8828\n",
      "Epoch 18/50\n",
      "4906/4906 [==============================] - 117s 24ms/step - loss: 0.7684 - acc: 0.8791 - val_loss: 0.7816 - val_acc: 0.8590\n",
      "Epoch 19/50\n",
      "4906/4906 [==============================] - 116s 24ms/step - loss: 0.7692 - acc: 0.8730 - val_loss: 0.7684 - val_acc: 0.8755\n",
      "Epoch 20/50\n",
      "4906/4906 [==============================] - 117s 24ms/step - loss: 0.7357 - acc: 0.8840 - val_loss: 0.7281 - val_acc: 0.8901\n",
      "Epoch 21/50\n",
      "4906/4906 [==============================] - 117s 24ms/step - loss: 0.7123 - acc: 0.8926 - val_loss: 0.7469 - val_acc: 0.8791\n",
      "Epoch 22/50\n",
      "4906/4906 [==============================] - 114s 23ms/step - loss: 0.7076 - acc: 0.8916 - val_loss: 0.7086 - val_acc: 0.8956\n",
      "Epoch 23/50\n",
      "4906/4906 [==============================] - 116s 24ms/step - loss: 0.7015 - acc: 0.8916 - val_loss: 0.7314 - val_acc: 0.8828\n",
      "Epoch 24/50\n",
      "4906/4906 [==============================] - 116s 24ms/step - loss: 0.6953 - acc: 0.8891 - val_loss: 0.6848 - val_acc: 0.8919\n",
      "Epoch 25/50\n",
      "4906/4906 [==============================] - 115s 23ms/step - loss: 0.6728 - acc: 0.8936 - val_loss: 0.6828 - val_acc: 0.8938\n",
      "Epoch 26/50\n",
      "4906/4906 [==============================] - 115s 23ms/step - loss: 0.6611 - acc: 0.8979 - val_loss: 0.7445 - val_acc: 0.8553\n",
      "Epoch 27/50\n",
      "4906/4906 [==============================] - 117s 24ms/step - loss: 0.7043 - acc: 0.8753 - val_loss: 0.6681 - val_acc: 0.8901\n",
      "Epoch 28/50\n",
      "4906/4906 [==============================] - 114s 23ms/step - loss: 0.6509 - acc: 0.8956 - val_loss: 0.6535 - val_acc: 0.8974\n",
      "Epoch 29/50\n",
      "4906/4906 [==============================] - 107s 22ms/step - loss: 0.6307 - acc: 0.9034 - val_loss: 0.6666 - val_acc: 0.9029\n",
      "Epoch 30/50\n",
      "4906/4906 [==============================] - 113s 23ms/step - loss: 0.6517 - acc: 0.8930 - val_loss: 0.6440 - val_acc: 0.9011\n",
      "Epoch 31/50\n",
      "4906/4906 [==============================] - 113s 23ms/step - loss: 0.6190 - acc: 0.9054 - val_loss: 0.6437 - val_acc: 0.9048\n",
      "Epoch 32/50\n",
      "4906/4906 [==============================] - 109s 22ms/step - loss: 0.6240 - acc: 0.9032 - val_loss: 0.6625 - val_acc: 0.8791\n",
      "Epoch 33/50\n",
      "4906/4906 [==============================] - 113s 23ms/step - loss: 0.6295 - acc: 0.8950 - val_loss: 0.6413 - val_acc: 0.8938\n",
      "Epoch 34/50\n",
      "4906/4906 [==============================] - 116s 24ms/step - loss: 0.6064 - acc: 0.9018 - val_loss: 0.6108 - val_acc: 0.9066\n",
      "Epoch 35/50\n",
      "4906/4906 [==============================] - 115s 23ms/step - loss: 0.5942 - acc: 0.9046 - val_loss: 0.6143 - val_acc: 0.9029\n",
      "Epoch 36/50\n",
      "4906/4906 [==============================] - 114s 23ms/step - loss: 0.5856 - acc: 0.9083 - val_loss: 0.6407 - val_acc: 0.8993\n",
      "Epoch 37/50\n",
      "4906/4906 [==============================] - 114s 23ms/step - loss: 0.6185 - acc: 0.8905 - val_loss: 0.6118 - val_acc: 0.8901\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "with tf.Session() as session:\n",
    "    K.set_session(session)\n",
    "    session.run(tf.global_variables_initializer())  \n",
    "    session.run(tf.tables_initializer())\n",
    "    history = model_useT.fit(np.asarray(X), y_encoded, epochs=50, batch_size=2048, validation_split = 0.1,\n",
    "                                 verbose = 1, callbacks = callbacks)\n",
    "    model_useT.save_weights('./quetopic_model_useT.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "intents, sub_intents, ques = data_unzip(\"TREC_10.label\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "definition.html\t\tsaved_model.pb\t  train_3000.label  TREC_10.label\r\n",
      "le_quetopic.pickle\ttrain_1000.label  train_4000.label  Untitled.ipynb\r\n",
      "quetopic_model_useT.h5\ttrain_2000.label  train_5500.label  variables\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_X = [cleanText(i) for i in ques]\n",
    "import pickle\n",
    "with open(\"le_quetopic.pickle\",\"rb\") as f:\n",
    "    le = pickle.load(f)\n",
    "test_y = le.transform(intents)\n",
    "import tensorflow as tf\n",
    "num_classes = le.classes_.shape[0]\n",
    "test_y_enc = tf.keras.backend.one_hot(test_y, num_classes)\n",
    "with tf.Session() as sess:\n",
    "    test_y_enc = sess.run(test_y_enc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as session:\n",
    "    K.set_session(session)\n",
    "    session.run(tf.global_variables_initializer())  \n",
    "    session.run(tf.tables_initializer())\n",
    "    pred = model_useT.predict(np.asarray(test_X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_X[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ABBR', 'DESC', 'ENTY', 'HUM', 'LOC', 'NUM'], dtype='<U4')"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.16691467, 0.15996051, 0.16240428, 0.169468  , 0.1740457 ,\n",
       "       0.16720681], dtype=float32)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 1., 0.], dtype=float32)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_y_enc[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def replace_contraction(text):\n",
    "    contraction_patterns = [ (r'won\\'t', 'will not'), (r'can\\'t', 'can not'), (r'i\\'m', 'i am'), (r'ain\\'t', 'is not'), (r'(\\w+)\\'ll', '\\g<1> will'), (r'(\\w+)n\\'t', '\\g<1> not'),\n",
    "                         (r'(\\w+)\\'ve', '\\g<1> have'), (r'(\\w+)\\'s', '\\g<1> is'), (r'(\\w+)\\'re', '\\g<1> are'), (r'(\\w+)\\'d', '\\g<1> would'), (r'&', 'and'), (r'dammit', 'damn it'), (r'dont', 'do not'), (r'wont', 'will not') ]\n",
    "    patterns = [(re.compile(regex), repl) for (regex, repl) in contraction_patterns]\n",
    "    for (pattern, repl) in patterns:\n",
    "        (text, count) = re.subn(pattern, repl, text)\n",
    "    return text\n",
    "def replace_links(text, filler=' '):\n",
    "        text = re.sub(r'((http|https)\\:\\/\\/)?[a-zA-Z0-9\\.\\/\\?\\:@\\-_=#]+\\.([a-zA-Z]){2,6}([a-zA-Z0-9\\.\\&\\/\\?\\:@\\-_=#])*',\n",
    "                      filler, text).strip()\n",
    "        return text\n",
    "def remove_numbers(text):\n",
    "    text = ''.join([i for i in text if not i.isdigit()])\n",
    "    return text\n",
    "def cleanText(text):\n",
    "    text = text.strip().replace(\"\\n\", \" \").replace(\"\\r\", \" \")\n",
    "    text = replace_contraction(text)\n",
    "    text = replace_links(text, \"link\")\n",
    "    text = remove_numbers(text)\n",
    "    text = re.sub(r'[,!@#$%^&*)(|/><\";:.?\\'\\\\}{]',\"\",text)\n",
    "    text = text.lower()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0315 20:28:07.462201 139888066234176 __init__.py:56] Some hub symbols are not available because TensorFlow version is less than 1.14\n"
     ]
    },
    {
     "ename": "UnsupportedHandleError",
     "evalue": "unsupported handle format '../module/module_useT'. No resolvers found that can successfully resolve it. If the handle points to the local filesystem, the error indicates that the module directory does not exist. Supported handle formats: URLs pointing to a TGZ  file (e.g. https://address/module.tgz), or Local File System directory file (e.g. /tmp/my_local_module).",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mUnsupportedHandleError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-733906ae779f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow_hub\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mhub\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0membed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"../module/module_useT\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mUseTEmbedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0membed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msignature\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"default\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mas_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"default\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sambit9238/.local/lib/python3.6/site-packages/tensorflow_hub/module.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, spec, trainable, name, tags)\u001b[0m\n\u001b[1;32m    145\u001b[0m     \"\"\"\n\u001b[1;32m    146\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_v1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_default_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 147\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mas_module_spec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    148\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trainable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sambit9238/.local/lib/python3.6/site-packages/tensorflow_hub/module.py\u001b[0m in \u001b[0;36mas_module_spec\u001b[0;34m(spec)\u001b[0m\n\u001b[1;32m     34\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m   \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstring_types\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mload_module_spec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m   \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Unknown module spec type: %r\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sambit9238/.local/lib/python3.6/site-packages/tensorflow_hub/module.py\u001b[0m in \u001b[0;36mload_module_spec\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mon\u001b[0m \u001b[0mfile\u001b[0m \u001b[0mhandling\u001b[0m \u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m   \"\"\"\n\u001b[0;32m---> 60\u001b[0;31m   \u001b[0mpath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mregistry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresolver\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mregistry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sambit9238/.local/lib/python3.6/site-packages/tensorflow_hub/registry.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mimpl\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mreversed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_impls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mimpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_supported\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mimpl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m     raise RuntimeError(\n\u001b[1;32m     44\u001b[0m         \"Missing implementation that supports: %s(*%r, **%r)\" % (\n",
      "\u001b[0;32m/home/sambit9238/.local/lib/python3.6/site-packages/tensorflow_hub/resolver.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, handle)\u001b[0m\n\u001b[1;32m    496\u001b[0m         \u001b[0;34m\"exist. Supported handle formats: URLs pointing to a TGZ  file \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    497\u001b[0m         \u001b[0;34m\"(e.g. https://address/module.tgz), or Local File System directory \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 498\u001b[0;31m         \"file (e.g. /tmp/my_local_module).\" % handle)\n\u001b[0m",
      "\u001b[0;31mUnsupportedHandleError\u001b[0m: unsupported handle format '../module/module_useT'. No resolvers found that can successfully resolve it. If the handle points to the local filesystem, the error indicates that the module directory does not exist. Supported handle formats: URLs pointing to a TGZ  file (e.g. https://address/module.tgz), or Local File System directory file (e.g. /tmp/my_local_module)."
     ]
    }
   ],
   "source": [
    "import tensorflow_hub as hub\n",
    "import tensorflow as tf\n",
    "embed = hub.Module(\"../module/module_useT\")\n",
    "def UseTEmbedding(x):\n",
    "    return embed(tf.squeeze(tf.cast(x, tf.string)), signature=\"default\", as_dict=True)[\"default\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow.keras as keras\n",
    "import numpy as np\n",
    "from keras.layers import Input, Lambda, Dense\n",
    "from keras.models import Model\n",
    "import keras.backend as K\n",
    "def build_model(): \n",
    "    input_text = Input(shape=(1,), dtype=\"string\")\n",
    "    embedding = Lambda(UseTEmbedding, output_shape=(512, ))(input_text)\n",
    "    dense1 = Dense(256, kernel_regularizer=keras.regularizers.l2(0.001), \\\n",
    "                   activation=tf.nn.relu)(embedding)\n",
    "    dense2 = Dense(256, kernel_regularizer=keras.regularizers.l2(0.001), \\\n",
    "                   activation=tf.nn.relu)(dense1)\n",
    "    pred = Dense(num_classes, activation='sigmoid')(dense2)\n",
    "    model = Model(inputs=[input_text], outputs=[pred])\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ip_text = ['''how are you?''',\n",
    "          '''where does he go?''',\n",
    "          '''What is your return policy?''']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ip_X = [cleanText(i) for i in ip_text]\n",
    "import pickle\n",
    "with open(\"le_quetopic.pickle\",\"rb\") as f:\n",
    "    le = pickle.load(f)\n",
    "num_classes = le.classes_.shape[0]\n",
    "model = build_model()\n",
    "pred = None\n",
    "with tf.Session() as session:\n",
    "    K.set_session(session)\n",
    "    session.run(tf.global_variables_initializer())  \n",
    "    session.run(tf.tables_initializer())\n",
    "    model.load_weights('./quetopic_model_useT.h5')\n",
    "    pred = model.predict(np.asarray(ip_X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ABBR', 'DESC', 'ENTY', 'HUM', 'LOC', 'NUM'], dtype='<U4')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.01795071, 0.6648252 , 0.05166404, 0.00378014, 0.01417793,\n",
       "       0.24760196], dtype=float32)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred[2]/sum(pred[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ABBR': 0.017950712,\n",
       " 'DESC': 0.6648252,\n",
       " 'ENTY': 0.051664036,\n",
       " 'HUM': 0.0037801398,\n",
       " 'LOC': 0.014177929,\n",
       " 'NUM': 0.24760196}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(zip(le.classes_, pred[2]/sum(pred[2])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = []\n",
    "for i in pred:\n",
    "    result.append([])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4, 3, 0, 2, 5, 1],\n",
       "       [0, 5, 2, 3, 1, 4]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred.argsort(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred[0].argsort()[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}