6_Self_Case_Study_2_Final_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Importing all necessary Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Embedding, LSTM, Bidirectional\n",
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "import tensorflow as tf\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tensorflow import keras\n",
    "\n",
    "# import tensorflow as tf\n",
    "from keras.layers import Dense, Conv2D, MaxPool2D , Flatten\n",
    "from keras.models import Model\n",
    "\n",
    "\n",
    "\n",
    "#importing tensorflow\n",
    "from tensorflow.keras.layers import Dense,Input,Activation,Dropout,Flatten,MaxPool1D,concatenate,Embedding,BatchNormalization\n",
    "from keras.layers import CuDNNLSTM,SpatialDropout1D,GlobalMaxPooling1D,GlobalAveragePooling1D,Concatenate,Add,CuDNNGRU\n",
    "from tensorflow.keras.models import Model,Sequential\n",
    "import random as rn\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import itertools\n",
    "import os\n",
    "import shutil\n",
    "import random\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "%matplotlib inline\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import nltk\n",
    "import spacy\n",
    "import string\n",
    "pd.options.mode.chained_assignment = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data = pd.read_csv(r\"C:\\Users\\Lenovo\\Downloads\\Self case study 2\\Data\\jigsaw-unintended-bias-in-toxicity-classification\\train.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Below files contain vocabulary and jigsaw all symbols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "  \n",
    "# Open the file in binary mode\n",
    "with open('preprocessing_data_final_pipeline.pkl', 'rb') as file:\n",
    "    # Call load method to deserialze\n",
    "    embeddings_index,jigsaw_symbols= pickle.load(file)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. <i> Converting emojis to meaningful word <i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/57580288/how-to-replace-emoji-to-word-in-a-text\n",
    "import emoji\n",
    "def emojis_to_word(text):\n",
    "    text = emoji.demojize(text, delimiters=(\"\", \"\"))\n",
    "    text = text.replace('_',' ')\n",
    "    return text\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. <i> Converting to Lower case <i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data['comment_text'] = Data['comment_text'].str.lower()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. <i>Decontraction of word<i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://www.geeksforgeeks.org/nlp-expand-contractions-in-text-processing/\n",
    "import contractions\n",
    "def decontraction(text):\n",
    "    expanded_words = []    \n",
    "    for word in text.split():\n",
    "      # using contractions.fix to expand the shotened words\n",
    "        expanded_words.append(contractions.fix(word))   \n",
    "\n",
    "    expanded_text = ' '.join(expanded_words)\n",
    "    return expanded_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4A. <i>Applying Lemmatization<i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "def Lemmatization(text):\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    text = lemmatizer.lemmatize(str(text))\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4B. <i>Applying Stemming <i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem.porter import PorterStemmer\n",
    "stemmer = PorterStemmer()\n",
    "def stem_words(text):\n",
    "    return \" \".join([stemmer.stem(word) for word in text.split()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5A. <i>Removing non english word <i>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> We will use custom vocab. glove vocab to drop non english word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## Glove file\n",
    "# from numpy import asarray\n",
    "# # load the whole embedding into memory\n",
    "# embeddings_index = dict()\n",
    "# f = open(r\"C:\\Users\\Lenovo\\Downloads\\glove.6B\\glove.6B.100d.txt\",encoding=\"utf8\")\n",
    "# for line in f:\n",
    "#     values = line.split()\n",
    "#     word = values[0]\n",
    "#     coefs = asarray(values[1:], dtype='float32')\n",
    "#     embeddings_index[word] = coefs\n",
    "# f.close()\n",
    "# print('Loaded %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Non_english_word_dropping(text):\n",
    "    final = []\n",
    "    text_temp = text.split()\n",
    "    for i in text_temp:\n",
    "        if i in embeddings_index:\n",
    "            final.append(i)\n",
    "    text = ' '.join(final)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5B. <i>Removing non english Character/symbols and stop words <i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "# \", \".join(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "179\n",
      "176\n"
     ]
    }
   ],
   "source": [
    "STOPWORDS = set(stopwords.words('english'))\n",
    "print(len(STOPWORDS))\n",
    "STOPWORDS = list(STOPWORDS - {'no','nor','not'})\n",
    "print(len(STOPWORDS))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stopwords(text):\n",
    "    \"\"\"custom function to remove the stopwords\"\"\"\n",
    "    return \" \".join([word for word in str(text).split() if word not in STOPWORDS])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_jigsaw_symbols(text):\n",
    "    \"\"\"custom function to remove the punctuation\"\"\"\n",
    "    return text.translate(str.maketrans('', '', jigsaw_symbols))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' lets write both in one Function'''\n",
    "def remove_jigsaw_symbol_stop_word(text):\n",
    "    ## Dont change order of calling function 1.remove_jigsaw_symbols and 2.remove_stopwords\n",
    "    text = remove_jigsaw_symbols(text)\n",
    "    text = remove_stopwords(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. <i>Removing http/https links <i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def removing_links(text):\n",
    "    text = re.sub(\"(http|https)://[\\w\\-]+(\\.[\\w\\-]+)+\\S*\", \"\", text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. <i>removal of Numbers,tabs and newline (\\t,\\n) <i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_num_tab_newline(text):\n",
    "    text = re.sub(r\"[\\d\\t\\n]*\", \"\", text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. <i>removal Punctuations(\\t,\\n) <i>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> we have already removed punctuation,number, when we removed jigsaw symbols. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. <i>removal extra space<i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_extra_speces(text):\n",
    "    text = re.sub(r\"[\\s\\s]+\", \" \", text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "# with open('preprocessing_data_final_pipeline.pkl', 'wb') as file:\n",
    "#     pickle.dump([embeddings_index,jigsaw_symbols], file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Writing single function for preprocessing the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessing_DL(Data):\n",
    "    ## Note : dont change preprocessing sequence\n",
    "    ''' Converting Emojis to word'''\n",
    "    Data[\"comment_text_pre\"] = Data[\"comment_text\"].apply(lambda text: emojis_to_word(text))\n",
    "    '''Converting in Lower case'''\n",
    "    Data[\"comment_text_pre\"] = Data['comment_text_pre'].str.lower()\n",
    "    '''Decontraction'''\n",
    "    Data[\"comment_text_pre\"] = Data[\"comment_text_pre\"].apply(lambda text: decontraction(text))\n",
    "    '''Lemmatization'''\n",
    "    Data[\"comment_text_pre\"] = Data[\"comment_text_pre\"].apply(lambda text: Lemmatization(text))\n",
    "    '''removing_links'''\n",
    "    Data[\"comment_text_pre\"] = Data[\"comment_text_pre\"].apply(lambda text: removing_links(text))\n",
    "    '''remove_jigsaw_symbol/character_stop_word'''\n",
    "    Data[\"comment_text_pre\"] = Data[\"comment_text_pre\"].apply(lambda text: remove_jigsaw_symbol_stop_word(text))\n",
    "    '''Non_english_word_dropping'''\n",
    "    Data[\"comment_text_pre\"] = Data[\"comment_text_pre\"].apply(lambda text: Non_english_word_dropping(text))\n",
    "    '''remove_num_tab_newline'''\n",
    "    Data[\"comment_text_pre\"] = Data[\"comment_text_pre\"].apply(lambda text: remove_num_tab_newline(text))\n",
    "    '''remove_extra_speces'''\n",
    "    Data[\"comment_text_pre\"] = Data[\"comment_text_pre\"].apply(lambda text: remove_extra_speces(text))\n",
    "    \n",
    "    return Data[\"comment_text_pre\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Loding train tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('train_tokenizer.pkl', 'rb') as file:\n",
    "    tokenizer,max_length = pickle.load(file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Loding best model: Bidirectional_LSTM_128_128\n",
    "* weighted AUC : 0.8946"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "filepath =r\"C:\\Users\\Lenovo\\Downloads\\Self case study 2\\Untitled Folder 1\\my_best_model_2.hdf5\"\n",
    "model = load_model(filepath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ##  Final Function \n",
    "* Input: data frame (single row)\n",
    "* Output : probability score  and class "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Final_1(text):\n",
    "#     data preprocessing\n",
    "    text = preprocessing_DL(text) # Inpu: pass text as single row pandas data frame Output: list[single value]\n",
    "    text.dropna(inplace=True)\n",
    "    test_sequences = tokenizer.texts_to_sequences(text)\n",
    "    test_padding = pad_sequences(test_sequences,padding='pre',maxlen=max_length)\n",
    "    prediction =  model.predict(test_padding) \n",
    "    if prediction > 0.6:\n",
    "        class_ = 'Toxic'\n",
    "    else:\n",
    "        class_ = 'Non Toxic'\n",
    "    return prediction,class_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ## Input :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {'comment_text': [\" 'This is so cool. Its like, would you want your mother to read this?? Really great idea, well done!',\"]}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ## Output :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {'comment_text': [\" 'This is so cool. Its like, would you want your mother to read this?? Really great idea, well done!',\"]}\n",
    "text = pd.DataFrame(d)\n",
    "predict_,class_ = Final_1(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Class: Non toixc \n",
      "predicted Class:  Non Toxic \n",
      "toxic score : 0.003443027\n"
     ]
    }
   ],
   "source": [
    "text = pd.DataFrame(d)\n",
    "predict_,class_ = Final_1(text)\n",
    "print(\"Original Class: Non toixc\",'\\npredicted Class: ',class_,'\\ntoxic score :',predict_[0][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ## Input :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {'comment_text':['hahahahahahahahhha suck it']}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ## Output :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Class: toixc \n",
      "predicted Class:  Toxic \n",
      "toxic score : 0.9230589\n"
     ]
    }
   ],
   "source": [
    "text = pd.DataFrame(d)\n",
    "predict_,class_ = Final_1(text)\n",
    "print(\"Original Class: toixc\",'\\npredicted Class: ',class_,'\\ntoxic score :',predict_[0][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> below data frame contain all text which contain identity group, we are checking how our modle is performing on these data set  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ## Input :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {'comment_text': ['i am man',\n",
    "                      'i am woman',\n",
    "                      'i am lesbian', \n",
    "                      'i am gay man',\n",
    "                      'i am dyke',\n",
    "                      'i am white man',\n",
    "                      'i am gay white man',\n",
    "                      'i am black man',\n",
    "                      'i am gay white man',\n",
    "                      'i am black man',\n",
    "                      'i am an gay white women',\n",
    "                      'i am gay black man',\n",
    "                      'i am black women',\n",
    "                      'i am gay black women ',\n",
    "                     'This is so cool. Its like, would you want your mother to read this?? Really great idea, well done!',\n",
    "                     \"This is such an urgent design problem; kudos to you for taking it on. Very impressive!\",\n",
    "                     \"hahahahahahahahhha suck it.\",\n",
    "                     \"It's ridiculous that these guys are being called protesters. Being armed is a threat of violence, which makes them terrorists.\",\n",
    "                     \"Angry trolls, misogynists and Racists, oh my. It doesn't take all of my 150 IQ to see the slant here.  it's the Diversity diode at work yet again. We can say anything that we want because we are Diversity. You on the other hand must only  say what we allow you to say. From now on, winning arguments against any member of diversity will be considered offensive language.  facts, cogent, linear posts and Math are now verboten.\"]}\n",
    "text = pd.DataFrame(d)\n",
    "predict_ = Final_1(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ## Output :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment_text</th>\n",
       "      <th>comment_text_pre</th>\n",
       "      <th>my score</th>\n",
       "      <th>bias score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i am man</td>\n",
       "      <td>man</td>\n",
       "      <td>0.023999</td>\n",
       "      <td>0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>i am woman</td>\n",
       "      <td>woman</td>\n",
       "      <td>0.020597</td>\n",
       "      <td>0.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i am lesbian</td>\n",
       "      <td>lesbian</td>\n",
       "      <td>0.308145</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i am gay man</td>\n",
       "      <td>gay man</td>\n",
       "      <td>0.403672</td>\n",
       "      <td>0.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i am dyke</td>\n",
       "      <td>dyke</td>\n",
       "      <td>0.128925</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>i am white man</td>\n",
       "      <td>white man</td>\n",
       "      <td>0.092565</td>\n",
       "      <td>0.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>i am gay white man</td>\n",
       "      <td>gay white man</td>\n",
       "      <td>0.512516</td>\n",
       "      <td>0.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>i am black man</td>\n",
       "      <td>black man</td>\n",
       "      <td>0.159100</td>\n",
       "      <td>0.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>i am gay white man</td>\n",
       "      <td>gay white man</td>\n",
       "      <td>0.512516</td>\n",
       "      <td>0.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>i am black man</td>\n",
       "      <td>black man</td>\n",
       "      <td>0.159100</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>i am an gay white women</td>\n",
       "      <td>gay white women</td>\n",
       "      <td>0.674933</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>i am gay black man</td>\n",
       "      <td>gay black man</td>\n",
       "      <td>0.579392</td>\n",
       "      <td>0.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>i am black women</td>\n",
       "      <td>black women</td>\n",
       "      <td>0.336949</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>i am gay black women</td>\n",
       "      <td>gay black women</td>\n",
       "      <td>0.714429</td>\n",
       "      <td>0.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>This is so cool. Its like, would you want your...</td>\n",
       "      <td>cool like would want mother read really great ...</td>\n",
       "      <td>0.003304</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>This is such an urgent design problem; kudos t...</td>\n",
       "      <td>urgent design problem kudos taking impressive</td>\n",
       "      <td>0.001713</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>hahahahahahahahhha suck it.</td>\n",
       "      <td>suck</td>\n",
       "      <td>0.923059</td>\n",
       "      <td>0.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>It's ridiculous that these guys are being call...</td>\n",
       "      <td>ridiculous guys called protesters armed threat...</td>\n",
       "      <td>0.838754</td>\n",
       "      <td>0.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Angry trolls, misogynists and Racists, oh my. ...</td>\n",
       "      <td>angry trolls misogynists racists oh not take i...</td>\n",
       "      <td>0.768202</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         comment_text  \\\n",
       "0                                            i am man   \n",
       "1                                          i am woman   \n",
       "2                                        i am lesbian   \n",
       "3                                        i am gay man   \n",
       "4                                           i am dyke   \n",
       "5                                      i am white man   \n",
       "6                                  i am gay white man   \n",
       "7                                      i am black man   \n",
       "8                                  i am gay white man   \n",
       "9                                      i am black man   \n",
       "10                            i am an gay white women   \n",
       "11                                 i am gay black man   \n",
       "12                                   i am black women   \n",
       "13                              i am gay black women    \n",
       "14  This is so cool. Its like, would you want your...   \n",
       "15  This is such an urgent design problem; kudos t...   \n",
       "16                        hahahahahahahahhha suck it.   \n",
       "17  It's ridiculous that these guys are being call...   \n",
       "18  Angry trolls, misogynists and Racists, oh my. ...   \n",
       "\n",
       "                                     comment_text_pre  my score  bias score  \n",
       "0                                                 man  0.023999        0.20  \n",
       "1                                               woman  0.020597        0.41  \n",
       "2                                             lesbian  0.308145        0.51  \n",
       "3                                             gay man  0.403672        0.57  \n",
       "4                                                dyke  0.128925        0.60  \n",
       "5                                           white man  0.092565        0.66  \n",
       "6                                       gay white man  0.512516        0.66  \n",
       "7                                           black man  0.159100        0.77  \n",
       "8                                       gay white man  0.512516        0.78  \n",
       "9                                           black man  0.159100        0.80  \n",
       "10                                    gay white women  0.674933        0.80  \n",
       "11                                      gay black man  0.579392        0.82  \n",
       "12                                        black women  0.336949        0.85  \n",
       "13                                    gay black women  0.714429        0.87  \n",
       "14  cool like would want mother read really great ...  0.003304        0.00  \n",
       "15      urgent design problem kudos taking impressive  0.001713        0.00  \n",
       "16                                               suck  0.923059        0.45  \n",
       "17  ridiculous guys called protesters armed threat...  0.838754        0.50  \n",
       "18  angry trolls misogynists racists oh not take i...  0.768202        0.60  "
      ]
     },
     "execution_count": 301,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text['my score']= predict_\n",
    "text['bias score'] =[0.20,0.41,0.51,0.57,0.60,0.66,0.66,0.77,0.78,0.80,0.80,0.82,0.85,0.87,0.0,0.0,0.45,0.5,0.6]\n",
    "text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Observation:\n",
    "* dropping data not good idea, using different data set might helpful\n",
    "* dont make model more complex.\n",
    "* maxpooling , average pooling is very helpful  when we want capture info from whols sequence rather, dont use last hidden state, bcoz this is classification task. not sequene translation.\n",
    "* after experimentation i found that , bert will help us to caputure whole sequence.\n",
    "    * EX. sequence contain any word except man or women  after or before the subgroup will give different result, so while creating embedding it always  good to use that techinique which will also preserve positionl information. (embedding layer or bidirectional LSTM or Bert)\n",
    "* it dosent make sense to pass bert emebdding[poold output] to LSTM layer, bcoz ,lstm we use when we pass sequence , in this case sequence is  contain only embeddings of 1 word which is [start] \n",
    "* concatinnating differt layers will improve results.\n",
    "* we have train more than 1 model . and we choose best model based on point estimate . which may be wrong ,average model may handle some cases where best model fail. always use CI  to choose base mdoel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}