notebooks/support_email_project_part_1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "support_email_project_part_1.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "76Zits4zxKTb"
      },
      "source": [
        "# Support Email Classification Project part 1\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jZQCgtzixRaB"
      },
      "source": [
        "## This notebook Contains the script to clean the raw text data into the readable format, Input is a dataframe with Columns as Request type and the Email body to be cleaned, output is a dataframe with cleaned email body"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sveHK-KHxm-P"
      },
      "source": [
        "### NOTE: To use this script the column name should be *'Request Type'* and *'Content'* (else rename them to above mentioned format) and also, drop other columns  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z-gX4AaVyIlQ"
      },
      "source": [
        "## Input Text Example"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "29dQq7RcyLDW"
      },
      "source": [
        "##### *Input Text*: &lt;!DOCTYPE html PUBLIC &quot;-//W3C//DTD XHTML 1.0 Transitional//EN&quot; &quot;http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd&quot;&gt;&#13;&#10;&lt;html&gt;&#13;&#10;    &lt;head&gt;&#13;&#10;        &lt;meta http-equiv=&quot;Content-Type&quot; content=&quot;text/html; charset=UTF-8&quot; /&gt;&#13;&#10;        &lt;title&gt;Booking reference number 280209715774&lt;/title&gt;&#13;&#10;&#9;&lt;/head&gt;&#13;&#10;    &lt;body&gt;&#13;&#10;    Hi,&lt;br /&gt;&#13;&#10;&lt;br /&gt;&#13;&#10;I have booked a flight for 4 passengers, booking reference number 280209715774. However the flight date is incorrect (I had selected 08.03.2020, but the date on the booking is now showing for 08.04.2020), I would like to change the date to the 08.03.2020 for all the passengers for the same timings. Please can you assist me, as the website keeps crashing and I am unable to do this online. &lt;br /&gt;&#13;&#10;&lt;br /&gt;&#13;&#10;You can contact me on +91 93183 69980 &lt;br /&gt;&#13;&#10;&lt;br /&gt;&#13;&#10;Kind Regards, &lt;br /&gt;&#13;&#10;Sukhvinder K Dhanjal&#13;&#10;    &lt;/body&gt;&#13;&#10;&lt;/html&gt;"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MEf3RPjeyOTa"
      },
      "source": [
        "## Cleaned Output Text Example"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bbuXcGilyRRI"
      },
      "source": [
        "public transitional quot www org content type quot content text title booking reference number title body book flight passenger book reference number flight date incorrect select date booking show change date passenger timing assist website keep crash unable regard sukhvinder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZWDWjFaGxIUw",
        "outputId": "c33a042e-28ab-4cd5-f8af-497637c4fb3b"
      },
      "source": [
        "#DATA CLEANING\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
        "import spacy\n",
        "from nltk.corpus import stopwords\n",
        "import nltk \n",
        "nltk.download('stopwords')\n",
        "stop_words = stopwords.words('english')\n",
        "stop_words.extend(['\\n'])\n",
        "#from nltk.stem.snowball import SnowballStemmer\n",
        "import re\n",
        "from nltk.tokenize.treebank import TreebankWordDetokenizer\n",
        "from gensim.utils import simple_preprocess\n",
        "import gensim\n",
        "import copy"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FYY1X9tIzFms"
      },
      "source": [
        "### Provide your input dataframe below"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "muEHwd7lunee"
      },
      "source": [
        "support_email_data= pd.read_csv(r'/content/drive/MyDrive/final_support_data.csv',encoding= 'latin-1')\n",
        "#support_email_data= pd.read_csv(r'/content/drive/MyDrive/CustomerService_01Dec2019_23March2020.csv',encoding= 'latin-1')"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "id": "nSEI5YT1u-OS",
        "outputId": "7b6d61c1-6d1e-42d1-e0e5-606e4d22abcf"
      },
      "source": [
        "support_email_data.head()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ï»¿Sales Inquiry Status</th>\n",
              "      <th>Sub Sales Enquiry</th>\n",
              "      <th>Interaction Id</th>\n",
              "      <th>Request_Type</th>\n",
              "      <th>Request Sub Type</th>\n",
              "      <th>Subject</th>\n",
              "      <th>content</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Resolved</td>\n",
              "      <td>Resolved on Email</td>\n",
              "      <td>24769821</td>\n",
              "      <td>Refunds</td>\n",
              "      <td>Refund not initiated/processed</td>\n",
              "      <td>About refunds</td>\n",
              "      <td>&lt;div dir=\"auto\"&gt;Greting of the dayÂ &lt;div dir=\"...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Resolved</td>\n",
              "      <td>Resolved on Email</td>\n",
              "      <td>24769820</td>\n",
              "      <td>Refunds</td>\n",
              "      <td>Refund - Direct Airline Cancellation</td>\n",
              "      <td>Refund issues</td>\n",
              "      <td>&amp;lt;div dir=&amp;quot;auto&amp;quot;&amp;gt;&amp;lt;div style=...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Dupe complaint</td>\n",
              "      <td>Dupe complaint</td>\n",
              "      <td>24769819</td>\n",
              "      <td>Others</td>\n",
              "      <td>Call Disconnected</td>\n",
              "      <td>Cancellation of ticket</td>\n",
              "      <td>&amp;lt;html&amp;gt;&amp;#13;&amp;#10;&amp;lt;head&amp;gt;&amp;#13;&amp;#10;&amp;l...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Resolved</td>\n",
              "      <td>Resolved on Email</td>\n",
              "      <td>24769816</td>\n",
              "      <td>Cancellation</td>\n",
              "      <td>Cancellation Charges</td>\n",
              "      <td>Fwd: Yatra MyBookings</td>\n",
              "      <td>&lt;div&gt;&lt;div dir=\"auto\"&gt;Hi team,&lt;/div&gt;&lt;/div&gt;&lt;div ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>24769812</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Refund confirmation</td>\n",
              "      <td>&lt;div dir=\"auto\"&gt;Dear team,&lt;div dir=\"auto\"&gt;Â Ia...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  ï»¿Sales Inquiry Status  ...                                            content\n",
              "0                Resolved  ...  <div dir=\"auto\">Greting of the dayÂ <div dir=\"...\n",
              "1                Resolved  ...  &lt;div dir=&quot;auto&quot;&gt;&lt;div style=...\n",
              "2          Dupe complaint  ...  &lt;html&gt;&#13;&#10;&lt;head&gt;&#13;&#10;&l...\n",
              "3                Resolved  ...  <div><div dir=\"auto\">Hi team,</div></div><div ...\n",
              "4                     NaN  ...  <div dir=\"auto\">Dear team,<div dir=\"auto\">Â Ia...\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hZzkiId8zMtU"
      },
      "source": [
        "# Cleaning Script\n",
        "## Steps:\n",
        "### 1. Removing null rows\n",
        "### 2. Beautiful soup and unescape are used to remove HTML part of the string \n",
        "### 3. Merging minority classes(Domestic, Ecash, Self D, Special service) to others\n",
        "### 4. Formed strings to tokens, followed by the steps of basic cleaning, stop  words removal, lemmatization and then returned the detokenized string "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lLLibO9HuiLI"
      },
      "source": [
        "#removing rows will null mail content and request type\n",
        "support_email_data = support_email_data.rename({'Request Type':'Request_Type'}, axis=1)\n",
        "support_email_data = support_email_data[support_email_data['Content'].notna()] \n",
        "support_email_data = support_email_data[support_email_data['Request_Type'].notna()] \n",
        "support_email_data = support_email_data.reset_index(drop=True)\n",
        "\n",
        "#Semi cleaning code for cleaning Content \n",
        "#Code to remove the garbage text and extracting out the required string\n",
        "from bs4 import BeautifulSoup\n",
        "from html import unescape\n",
        "\n",
        "support_email_data['semi_clean_content'] = '' #Adding a column for clean text to the dataframe \n",
        "for i in range(len(support_email_data)):\n",
        "    toxic_string= support_email_data['Content'][i]\n",
        "    html_str= unescape(toxic_string)\n",
        "    soup= BeautifulSoup(html_str)\n",
        "    clean_str= soup.get_text()\n",
        "    support_email_data['semi_clean_content'][i]= clean_str\n",
        "\n",
        "#Removing rows with null semi clean content\n",
        "support_email_data = support_email_data[support_email_data['semi_clean_content'].notna()] \n",
        "support_email_data = support_email_data.reset_index(drop=True)\n",
        "\n",
        "# Assigning the 4 minority to Others\n",
        "def merging_minority(df):\n",
        "    for i in range(len(df)):\n",
        "        if(df['Request_Type'][i] =='Domestic'):\n",
        "           df['Request_Type'][i] ='Others' \n",
        "        elif(df['Request_Type'][i] =='Ecash'):\n",
        "           df['Request_Type'][i] ='Others' \n",
        "        elif(df['Request_Type'][i] =='Self D'):\n",
        "           df['Request_Type'][i] ='Others' \n",
        "        elif(df['Request_Type'][i] =='Special service'):\n",
        "           df['Request_Type'][i] ='Others' \n",
        "    return df\n",
        "\n",
        "#Assignining random class content to others class\n",
        "def merging_minority_2(df):\n",
        "    for i in range(len(df)):\n",
        "        if(df['Request_Type'][i]!='Refunds' and df['Request_Type'][i]!='Eticket/Voucher' and df['Request_Type'][i]!='Booking information' and df['Request_Type'][i]!='Cancellation' and df['Request_Type'][i]!='Others' and df['Request_Type'][i]!='Amendment' and df['Request_Type'][i]!='Promotions' and df['Request_Type'][i]!='Booking Query' and df['Request_Type'][i]!='Website Error'):\n",
        "            df['Request_Type'][i]='Others'\n",
        "    return df\n",
        "\n",
        "support_email_data_9_col = copy.deepcopy(support_email_data) \n",
        "support_email_data_9_col = merging_minority(support_email_data_9_col)\n",
        "support_email_data_9_col = merging_minority_2(support_email_data_9_col)\n",
        "\n",
        "\n",
        "#Choosing required columns of the dataframe \n",
        "support_email_data_9_col= support_email_data_9_col[['Request_Type','semi_clean_content','Content']]\n",
        "#support_email_data_9_col.isnull().sum()\n",
        "\n",
        "#Cleaning Phase 2\n",
        "#Tokenising and cleaning the dataset \n",
        "# Convert email body to list\n",
        "data = support_email_data_9_col.semi_clean_content.values.tolist() #Data is a list of strings \n",
        "\n",
        "# tokenize - break down each sentence into a list of words\n",
        "import gensim\n",
        "def sent_to_words(sentences):\n",
        "    for sentence in sentences:\n",
        "        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))  # deacc=True removes punctuations\n",
        "\n",
        "#Calling above function\n",
        "data_words = list(sent_to_words(data))       \n",
        "################################################################################\n",
        "#Customly cleaning the dataset of tokens\n",
        "def data_cleaning(transactions):\n",
        "    #DATA CLEANING(REMOVING NAN, SPEACIAL CHARS AND NUMS, SINGLE CHARS, LEADING AND TRAILING SPACES)\n",
        "    for i in range(len(transactions)):\n",
        "        transactions[i] = [x for x in transactions[i] if x != '']  #Removing empty strings from the Lists of transactins\n",
        "        transactions[i] = [re.sub('[^a-zA-Z]+', ' ', _) for _ in transactions[i]] #Removing Speacial chars and nums\n",
        "        transactions[i]=  [re.sub(r'\\b\\w{1,2}\\b','', k) for k in transactions[i]] #Removing single characters\n",
        "        transactions[i] = [re.sub(\"^\\s+|\\s+$\", \"\", j, flags=re.UNICODE) for j in transactions[i]] # Removing both leading and trailing spaces\n",
        "    #CONVERTING ELEMENTS OF LIST TO LOWER CASE\n",
        "    transactions= [[x.lower() for x in subl] for subl in transactions]\n",
        "    #transactions = [subl for subl in transactions if len(subl) > 1]\n",
        "    transactions= [list(filter(None, sublist)) for sublist in transactions]\n",
        "    return transactions\n",
        "\n",
        "#Calling above function \n",
        "data_words_cleaned= data_cleaning(data_words)\n",
        "################################################################################\n",
        "# REMOVE stop_words and lemmatize\n",
        "# remove stop_words, make bigrams and lemmatize\n",
        "def remove_stopwords(texts):\n",
        "    return [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts]\n",
        "\n",
        "def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):\n",
        "    texts_out = []\n",
        "    for sent in texts:\n",
        "        doc = nlp(\" \".join(sent))\n",
        "        texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])\n",
        "    return texts_out\n",
        "# Remove Stop Words\n",
        "data_words_nostops = remove_stopwords(data_words_cleaned)\n",
        "# Initialize spacy 'en' model, keeping only tagger component (for efficiency)\n",
        "nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])\n",
        "# Do lemmatization keeping only noun, adj, vb, adv\n",
        "data_lemmatized = lemmatization(data_words_nostops, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])\n",
        "\n",
        "#Adding a column for the cleaned text \n",
        "from nltk.tokenize.treebank import TreebankWordDetokenizer\n",
        "support_email_data_9_col_cleaned = pd.DataFrame(columns=['cleaned_body'], index=support_email_data_9_col.index)\n",
        "for i in range(0, len(data_lemmatized)):\n",
        "    support_email_data_9_col_cleaned['cleaned_body'][i]= TreebankWordDetokenizer().detokenize(data_lemmatized[i])\n",
        "\n",
        "support_email_data_9_col['cleaned_body'] = support_email_data_9_col_cleaned['cleaned_body'].values\n",
        "\n",
        "#Removing rows with null cleaned_body and null Request_Type\n",
        "support_email_data_9_col = support_email_data_9_col[support_email_data_9_col['cleaned_body'].notna()] \n",
        "support_email_data_9_col = support_email_data_9_col[support_email_data_9_col['Request_Type'].notna()] \n",
        "support_email_data_9_col.reset_index(drop=True, inplace=True)\n",
        "\n",
        "support_email_data_9_col= support_email_data_9_col[['Request_Type','cleaned_body']]\n",
        "support_email_data_9_col.to_csv('cleaning_script_testing.csv', index= False)"
      ],
      "execution_count": 11,
      "outputs": []
    }
  ]
}