Text Cleaning and BoW.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing the libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import nltk\n",
    "import re\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from nltk.stem.porter import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing the dataset\n",
    "import sqlite3\n",
    "con = sqlite3.connect('./database.sqlite')\n",
    "#filtering only positive and negative reviews\n",
    "data = pd.read_sql_query(\"\"\"\n",
    "SELECT *\n",
    "FROM Reviews\n",
    "WHERE Score != 3\n",
    "\"\"\", con)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[['Text', 'Score']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_score(score):\n",
    "    if score>3:\n",
    "        return 'Positive'\n",
    "    else:\n",
    "        return 'Negative'\n",
    "data['Score'] = data['Score'].apply(map_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I have bought several of the Vitality canned d...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Product arrived labeled as Jumbo Salted Peanut...</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>This is a confection that has been around a fe...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>If you are looking for the secret ingredient i...</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Great taffy at a great price.  There was a wid...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text     Score\n",
       "0  I have bought several of the Vitality canned d...  Positive\n",
       "1  Product arrived labeled as Jumbo Salted Peanut...  Negative\n",
       "2  This is a confection that has been around a fe...  Positive\n",
       "3  If you are looking for the secret ingredient i...  Negative\n",
       "4  Great taffy at a great price.  There was a wid...  Positive"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#deduplication \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TEXT PREPROCESSING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to C:\\Users\\TANMAY\n",
      "[nltk_data]     LATA\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n",
      "***************************************\n",
      "tasti\n"
     ]
    }
   ],
   "source": [
    "#Getting Stopwords\n",
    "import string\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "stop = stopwords.words('english')\n",
    "sno = nltk.stem.SnowballStemmer('english')\n",
    "print (stop)\n",
    "print('***************************************')\n",
    "print(sno.stem('tasty'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Functions for cleaning HTML tags and punctuation \n",
    "def clean_html(sentence):\n",
    "    cleanr = re.compile('<.*?>')\n",
    "    cleantext = re.sub(cleanr, ' ', sentence)\n",
    "    return cleantext\n",
    "def clean_punc(word):\n",
    "    cleaned = re.sub(r'[?|!|\\'|#]', r'', word)\n",
    "    cleaned = re.sub(r'[.|,|)|(|\\|/]', r' ', cleaned)\n",
    "    return cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a list of filtered sentences:\n",
    "final_string = []\n",
    "s = ''\n",
    "for sentence in data['Text'].values:\n",
    "    filtered_sentence = []\n",
    "    sentence = clean_html(sentence)\n",
    "    for word in sentence.split():\n",
    "        for cleaned_word in clean_punc(word).split():\n",
    "            if (cleaned_word.isalpha() and (len(cleaned_word) > 2) and cleaned_word not in stop):\n",
    "                s = (sno.stem(cleaned_word.lower())).encode('utf8')\n",
    "                filtered_sentence.append(s)\n",
    "            else:\n",
    "                continue\n",
    "                 \n",
    "    strl = b' '.join(filtered_sentence)\n",
    "    final_string.append(strl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Cleaned Text'] = final_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Score</th>\n",
       "      <th>Cleaned Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I have bought several of the Vitality canned d...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>b'bought sever vital can dog food product foun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Product arrived labeled as Jumbo Salted Peanut...</td>\n",
       "      <td>Negative</td>\n",
       "      <td>b'product arriv label jumbo salt peanut peanut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>This is a confection that has been around a fe...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>b'this confect around centuri light pillowi ci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>If you are looking for the secret ingredient i...</td>\n",
       "      <td>Negative</td>\n",
       "      <td>b'look secret ingredi robitussin believ found ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Great taffy at a great price.  There was a wid...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>b'great taffi great price there wide assort yu...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text     Score  \\\n",
       "0  I have bought several of the Vitality canned d...  Positive   \n",
       "1  Product arrived labeled as Jumbo Salted Peanut...  Negative   \n",
       "2  This is a confection that has been around a fe...  Positive   \n",
       "3  If you are looking for the secret ingredient i...  Negative   \n",
       "4  Great taffy at a great price.  There was a wid...  Positive   \n",
       "\n",
       "                                        Cleaned Text  \n",
       "0  b'bought sever vital can dog food product foun...  \n",
       "1  b'product arriv label jumbo salt peanut peanut...  \n",
       "2  b'this confect around centuri light pillowi ci...  \n",
       "3  b'look secret ingredi robitussin believ found ...  \n",
       "4  b'great taffi great price there wide assort yu...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bag of Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vect = CountVectorizer()\n",
    "bow_data = count_vect.fit_transform(data['Cleaned Text'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<525814x70780 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 17378570 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow_data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}