movies_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dfae49b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\gdiva\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\gdiva\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "nltk.download('wordnet')\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "972cfaf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_df = pd.read_csv(\"IMDB Dataset.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9769a874",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6087</th>\n",
       "      <td>Despite what its critics ensue, I enjoyed imme...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30312</th>\n",
       "      <td>I fail to see how anyone who has actually read...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30193</th>\n",
       "      <td>This is a fascinating film--especially to old ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35727</th>\n",
       "      <td>In general I like dinosaur movies but this one...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44441</th>\n",
       "      <td>The Fiendish Plot of Dr. Fu Manchu starring Pe...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  review sentiment\n",
       "6087   Despite what its critics ensue, I enjoyed imme...  positive\n",
       "30312  I fail to see how anyone who has actually read...  negative\n",
       "30193  This is a fascinating film--especially to old ...  negative\n",
       "35727  In general I like dinosaur movies but this one...  negative\n",
       "44441  The Fiendish Plot of Dr. Fu Manchu starring Pe...  negative"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5cd4e53f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 2)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cb950e79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_df[\"review\"].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e00dc36c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "418"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_df[\"review\"].duplicated().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6412ce5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_df.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c4127668",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49582, 2)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d1f3865c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"One of the other reviewers has mentioned that after watching just 1 Oz episode you'll be hooked. They are right, as this is exactly what happened with me.<br /><br />The first thing that struck me about Oz was its brutality and unflinching scenes of violence, which set in right from the word GO. Trust me, this is not a show for the faint hearted or timid. This show pulls no punches with regards to drugs, sex or violence. Its is hardcore, in the classic use of the word.<br /><br />It is called OZ as that is the nickname given to the Oswald Maximum Security State Penitentary. It focuses mainly on Emerald City, an experimental section of the prison where all the cells have glass fronts and face inwards, so privacy is not high on the agenda. Em City is home to many..Aryans, Muslims, gangstas, Latinos, Christians, Italians, Irish and more....so scuffles, death stares, dodgy dealings and shady agreements are never far away.<br /><br />I would say the main appeal of the show is due to the fact that it goes where other shows wouldn't dare. Forget pretty pictures painted for mainstream audiences, forget charm, forget romance...OZ doesn't mess around. The first episode I ever saw struck me as so nasty it was surreal, I couldn't say I was ready for it, but as I watched more, I developed a taste for Oz, and got accustomed to the high levels of graphic violence. Not just violence, but injustice (crooked guards who'll be sold out for a nickel, inmates who'll kill on order and get away with it, well mannered, middle class inmates being turned into prison bitches due to their lack of street skills or prison experience) Watching Oz, you may become comfortable with what is uncomfortable viewing....thats if you can get in touch with your darker side.\""
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_df.review[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3747e58b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_data(text) :\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"<.*?>\", \" \", text)\n",
    "    text = text.translate(str.maketrans('', '',string.punctuation))\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    a = []\n",
    "    for w in text.split() : \n",
    "        if w not in stopwords.words('english') :\n",
    "            if w.isnumeric()==False :\n",
    "                a.append(lemmatizer.lemmatize(w))\n",
    "    return \" \".join(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6bad1578",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "one reviewer mentioned watching oz episode youll hooked right exactly happened first thing struck oz brutality unflinching scene violence set right word go trust show faint hearted timid show pull punch regard drug sex violence hardcore classic use word called oz nickname given oswald maximum security state penitentary focus mainly emerald city experimental section prison cell glass front face inwards privacy high agenda em city home manyaryans muslim gangsta latino christian italian irish moreso scuffle death stare dodgy dealing shady agreement never far away would say main appeal show due fact go show wouldnt dare forget pretty picture painted mainstream audience forget charm forget romanceoz doesnt mess around first episode ever saw struck nasty surreal couldnt say ready watched developed taste oz got accustomed high level graphic violence violence injustice crooked guard wholl sold nickel inmate wholl kill order get away well mannered middle class inmate turned prison bitch due lack street skill prison experience watching oz may become comfortable uncomfortable viewingthats get touch darker side\n"
     ]
    }
   ],
   "source": [
    "print(clean_data(movie_df.review[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8893d572",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "movie_df[\"review\"] = movie_df[\"review\"].apply(clean_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d667829e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one reviewer mentioned watching oz episode you...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>wonderful little production filming technique ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>thought wonderful way spend time hot summer we...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>basically there family little boy jake think t...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>petter matteis love time money visually stunni...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment\n",
       "0  one reviewer mentioned watching oz episode you...  positive\n",
       "1  wonderful little production filming technique ...  positive\n",
       "2  thought wonderful way spend time hot summer we...  positive\n",
       "3  basically there family little boy jake think t...  negative\n",
       "4  petter matteis love time money visually stunni...  positive"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "24cd5168",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorize = CountVectorizer(max_features=10000)\n",
    "X = vectorize.fit_transform(movie_df[\"review\"]).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "84208679",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49582, 10000)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "36b049fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['positive', 'negative'], dtype=object)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_df[\"sentiment\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c109e564",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = []\n",
    "for i in movie_df[\"sentiment\"] :\n",
    "    if i==\"positive\" :\n",
    "        Y.append(1)\n",
    "    else :\n",
    "        Y.append(0)\n",
    "        \n",
    "Y = np.array(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "bddde4ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49582,)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4be67f1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 0, 1])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "0c871910",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "f714e6bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X,Y,test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "833e2008",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(39665, 10000)\n",
      "(9917, 10000)\n",
      "(39665,)\n",
      "(9917,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9f2bd5fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c09a5627",
   "metadata": {},
   "outputs": [],
   "source": [
    "nb1 = GaussianNB()\n",
    "nb2 = MultinomialNB()\n",
    "nb3 = BernoulliNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d22f04ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>BernoulliNB()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">BernoulliNB</label><div class=\"sk-toggleable__content\"><pre>BernoulliNB()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "BernoulliNB()"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb1.fit(X_train,y_train)\n",
    "nb2.fit(X_train,y_train)\n",
    "nb3.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "368729ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred1 = nb1.predict(X_test)\n",
    "y_pred2 = nb2.predict(X_test)\n",
    "y_pred3 = nb3.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c0ca2744",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "81089edb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7291519612786125\n",
      "0.8507613189472623\n",
      "0.8582232529998992\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_pred1,y_test))\n",
    "print(accuracy_score(y_pred2,y_test))\n",
    "print(accuracy_score(y_pred3,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "b911ebb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "f76a8215",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['model.pkl']"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(vectorize, 'vectorizer.pkl')\n",
    "joblib.dump(nb3, 'model.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ca71aa7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}