sentiment-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Phân tích tình cảm khi đánh giá phim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import re\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.svm import LinearSVC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_train = []\n",
    "for line in open('./aclImdb/movie_data/full_train.txt', 'r'):\n",
    "    \n",
    "    reviews_train.append(line.strip())\n",
    "    \n",
    "reviews_test = []\n",
    "for line in open('./aclImdb/movie_data/full_test.txt', 'r'):\n",
    "    \n",
    "    reviews_test.append(line.strip())\n",
    "    \n",
    "target = [1 if i < 12500 else 0 for i in range(25000)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reviews data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'When I first read Armistead Maupins story I was taken in by the human drama displayed by Gabriel No one and those he cares about and loves. That being said, we have now been given the film version of an excellent story and are expected to see past the gloss of Hollywood...<br /><br />Writer Armistead Maupin and director Patrick Stettner have truly succeeded! <br /><br />With just the right amount of restraint Robin Williams captures the fragile essence of Gabriel and lets us see his struggle with issues of trust both in his personnel life(Jess) and the world around him(Donna).<br /><br />As we are introduced to the players in this drama we are reminded that nothing is ever as it seems and that the smallest event can change our lives irrevocably. The request to review a book written by a young man turns into a life changing event that helps Gabriel find the strength within himself to carry on and move forward.<br /><br />It\\'s to bad that most people will avoid this film. I only say that because the average American will probably think \"Robin Williams in a serious role? That didn\\'t work before!\" PLEASE GIVE THIS MOVIE A CHANCE! Robin Williams touches the darkness we all must find and go through in ourselves to be better people. Like his movie One Hour Photo he has stepped up as an actor and made another quality piece of art.<br /><br />Oh and before I forget, I believe Bobby Cannavale as Jess steals every scene he is in. He has the 1940\\'s leading man looks and screen presence. It\\'s this hacks opinion he could carry his own movie right now!!<br /><br />S~'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_train[10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "REPLACE_NO_SPACE = re.compile(\"(\\.)|(\\;)|(\\:)|(\\!)|(\\?)|(\\,)|(\\\")|(\\()|(\\))|(\\[)|(\\])|(\\d+)\")\n",
    "REPLACE_WITH_SPACE = re.compile(\"(<br\\s*/><br\\s*/>)|(\\-)|(\\/)\")\n",
    "NO_SPACE = \"\"\n",
    "SPACE = \" \"\n",
    "\n",
    "def preprocess_reviews(reviews):\n",
    "    \n",
    "    reviews = [REPLACE_NO_SPACE.sub(NO_SPACE, line.lower()) for line in reviews]\n",
    "    reviews = [REPLACE_WITH_SPACE.sub(SPACE, line) for line in reviews]\n",
    "    \n",
    "    return reviews\n",
    "\n",
    "reviews_train_clean = preprocess_reviews(reviews_train)\n",
    "reviews_test_clean = preprocess_reviews(reviews_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reviews data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"when i first read armistead maupins story i was taken in by the human drama displayed by gabriel no one and those he cares about and loves that being said we have now been given the film version of an excellent story and are expected to see past the gloss of hollywood writer armistead maupin and director patrick stettner have truly succeeded  with just the right amount of restraint robin williams captures the fragile essence of gabriel and lets us see his struggle with issues of trust both in his personnel lifejess and the world around himdonna as we are introduced to the players in this drama we are reminded that nothing is ever as it seems and that the smallest event can change our lives irrevocably the request to review a book written by a young man turns into a life changing event that helps gabriel find the strength within himself to carry on and move forward it's to bad that most people will avoid this film i only say that because the average american will probably think robin williams in a serious role that didn't work before please give this movie a chance robin williams touches the darkness we all must find and go through in ourselves to be better people like his movie one hour photo he has stepped up as an actor and made another quality piece of art oh and before i forget i believe bobby cannavale as jess steals every scene he is in he has the 's leading man looks and screen presence it's this hacks opinion he could carry his own movie right now s~\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_train_clean[10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.01: 0.87264\n",
      "Accuracy for C = 0.05: 0.88384\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.25: 0.8832\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.5: 0.8808\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 1: 0.8784\n",
      "Final Accuracy: 0.88176\n"
     ]
    }
   ],
   "source": [
    "baseline_vectorizer = CountVectorizer(binary = True)\n",
    "baseline_vectorizer.fit(reviews_train_clean)\n",
    "X_baseline = baseline_vectorizer.transform(reviews_train_clean)\n",
    "X_test_baseline = baseline_vectorizer.transform(reviews_test_clean)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X_baseline, target, train_size = 0.75\n",
    ")\n",
    "\n",
    "for c in [0.01, 0.05, 0.25, 0.5, 1]:\n",
    "    \n",
    "    lr = LogisticRegression(C = c)\n",
    "    lr.fit(X_train, y_train)\n",
    "    print (\"Accuracy for C = %s: %s\" \n",
    "           % (c, accuracy_score(y_val, lr.predict(X_val))))\n",
    "    \n",
    "#     Accuracy for C=0.01: 0.87472\n",
    "#     Accuracy for C=0.05: 0.88368\n",
    "#     Accuracy for C=0.25: 0.88016\n",
    "#     Accuracy for C=0.5: 0.87808\n",
    "#     Accuracy for C=1: 0.87648\n",
    "\n",
    "final_model = LogisticRegression(C = 0.05)\n",
    "final_model.fit(X_baseline, target)\n",
    "print (\"Final Accuracy: %s\" % accuracy_score(target, final_model.predict(X_test_baseline)))\n",
    "# Final Accuracy: 0.88128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Removing stop words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cài đặt stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /Users/admin/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.01: 0.87376\n",
      "Accuracy for C = 0.05: 0.88304\n",
      "Accuracy for C = 0.25: 0.87888\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.5: 0.87728\n",
      "Accuracy for C = 1: 0.87648\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "\n",
    "english_stop_words = stopwords.words('english')\n",
    "def remove_stop_words(corpus):\n",
    "    removed_stop_words = []\n",
    "    for review in corpus:\n",
    "        removed_stop_words.append(\n",
    "            ' '.join([word for word in review.split() \n",
    "                      if word not in english_stop_words])\n",
    "        )\n",
    "    return removed_stop_words\n",
    "\n",
    "no_stop_words_train = remove_stop_words(reviews_train_clean)\n",
    "no_stop_words_test = remove_stop_words(reviews_test_clean)\n",
    "\n",
    "cv = CountVectorizer(binary = True)\n",
    "cv.fit(no_stop_words_train)\n",
    "X = cv.transform(no_stop_words_train)\n",
    "X_test = cv.transform(no_stop_words_test)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X, target, train_size = 0.75\n",
    ")\n",
    "\n",
    "for c in [0.01, 0.05, 0.25, 0.5, 1]:\n",
    "    \n",
    "    lr = LogisticRegression(C = c)\n",
    "    lr.fit(X_train, y_train)\n",
    "    print(\"Accuracy for C = %s: %s\" % (c, accuracy_score(y_val, lr.predict(X_val))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.01: 0.87312\n",
      "Accuracy for C = 0.05: 0.8776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.25: 0.87504\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.5: 0.86992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 1: 0.86784\n",
      "Final Accuracy: 0.87748\n"
     ]
    }
   ],
   "source": [
    "def get_stemmed_text(corpus):\n",
    "    from nltk.stem.porter import PorterStemmer\n",
    "    stemmer = PorterStemmer()\n",
    "\n",
    "    return [' '.join([stemmer.stem(word) for word in review.split()]) for review in corpus]\n",
    "\n",
    "stemmed_reviews_train = get_stemmed_text(reviews_train_clean)\n",
    "stemmed_reviews_test = get_stemmed_text(reviews_test_clean)\n",
    "\n",
    "cv = CountVectorizer(binary = True)\n",
    "cv.fit(stemmed_reviews_train)\n",
    "X = cv.transform(stemmed_reviews_train)\n",
    "X_test = cv.transform(stemmed_reviews_test)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X, target, train_size = 0.75\n",
    ")\n",
    "\n",
    "for c in [0.01, 0.05, 0.25, 0.5, 1]:\n",
    "    \n",
    "    lr = LogisticRegression(C = c)\n",
    "    lr.fit(X_train, y_train)\n",
    "    print (\"Accuracy for C = %s: %s\" % (c, accuracy_score(y_val, lr.predict(X_val))))\n",
    "    \n",
    "final_stemmed = LogisticRegression(C = 0.05)\n",
    "final_stemmed.fit(X, target)\n",
    "print (\"Final Accuracy: %s\" % accuracy_score(target, final_stemmed.predict(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.01: 0.87296\n",
      "Accuracy for C = 0.05: 0.88096\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.25: 0.88016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.5: 0.87616\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 1: 0.872\n",
      "Final Accuracy: 0.87436\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "def get_lemmatized_text(corpus):\n",
    "    \n",
    "    from nltk.stem import WordNetLemmatizer\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    return [' '.join([lemmatizer.lemmatize(word) for word in review.split()]) for review in corpus]\n",
    "\n",
    "lemmatized_reviews_train = get_lemmatized_text(reviews_train_clean)\n",
    "lemmatized_reviews_test = get_lemmatized_text(reviews_test_clean)\n",
    "\n",
    "cv = CountVectorizer(binary = True)\n",
    "cv.fit(lemmatized_reviews_train)\n",
    "X = cv.transform(lemmatized_reviews_train)\n",
    "X_test = cv.transform(lemmatized_reviews_test)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X, target, train_size = 0.75\n",
    ")\n",
    "\n",
    "for c in [0.01, 0.05, 0.25, 0.5, 1]:\n",
    "    \n",
    "    lr = LogisticRegression(C=c)\n",
    "    lr.fit(X_train, y_train)\n",
    "    print (\"Accuracy for C = %s: %s\" % (c, accuracy_score(y_val, lr.predict(X_val))))\n",
    "    \n",
    "final_lemmatized = LogisticRegression(C=0.25)\n",
    "final_lemmatized.fit(X, target)\n",
    "print (\"Final Accuracy: %s\" % accuracy_score(target, final_lemmatized.predict(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### N-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.01: 0.88\n",
      "Accuracy for C = 0.05: 0.89008\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.25: 0.89024\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.5: 0.89088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 1: 0.89056\n",
      "Final Accuracy: 0.89784\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "ngram_vectorizer = CountVectorizer(binary = True, ngram_range = (1, 2))\n",
    "ngram_vectorizer.fit(reviews_train_clean)\n",
    "X = ngram_vectorizer.transform(reviews_train_clean)\n",
    "X_test = ngram_vectorizer.transform(reviews_test_clean)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X, target, train_size = 0.75\n",
    ")\n",
    "\n",
    "for c in [0.01, 0.05, 0.25, 0.5, 1]:\n",
    "    \n",
    "    lr = LogisticRegression(C = c)\n",
    "    lr.fit(X_train, y_train)\n",
    "    print (\"Accuracy for C = %s: %s\" % (c, accuracy_score(y_val, lr.predict(X_val))))\n",
    "    \n",
    "# Accuracy for C=0.01: 0.88416\n",
    "# Accuracy for C=0.05: 0.892\n",
    "# Accuracy for C=0.25: 0.89424\n",
    "# Accuracy for C=0.5: 0.89456\n",
    "# Accuracy for C=1: 0.8944\n",
    "    \n",
    "final_ngram = LogisticRegression(C = 0.5)\n",
    "final_ngram.fit(X, target)\n",
    "print (\"Final Accuracy: %s\" % accuracy_score(target, final_ngram.predict(X_test)))\n",
    "\n",
    "# Final Accuracy: 0.898"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Word counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.01: 0.88192\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.05: 0.88992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.25: 0.88672\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.5: 0.88304\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 1: 0.87952\n",
      "Final Accuracy: 0.88164\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "wc_vectorizer = CountVectorizer(binary = False)\n",
    "wc_vectorizer.fit(reviews_train_clean)\n",
    "X = wc_vectorizer.transform(reviews_train_clean)\n",
    "X_test = wc_vectorizer.transform(reviews_test_clean)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X, target, train_size = 0.75, \n",
    ")\n",
    "\n",
    "for c in [0.01, 0.05, 0.25, 0.5, 1]:\n",
    "    \n",
    "    lr = LogisticRegression(C = c)\n",
    "    lr.fit(X_train, y_train)\n",
    "    print (\"Accuracy for C = %s: %s\" % (c, accuracy_score(y_val, lr.predict(X_val))))\n",
    "    \n",
    "# Accuracy for C=0.01: 0.87456\n",
    "# Accuracy for C=0.05: 0.88016\n",
    "# Accuracy for C=0.25: 0.87936\n",
    "# Accuracy for C=0.5: 0.87936\n",
    "# Accuracy for C=1: 0.87696\n",
    "    \n",
    "final_wc = LogisticRegression(C = 0.05)\n",
    "final_wc.fit(X, target)\n",
    "print (\"Final Accuracy: %s\" % accuracy_score(target, final_wc.predict(X_test)))\n",
    "\n",
    "# Final Accuracy: 0.88184"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.01: 0.79184\n",
      "Accuracy for C = 0.05: 0.82864\n",
      "Accuracy for C = 0.25: 0.86288\n",
      "Accuracy for C = 0.5: 0.87808\n",
      "Accuracy for C = 1: 0.88512\n",
      "Final Accuracy: 0.88204\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer()\n",
    "tfidf_vectorizer.fit(reviews_train_clean)\n",
    "X = tfidf_vectorizer.transform(reviews_train_clean)\n",
    "X_test = tfidf_vectorizer.transform(reviews_test_clean)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X, target, train_size = 0.75\n",
    ")\n",
    "\n",
    "for c in [0.01, 0.05, 0.25, 0.5, 1]:\n",
    "    \n",
    "    lr = LogisticRegression(C = c)\n",
    "    lr.fit(X_train, y_train)\n",
    "    print (\"Accuracy for C = %s: %s\" % (c, accuracy_score(y_val, lr.predict(X_val))))\n",
    "\n",
    "# Accuracy for C=0.01: 0.79632\n",
    "# Accuracy for C=0.05: 0.83168\n",
    "# Accuracy for C=0.25: 0.86768\n",
    "# Accuracy for C=0.5: 0.8736\n",
    "# Accuracy for C=1: 0.88432\n",
    "    \n",
    "final_tfidf = LogisticRegression(C=1)\n",
    "final_tfidf.fit(X, target)\n",
    "print (\"Final Accuracy: %s\" % accuracy_score(target, final_tfidf.predict(X_test)))\n",
    "\n",
    "# Final Accuracy: 0.882"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support vector machines (SVM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.01: 0.88496\n",
      "Accuracy for C = 0.05: 0.88272\n",
      "Accuracy for C = 0.25: 0.88048\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/svm/_base.py:985: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.5: 0.88064\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/admin/opt/anaconda3/lib/python3.9/site-packages/sklearn/svm/_base.py:985: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 1: 0.88048\n",
      "Final Accuracy: 0.8974\n"
     ]
    }
   ],
   "source": [
    "ngram_vectorizer = CountVectorizer(binary = True, ngram_range = (1, 2))\n",
    "ngram_vectorizer.fit(reviews_train_clean)\n",
    "X = ngram_vectorizer.transform(reviews_train_clean)\n",
    "X_test = ngram_vectorizer.transform(reviews_test_clean)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X, target, train_size = 0.75\n",
    ")\n",
    "\n",
    "for c in [0.01, 0.05, 0.25, 0.5, 1]:\n",
    "    \n",
    "    svm = LinearSVC(C = c)\n",
    "    svm.fit(X_train, y_train)\n",
    "    print (\"Accuracy for C = %s: %s\" % (c, accuracy_score(y_val, svm.predict(X_val))))\n",
    "    \n",
    "# Accuracy for C=0.01: 0.89104\n",
    "# Accuracy for C=0.05: 0.88736\n",
    "# Accuracy for C=0.25: 0.8856\n",
    "# Accuracy for C=0.5: 0.88608\n",
    "# Accuracy for C=1: 0.88592\n",
    "    \n",
    "final_svm_ngram = LinearSVC(C = 0.01)\n",
    "final_svm_ngram.fit(X, target)\n",
    "print (\"Final Accuracy: %s\" % accuracy_score(target, final_svm_ngram.predict(X_test)))\n",
    "\n",
    "# Final Accuracy: 0.8974"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for C = 0.001: 0.89072\n",
      "Accuracy for C = 0.005: 0.89584\n",
      "Accuracy for C = 0.01: 0.89728\n",
      "Accuracy for C = 0.05: 0.89744\n",
      "Accuracy for C = 0.1: 0.89728\n",
      "Final Accuracy: 0.90064\n"
     ]
    }
   ],
   "source": [
    "stop_words = ['in', 'of', 'at', 'a', 'the']\n",
    "ngram_vectorizer = CountVectorizer(binary = True, ngram_range=(1, 3), stop_words = stop_words)\n",
    "ngram_vectorizer.fit(reviews_train_clean)\n",
    "X = ngram_vectorizer.transform(reviews_train_clean)\n",
    "X_test = ngram_vectorizer.transform(reviews_test_clean)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X, target, train_size = 0.75\n",
    ")\n",
    "\n",
    "for c in [0.001, 0.005, 0.01, 0.05, 0.1]:\n",
    "    \n",
    "    svm = LinearSVC(C = c)\n",
    "    svm.fit(X_train, y_train)\n",
    "    print (\"Accuracy for C = %s: %s\" % (c, accuracy_score(y_val, svm.predict(X_val))))\n",
    "    \n",
    "# Accuracy for C=0.001: 0.88784\n",
    "# Accuracy for C=0.005: 0.89456\n",
    "# Accuracy for C=0.01: 0.89376\n",
    "# Accuracy for C=0.05: 0.89264\n",
    "# Accuracy for C=0.1: 0.8928\n",
    "    \n",
    "final = LinearSVC(C = 0.01)\n",
    "final.fit(X, target)\n",
    "print (\"Final Accuracy: %s\" % accuracy_score(target, final.predict(X_test)))\n",
    "\n",
    "# Final Accuracy: 0.90064"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top postitive and negative features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('excellent', 0.2293214987641556)\n",
      "('perfect', 0.18456042365581982)\n",
      "('great', 0.17897486004133786)\n",
      "('wonderful', 0.16014961449812032)\n",
      "('amazing', 0.15411677897924775)\n",
      "('superb', 0.14690756699341834)\n",
      "('enjoyable', 0.1434676310815318)\n",
      "('best', 0.13042556385856732)\n",
      "('today', 0.12939427031366205)\n",
      "('fun', 0.1268216716662226)\n",
      "('enjoyed', 0.12571365817254648)\n",
      "('brilliant', 0.11940920162043987)\n",
      "('must see', 0.11784615602851584)\n",
      "('loved', 0.11677477244895866)\n",
      "('fantastic', 0.11412472104556672)\n",
      "('liked', 0.11097599417543294)\n",
      "('incredible', 0.1091168675604505)\n",
      "('funniest', 0.10882618225652285)\n",
      "('wonderfully', 0.1086218719859292)\n",
      "('better than', 0.1080799851527195)\n",
      "('beautiful', 0.10396791241865445)\n",
      "('rare', 0.10258574651354867)\n",
      "('bit', 0.10257837211862245)\n",
      "('love', 0.10257485931078715)\n",
      "('watch it', 0.10204503737271711)\n",
      "('job', 0.10185731017300928)\n",
      "('well worth', 0.10094752186347804)\n",
      "('highly', 0.10092562526766095)\n",
      "('moving', 0.09797527560684223)\n",
      "('recommended', 0.09790959722383555)\n",
      "\n",
      "\n",
      "\n",
      "('worst', -0.35899087247793776)\n",
      "('awful', -0.2550575103563821)\n",
      "('boring', -0.24068184789033847)\n",
      "('waste', -0.23683697158317385)\n",
      "('bad', -0.22181965386607716)\n",
      "('poor', -0.2019393520225097)\n",
      "('terrible', -0.1998446440879541)\n",
      "('dull', -0.18413720503012201)\n",
      "('poorly', -0.17534068442486253)\n",
      "('disappointment', -0.17488533870920323)\n",
      "('disappointing', -0.16942339503045203)\n",
      "('unfortunately', -0.1565966003651993)\n",
      "('worse', -0.15637532965406795)\n",
      "('stupid', -0.15564805017307448)\n",
      "('horrible', -0.15262791867192896)\n",
      "('mess', -0.14870509402474413)\n",
      "('nothing', -0.13927140274647487)\n",
      "('lame', -0.13907836677997942)\n",
      "('lacks', -0.13711889620348275)\n",
      "('save', -0.13563660680827835)\n",
      "('oh', -0.13445025500544758)\n",
      "('avoid', -0.13014764256858216)\n",
      "('ridiculous', -0.1290266217566986)\n",
      "('annoying', -0.12751794712607167)\n",
      "('script', -0.12592634026767402)\n",
      "('weak', -0.12541434741034982)\n",
      "('fails', -0.12449544032212091)\n",
      "('badly', -0.12307914249631512)\n",
      "('not good', -0.12065431705861535)\n",
      "('laughable', -0.11883430382766945)\n"
     ]
    }
   ],
   "source": [
    "feature_to_coef = {\n",
    "    word: coef for word, coef in zip(\n",
    "        ngram_vectorizer.get_feature_names(), final.coef_[0]\n",
    "    )\n",
    "}\n",
    "\n",
    "for best_positive in sorted(\n",
    "    feature_to_coef.items(), \n",
    "    key = lambda x: x[1], \n",
    "    reverse = True)[:30]:\n",
    "    print(best_positive)\n",
    "    \n",
    "print(\"\\n\\n\")\n",
    "for best_negative in sorted(\n",
    "    feature_to_coef.items(), \n",
    "    key = lambda x: x[1])[:30]:\n",
    "    print(best_negative)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}