MKII_AMZReviews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "pd.set_option('display.max_columns', 1000000)\n",
    "pd.set_option('display.max_rows', 1000000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(455000, 13)\n"
     ]
    }
   ],
   "source": [
    "#Remove Na values\n",
    "data = pd.read_csv('Amazon.csv')\n",
    "print(data.shape)\n",
    "data.Summary = data.Summary.fillna('0')\n",
    "data.ProfileName = data.ProfileName.fillna(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ianssmith/anaconda3/lib/python3.5/site-packages/ipykernel/__main__.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "#Various text features from review text\n",
    "data['hasSC'] = data['Text'].str.contains(';')\n",
    "data['hasbroken'] = data['Text'].str.contains('broken')\n",
    "data['hasdidnt'] = data['Text'].str.contains('didn\\'t')\n",
    "data['hasperfect'] = data['Text'].str.contains('perfect')\n",
    "data['haslove'] = data['Text'].str.contains('love')\n",
    "data['hasEP'] = data['Text'].str.contains('!')\n",
    "data['hasdash'] = data['Text'].str.contains('-')\n",
    "data['hasChocolate'] = data['Text'].str.contains('chocolate')\n",
    "data['hasCoffee'] = data['Text'].str.contains('coffee')\n",
    "data['hasRec'] = data['Text'].str.contains('recommend')\n",
    "data['hasTerrible'] = data['Text'].str.contains('terrible')\n",
    "data['sumEP'] = data['Summary'].str.contains('!')\n",
    "data['sumPeriod'] = data['Summary'].str.contains('.')\n",
    "data['nameSpace'] = data['ProfileName'].str.contains(' ')\n",
    "data['sumLen'] = data['Summary'].str.len()\n",
    "data['reviewLen'] = data['Text'].str.len()\n",
    "#Has the review writer written more than one review\n",
    "data['duplicateIds'] = data.UserId.duplicated()\n",
    "data['nameLen'] = data['ProfileName'].str.len()\n",
    "#the difference in character length between summary and text\n",
    "data['sumtexdif'] = data['Text'].str.len() - data['Summary'].str.len()\n",
    "data.sumtexdif[data.sumtexdif < 0] = 0\n",
    "\n",
    "data['sumComma'] = data['Summary'].str.contains(',')\n",
    "data['sumColon'] = data['Summary'].str.contains(':')\n",
    "data['sumSC'] = data['Summary'].str.contains(';')\n",
    "data['sumDash'] = data['Summary'].str.contains('-')\n",
    "data['hasComma'] = data['Text'].str.contains(',')\n",
    "data['hasColon'] = data['Text'].str.contains(':')\n",
    "#Include values of reviews written per user and per product\n",
    "data['ProductCounts'] = data.groupby('ProductId')['ProductId'].transform('count')\n",
    "data['UserCounts'] = data.groupby('UserId')['UserId'].transform('count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "###Reshaping and combining features for model input\n",
    "XScore = data.iloc[:, 7].values.reshape(data.shape[0], 1)\n",
    "\n",
    "hasSC = data.iloc[:, 13].values.reshape(data.shape[0], 1)\n",
    "hasbroken = data.iloc[:, 14].values.reshape(data.shape[0], 1)\n",
    "hasdidnt = data.iloc[:, 15].values.reshape(data.shape[0], 1)\n",
    "hasperfect = data.iloc[:, 16].values.reshape(data.shape[0], 1)\n",
    "haslove = data.iloc[:, 17].values.reshape(data.shape[0], 1)\n",
    "hasEP = data.iloc[:, 18].values.reshape(data.shape[0], 1)\n",
    "hasdash = data.iloc[:, 19].values.reshape(data.shape[0], 1)\n",
    "hasChocolate = data.iloc[:, 20].values.reshape(data.shape[0],1)\n",
    "hasCoffee = data.iloc[:, 21].values.reshape(data.shape[0],1)\n",
    "hasRec = data.iloc[:, 22].values.reshape(data.shape[0],1)\n",
    "hasTerrible = data.iloc[:, 23].values.reshape(data.shape[0],1)\n",
    "sumEP = data.iloc[:, 24].values.reshape(data.shape[0],1)\n",
    "sumPeriod = data.iloc[:, 25].values.reshape(data.shape[0],1)\n",
    "nameSpace = data.iloc[:, 26].values.reshape(data.shape[0],1)\n",
    "sumLen = data.iloc[:, 27].values.reshape(data.shape[0],1)\n",
    "XreviewLen = data.iloc[:, 28].values.reshape(data.shape[0], 1)\n",
    "XIddups = data.iloc[:, 29].values.reshape(data.shape[0], 1)\n",
    "nameLen = data.iloc[:, 30].values.reshape(data.shape[0], 1)\n",
    "sumtexdif = data.iloc[:, 31].values.reshape(data.shape[0], 1)\n",
    "sumComma = data.iloc[:, 32].values.reshape(data.shape[0],1)\n",
    "sumColon = data.iloc[:, 33].values.reshape(data.shape[0],1)\n",
    "sumSC = data.iloc[:, 34].values.reshape(data.shape[0],1)\n",
    "sumDash = data.iloc[:, 35].values.reshape(data.shape[0],1)\n",
    "hasComma = data.iloc[:, 36].values.reshape(data.shape[0],1)\n",
    "hasColon = data.iloc[:, 37].values.reshape(data.shape[0],1)\n",
    "ProductCounts = data.iloc[:, 38].values.reshape(data.shape[0],1)\n",
    "UserCounts = data.iloc[:, 39].values.reshape(data.shape[0],1)\n",
    "\n",
    "Xtoadd = np.concatenate(( hasSC, hasbroken, hasdidnt, hasperfect, \n",
    "                         haslove, hasEP, hasdash, XScore, XreviewLen,\n",
    "                         XIddups, hasChocolate, hasCoffee, hasRec,\n",
    "                         hasTerrible, sumtexdif, sumEP, sumPeriod,\n",
    "                         sumLen, nameLen, nameSpace, sumComma, sumColon,\n",
    "                         sumSC, sumDash, hasComma, hasColon , ProductCounts, UserCounts), axis=1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Function to show model results\n",
    "def print_results():\n",
    "    print('Error rate on training set: ')\n",
    "    print((y_train != y_pred).sum() / X_train.shape[0])\n",
    "    print('Accuracy rate on training set: ')\n",
    "    print(1 - (y_train != y_pred).sum() / X_train.shape[0])\n",
    "    print('True positive rate on training tet:')\n",
    "    print(((y_train==True) & (y_pred==True)).sum() / y_train.sum())\n",
    "    print('**************')\n",
    "    print('Error rate on test set: ')\n",
    "    print((y_test != y_pred_test).sum() / X_test.shape[0])\n",
    "    print('Accuracy rate on test set: ')\n",
    "    print(1 - (y_test != y_pred_test).sum() / X_test.shape[0])\n",
    "    print('True positive rate on test set')\n",
    "    print(((y_test==True) & (y_pred_test==True)).sum() / y_test.sum())\n",
    "    print('True negative rate on test set')\n",
    "    print(((y_test==False) & (y_pred_test==False)).sum() / (y_test.shape[0] - y_test.sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk.stem\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "english_stemmer = nltk.stem.SnowballStemmer('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class StemmedCountVectorizer(CountVectorizer):\n",
    "    def build_analyzer(self):\n",
    "        analyzer = super(StemmedCountVectorizer, self).build_analyzer()\n",
    "        return lambda doc: (english_stemmer.stem(w) for w in analyzer(doc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vectorizer = StemmedCountVectorizer(min_df=1,\n",
    "stop_words='english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Run stemming vectorizer on text based elements of data to add into feature set\n",
    "Xstemmer = vectorizer.fit_transform(data.Text)\n",
    "XstemmerSum = vectorizer.fit_transform(data.Summary)\n",
    "Xstemname = vectorizer.fit_transform(data.ProfileName)\n",
    "Xstemuid = vectorizer.fit_transform(data.UserId)\n",
    "Xstempid = vectorizer.fit_transform(data.ProductId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Convert features\n",
    "from scipy.sparse import csr_matrix, hstack\n",
    "XtoaddSparse = csr_matrix(Xtoadd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Combine features\n",
    "Xfinal = hstack([Xstemmer, XstemmerSum, Xstemname, Xstemuid, Xstempid, XtoaddSparse])\n",
    "X = csr_matrix(Xfinal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Specify y value\n",
    "y = data.iloc[:, 12].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Split data for training and testing\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error rate on training set: \n",
      "0.392913657771\n",
      "Accuracy rate on training set: \n",
      "0.607086342229\n",
      "True positive rate on training tet:\n",
      "0.647795763733\n",
      "**************\n",
      "Error rate on test set: \n",
      "0.408131868132\n",
      "Accuracy rate on test set: \n",
      "0.591868131868\n",
      "True positive rate on test set\n",
      "0.630758469072\n",
      "True negative rate on test set\n",
      "0.588791474627\n"
     ]
    }
   ],
   "source": [
    "#Train and predict; Print results\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "clf = MultinomialNB()\n",
    "clf.fit(X_train, y_train)\n",
    "y_pred = clf.fit(X_train, y_train).predict(X_train)\n",
    "y_pred_test = clf.predict(X_test)\n",
    "print_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}