01A - SciPy - TFIDF - Doc2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Import, clean and split data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "raw/aggressive_dedup.json\n",
      "54.28077760338783  GB\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "import re\n",
    "import time\n",
    "from bs4 import BeautifulSoup \n",
    "\n",
    "# Should have 82.83 million reviews\n",
    "filename = \"raw/aggressive_dedup.json\"\n",
    "print(filename)\n",
    "print(os.path.getsize(filename)/(1024*1024*1024), \" GB\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Raw:\n",
      "I am addicted to this show. If not for amazon I would have to wait until the season was over befor watching it.\n",
      "Clean:\n",
      "i am addicted to this show . if not for amazon i would have to wait until the season was over befor watching it .\n",
      " 5.0\n",
      "Raw:\n",
      "The product delivered what it promised...I do see/feel thicker hair on initial application. Has a money back quanantee on it, but have already purchased 2nd bottle!\n",
      "Clean:\n",
      "the product delivered what it promised . . . i do see/feel thicker hair on initial application . has a money back quanantee on it , but have already purchased 2nd bottle !\n",
      " 5.0\n",
      "Raw:\n",
      "They don't glow. I was 100% disapointed. I thought that they were going to be cool and glow. I guess I thought wrong.\n",
      "Clean:\n",
      "they don ' t glow . i was 100% disapointed . i thought that they were going to be cool and glow . i guess i thought wrong .\n",
      " 1.0\n",
      "Raw:\n",
      "Very good quality. Expected different colorization. But definitely worth the money  I would recommend to anyone that has a rustic themed room\n",
      "Clean:\n",
      "very good quality . expected different colorization . but definitely worth the money i would recommend to anyone that has a rustic themed room\n",
      " 5.0\n",
      "Raw:\n",
      "i love this knife, even my husband loves this knife. its really sharp and cuts like a dream. want one in every color\n",
      "Clean:\n",
      "i love this knife , even my husband loves this knife . its really sharp and cuts like a dream . want one in every color\n",
      " 5.0\n"
     ]
    }
   ],
   "source": [
    "def line_feeder(fname):\n",
    "    with open(fname, 'r', encoding='utf-8') as f:\n",
    "        for line in f:\n",
    "            js_out = json.loads(line) \n",
    "            yield js_out   \n",
    "        \n",
    "def clean_review(review):\n",
    "    temp = BeautifulSoup(review, \"lxml\").get_text()\n",
    "    punctuation = \"\\\"'.,?!:;(){}[]\"\n",
    "    for char in punctuation:\n",
    "        temp = temp.replace(char, ' ' + char + ' ')\n",
    "    words = \" \".join(temp.lower().split()) + \"\\n\"\n",
    "    return words\n",
    "\n",
    "def example(start=200, cut=5):\n",
    "    for c,x in enumerate(line_feeder(filename)):\n",
    "        if c > start:\n",
    "            rev, rating = clean_review(x[\"reviewText\"]), x[\"overall\"]\n",
    "            print(\"Raw:\")\n",
    "            print(x[\"reviewText\"]), x[\"overall\"]\n",
    "            print(\"Clean:\")\n",
    "            print(rev, rating)          \n",
    "            if c == start+cut:\n",
    "                return\n",
    "        \n",
    "example()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "![alt text](https://github.com/ilkarman/Bangalore_Sentiment/blob/master/sample_rev.PNG?raw=true \"Review\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "good_rev = []\n",
    "bad_rev = []\n",
    "neut_rev = []\n",
    "error_rev = []\n",
    "\n",
    "gr = open('good_reviews.txt', 'w', encoding='utf-8')\n",
    "br = open('bad_reviews.txt', 'w', encoding='utf-8')\n",
    "nt = open('neutral_reviews.txt', 'w', encoding='utf-8')\n",
    "er = open('error_reviews.txt', 'w', encoding='utf-8')\n",
    "\n",
    "chunks = 0\n",
    "stime = time.time()\n",
    "for x in line_feeder(filename):\n",
    "    \n",
    "    chunks += 1\n",
    "    rev, rating = clean_review(x[\"reviewText\"]), x[\"overall\"]\n",
    "    \n",
    "    if not len(rev) > 10:\n",
    "        # Fewer than 10 characters not meangingful\n",
    "        error_rev.append(rev)\n",
    "    else:\n",
    "        # Review long enough to consider\n",
    "        if rating in [4,5]:\n",
    "            good_rev.append(rev)\n",
    "        elif rating in [1,2]:\n",
    "            bad_rev.append(rev)\n",
    "        else:\n",
    "            neut_rev.append(rev)\n",
    "            \n",
    "    # Chunk every N=1000*000 reviews\n",
    "    # Limited by IO, disk = 96%\n",
    "    # Takes 305 seconds for 1mill, so around 420 minutes = 7 hours\n",
    "    if chunks % (1000*1000) == 0:\n",
    "        print(\"Processed: %d records\" % chunks)\n",
    "        print(\"Elapsed: %.2f\" % (time.time() - stime))\n",
    "\n",
    "        gr.writelines(good_rev)\n",
    "        br.writelines(bad_rev)\n",
    "        nt.writelines(neut_rev)\n",
    "        er.writelines(error_rev)\n",
    "\n",
    "        good_rev = []\n",
    "        bad_rev = []\n",
    "        neut_rev = []\n",
    "        error_rev = []\n",
    "            \n",
    "# Any remaining\n",
    "gr.writelines(good_rev)\n",
    "gr.close()\n",
    "br.writelines(bad_rev)\n",
    "br.close()\n",
    "nt.writelines(neut_rev)\n",
    "nt.close()\n",
    "er.writelines(error_rev)\n",
    "er.close()\n",
    "\n",
    "del good_rev\n",
    "del bad_rev\n",
    "del neut_rev\n",
    "del error_rev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Check sizes\n",
    "def file_len(fname):\n",
    "    with open(fname) as f:\n",
    "        for i, l in enumerate(f):\n",
    "            pass\n",
    "    return i + 1\n",
    "\n",
    "# Should add up:\n",
    "# print(\"Raw contains %d lines\" % file_len(filename))\n",
    "\n",
    "# We have 64,439,865 good reviews\n",
    "# print(\"Good contains %d lines\" % file_len('good_reviews.txt'))\n",
    "# We have 10,961,504 bad reviews\n",
    "# print(\"Bad contains %d lines\" % file_len('bad_reviews.txt'))\n",
    "\n",
    "\n",
    "# print(\"Neutral contains %d lines\" % file_len('neutral_reviews.txt'))\n",
    "# print(\"Short contains %d lines\" % file_len('error_reviews.txt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Train/Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 1 mill \n",
    "_SAMPLE_SIZE = 1000*1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Split data into train and test (also use subsample):\n",
    "import random\n",
    "\n",
    "def train_test_split(train_ratio=0.5):\n",
    "    # Train -> true\n",
    "    return random.uniform(0,1) <= train_ratio\n",
    "\n",
    "def line_feeder(fname, cutoff):\n",
    "    i = 0\n",
    "    with open(fname, 'r', encoding='utf-8') as f:\n",
    "        for line in f:\n",
    "            yield line\n",
    "            i+=1\n",
    "            if i == cutoff:\n",
    "                break\n",
    "            \n",
    "def split_data(dataname, sample_size, train_ratio):\n",
    "    with open('train_' + dataname, 'w', encoding='utf-8') as tr:\n",
    "        with open('test_' + dataname, 'w', encoding='utf-8') as te:\n",
    "            for line in line_feeder(dataname, sample_size):\n",
    "                if train_test_split(0.5):\n",
    "                    tr.write(line)\n",
    "                else:\n",
    "                    te.write(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# I wanted a quick cut so I go by the first _SAMPLE_SIZE reviews, perhaps\n",
    "# a better approach is to create a probability = _SAMPLE_SIZE/full_size\n",
    "# and keep if random <= prob. and thus sample all lines\n",
    "# however that would take a bit longer so have omitted.\n",
    "\n",
    "split_data(dataname = 'good_reviews.txt', sample_size = _SAMPLE_SIZE, train_ratio = 0.5)\n",
    "split_data(dataname = 'bad_reviews.txt', sample_size = _SAMPLE_SIZE, train_ratio = 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sources = {'test_bad_reviews.txt':'TE_B',\n",
    "           'test_good_reviews.txt':'TE_G',\n",
    "           'train_bad_reviews.txt':'TR_B',\n",
    "           'train_good_reviews.txt':'TR_G'}"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}