datapreprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/praphul/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import jsonlines\n",
    "import numpy as np\n",
    "import random\n",
    "import pickle as pkl\n",
    "import re\n",
    "import sklearn \n",
    "import tensorflow as tf\n",
    "import pickle as pkl\n",
    "import glob\n",
    "from nltk import word_tokenize   \n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Muslims BUSTED: They Stole Millions In Gov’t B...</td>\n",
       "      <td>Print They should pay all the back all the mon...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Re: Why Did Attorney General Loretta Lynch Ple...</td>\n",
       "      <td>Why Did Attorney General Loretta Lynch Plead T...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BREAKING: Weiner Cooperating With FBI On Hilla...</td>\n",
       "      <td>Red State : \\nFox News Sunday reported this mo...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PIN DROP SPEECH BY FATHER OF DAUGHTER Kidnappe...</td>\n",
       "      <td>Email Kayla Mueller was a prisoner and torture...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FANTASTIC! TRUMP'S 7 POINT PLAN To Reform Heal...</td>\n",
       "      <td>Email HEALTHCARE REFORM TO MAKE AMERICA GREAT ...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0  Muslims BUSTED: They Stole Millions In Gov’t B...   \n",
       "1  Re: Why Did Attorney General Loretta Lynch Ple...   \n",
       "2  BREAKING: Weiner Cooperating With FBI On Hilla...   \n",
       "3  PIN DROP SPEECH BY FATHER OF DAUGHTER Kidnappe...   \n",
       "4  FANTASTIC! TRUMP'S 7 POINT PLAN To Reform Heal...   \n",
       "\n",
       "                                                text  label  \n",
       "0  Print They should pay all the back all the mon...    1.0  \n",
       "1  Why Did Attorney General Loretta Lynch Plead T...    1.0  \n",
       "2  Red State : \\nFox News Sunday reported this mo...    1.0  \n",
       "3  Email Kayla Mueller was a prisoner and torture...    1.0  \n",
       "4  Email HEALTHCARE REFORM TO MAKE AMERICA GREAT ...    1.0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = pd.read_csv('./kaggle_fake.csv')\n",
    "df1 = df1[['title','text']]\n",
    "df1['label'] = pd.Series(np.ones([df1.shape[0]]))\n",
    "df1[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Muslims BUSTED: They Stole Millions In Gov’t Benefits'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_data = df1.as_matrix(columns=None)\n",
    "pos_data = list(pos_data)\n",
    "pos_data[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Climate Engineering And Microwaved Skies'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_data[3890][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"geoengineeringwatch.org \\nWe are all swimming in a sea of microwave transmissions that are decimating the climate system and are extremely harmful to all life forms. What you don't know can hurt you. Many are now finally beginning to look up and take notice of the ongoing atmospheric aerosol spraying that is occurring in skies all over the globe. Incredibly anomalous cloud shapes and forms which can only be considered engineered have been photographed from satellite. Extremely powerful microwave transmissions are a major aspect of the climate engineering assault that often goes unnoticed by many, even within the ranks of the anti-climate engineering movement. Radio frequency transmissions are utilized by the climate engineers to manipulate the aircraft sprayed heavy metal and chemical particulates. Like sound waves, microwave transmissions ripple through the atmosphere, leaving their telltale signature on the geoengineering aerosol laced cloud cover. Radio frequency transmissions emanating from the French Southern and Antarctic Lands, captured on satellite imagery. French Southern and Antarctic Lands Palm Springs, California. Photo credit: Ron Morgan Port Washington, New York. Photo credit: GeoengineeringWatch.org Palm Springs, California. Photo credit: Ron Morgan Morongo Valley, California. Photo credit: Ron Morgan Palm Springs, California. Photo credit: Ron Morgan Morongo Valley, California. Photo credit: Ron Morgan Morongo Valley, California. Photo credit: Ron Morgan Cantabria, Spain. Photo credit: Alberto Ibañez Morongo Valley, California. Photo credit: Ron Morgan Morongo Valley, California. Photo credit: Ron Morgan Honolulu, Hawaii. Photo credit: Irmina Bernal\""
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_data[3890][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# f = open('raw_data/pos_data', 'wb')\n",
    "# pkl.dump(pos_data, f)\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Worcester breakfast club for veterans gives hu...</td>\n",
       "      <td>VETERANS saluted Worcester's first ever breakf...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Jumpshot Gives Marketers Renewed Visibility In...</td>\n",
       "      <td>New Product Gives Marketers Access to Real Key...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Return Of The Nike Air Max Sensation Has 8...</td>\n",
       "      <td>Home »\\rStyle » The Return Of The Nike Air Max...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>This New Dating App Will Ruin Your Internet Game</td>\n",
       "      <td>NYMag.com Daily Intelligencer Vulture The Cut ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Pay up or face legal action: DBKL</td>\n",
       "      <td>KUALA LUMPUR, Sept 15 (MySinchew) -- The Kuala...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0  Worcester breakfast club for veterans gives hu...   \n",
       "1  Jumpshot Gives Marketers Renewed Visibility In...   \n",
       "2  The Return Of The Nike Air Max Sensation Has 8...   \n",
       "3   This New Dating App Will Ruin Your Internet Game   \n",
       "4                  Pay up or face legal action: DBKL   \n",
       "\n",
       "                                                text  label  \n",
       "0  VETERANS saluted Worcester's first ever breakf...      0  \n",
       "1  New Product Gives Marketers Access to Real Key...      0  \n",
       "2  Home »\\rStyle » The Return Of The Nike Air Max...      0  \n",
       "3  NYMag.com Daily Intelligencer Vulture The Cut ...      0  \n",
       "4  KUALA LUMPUR, Sept 15 (MySinchew) -- The Kuala...      0  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# temp_dict = {'title':[], 'text':[], 'label':[]}\n",
    "# with jsonlines.open('./sample-1M.jsonl') as reader:\n",
    "#     for obj in reader:\n",
    "#         temp_dict['title'].append(obj['title'])\n",
    "#         temp_dict['text'].append(obj['content'])\n",
    "#         temp_dict['label'].append(0)\n",
    "\n",
    "# df2 = pd.DataFrame(temp_dict)\n",
    "# df2 = df2[['title','text','label']]\n",
    "# df2[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Jumpshot Gives Marketers Renewed Visibility Into Paid and Organic Keywords With Launch of Jumpshot Elite'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# neg_data = df2.as_matrix(columns=None)\n",
    "# neg_data = list(neg_data)\n",
    "# neg_data[1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# random.shuffle(neg_data)\n",
    "# neg_data = neg_data[:20000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# type(neg_data[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# f = open('raw_data/neg_data', 'wb')\n",
    "# pkl.dump(neg_data, f)\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('raw_data/neg_data', 'rb')\n",
    "neg_data= pkl.load(f)\n",
    "f.close()\n",
    "\n",
    "f = open('raw_data/pos_data', 'rb')\n",
    "pos_data= pkl.load(f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def filter(line, whitelist):\n",
    "    ##############################################################\n",
    "    #   Filters out all characters which are not in whitelist    #\n",
    "    ##############################################################\n",
    "\n",
    "    return ''.join([ch for ch in line if ch in whitelist])\n",
    "\n",
    "\n",
    "WHITELIST1 = \"0123456789abcdefghijklmnopqrstuvwxyz' \"\n",
    "WHITELIST2 = \"0123456789abcdefghijklmnopqrstuvwxyz' \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for data in pos_data:\n",
    "    if(type(data[0])!=str):\n",
    "        data[0]=str(data[0])\n",
    "    if(type(data[1])!=str):\n",
    "        data[1]=str(data[1])\n",
    "    \n",
    "    data[0] = filter(data[0].lower(), WHITELIST2)\n",
    "    data[1] = filter(data[1].lower(), WHITELIST1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'red state  fox news sunday reported this morning that anthony weiner is cooperating with the fbi which has reopened yes lefties reopened the investigation into hillary clintons classified emails watch as chris wallace reports the breaking news during the panel segment near the end of the show and the news is breaking while were on the air our colleague bret baier has just sent us an email saying he has two sources who say that anthony weiner who also had coownership of that laptop with his estranged wife huma abedin is cooperating with the fbi investigation had given them the laptop so therefore they didnt need a warrant to get in to see the contents of said laptop pretty interesting development targets of federal investigations will often cooperate hoping that they will get consideration from a judge at sentencing given weiners wellknown penchant for lying its hard to believe that a prosecutor would give weiner a deal based on an agreement to testify unless his testimony were very strongly corroborated by hard evidence but cooperation can take many forms  and as wallace indicated on this mornings show one of those forms could be signing a consent form to allow   the contents of devices that they could probably get a warrant for anyway well see if weiners cooperation extends beyond that more related'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_data[2][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for data in neg_data:\n",
    "    if(type(data[0])!=str):\n",
    "        data[0]=str(data[0])\n",
    "    if(type(data[1])!=str):\n",
    "        data[1]=str(data[1])\n",
    "    \n",
    "    data[0] = filter(data[0].lower(), WHITELIST2)\n",
    "    data[1] = filter(data[1].lower(), WHITELIST1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"popular pro audio publication tape op reports record growth in their print circulation  sacramento ca prweb september 08 2015  at a time when print media is considered dead or on the decline tape op magazines circulation has increased from 42500 to 45000 qualified readers in 2015  tape op an independently owned publication is beloved in pro audio circles because it is known as the creative music recording magazine with a very clear mission tape op is about learning how to make great recordings with whatever tools you have access to    publisher john baccigaluppi says from the beginning tape ops circulation strategy has been to target the individual end user and reader and focus on our subscription base while we are confident that print media remains vital and the best value for advertisers newsstand sales and traditional print distribution are fading and inefficient with our coordinated online and social media presence tied into the print edition of tape op it's allowed us to grow our highly targeted subscriber base to over 45000 subscribers who we can directly interact with and respond to  tape op magazine began as a zine in 1996 and has grown to a bimonthly magazine with 45000 subscribers of its print edition and 70000 of its downloadable digital version  the largest readership of any music recording magazine in the world tape op is owned and run by two studio ownerproducerengineers larry crane and john baccigaluppi and its contributors include active recordists across the world tape op is focused on learning how to make great recordings with whatever tools the user has access to and the magazines real life tone holds true as its written by recording engineers for recording engineers through long form interviews gear reviews and varying thoughts and opinions tape op encourages creativity and innovation while celebrating the art of recording music  in this era of media overload i think people appreciate authenticity john and i are the owners of tape op there is not a corporate umbrella over us and we are able to operate independently we're not solely journalists we're active engineers and producers writing about what we do everyday editor and founder crane shares i think all of this contributes to the large and loyal readership the magazine has developed  readers regularly post photos when they get their new issue we love that  it is not just readers happy with tape op  their advertisers are quick to sing their praises  adam audio has kept our advertising package with tape op because we see real results from appearing in this publication  they do a great job of covering old and new technology and showing that each has value in the studio said david hetrick the speaker manufacturers north american ceo  so deep is the abiding love for tape op magazine that oregonbased fort george brewing announced in april that they have created a special brew overdub ipa in the magazines honor a good beer is like a good sound recording a spokesman for the company theorizes it is balanced and pristine  soft and subdued when it needs to be and in your face and crankin when it must  read full article       for more information about tape op magazine visit them online at tapeopcom  for the original version on prweb visit httpwwwprwebcomreleases201509prweb12940391htm\""
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neg_data[89][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def preprocess(string): \n",
    "    string = re.sub(\"\\.\\.\\.\", \" \\. \", string)\n",
    "    string = re.sub(r\"won't\", \"will not\", string)\n",
    "    string = re.sub(r\"n\\'t\", \" not\", string)\n",
    "    string = re.sub(r\"n't\", \" not \", string)\n",
    "    string = re.sub(r\"\\'re\", \" are\", string)\n",
    "    string = re.sub(r\"\\'s\", \" is\", string)\n",
    "    string = re.sub(r\"\\'d\", \" would\", string)\n",
    "    string = re.sub(r\"\\'ll\", \" will\", string)\n",
    "    string = re.sub(r\"\\'t\", \" not\", string)\n",
    "    string = re.sub(r\"\\'ve\", \" have\", string)\n",
    "    string = re.sub(r\"\\'m\", \" am\", string)\n",
    "    string = re.sub(r\"\\'d've\", \" would have\", string)\n",
    "    string = re.sub(r\"\\'d'y\", \" do you\", string)\n",
    "    string = re.sub(r\"'\", \" \", string)\n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for data in pos_data:\n",
    "    if(type(data[0])!=str):\n",
    "        data[0]=str(data[0])\n",
    "    if(type(data[1])!=str):\n",
    "        data[1]=str(data[1])\n",
    "    \n",
    "    data[0] = preprocess(data[0])\n",
    "    data[1] = preprocess(data[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for data in neg_data:\n",
    "    if(type(data[0])!=str):\n",
    "        data[0]=str(data[0])\n",
    "    if(type(data[1])!=str):\n",
    "        data[1]=str(data[1])\n",
    "    \n",
    "    data[0] = preprocess(data[0])\n",
    "    data[1] = preprocess(data[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# full_data = pos_data + neg_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for data in full_data:\n",
    "#     data[0] = data[0].split()\n",
    "#     data[1] = data[1].split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# data_final = []\n",
    "\n",
    "# for data in full_data:\n",
    "#     temp1 = []\n",
    "#     for i in range(22):\n",
    "#         if(i<len(data[0])):\n",
    "#             temp1.append(data[0][i])\n",
    "#         else:\n",
    "#             temp1.append('PAD')\n",
    "#     temp2 = []\n",
    "#     for j in range(250):\n",
    "#         if(j<len(data[1])):\n",
    "#             temp2.append(data[1][j])\n",
    "#         else:\n",
    "#             temp2.append('PAD')\n",
    "#     data_final.append(np.array([temp1, temp2, data[2]]))\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('data_final', 'wb')\n",
    "pkl.dump(data_final, f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for data in pos_data:\n",
    "    data[1] = re.split(\"[\\.?!]\", data[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['complains obama is not hawkish enough   in a new interview with britains sky news former nato secretarygeneral anders fogh rasmussen brought out the old narrative of america as the worlds policeman but with a lot more upbeat of an attitude about it than one would generally see',\n",
       " ' rasmussen criticized president obama for not being hawkish enough saying his successor needs to be much more interventionist and declaring  we need america as the worlds policeman  adding that the us needs to restore international law and order through wars',\n",
       " ' rasmussen who was always a relative hawk in the post but seems to have taken it to an entirely new level set out a series of things the us needs to fix militarily including iraq syria libya russia china and north korea',\n",
       " ' this of course closely mirrors recent pentagon talk of wars in the decades to come',\n",
       " ' the timing of his calls for extreme us bellicosity are centered on trying to influence the upcoming us election in favor of democratic nominee hillary clinton who has campaigned heavily on picking fights in syria and against russia',\n",
       " ' rasmussen underscored this fact by declaring donald trump who openly said the us cannot be the worlds police as very dangerous for the world',\n",
       " '  ']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_data[890][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for data in neg_data:\n",
    "    data[1] = re.split(\"[\\.?!]\", data[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['monday september 28 2015foot fetish thighhigh chryslerbuilding boots im in love with the paul andrew chryslerbuilding boots that i saw in a bergdorf goodman catalog',\n",
       " ' maybe i can wear them with my zang toi gown that depicts the time warner center and the hearst tower  zang toi gown in 2009',\n",
       " ' click for original post',\n",
       " '  and accessorize with my taxi and passenger maneater ring which is engraved with the brooklyn bridge',\n",
       " ' click to see on my website',\n",
       " ' what too much',\n",
       " ' but i love new york',\n",
       " ' categories bergdorf goodman boots foot fetish my designs rings shoes zang toi']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neg_data[876][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data = []\n",
    "for data in neg_data:\n",
    "    train_data.append(data)\n",
    "for data in pos_data:\n",
    "    train_data.append(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_data[1][1][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(len(train_data)):\n",
    "    train_data[i][0] = str(train_data[i][0]).split()\n",
    "    for k in range(len(train_data[i][1])):\n",
    "        train_data[i][1][k] = str(train_data[i][1][k]).split() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['citizenship', 'provisions', 'discriminate', 'against', 'women']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[90][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# f = open('processed_data/train_data', 'rb')\n",
    "# train_data = pkl.load(f)\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data[98][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "wordlist = []\n",
    "for i in range(len(train_data)):\n",
    "    for j in train_data[i][0]:\n",
    "        wordlist.append(j)\n",
    "    for k in train_data[i][1]:\n",
    "        for w in k:\n",
    "            wordlist.append(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "wordlist[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "word_count = Counter(wordlist)\n",
    "word_count = word_count.most_common()\n",
    "len(word_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab = [w for w, i in word_count]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('../embeddings/vocab_glove', 'rb')\n",
    "vocab_glove = pkl.load(f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_vocab = []\n",
    "for word in vocab:\n",
    "    if(word in vocab_glove):\n",
    "        final_vocab.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('processed_data/final_vocab', 'wb')\n",
    "pkl.dump(final_vocab, f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# f = open('processed_data/final_vocab', 'rb')\n",
    "# final_vocab = pkl.load(f)\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(final_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_final = []\n",
    "pad_sent = []\n",
    "for i in range(23):\n",
    "    pad_sent.append('PAD')\n",
    "    \n",
    "for data in train_data:\n",
    "    temp1 = []\n",
    "    for i in range(22):\n",
    "        if(i<len(data[0])):\n",
    "            temp1.append(data[0][i])\n",
    "        else:\n",
    "            temp1.append('PAD')\n",
    "    temp2 = []\n",
    "    for i in range(10):\n",
    "        if(i<len(data[1])):\n",
    "            temp3 = []\n",
    "            for j in range(23):\n",
    "                if(j<len(data[1][i])):\n",
    "                    temp3.append(data[1][i][j])\n",
    "                else:\n",
    "                    temp3.append('PAD')\n",
    "            temp2.append(temp3)\n",
    "        else:\n",
    "            temp2.append(pad_sent)\n",
    "            \n",
    "    train_final.append(np.array([temp1, temp2, data[2]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_final[13][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('processed_data/train_final', 'wb')\n",
    "pkl.dump(train_final, f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}