prep_text_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to /Users/aaksha/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "#nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "REPLACE_BY_SPACE_RE = re.compile('[/(){}\\[\\]\\|@,;]')\n",
    "BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')\n",
    "STOPWORDS = set(stopwords.words('english'))\n",
    "stemmer = nltk.stem.WordNetLemmatizer()\n",
    "def text_prepare(text):\n",
    "    \"\"\"\n",
    "        text: a string\n",
    "        \n",
    "        return: modified initial string\n",
    "    \"\"\"\n",
    "    text = text.lower() # lowercase text\n",
    "    text = re.sub(REPLACE_BY_SPACE_RE,' ',text) # replace REPLACE_BY_SPACE_RE symbols by space in text\n",
    "    text = re.sub(BAD_SYMBOLS_RE,' ',text) # delete symbols which are in BAD_SYMBOLS_RE from text\n",
    "    text_words=text.split(' ')\n",
    "    #lowercase,remove urls,delete stopwords and blanks\n",
    "    resultwords= []\n",
    "    for word in text_words:\n",
    "        if len(word)<=0: continue\n",
    "        if word in STOPWORDS: continue\n",
    "        if word.find('http')==0: word='url'\n",
    "        resultwords.append(word)\n",
    "        \n",
    "    #resultwords  = [word for word in text_words if word.lower() not in STOPWORDS and len(word)>0] # delete stopwords from text\n",
    "    final_words = [stemmer.lemmatize(token) for token in resultwords] #lemmatize words\n",
    "    text = ' '.join(final_words) \n",
    "    return text\n",
    "\n",
    "#make url extraction better, later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "review_data=pandas.read_pickle('data/reviews_selected.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3257815, 13)\n",
      "Index(['review_id', 'user_id', 'business_id', 'stars', 'useful', 'funny',\n",
      "       'cool', 'text', 'date', 'name', 'attributes.RestaurantsPriceRange2',\n",
      "       'review_count', 'date2'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(review_data.shape)\n",
    "print(review_data.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Raw\n",
      "In retrospect, I should have known better than to step foot into Amy's Baking Company (a.k.a. ABC Bistro).  \n",
      "\n",
      "-8 pm on a Saturday night, three tables are occupied\n",
      "-a sign on the door indicates they're looking for an experienced line cook, a dishwasher, waitresses, you name it\n",
      "-Pita Jungle, just a stone's throw away, is packed to the gills\n",
      "\n",
      "These are all bad signs.  They go ignored, however, because I'd eaten one thing all day, and I had drove here to try their pizza.  Never, ever again.\n",
      "\n",
      "The waitress brings me out a water and a menu to my table for one  on the patio, which I had all to myself.  Browsing the apps, I was pleasantly surprised to see several things priced $3, which seemed like a bargain, given the high price of everything else.  $3 tapanade sounded like a good deal, but I was informed that it didn't come with anything - it was $3 extra for bread.  Since I was solo, I decided to skip it and instead pay $14 for their 12\" margherita pizza.  \n",
      "\n",
      "About three minutes later, the waitress drops off my pizza.  \"Your pizza\", she says, leaves a plate and off she goes.  I sat for a moment, confused as to how things happened so quickly.  I try to grab a slice of the pizza, but it was so blazing hot that I wasn't even going to attempt to handle it.  I waited for a few minutes to let it cool off, only to discover that not only had it cooled enough to handle, but was actually well-cooled and most likely reheated.  After a closer inspection of the pie in front of me, it was evident that it had been reheated, as there were smaller tomatoes that had spent their time in the oven, and larger tomatoes that had barely been kissed by the heat of the oven.\n",
      "\n",
      "I took a bite, and was immediately underwhelmed.  The crust had very little character, was slightly sweet but had that store-bought quality to it.  The pesto tasted okay, but the tomatoes were completely tasteless and overall, it just fell flat.  It's margherita - the ingredients need to shine to make such a simple pizza.  These ingredients were sub-par.  After two small pieces, I decided I was wasting my calories and just gave up on it.\n",
      "\n",
      "So I sat outside, not eating, and sat.  And sat.  Where the hell was the waitress?  I glanced inside a few times, hoping to catch her eye, but she must have been occupied elsewhere.  \n",
      "\n",
      "The owner comes out.  \"How'd you like your pizza?\"  Instead of immediately responding, I asked how it had come out so quickly.  In short, he told me another table had ordered it, decided they didn't want it, and it sat in the kitchen for two minutes, who in turn decided to send it out to me since I didn't order anything else.  \n",
      "\n",
      "Me: \"Well, it didn't really taste fresh\".\n",
      "Him: \"No, no, our pizzas are the best.  Ask our customers.  You're the first person since we've opened to ever not like our pizzas\".  \n",
      "\n",
      "He got very defensive about the pizza, but I hadn't really launched a harsh criticism on the pizza, just said I didn't really enjoy it.  So I sat some more, with an empty drink, and realized they wanted me gone.  The owner wouldn't make eye contact with me.  The server never came back out asking if I wanted something else.  And they still hadn't refilled my drink.\n",
      "\n",
      "So I sat on a patio, alone, and decided to have a smoke.  The waitress comes out...\n",
      "\n",
      "Her: \"Sir, there's a no smoking sign over there, I was told to tell you to not to smoke here.\"  \n",
      "Me: \"Is this the last f*** you in this experience?\"\n",
      "Her: \"Yes, I think so\".\n",
      "\n",
      "She walks away.  I walk inside, pay the bill, and leave.  No apologies, no discounts on the bill, nothing.  $18 for weak iced tea and shitty pizza.\n",
      "\n",
      "This is arrogance in its worst form.  I can forgive bad food, but I cannot forgive misplaced arrogance and the blatant dismissal of a customer, whether you agree with them or not.  \n",
      "\n",
      "Perhaps the sign on the door should also say, \"Wanted: New owner\".  \n",
      "\n",
      "I cannot, for the life of me, recommend this restaurant to anyone.\n",
      "\n",
      "---\n",
      "UPDATE:\n",
      "Since Yelp's 5000 character limit will not let me post the full response by the owner (Amy B.), here is a link to the comments:\n",
      "http://blogs.phoenixnewtimes.com/bella/2010/08/ouch_todays_hard_lesson_on_yel.php\n",
      "Prepped\n",
      "retrospect known better step foot amy baking company k abc bistro 8 pm saturday night three table occupied sign door indicates looking experienced line cook dishwasher waitress name pita jungle stone throw away packed gill bad sign go ignored however eaten one thing day drove try pizza never ever waitress brings water menu table one patio browsing apps pleasantly surprised see several thing priced 3 seemed like bargain given high price everything else 3 tapanade sounded like good deal informed come anything 3 extra bread since solo decided skip instead pay 14 12 margherita pizza three minute later waitress drop pizza pizza say leaf plate go sat moment confused thing happened quickly try grab slice pizza blazing hot even going attempt handle waited minute let cool discover cooled enough handle actually well cooled likely reheated closer inspection pie front evident reheated smaller tomato spent time oven larger tomato barely kissed heat oven took bite immediately underwhelmed crust little character slightly sweet store bought quality pesto tasted okay tomato completely tasteless overall fell flat margherita ingredient need shine make simple pizza ingredient sub par two small piece decided wasting calorie gave sat outside eating sat sat hell waitress glanced inside time hoping catch eye must occupied elsewhere owner come like pizza instead immediately responding asked come quickly short told another table ordered decided want sat kitchen two minute turn decided send since order anything else well really taste fresh pizza best ask customer first person since opened ever like pizza got defensive pizza really launched harsh criticism pizza said really enjoy sat empty drink realized wanted gone owner make eye contact server never came back asking wanted something else still refilled drink sat patio alone decided smoke waitress come sir smoking sign told tell smoke last f experience yes think walk away walk inside pay bill leave apology discount bill nothing 18 weak iced tea shitty pizza arrogance worst form forgive bad food cannot forgive misplaced arrogance blatant dismissal customer whether agree perhaps sign door also say wanted new owner cannot life recommend restaurant anyone update since yelp 5000 character limit let post full response owner amy b link comment url blog phoenixnewtimes com bella 2010 08 ouch_todays_hard_lesson_on_yel php\n"
     ]
    }
   ],
   "source": [
    "for i in range(1,10):\n",
    "    raw=review_data.iloc[i]['text']\n",
    "    prepped=text_prepare(raw)\n",
    "    print(\"Raw\")\n",
    "    print(raw)\n",
    "    print(\"Prepped\")\n",
    "    print(prepped)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This took a good 10-15 mins. to run\n",
    "review_data['text_prep']=review_data['text'].apply(text_prepare)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "review_data_drop_raw=review_data.drop(['text'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['review_id', 'user_id', 'business_id', 'stars', 'useful', 'funny',\n",
      "       'cool', 'date', 'name', 'attributes.RestaurantsPriceRange2',\n",
      "       'review_count', 'date2', 'text_prep'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print (review_data_drop_raw.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "review_data_drop_raw.to_pickle(\"data/reviews_prepped.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}