notebooks/17_recommenders.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "# \"\"\"\n",
    "# Class 15: Recommendation Engines\n",
    "# Content based and Collaborative based filtering\n",
    "# Jaccard Similarity\n",
    "# Modified KNN Algorithm\n",
    "# \"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "########################################\n",
    "## Collaborative-Based User Filtering ##\n",
    "########################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#read in brands data\n",
    "user_brands = pd.read_csv('../data/user_brand.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Target                        1866\n",
       "Old Navy                      1200\n",
       "Home Depot                    1186\n",
       "Kohl's                        1157\n",
       "Banana Republic                932\n",
       "Nordstrom                      904\n",
       "Gap                            860\n",
       "Crate & Barrel                 816\n",
       "Express                        785\n",
       "KitchenAid                     700\n",
       "J.Crew                         569\n",
       "Container Store                564\n",
       "Steve Madden                   539\n",
       "Guess                          509\n",
       "Cuisinart                      506\n",
       "Nine West                      489\n",
       "Calvin Klein                   476\n",
       "Levi's                         472\n",
       "Converse                       456\n",
       "New Balance                    438\n",
       "BCBGMAXAZRIA                   429\n",
       "Restoration Hardware           410\n",
       "Lacoste                        405\n",
       "Kenneth Cole                   366\n",
       "Kate Spade                     354\n",
       "Puma                           350\n",
       "Melissa & Doug                 335\n",
       "DKNY                           328\n",
       "Last Call by Neiman Marcus     308\n",
       "Crocs                          293\n",
       "                              ... \n",
       "UNIQLO                           1\n",
       "Janie and Jack                   1\n",
       "Billionaire Boys Club            1\n",
       "Discovery Channel Store          1\n",
       "Barneys                          1\n",
       "Keds                             1\n",
       "Black Diamond                    1\n",
       "Lord & Taylor                    1\n",
       "Boden                            1\n",
       "Vince Camuto                     1\n",
       "Crate and Barrel                 1\n",
       "Bloomingdale's                   1\n",
       "Amazon.com                       1\n",
       "Grand                            1\n",
       "ZOO                              1\n",
       "Joseph Abboud                    1\n",
       "Hugo Boss                        1\n",
       "Chanel                           1\n",
       "Sky                              1\n",
       "Gymboree                         1\n",
       "Marmot                           1\n",
       "Stride Rite                      1\n",
       "Oakley                           1\n",
       "DC                               1\n",
       "J Jill                           1\n",
       "Carter                           1\n",
       "Brooks Brothers                  1\n",
       "David Tutera                     1\n",
       "Barneys Warehouse                1\n",
       "Walk-Over                        1\n",
       "Name: Store, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#look at count of stores\n",
    "user_brands.Store.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Series of user IDs, note the duplicates\n",
    "user_ids = user_brands.ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    80002\n",
       "1    80002\n",
       "2    80010\n",
       "3    80010\n",
       "4    80010\n",
       "Name: ID, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_ids.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# EXERCISE \n",
    "# create a dictionary called brandsfor that takes in a user id and outputs a list of the brands they like\n",
    "# brandsfor = ANSWER HEREEEEEE\n",
    "\n",
    "# key == a user ID, and the value is a \n",
    "# value == list of stores that the user \"likes\"\n",
    "# example brandsfor[83065] == [\"Kohl's\", 'Target']\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ANSWER\n",
    "\n",
    "brandsfor = dict(user_brands.groupby('ID').Store.apply(lambda x:list(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Kohl's\", 'Target']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "brandsfor[83065]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# try it out. User 83065 likes Kohl's and Targe\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hanky Panky',\n",
       " 'Betsey Johnson',\n",
       " 'Converse',\n",
       " 'Steve Madden',\n",
       " 'Old Navy',\n",
       " 'Target',\n",
       " 'Nordstrom']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# User 82983 likes many more!\n",
    "brandsfor[82983]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "########################\n",
    "## Jaccard Similarity ##\n",
    "########################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#\n",
    "#The Jaccard Similarity allows us to compare two sets\n",
    "#If we regard people as merely being a set of brands they prefer\n",
    "#the Jaccard Similarity allows us to compare people\n",
    "#\n",
    "#Example. the jaccard similarty between user 82983 and 83065 is .125\n",
    "#            because\n",
    "#             brandsfor['83065'] == [\"Kohl's\", 'Target']\n",
    "#             brandsfor['82983'] == ['Hanky Panky', 'Betsey Johnson', 'Converse', 'Steve Madden', 'Old Navy', 'Target', 'Nordstrom']\n",
    "\n",
    "# the intersection of these two sets is just set(\"Target\")\n",
    "# the union of the two sets is set(['Target', 'Hanky Panky', 'Betsey Johnson', 'Converse', 'Steve Madden', 'Old Navy', 'Target', 'Nordstrom'])\n",
    "# so the len(intersection) / len(union) = 1 / 8 == .125\n",
    "\n",
    "# EXERCISE: what is the Jaccard Similarity \n",
    "#           between user 82956 and user 82963?\n",
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ANSWER HERE\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# 0.3333333333333333"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# '''\n",
    "# EXERCISE: Complete the jaccard method below.\n",
    "#           It should take in a list of brands, and output the \n",
    "#           jaccard similarity between them\n",
    "\n",
    "# This should work with anything in the set, for example\n",
    "# jaccard([1,2,3], [2,3,4,5,6])  == .3333333\n",
    "\n",
    "# HINT: set1 & set2 is the intersection\n",
    "#       set1 | set2 is the union\n",
    "\n",
    "# '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def jaccard(first, second):\n",
    "  first = set(first)\n",
    "  second = set(second)\n",
    "  # the line below should be changed\n",
    "  # ANSWER HEREEEEEE\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def jaccard(first, second):\n",
    "  first = set(first)\n",
    "  second = set(second)\n",
    "  return len(first & second) / float(len(first | second))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3333333333333333"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# try it out!\n",
    "brandsfor[83065] # brands for user 83065\n",
    "brandsfor[82983] # brands for user 82983\n",
    "jaccard(brandsfor[83065], brandsfor[82983])\n",
    "jaccard(brandsfor[82956], brandsfor[82963])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#######################\n",
    "### Our Recommender ###\n",
    "#######################\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# '''\n",
    "# Our recommender will be a modified KNN collaborative algorithm.\n",
    "# Input: A given user's brands that they like\n",
    "# Output: A set (no repeats) of brand recommendations based on\n",
    "#         similar users preferences\n",
    "\n",
    "# 1. When a user's brands are given to us, we will calculate the input user's\n",
    "# jaccard similarity with every person in our brandsfor dictionary\n",
    "\n",
    "# 2. We will pick the K most similar users and recommend\n",
    "# the brands that they like that the given user doesn't know about\n",
    "\n",
    "# EXAMPLE:\n",
    "# Given User likes ['Target', 'Old Navy', 'Banana Republic', 'H&M']\n",
    "# Outputs: ['Forever 21', 'Gap', 'Steve Madden']\n",
    "# '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "given_user = ['Target', 'Old Navy', 'Banana Republic', 'H&M']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#similarty between user 83065 and given user\n",
    "brandsfor[83065]\n",
    "jaccard(brandsfor[83065], given_user) \n",
    "# should be 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# '''\n",
    "# EXERCISE\n",
    "#     Find the similarty between given_user and ALL of our users\n",
    "#     output should be a dictionary where\n",
    "#     the key is a user id and the value is the jaccard similarity\n",
    "# {...\n",
    "#  {81920: 0.07142857142857142,\n",
    "#  81921: 0.25,\n",
    "#  90114: 0.18181818181818182,\n",
    "#  81923: 0.25,\n",
    "#  ...}\n",
    "# '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ANSWER HEREEEEEE\n",
    "similarities\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.07142857142857142"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ANSWER\n",
    "similarities = {}\n",
    "for user_id, brands in brandsfor.iteritems():\n",
    "    similarities[user_id] = jaccard(given_user, brands)\n",
    "similarities[81920]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "K = 5 #number of similar users to look at\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Now for the top K most similar users, let's aggregate the brands they like.\n",
    "# I sort by the jaccard similarty so most similar users are first\n",
    "# I use the sorted method, but because I'm dorting dictionaries\n",
    "# I specify the \"key\" as the value of the dictionary\n",
    "# the key is what the list should sort on\n",
    "# so the most similar users end up being on top\n",
    "# ANSWER HEREEEEEE\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(81012, 0.75), (82970, 0.6), (91362, 0.6), (84807, 0.6), (88549, 0.6)]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# list of K similar users' IDs\n",
    "most_similar_users = sorted(similarities.items(), key = lambda x: x[1], reverse=True)[:K]\n",
    "most_similar_users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Banana Republic', 'Old Navy', 'Target']"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's see what some of the most similar users likes\n",
    "brandsfor[most_similar_users[0][0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Steve Madden', 'Banana Republic', 'Old Navy', 'Target']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "brandsfor[most_similar_users[3][0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81012\n",
      "82970\n",
      "91362\n",
      "84807\n",
      "88549\n"
     ]
    }
   ],
   "source": [
    "# Aggregate all brands liked by the K most similar users into a single set\n",
    "brands_to_recommend = set()\n",
    "for user in most_similar_users:\n",
    "    print user[0]\n",
    "    # ANSWER HEREEEEEE\n",
    "    # add to the set of brands_to_recommend\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81012\n",
      "82970\n",
      "91362\n",
      "84807\n",
      "88549\n"
     ]
    }
   ],
   "source": [
    "# Aggregate all brands liked by the K most similar users into a single set\n",
    "# be sure to not include brands the the given user is already known to like1\n",
    "brands_to_recommend = []\n",
    "for user in most_similar_users:\n",
    "    print user[0]\n",
    "    for brand in brandsfor[user[0]]:\n",
    "        if brand not in given_user:\n",
    "            brands_to_recommend.append(brand)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Gap', 'Gap', 'Steve Madden', 'Forever 21']"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "brands_to_recommend\n",
    "# UH OH WE HAVE DUPLICATES. Banana Republic, Old Navy, Target are all repeats.\n",
    "# let's caculate a \"score\" of recommendation\n",
    "# We will define the score as being the number of times\n",
    "# a brand appears within the first K users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Forever 21': 1, 'Gap': 2, 'Steve Madden': 1}"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "dict(Counter(brands_to_recommend))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#################################\n",
    "#### Collaborative Item based ###\n",
    "#################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# '''\n",
    "# We can also define a similary between items using jaccard similarity.\n",
    "# We can say that the similarity between two items is the jaccard similarity\n",
    "# between the sets of people who like the two brands.\n",
    "\n",
    "# Example: similarity of Gap to Target is:\n",
    "# '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# filter users by liking Gap\n",
    "gap_lovers = set(user_brands['Gap' == user_brands.Store].ID)\n",
    "old_navy_lovers = set(user_brands['Old Navy' == user_brands.Store].ID)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.35437212360289283"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# similarty between Gap and Old Navy\n",
    "jaccard(gap_lovers, old_navy_lovers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.21257750221434898"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "guess_lovers = set(user_brands['Guess' == user_brands.Store].ID)\n",
    "# similarty between Gap andGuess\n",
    "jaccard(guess_lovers, gap_lovers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2068654019873532"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calvin_lovers = set(user_brands['Calvin Klein' == user_brands.Store].ID)\n",
    "# similarty between Gap and Calvin Klein\n",
    "jaccard(calvin_lovers, gap_lovers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [sfdat26-env]",
   "language": "python",
   "name": "Python [sfdat26-env]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}