Recommendation System.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import sys\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies=pd.read_csv('movies.csv')\n",
    "tags=pd.read_csv('tags.csv')\n",
    "ratings=pd.read_csv('ratings.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId                               title  \\\n",
       "0        1                    Toy Story (1995)   \n",
       "1        2                      Jumanji (1995)   \n",
       "2        3             Grumpier Old Men (1995)   \n",
       "3        4            Waiting to Exhale (1995)   \n",
       "4        5  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                        genres  \n",
       "0  Adventure|Animation|Children|Comedy|Fantasy  \n",
       "1                   Adventure|Children|Fantasy  \n",
       "2                               Comedy|Romance  \n",
       "3                         Comedy|Drama|Romance  \n",
       "4                                       Comedy  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>tag</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>60756</td>\n",
       "      <td>funny</td>\n",
       "      <td>1445714994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>60756</td>\n",
       "      <td>Highly quotable</td>\n",
       "      <td>1445714996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>60756</td>\n",
       "      <td>will ferrell</td>\n",
       "      <td>1445714992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>89774</td>\n",
       "      <td>Boxing story</td>\n",
       "      <td>1445715207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>89774</td>\n",
       "      <td>MMA</td>\n",
       "      <td>1445715200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId              tag   timestamp\n",
       "0       2    60756            funny  1445714994\n",
       "1       2    60756  Highly quotable  1445714996\n",
       "2       2    60756     will ferrell  1445714992\n",
       "3       2    89774     Boxing story  1445715207\n",
       "4       2    89774              MMA  1445715200"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964982703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964981247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964982224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "      <td>5.0</td>\n",
       "      <td>964983815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>5.0</td>\n",
       "      <td>964982931</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating  timestamp\n",
       "0       1        1     4.0  964982703\n",
       "1       1        3     4.0  964981247\n",
       "2       1        6     4.0  964982224\n",
       "3       1       47     5.0  964983815\n",
       "4       1       50     5.0  964982931"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies['genres']=movies['genres'].str.replace('|',' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9742"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#how many movies are in this dataset\n",
    "len(movies.movieId.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9724"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#how many movies have been rated\n",
    "len(ratings.movieId.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#limit ratings to user ratings that have rated more than 55 movies --\n",
    "#otherwise it becomes impossible to pivot the rating dataframe later for collaborative filtering.\n",
    "\n",
    "ratings_f = ratings.groupby('userId').filter(lambda x: len(x) >= 55)\n",
    "\n",
    "#list the movie titles that survive the filtering\n",
    "movie_list_rating=ratings_f.movieId.unique().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98.7990145760624"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we have kept 98.8% of the original movie title in ratings data frame\n",
    "len(ratings_f.movieId.unique())/len(movies.movieId.unique())*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59.67213114754099"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#but only 59.6% of the users\n",
    "len(ratings_f.userId.unique())/len(ratings.userId.unique())*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#filtering the movies dataframe\n",
    "movies=movies[movies.movieId.isin(movie_list_rating)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure Animation Children Comedy Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure Children Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy Romance</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId                    title  \\\n",
       "0        1         Toy Story (1995)   \n",
       "1        2           Jumanji (1995)   \n",
       "2        3  Grumpier Old Men (1995)   \n",
       "\n",
       "                                        genres  \n",
       "0  Adventure Animation Children Comedy Fantasy  \n",
       "1                   Adventure Children Fantasy  \n",
       "2                               Comedy Romance  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#map movie to id\n",
    "Mapping_file= dict(zip(movies.title.tolist(),movies.movieId.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags.drop(['timestamp'],1, inplace=True)\n",
    "ratings_f.drop(['timestamp'],1,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge the movies and the tags data frame create a metdata tag for each movie:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>userId</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure Animation Children Comedy Fantasy</td>\n",
       "      <td>336.0</td>\n",
       "      <td>pixar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure Animation Children Comedy Fantasy</td>\n",
       "      <td>474.0</td>\n",
       "      <td>pixar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure Animation Children Comedy Fantasy</td>\n",
       "      <td>567.0</td>\n",
       "      <td>fun</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId             title                                       genres  \\\n",
       "0        1  Toy Story (1995)  Adventure Animation Children Comedy Fantasy   \n",
       "1        1  Toy Story (1995)  Adventure Animation Children Comedy Fantasy   \n",
       "2        1  Toy Story (1995)  Adventure Animation Children Comedy Fantasy   \n",
       "\n",
       "   userId    tag  \n",
       "0   336.0  pixar  \n",
       "1   474.0  pixar  \n",
       "2   567.0    fun  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#create a mixed dataframe of movies title, genres\n",
    "#and all user tags given to each movie\n",
    "mixed=pd.merge(movies, tags,on='movieId', how='left')\n",
    "mixed.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>metadata</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>pixar pixar fun Adventure Animation Children C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>fantasy magic board game Robin Williams game A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>moldy old Comedy Romance</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId                    title  \\\n",
       "0        1         Toy Story (1995)   \n",
       "1        2           Jumanji (1995)   \n",
       "2        3  Grumpier Old Men (1995)   \n",
       "\n",
       "                                            metadata  \n",
       "0  pixar pixar fun Adventure Animation Children C...  \n",
       "1  fantasy magic board game Robin Williams game A...  \n",
       "2                           moldy old Comedy Romance  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create metadata from tags and genres\n",
    "mixed.fillna(\"\",inplace=True)\n",
    "mixed=pd.DataFrame(mixed.groupby('movieId')['tag'].apply(lambda x: \"%s\" % ' '.join(x)))\n",
    "Final=pd.merge(movies, mixed, on='movieId', how='left')\n",
    "Final['metadata']=Final[['tag','genres']].apply(lambda x:' '.join(x),axis=1)\n",
    "Final[['movieId','title','metadata']].head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9625, 5)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Final.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating a content latent matrix from movie metadata:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tf-idf vectors and truncated SVD: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9625, 1675)\n"
     ]
    }
   ],
   "source": [
    "#Term frequency and inverse document frequency\n",
    "\n",
    "# Purpose of Tf-idf is to highlight words which are frequent in a document but not across documents\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tfidf=TfidfVectorizer(stop_words='english')\n",
    "tfidf_matrix=tfidf.fit_transform(Final['metadata'])\n",
    "tfidf_df=pd.DataFrame(tfidf_matrix.toarray(), index=Final.index.tolist())\n",
    "print(tfidf_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1665</th>\n",
       "      <th>1666</th>\n",
       "      <th>1667</th>\n",
       "      <th>1668</th>\n",
       "      <th>1669</th>\n",
       "      <th>1670</th>\n",
       "      <th>1671</th>\n",
       "      <th>1672</th>\n",
       "      <th>1673</th>\n",
       "      <th>1674</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1675 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0     1     2     3     4     5     6     7     8     9     ...  1665  \\\n",
       "0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "1   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "2   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "3   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "4   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "\n",
       "   1666  1667  1668  1669  1670  1671  1672  1673  1674  \n",
       "0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "1   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "2   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "3   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "4   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "\n",
       "[5 rows x 1675 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The first 200 components explain over 50% of the varience:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEJCAYAAAB7UTvrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmNklEQVR4nO3deZxcVZn/8c83CSGEDlvSyBIgEBbBZRxsMT8HUUdkE4jMOAoqioqMRAbHJRGBAUGZ0UQReQ1hUVDGhUVxiYADDio6jiwdRCSsCYIEAiQhQDoQQsLz++Pc6lRXV3XfTvrW0vV9v171qqpzb1U9fbv7PnXvuec5igjMzMwARjU6ADMzax5OCmZm1stJwczMejkpmJlZLycFMzPrNabRAWyMSZMmxZQpUxodhplZS5k/f/6yiOistqylk8KUKVPo7u5udBhmZi1F0iO1lvn0kZmZ9XJSMDOzXk4KZmbWy0nBzMx6OSmYmTWLnh4480yYNAmkwW+TJqX1e3qGLQQnBTOzanp64POfh/Hj8+2gh+M2YQKcfTYsX54vxuXLYfZsmDZt2BKDk4KZtZZ67awnTIAvfxleeKHRP/HAVq+GRYtgzpxheTsnBTMbfkM9DTISd9b1tHo1XHjhsLxVSw9eM7Nh0NMD55wD3/iGd7StLO8pp0E4KZi1op6edLrggguGbWdgLW7ixGF5GycFs3rzDt2G27hxcOKJw/JWTgpmG8I7dmsW48bB1Kkwc+awvJ2TgrUvn0u3VjdpEsyYkRJCR8ewvKWTgo0c3snbSDF+PJx8Mpx22rDt7PNyUrDm5NMz1iwauINuhNxJQdJE4DBg+4iYLWkHYFRELC4sOht5/G3ehkub7azrJVdSkPQW4BqgG/g7YDawB/BZ4IjCorPW4B291eIdd8vJe6RwHvDeiLhJ0oqs7VZgv0KisubhHX778A7cyJ8UpkTETdnjyO7XDOH11oy8wx8ZCrgCxdpX3p36PZIOjogbytoOBP5cQEw2XLzTb37eoVuTyZsUPgNcK+k6YDNJF5P6EqYXFpkNzlfoNAfv2G0EyZUUIuIWSa8FPgBcBjwK7Ocrj+rA3/aL53PpZr3yXn20KbA0ImaXtW0iadOIeLGw6NqJd/7Dxzt5sw2Wdz6FXwKvr2h7PXBDlXVtMNUmCXGN+NomTYIzzoCVKyFi8NuqVfAf/+GEYLYB8vYpvIZ0CWq524C/Gd5wRrDSkcB556UJMdqdv82bNaW8SeFZ4BXAE2VtrwBWDXtEI00pGcyeDS+/3OhoiuUdvVnLy5sUrgF+IOlk4CFgKnAucHVRgY0ITzwBr30tLF3a6Eg2nnf4Zm0hb5/CacC9pFNGK4FbgPuBUwuKq/X19MBrXtMaCWH8eDjllIHP2fs8vVlbyHtJ6mrgE5JOAiYByyIiBnlZ++rpgXe+E5Yta3Qkib/lm1lOQ6mSuiWwF9CRPQcgIn5VSGStqqcHpk2DBQvq+7keQGVmwyDvOIXjgAuAHuD5skUB7Db8YbWwc86B++8f/vf1t30zq4O8RwrnAO+OiF8UGUzLK5WdWLduw9/D3/jNrIHyJoUxwI1FBjIiDDUh+Nu/mTWZvEnhK8Dpkr4YESP8YvsN1NMDX/1qvnUnTYI//xm2267YmMzMhihvUvgUsB0wS1KfcpwRsfOwR9VqSp3Lzz8/+LoHHADXXecjAzNrSnmTwgcKjaLVzZkDixYNvt7o0U4IZtbU8o5TuHljP0jSIcA3gNHAtyLiyxXLdwYuB7bK1jklIq7f2M+ti7lzB69nNHq0O4/NrOkNZZzC64A3kwavqdQeEWfkeO1o0iWt7wAWA7dLmhcR95StdjpwdURcKGkf4HpgSt74GirPBDevfGXqUDYza2K5ylxIOgH4PfD3wOdIVVM/A+ye83P2AxZGxEMRsQa4kv6ztgWwRfZ4S+DxnO/dWD09MGqQzTh+PNxyi48SzKzp5a19NAs4JCKOAl7I7t8NvJTz9TuSZmsrWZy1lfsC8AFJi0lHCf+S870ba86cgZePGQOf/awTgpm1hLxJYduI+F32+GVJo7KBbEcMYyzHAN+JiMnAYcB3JfWLT9IJkroldS9thmJzc+cOPDYhIvUlmJm1gLxJYbGkKdnjB4Dpkt4MrMn5+seAncqeT87ayn2UrBR3RPwBGEfqv+gjIi6JiK6I6Ors7Mz58QUarD/h5Zd9lGBmLSNvUpgN7J09Phv4HvAr4Kycr78d2EPSrpLGAkcD8yrW+SvwdgBJe5OSQhMcCgygpwc222zgdSb1y2tmZk0r7yWp3yl7/AtJWwNjI6In5+vXZmW3byBdbnpZRCyQdDbQHRHzSB3X35T0KVKn83FNXZ67NGDtxRdrrzNuHJx4Yv1iMjPbSDWTgiSVdspVzu2vBdZmfQu5yl5kYw6ur2g7o+zxPcDf5Q284UoD1mr1J4wZA1Onuj/BzFrKQEcKz7L+EtG1pG/v5ZS1jS4gruY32IC1sWN9GaqZtZyBksKryh7vWnQgLWewDubVq50QzKzl1EwKEfEo9I5Gvhw4OCIGOIHeZrbZZuDEMHFi/WIxMxsmg159FBHrSEcKea9UGvl6ekCqvdwdzGbWovLu6M8CLpS0i6TRkkaVbkUG17TmzIGVK2svnzDBHcxm1pLyFsT7VnZ/bFlb+3Y0z5078KWoEe5PMLOWlDcpuKO53GCdzE8/XZ84zMyGWd7Ba48UHUhLmTgRli0beLmZWQsaynwKRwJvof98Ch8sIK7m1dMD++wDv/1t9eXuZDazFpZ3PoUzgYuz9f8JWA4cDDxTWGTNqFTa4tZbqy8fN86jmM2speW9eugjwDsi4lPAmuz+CFplZrThUiptUauTeb/9PIrZzFpa3qSwVUTcnT1eI2mTiLiNdDqpfQxW2uLee50QzKyl5e1TWCTpVRGxALgbOFHSCmBFcaE1ocGuOsozV7OZWRPLmxROB0qX1JwC/ADoAGYUEVTT8lVHZjbC5Tp9FBHXR8Rvs8e3RcTuEbFdRPy42PCazPHHp5LY1fiqIzMbAfJeffRTSf8kaVzRATWtnh746U+rz58gwa67+qojM2t5eTuabwZmAk9KulzSwW1X92jOHHj44VTCotKoUTB9ujuZzazl5T199PWI2A/oAh4CzgMel3R+gbE1l4GuPFq3Di69tL7xmJkVYEjf9iPiwYg4CzgauAv4RCFRNSNfeWRmbSB3UpA0VdLpkhYAvwQepJ3GKQx2ZZGvPDKzESBvR/PtwB3AnsBngR0i4hMR8b9FBtc0SvWOavGVR2Y2QuQdpzAH+HlEvFBkME2pVO9o4cLqy13vyMxGkLwdzVe3ZUIA1zsys7bSXpeVbgjXOzKzNuKkMBhfdWRmbaRmUpB0Utnj3esTThPyVUdm1kYGOlI4p+zxHUUH0rRmzEidydX4qiMzG2EGuvroIUlfAxYAm0j6SLWVIuKyQiJrFjNnwjXXwAMPwEsvrW/3VUdmNgINlBTeC8wCjgE2AY6tsk4AIzspdHSkq4sOPxxuvjkVv5s0KR0hzJzpTmYzG1FqJoWIeAA4HkDSTRHx9rpF1Ux6etJlqX/4Q3o+caITgpmNWLkGr0XE2yWNAd4E7AgsBv4QEWuLDK7hSgPXFi2CNWtS27JlMHt2OqXk8QlmNsLkLXOxF3Avaca1k4ErgPsk7V1gbI1XGrhWOU5h9erUPmdOY+IyMytI3nEKFwKXADtFxP+LiMnARcDcwiJrBgMNXFu9Gi68sL7xmJkVLG9SeB1wbkSfGWbOy9pHLg9cM7M2kzcpPE7/MtlvztpHLg9cM7M2kzcpnArMk3SlpK9IuhKYl7WPXB64ZmZtJm+V1HnAvsDdwITs/vUR8bMCY2u8mTPTADWpb7sHrpnZCJW7IF5EPBARX4qIGdn9A0P5IEmHSLpf0kJJp9RY5z2S7pG0QNIPhvL+hejogN/8BiJg/HgYNQo6O2HWLF+OamYjUt5JdjaKpNHABcA7SGMcbpc0LyLuKVtnD+DzwN9FxApJ29YjtppKg9bOPz89Hz0aTj/dg9bMbESrV+ns/YCFEfFQRKwBrgSmV6zzMeCCiFgBEBFP1Sm2/kqD1mbPhmeeSW0rV6bn06al5WZmI1C9ksKOwKNlzxdnbeX2BPaU9HtJt0g6pNobSTpBUrek7qVLlxYTrQetmVmbyjui+fwa7ecNYyxjgD2At5KK8H1T0laVK0XEJRHRFRFdnZ2dw/jxZTxozczaVN4jheNqtFernFrNY8BOZc8nZ23lFgPzIuKliPgL8AApSdSfB62ZWZsasKO5bA6FMVXmU9gNWJbzc24H9pC0KykZHA28r2Kdn5KOEL4taRLpdNJDOd9/eE2cmArfDbTczGwEGuzqo9KRwFj6HhUE8CTwoTwfEhFrs+k9bwBGA5dFxAJJZwPd2TiIG4CDJN0DrANmRkRjvpLPmJE6laudQvKgNTMbwdS3nFGNlaQvRcTpdYhnSLq6uqK7u3v437h09VGt2dY8RsHMWpik+RHRVW1Z3hHNp2dvtK2k3cpvwxlo0yjNtvaGN6TnHrRmZm0i1+A1SQeTpt3cvmJRkE4HjRylQWtz56Z+hdGj4bTTPGjNzNpC3quP5gJfBDaPiFFlt5GXEEqD1kodzevWedCambWNvElha+DiiHihyGAazoPWzKzN5U0KlwIfLjKQpuBBa2bW5vIWxJsGnJxVN32ifEFEHDDsUTWKB62ZWZvLmxS+ld1GNg9aM7M2lyspRMTlRQfSFDxozczaXN6CeJL0MUm/knRX1naApPcUG16dlWZaq5yC0zOtmVmbyNvRfDbwUeASYOesbTHwuSKCapjSoLVZs9ZPwelBa2bWRoZSJfXwiLiSNGAN4C+kongjS0cHnHRSmoLzvPPgqafgrLOcEMysLeRNCqOB0sitUlLoKGsbWR58MN3vvntj4zAzq7O8SeF64FxJm0LqYyCNcP55UYE1RE8PnHkmHHxwen7ssem5RzKbWZvImxQ+Tap79CywJekIYRdGUp9CeYmLUhJYscIlLsysreStkvpcRBxF6mSeBkyNiKMiYmWh0dWTS1yYmeW+JPUgSXtGxFMRcXtEPCFpL0nvKDrAunGJCzOz3KePLgAqjwpWZu0jg0tcmJnlTgrbRsSSirYlwHbDHE/jDFbCwiUuzKwN5E0KD0n6+4q2t5LGKowMM2b0H8lc4hIXZtYm8hbE+wLwY0mXAouAqaRS2iOnnPbMmXDNNWmMwpo169td4sLM2kjeq49+BhwEbA68M7s/OGsfGUolLt75zvRccokLM2s7gx4pSBoNPADsExEfLz6kBuroSGMSfvITePZZmDCh0RGZmdXVoEcKEbEOWAfUOOE+wixeDFtu6YRgZm0pb5/CecDVkv6dVB21VP+IiHiogLgaZ/FimDy50VGYmTVE3qTwn9l95WC1IBXLGzkefdRJwczaVt6O5lE1biMrIYCPFMysreUdpwCApJ0kTSsqmIYpVUft7IQnnoArrnB1VDNrS3lrH+0s6ffAfcD/ZG3vlvStIoOri/LqqMuWpbbnn3d1VDNrS3mPFC4GrgMmAC9lbb+kfx9D63F1VDOzXnmTwn7AlyPiZbIrjyKiNLdCa3N1VDOzXnmTwpNAn7kpJe0D/HXYI6o3V0c1M+uVNyl8FbhW0oeBMZKOAa4CvlJYZPXi6qhmZr3yXpJ6GTAT+CfgUeBDwL9FxPcLjK0+XB3VzKyXImLwtZpUV1dXdHd3b9yblK4+quxsLlVHdTE8MxthJM2PiK5qy3KPU5D0EUm/lLQgu/+oJA1fmA1Sqo46axaMzsbiuTqqmbWpXEcKkmYD00k1kB4BdgFOBn4eEbOKDHAgw3KkUG6rreCDH4Tzzx++9zQzazIDHSnkrX10HLBvRCwue9NrgTuAhiWFYfX886lc9g47NDoSM7OGyXv6aGV2q2x7Lu8HSTpE0v2SFko6ZYD1/lFSSKqaxQqzJJuCevvt6/qxZmbNJG9SOI80Hec7JO0t6SDgh8DXJe1WutV6cTZRzwXAocA+wDHZOIfK9SYAnwRuHeLPsfEefzzd+0jBzNpY3tNH38ju31bR/nagdAJ+oDLa+wELS3MvSLqS1EdxT8V6XySNfaj/hMg+UjAz2+jS2XnLaO9IGt9Qsjhr6yVpX2CniLhuoFgknSCpW1L30qVL84Sfj48UzMyGVjq7KJJGAecCnxls3Yi4JCK6IqKrs7Nz4z+8VDb71FPT8732ctlsM2tb9UoKjwE7lT2fnLWVTABeDfxG0sPANGBe4Z3N5WWzX3ghtS1b5rLZZta26pUUbgf2kLSrpLHA0cC80sKIeDYiJkXElIiYAtwCHBkRwzgIoQqXzTYz66MuSSEi1gInATcA9wJXR8QCSWdLOrIeMVTlstlmZn3kvfoISZuQTuvsEBFXSdocICJW5Xl9RFwPXF/RdkaNdd+aN66N4rLZZmZ95J2O8zXAA8A3gUuz5rcAlxUUV324bLaZWR95Tx9dCJwREa9k/XScNwP7FxJVvbhstplZH3mTwquA72WPS9NxrgI2KyKoupk5M5XHHju2b3upbPbM+o+hMzNrpLxJ4WHg9eUNkvYDFg53QHVVKpt9ZNbXLblstpm1tbwdzf8GXCfpImCspM8DHwc+Vlhk9dLRAfvvDz/6ESxd6n4EM2trectcXAscAnSS+hJ2Af4hIm4sMLb6WbIknULaZptGR2Jm1lC5jhQkTYqIPwIzCo6nMR5/PBXCGwETyZmZbYy8fQp/lXS9pPeXxieMKEuWuDqqmRn5k8LOwLXAicATkq6QdISk3IPfmtrjj7s6qpkZ+fsUlkXE3IjYn1S47k/AOcCSIoOrmyVLnBTMzNiw2kfbAq8AJgHPDGs09VQqmT1pEqxYAZdd5pLZZtb28pa52EfSFyUtBH6aNb8rIvYoLLIilZfMLtU3ev55l8w2s7aX90jh98D2wD+TZkf7VETcVlxYBXPJbDOzqhQRg68kjY2INXWIZ0i6urqiu3sDplzo7EyT6Qy0/KmnNjwwM7MmJml+RFSdxKzm1UOSjo2I72ZPP6Aa1/BHROtVSnXJbDOzqga6pPQYoJQUjq2xTtCK5bMnThz4SMGlLsysTdVMChFxWNnjt9UnnDqZMSN1Klebdc0ls82sjeW9+uiPNdqLnUO5KKWS2ZVzKbhktpm1ubxXH+1e2aDUybDb8IZTJ6WS2bNmrZ9LwSWzzcwGLogn6b+yh2PLHpdMARYUEVRddHTAWWfBjTfChAnp3syszQ1Wu2hRjcdBGrvww2GPqN6efhqmTGl0FGZmTWHApBARZwFIuiUibqhPSHW2fLnnUTAzy+SqchoRN0gaC+xFqnmksmW/Kii24r38cqp75KRgZgbkn2Rnf9Kpok2BLYDngAnAo7RqZzPAc8+lxOBxCWZmQP6rj74OzI6IbYCV2f0XgbmFRVYPpZHLPlIwMwPyJ4U9gW9UtH0Z+NTwhlNnTz+d7p0UzMyA/EnhWdJpI4AlkvYBtgZa+4J+JwUzsz7yJoUfA6WyF5cBvwbmAz8qIqi6KZ0+cp+CmRmQ/+qjfy17/FVJt5KOElr7MlUfKZiZ9ZErKVSKiN8NdyANUUoKW2/d2DjMzJrEQPMp/I40cnlAEXHAsEZUT08/DVtuCWM2KDeamY04A+0Nv1W3KBrFo5nNzPoYaD6Fy+sZSF319KR5mK+6Cl56KVVInTEjlcx2hVQza2N5RzR/pNaylpuOs6cHpk2DRYtSQoA0C9vs2XDNNS6dbWZtLe/J9MrpOLcDppIqpbZWUpgzJyWEylnXVq9O7XPmpJLaZmZtSBGD9iVXf2E6etg7Iho2TVlXV1d0dw9x8rfOzoHnZ+7shKee2rjAzMyamKT5EdFVbVnewWvVfAf46BCCOETS/ZIWSjqlyvJPS7pH0l2SbpK0y0bEVltpwNqGLjczG8HyztE8quLWAZwAPJPz9aOBC4BDgX2AY7JSGeX+CHRFxGtJI6Vn5/wZhmaw0cse3WxmbSzvkcJa4KWy27PAqcCJOV+/H7AwIh6KiDXAlcD08hUi4tcR8Xz29BZgcs73HpoZM2DcuOrLxo2DE/P+SGZmI0/ejuZdK56viogBTsz3syNp7oWSxcAbB1j/o8AvhvD++c2cma4yWrgQXnxxffu4cTB1alpuZtamch0pRMQjFbehJIQhkfQBoAuYU2P5CZK6JXUvXbp06B/Q0ZEuOz3++NIbps7lWbN8OaqZtb28fQo7S7pU0h2SHii/5fycx4Cdyp5PztoqP+dA4DTgyIh4sXI5QERcEhFdEdHV2dmZ8+MrdHTAhz+cHv/sZ+lqo7POckIws7aX9/TRD4H7gDOAFzbgc24H9pC0KykZHA28r3wFSX8LXAwcEhHFXxPa05PunQjMzHrlTQqvBP5fRLy8IR8SEWslnUQqtT0auCwiFkg6G+iOiHmk00UdwA8lAfw1Io7ckM/LpZQUNt+8sI8wM2s1eZPCz4G3kCbX2SARcT1wfUXbGWWPD9zQ994gPlIwM+snb1I4Gfg/SYuAJ8sXRETNukhNbdWqdO+kYGbWK29S+DawDriXDetTaD4+UjAz6ydvUvh7YIeIWFlkMHXlPgUzs37yjmi+CxhZ9R96etKMa2PHNjoSM7OmkfdI4VfAjZK+Tf8+hdYqnV2yalU6dZSudDIzM/Inhf1J4wsOqmgPWm0+hZKeHvcnmJlVyJUUIuJtRQdSdz097k8wM6uQdzrOmn0PGzqgreF8pGBm1k/e00drSaeKqhk9TLHUl5OCmVk/G1o6e3vgFNJI59a0ahVst12jozAzayp5+xQeqWh6RNKHSIXuLh32qOrBfQpmZv1szBzNWwAbWLu6Cfj0kZlZP3k7mr9L3z6F8cABwPeKCKounBTMzPrJ26ewsOL5KuCiiPifYY6nPiLWD14zM7NeefsUzio6kLpaswbWrnWfgplZhbzTcZ4v6U0VbW+SdF4hURXNFVLNzKrK29F8DNBd0Tafiik1W4aTgplZVXmTQlRZd/QQXt9cnBTMzKrKu1P/HfClUrmL7P4LWXvrKc265j4FM7M+8l599EngWmCJpEeAnYElwBFFBVYoHymYmVWV60ghIhYD+wLTgTnAu4DXZ+2to6cHzjwTjjoqPX/Xu9LzUpIwM2tziqhV5675dXV1RXd3Zf93DT09MG0aLFoEq1evbx83DqZOhVtu8ZGDmbUFSfMjoqvastbsKN4Qc+b0TwiQni9alJabmbW59kkKc+f2Twglq1fDhRfWNx4zsybUPklh+fKNW25m1gbaJylMnLhxy83M2kD7JIUZM1KncjXjxsGJJ9Y3HjOzJtQ+SWHmzHSVUWViKF19NHNmY+IyM2si7ZMUOjrSZaezZkFnJ4wale5nzfLlqGZmmfYZp2BmZoDHKZiZWU5OCmZm1stJwczMejkpmJlZr5buaJa0FHhkA18+CVg2jOEMp2aNzXENjeMaumaNbaTFtUtEdFZb0NJJYWNI6q7V+95ozRqb4xoaxzV0zRpbO8Xl00dmZtbLScHMzHq1c1K4pNEBDKBZY3NcQ+O4hq5ZY2ubuNq2T8HMzPpr5yMFMzOr4KRgZma92jIpSDpE0v2SFko6pYFx7CTp15LukbRA0iez9i9IekzSndntsAbE9rCkP2ef3521bSPpl5IezO63rnNMe5VtkzslPSfpXxu1vSRdJukpSXeXtVXdRkrOz/7m7pK0b53jmiPpvuyzfyJpq6x9iqQXyrbdRXWOq+bvTtLns+11v6SDi4prgNiuKovrYUl3Zu112WYD7B+K/RuLiLa6AaOBRcBuwFjgT8A+DYple2Df7PEE4AFgH+ALwGcbvJ0eBiZVtM0GTskenwJ8pcG/xyeAXRq1vYADgH2BuwfbRsBhwC8AAdOAW+sc10HAmOzxV8rimlK+XgO2V9XfXfZ/8CdgU2DX7H92dD1jq1j+NeCMem6zAfYPhf6NteORwn7Awoh4KCLWAFcC0xsRSEQsiYg7sscrgXuBHRsRS07Tgcuzx5cD72pcKLwdWBQRGzqifaNFxG+Bpyuaa22j6cB/RXILsJWk7esVV0TcGBFrs6e3AJOL+OyhxjWA6cCVEfFiRPwFWEj63617bJIEvAe4oqjPrxFTrf1DoX9j7ZgUdgQeLXu+mCbYEUuaAvwtcGvWdFJ2CHhZvU/TZAK4UdJ8SSdkba+IiCXZ4yeAVzQgrpKj6ftP2ujtVVJrGzXT391HSN8oS3aV9EdJN0t6cwPiqfa7a6bt9WbgyYh4sKytrtusYv9Q6N9YOyaFpiOpA7gG+NeIeA64EJgKvA5YQjp0rbf9I2Jf4FDgE5IOKF8Y6Xi1IdczSxoLHAn8MGtqhu3VTyO3US2STgPWAt/PmpYAO0fE3wKfBn4gaYs6htSUv7sKx9D3C0hdt1mV/UOvIv7G2jEpPAbsVPZ8ctbWEJI2If3Cvx8RPwaIiCcjYl1EvAx8kwIPm2uJiMey+6eAn2QxPFk6HM3un6p3XJlDgTsi4sksxoZvrzK1tlHD/+4kHQccDrw/25mQnZ5Znj2eTzp3v2e9Yhrgd9fw7QUgaQzwD8BVpbZ6brNq+wcK/htrx6RwO7CHpF2zb5xHA/MaEUh2rvJS4N6IOLesvfw84FHA3ZWvLTiuzSVNKD0mdVLeTdpOH8pW+xDws3rGVabPN7dGb68KtbbRPOCD2RUi04Bny04BFE7SIcAs4MiIeL6svVPS6OzxbsAewEN1jKvW724ecLSkTSXtmsV1W73iKnMgcF9ELC411Gub1do/UPTfWNE96M14I/XSP0DK8Kc1MI79SYd+dwF3ZrfDgO8Cf87a5wHb1zmu3UhXfvwJWFDaRsBE4CbgQeB/gG0asM02B5YDW5a1NWR7kRLTEuAl0vnbj9baRqQrQi7I/ub+DHTVOa6FpPPNpb+zi7J1/zH7Hd8J3AEcUee4av7ugNOy7XU/cGi9f5dZ+3eAj1esW5dtNsD+odC/MZe5MDOzXu14+sjMzGpwUjAzs15OCmZm1stJwczMejkpmJlZLycFGxJJ75d0Yx0+Z4qkyAYPNZSk30g6vtFxmNWDk4L1I2l/Sf8n6VlJT0v6vaQ3AETE9yPioEbHaK1NqWT29xodh/XX8G9h1lyyGi7XAicCV5PKi78ZeLGRcQ2FpDGxviKomQ2BjxSs0p4AEXFFpJo0L0Qqu3wXpPo5kv63tHJ2iufj2YQfz0i6IBuej6TRkr4maZmkv0g6qfyUkNLEJQeWvVfNb4+SPizpXkkrJT0k6Z/Llr1V0mJJn5P0BPDtitdumsX26rK2TqWJUraVtLWkayUtlbQie1y1tHRljJWnuSRtKelSSUuUJo/5UqkkQpX3Gi3pVEmLsp9rvqSdsmVvknR7drR2u6Q3lb3uN9n7/p+kHkk/lzRR0veVJh66XamqZvnv6ORsuy1TmnBnVLZslKTTJT2iNMnMf0nasuJn+5Ckv2avPa3sfUdJOiWLf7mkqyVtM9hrlUpunAq8N4v/T1n7cVmMK7O/l/dX225WLCcFq/QAsE7S5ZIOVb4y1IcDbwBeS6o7X5ol62Ok4nWvI01g8q6NiOup7HO2AD4MfF19Z5baDtiGNOnOCeUvjIgXgR+TaiaVvAe4OVLBv1GkRLILsDPwAvCfGxjnd0hVSHcnlTo+CKjVH/HpLKbDsp/rI8Dz2Y71OuB8UkmDc4HrJE0se+3RwLGk0shTgT9kP8M2pLr7Z1Z81lFAF+n3MD37LIDjstvbSOVNOuj/s+8P7EWaw+IMSXtn7f9C+p2+BdgBWEEqszDgayPiv4F/B66KiI6I+BulGlvnk8pZTADeRCrrYPVWZD0R31rzBuxN2rktJu3g5pFquEPagfxv2bpBKrNden4162eF+hXwz2XLDszWL80A9jBwYNnyLwDfyx5PKV+3Sow/BT6ZPX4rsAYYN8DPdCBpUp7S898DH6yx7uuAFWXPfwMcXxljZZykuvYvApuVLT8G+HWNz7kfmF6l/Vjgtoq2PwDHlcVzWtmyrwG/KHt+BHBnxe/okLLnM4Cbssc3ATPKlu1Fqv8zpuxnm1y2/Dbg6OzxvcDby5ZtP4TXVm7HzYFnSHWFNqvcJr7V7+YjBesnIu6NiOMiYjLwatK3wPMGeMkTZY+fJ33bJHtd+aQf5Y+HJDtquUWp4/sZ0rfrSWWrLI2I1QO8xa+B8ZLemJ1aeR2pJDiSxku6ODuF8hzwW9KsVVVP+wxgF2ATYEl2uuoZ4GJg2xrr70QqXlZpB6ByRrlH6DthypNlj1+o8ryDvsq3/SPZZ1T7rEdYn+BKav1+dwF+Uvaz3gusy/naPiJiFfBe4OOk7XedpFdWW9eK5aRgA4qI+0hHDa8eZNVqltB32sedKpavAsaXPd+u2ptI2pRUU/6rpCOWrYDrSVUhe0MdKJCIWEc6ijkmu10baYpDgM+QviG/MSK2IM3XS8X754n5UdKRwqSI2Cq7bRERr6oR1qOkUz+VHiftcMvtzMbNJ1C+7XfOPqPaZ+1MOjosTzK1PEo63bNV2W1cZHNxDKLf7ysiboiId5COOO4jza9gdeakYH1IeqWkz5Q6WrOOz2NI8/oO1dXAJyXtKGkr4HMVy+8k1czfRFIX8O4a7zOWNIH7UmCtpENJ5+qH6gekb6Pvzx6XTCB9u34mO59feT6+MuYDJO2cdch+vrQgUu36G4GvSdoi64idKuktNd7rW8AXJe2h5LVZv8H1wJ6S3idpjKT3kiZsv3YDfuaSmUod6jsBn2T9pDFXAJ9Sml+kg/Xn+vNcvXURcI6kXaC38z7vfOdPAlPKOrxfIWl61rfwItADvJz7p7Nh46RglVYCbwRulbSKlAzuJn2bHqpvknaSdwF/JO3s1pJOMQD8G+mb8grgLPruqHtl3+hPJiWZFcD72ICJkSLiVtI3/R3oO0fxecBmwDLSz/vfA7zHL0k71LuA+fTfUX+QlMTuyWL9EembbzXnkn6mG4HnSBOqbBZpVq/DSdt8OWlynMMjYlm+n7Sqn2Xx3knqxL40a7+MNKfBb4G/AKtJHch5fIP0e7hR0krStntjzteWplJdLukO0r7o06Qjl6dJndcn5nwvG0aeT8HqJvuGf1FEVJ4asQJJCmCPiFjY6Fis+flIwQojaTNJh2WnQHYknZb5SaPjMrPanBSsSCKdFlpBOn10L3BGQyMyswH59JGZmfXykYKZmfVyUjAzs15OCmZm1stJwczMejkpmJlZr/8P5Uj9cBlwP7UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compress with SVD (singular value decomposition)\n",
    "# SVD works very well for the sparse matrix\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "svd = TruncatedSVD(n_components=200)\n",
    "latent_matrix = svd.fit_transform(tfidf_df)\n",
    "# plot var explained to see what latent dimensions to use\n",
    "explained = svd.explained_variance_ratio_.cumsum()\n",
    "plt.plot(explained,'.-',ms=16,color='red')\n",
    "plt.xlabel('Singular value components', fontsize=12)\n",
    "plt.ylabel('cumulative percent of variance', fontsize=12)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#number of latent dimensions to keep\n",
    "n=200\n",
    "latent_matrix_1_df=pd.DataFrame(latent_matrix[:,0:n], index=Final.title.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9625, 200)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# our content latent matrix:\n",
    "latent_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# Creating a collaborative latent matrix from user ratings:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating\n",
       "0       1        1     4.0\n",
       "1       1        3     4.0\n",
       "2       1        6     4.0\n",
       "3       1       47     5.0\n",
       "4       1       50     5.0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_f.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_f1 = pd.merge(movies[['movieId']], ratings_f, on='movieId', how='right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>userId</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92724</th>\n",
       "      <td>160341</td>\n",
       "      <td>610</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92725</th>\n",
       "      <td>160527</td>\n",
       "      <td>610</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92726</th>\n",
       "      <td>160836</td>\n",
       "      <td>610</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92727</th>\n",
       "      <td>163937</td>\n",
       "      <td>610</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92728</th>\n",
       "      <td>163981</td>\n",
       "      <td>610</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>92729 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       movieId  userId  rating\n",
       "0            1       1     4.0\n",
       "1            1       7     4.5\n",
       "2            1      15     2.5\n",
       "3            1      17     4.5\n",
       "4            1      18     3.5\n",
       "...        ...     ...     ...\n",
       "92724   160341     610     2.5\n",
       "92725   160527     610     4.5\n",
       "92726   160836     610     3.0\n",
       "92727   163937     610     3.5\n",
       "92728   163981     610     3.5\n",
       "\n",
       "[92729 rows x 3 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_f2 = ratings_f1.pivot(index = 'movieId', columns='userId', values = 'rating').fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>userId</th>\n",
       "      <th>1</th>\n",
       "      <th>4</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>...</th>\n",
       "      <th>600</th>\n",
       "      <th>601</th>\n",
       "      <th>602</th>\n",
       "      <th>603</th>\n",
       "      <th>604</th>\n",
       "      <th>605</th>\n",
       "      <th>606</th>\n",
       "      <th>607</th>\n",
       "      <th>608</th>\n",
       "      <th>610</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movieId</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 364 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "userId   1    4    6    7    10   11   15   16   17   18   ...  600  601  602  \\\n",
       "movieId                                                    ...                  \n",
       "1        4.0  0.0  0.0  4.5  0.0  0.0  2.5  0.0  4.5  3.5  ...  2.5  4.0  0.0   \n",
       "2        0.0  0.0  4.0  0.0  0.0  0.0  0.0  0.0  0.0  3.0  ...  4.0  0.0  4.0   \n",
       "3        4.0  0.0  5.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...  0.0  0.0  0.0   \n",
       "\n",
       "userId   603  604  605  606  607  608  610  \n",
       "movieId                                     \n",
       "1        4.0  3.0  4.0  2.5  4.0  2.5  5.0  \n",
       "2        0.0  5.0  3.5  0.0  0.0  2.0  0.0  \n",
       "3        0.0  0.0  0.0  0.0  0.0  2.0  0.0  \n",
       "\n",
       "[3 rows x 364 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_f2.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9625, 364)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_f2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9625"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ratings_f.movieId.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import TruncatedSVD\n",
    "scd = TruncatedSVD(n_components=200)\n",
    "latent_matrix_2 = svd.fit_transform(ratings_f2)\n",
    "latent_matrix_2_df = pd.DataFrame(latent_matrix_2, index=Final.title.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9625, 200)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "latent_matrix_2_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEJCAYAAAB7UTvrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAApdElEQVR4nO3deZhcdZn28e+dBkRIEEhH1oQEDEhUBp0WcVTcgAEXMur7aoKjgksURJhBwCDKNsOwBFHnNbIzIoqAuBAFBBeQcUHTKKIsgRDBhDUJIDQQIPC8f5xTnUqlqvtX3XVq6bo/11VXqs45dc7Tpzv11G9XRGBmZgYwrtUBmJlZ+3BSMDOzQU4KZmY2yEnBzMwGOSmYmdmg9VodwGj09vbG1KlTWx2GmVlHuemmm1ZExKRq+zo6KUydOpX+/v5Wh2Fm1lEk3Vtrn6uPzMxskJOCmZkNclIwM7NBTgpmZjbIScHMrFMMDMBxx8GkSTBuXPbvccdl2xvEScHMrN0MDMDRR8NGG4G05jFhApx4IqxYARHZv6edBrvv3rDE4KRgZtZK1RLAhAlwyinw9NPDv3/VKrj7bpg3ryHhOCmYmTXDUN/+UxNALatWwZlnNiTMpiUFSftIWiRpsaS5VfZvJ+nnkm6RdL2kbZsVm5lZQ5Xq/nt7G/vhP5SVKxtymqYkBUk9wHxgX2AGMFvSjIrDTge+GRG7ACcCJzcjNjOzUassBZTq/hv0QZ1k4sSGnKZZ01zsBiyOiCUAki4BZgK3lR0zAzg8f34d8MMmxWZmlm5gIKu/nz+/uR/6Q9lwQzjooIacqlnVR9sAS8teL8u3lfsT8N78+XuACZLWSX2S5kjql9S/fPnyQoI1MwNqVwM1uxQwlPXWgx12gCOPbMjp2qmh+QjgzZL+CLwZuA94vvKgiDgnIvoiom/SpKqT/JmZjUw7VAPVo6cHjjgCbrwRxo9vyCmbVX10HzC57PW2+bZBEXE/eUlB0njgfRHxWJPiM7Nu047VQKk22ggOPRSOOaZhyaAkOSlIWh/YHdg6Ii6VtDFARDyZ8PaFwHRJ08iSwSxg/4rz9wKPRMQLwNHABamxmZkNa2AATjoJvvrV4noANVpvLxx8cFY11OAP/1qSkoKkVwELgGfIvuVfSlbF8xHgA8O9PyJWSzoEuAboAS6IiFslnQj0R8QC4C3AyZICuAH4dP0/jplZmVIi+MpXsr787azAb//1UEQMf5D0K+DsiLhI0qMRsVleUrgzIiobjJumr68vvMiOmQGdUx3UBh/+km6KiL5q+1Krj14BfCt/HpBVG0l6cQPiMzOrT6ckAGhJFdBopCaFe4B/BAa/lkvaDVhcQExmZusqJYKvfQ0eeaTV0VTXBqWA0UpNCl8ErpR0FrCBpKOBTwGfKCwyM+tu7dwwLGX/TpzYUaWAFElJISJ+LGkfsiTwS2A74L0RcVORwZlZl2nnhuEOqwYaqeQuqRHxR+DgAmMxs27UjolgDFQDjVTSiGZJ35f0poptb5J0eTFhmdmYNNTsoa1MCL29cOyx8MQT2eI1Tz4JJ5/cdQkB0ksKbwb+b8W23+JJ68xsOO3WQNzFpYAUqUlhFbAx8HjZtvHAcw2PyMw6nxNBx0pNCtcAZ0v6ZEQ8LmkT4GvAT4oLzcw6QjuNGRjDvYKaJTUpfJZs8Nojkh4BNgeuBj5UVGBm1sbaqSQwcSJ8+tNOAA2S2iX1UeCdkrYkm+10aUQ8WGhkZtZe2ikRdEn30Faod+rsF4CVwEaStgcoraZmZmNQOyQCtwc0VeosqfsA5wNbVewKsllPzWyscCLoaqklhfnAfwAXRkSbjTc3s1FzIrBcalLYjGzq7OHn2TazztDqROAG4raUmhTOBw7Eq6GZdbZWJwI3ELe91KSwO3CopLnAWr2OImKPhkdlZo3TyrmFXCXUcVKTwnn5w8w6QasSgauEOl7qOIULR3uhvAfTV8l6K50XEadU7J8CXAhsmh8zNyKuGu11zbqGE4E1QPI4BUlbALsBvYBK2yNi2HYGST1kPZj2ApYBCyUtiIjbyg77AnBZRJwpaQZwFTA1NT6zruREYA2WOk7hX8imubiLbL3mW4FXAr8irfF5N2BxaaCbpEuAmUB5Ughgk/z5S4D7U2Iz60qlZHDaafDCC825phNBV0gtKfwncGBEfFfSoxHxakkHkiWIFNsAS8teLwNeV3HM8cC1kj5DNiPrntVOJGkOMAdgypQpiZc3GwNa0XPIiaDrJC2yA0yJiO9WbLsQ+HADY5kNfCMitgXeAVwkaZ34IuKciOiLiL5JkyY18PJmbai0KM3EidliNCeeWHxCmDhxzYIzK1bACSc4IXSR1JLCw5K2iIiHgHskvR5YQfoUF/eRTaRXsm2+rdzHgH0AIuK3kjYka794OPEaZmNDK0oEHj9gudSkcC7wRuB7wJeB68gmx/tS4vsXAtMlTSNLBrOA/SuO+RvwduAbknYGNgSWJ57fbGx48EHYZRdYXvCfvscPWA2pXVJPLXv+TUnXAxtHxO2J718t6RCyxXp6gAsi4lZJJwL9EbGAbM2GcyX9O1mj8wGeVsPGvGYuUONEYAnqnTobgIj42wjecxVZN9PybceWPb8NeMNI4jHrKM2sHnIisDrVTAqSbo+InfPnS8m+va8jItwFyCxVM6qHnAhsFIYqKXyi7Pm/Fh2I2ZjVrAFmvb3w5z/DllsWdw0b82omhYj4FQyORv4oMCcinmlWYGYdrZkjjT2WwBpo2DaFiHhe0t5kvY3MbCjNGmnsRGAFSW1o/jJwgqTjIuK5IgMy6zjNajh2IrAmSE0KnwG2BA6XtJyyRmc3NFtXK7rhuKcnSwJuNLYmSU0Kbmg2K2lGe4FLBdYiqYPXfll0IGZtr+j2AicCawP1rKewK/Am1l1P4dha7zHreEW3F7h6yNpM6noKc8gam68F9gWuBvYGriguNLMWKpUK5s2D559v/Pk9wMzaVGpJ4Shgn4j433w9hfdI2pdsYjuzsaHoUoETgXWA1KTw0oj43/z5C5LGRcTVkr5dVGBmTVVkLyKPNLYOkrrIzjJJU/PndwIzJb0JeLaQqMyaZWAAjj4att668QmhtFjNX//qhGAdI7WkcBqwM3APcCJwObABcGgxYZkVqMhqIjccW4dL7ZL6jbLnV0vaDNggIgaKCsysEEVVE7m9wMaIpOojSV+R9NrS64h41gnBOkpR1US9vfDAA/Dkk3DyyU4I1vFS2xQEXCHpLkknSNqpyKDMGqJy0ftTToFGLebX2+v2AhuTUquPDsuXyXw7MBu4UdIS4NsRcUaRAZqNSBHVRO5FZF0gtaRARLwQET+NiI8CrwRWAvNS3y9pH0mLJC2WNLfK/i9Lujl/3CnpsdRzmw0qoprIvYisi9QzzcXGwHvISgpvAX4JfCTxvT3AfGAvYBmwUNKCfF1mACLi38uO/wzw6tTYrMsV0ZtIgs99zg3H1nVSp7n4Ltn0Fn8AvgN8JCJW1HGd3YDFEbEkP98lwEzgthrHzwaOq+P81q1cTWTWUKnVRwuBGRGxR0ScWWdCANgGWFr2elm+bR2StgOmAb+o8xrWTVxNZFaI1Ibm04oOpMws4PKIqDoLWT453xyAKVO8vk/XafREda4mMltLcpvCKN0HTC57vW2+rZpZwKdrnSgizgHOAejr62tQ/0Jra0WNQHY1kdk6mpUUFgLTJU0jSwazgP0rD5L0cmAz4LdNisvaXRFtBl7Mxqymmm0KkvYre77+aC4SEauBQ4BrgNuByyLiVkknll+HLFlcEtGoEUbWsRrdZiDB3LnwxBOwYgWccIITglkVqvX5K+nxiNik8nk76evri/7+/laHYY3W6NKBq4nM1iLppojoq7ZvqOqjByUdQtZtdD1Jb6VsGc6SiHAvIWuMUiPyqac2ZjoKVxOZ1W2opHAA2TTZh5FNk31BlWMC2L7xYVnXaHQjsqeuNhuVmkkhIn4D7AkgaXFEvKxpUVl3aGQ1kbuWmjVE6jiFlwFImkI26GxZRCwd+l1mNTS6mshtBmYNkzrNxZbApcDrySbCmyjpRmBWRNxfYHw21jSydOA2A7OGS53m4izgT8BmEbEV2ViCP+bbzYbXqC6m7lpqVqjUwWtvBLaKiOcAIuJJSUdRe1Sy2RqNKh24msiscKklhUeBGRXbdgIea2g0NrY0unTgierMCpdaUjgN+Jmk84F7ge2AA4EvFhWYdTiXDsw6UlJJISLOBT4A9ALvzv/dP5+czmyNRpUOPI21WUskT4iXj1z26GWrrhFTWnusgVnLNWuWVBvLGlFV5Gois7aQ2tBstq5GVBW5EdmsrbikYCPj0oHZmJRUUpB0RI3thzc2HGt7Lh2YjWmpJYVjgdOrbP8CcEbjwrG25tKB2Zg3ZFKQ9Lb8aU+V9RS2B54oKjBrI42YwM5TWpt1hOFKCufn/27I2uspBPAg8JkigrI2MtrSgbuZmnWUIZNCREwDkPTNiPhwc0KyttCI0oGrisw6TuqI5sGEIGlc+SP1QpL2kbRI0mJJc2sc835Jt0m6VdLFqee2BnvwQdh+ezjllJElBDckm3Ws1PUUXgPMB3Yhq0qCrH0hgJ6E9/fk798LWAYslLQgIm4rO2Y6cDTwhoh4VNJL6/lBrAFcOjDreqm9jy4EfgR8FHhqBNfZDVgcEUsAJF0CzARuKzvmE8D8iHgUICIeHsF1bKTcdmBmpCeF7YBjIka8duI2QPnyncuA11UcsyOApF+TlT6Oj4ifVJ5I0hxgDsCUKVNGGI4NcunAzMqktgn8ANi7yEDIEtR04C3AbOBcSZtWHhQR50REX0T0TZo0qeCQxji3HZhZhdSSwobADyT9iqwr6qDEXkn3AZPLXm/Luqu2LQN+l6/u9ldJd5IliYWJMVoqlw7MrIbUpHAba9f/12shMF3SNLJkMAvYv+KYH5KVEP5HUi9ZddKSUVzTqnHbgZkNISkpRMQJo7lIRKyWdAhwDVl7wQURcaukE4H+iFiQ79tb0m3A88CREbFyNNe1CgMD8KpXZQvej4RLB2ZjnlLbjiXtRfYN/6UR8W5JfcAm+eI7LdHX1xf9/f2tunxnGRiAd74Tbrih/ve6dGA2pki6KSL6qu1LnSX1M8CZwF3AHvnmp4H/bEiEVpzSrKabbjqyhNDbC/ffDyef7IRg1gVSex/9G7BnRJwCvJBvuwPYqYigrEHKexfVu0SmexaZdaXUhuYJrBlnUKpvWh94tuER2eiNtneR2w7MulZqSeEGoHK+okOB6xobjo3aaMYeuHRg1vVSSwqfAX4k6RPABEmLyNZSeFdhkVn9RtO7yKUDMyO9S+oDkl4LvJZsyoulwO8j4oWh32lNU+pdNJKEsMcecOWVbkg2s+RZUncFVkbE74Hf59smS9o8Iv5UYHyWYqQD0tZbD3bayQnBzAaltil8i6xhudwGwEWNDcfqUupuuvXW9ScECY44Am680QnBzAaltilMKU17XRIRd0ua2viQLMnAAOy+O9xxR/0Nym4/MLMaUksKy/KFdgblr+9vfEiW5KSTYNGi+sYfuHeRmQ0jtaTwZeAKSacBdwM7AEcAJxUVmNVQGoNwyin1vc+lAzNLkNr76FxJjwEfI5sCeynw2Yi4vMDYrNJIG5Tdu8jMEg2bFPL1lX8O/HNEfLf4kKyqkY5BmDTJCcHMkg3bphARzwPTABUfjlU10jEIvb1wyy1OCGaWLLWh+QTgLEnbSeqRNK70KDI4Y820FfXOcOoGZTMbgdSG5vPyfz9Utk1kk+P1NDQiW2MkVUY9PXDkkdlU12ZmdUpNCtMKjcLWNZIqo9II5WOOKS4uMxvTUnsf3QuQVxdtEREPFBpVtxtJL6PSCGWvjmZmo5C68tqmki4GVgGL8237SUpeeU3SPpIWSVosqXIabiQdIGm5pJvzx8dTzz2mlKqM6kkIXh3NzBoktaH4LODvZDOklhbW+S3wgZQ3591a5wP7AjOA2ZJmVDn00ojYNX+cV2X/2DaSKqM99nCDspk1TGqbwtuBrSPiOUkBEBHLJb008f27AYtL8ydJugSYCdxWb8Bj1kiqjDwGwcwaLLWk8Hegt3yDpClAatvCNqxZzhNgWb6t0vsk3SLpckmTq51I0hxJ/ZL6l9c7srddjbTKyGMQzKzBUpPCecD3JL0VGCfp9cCFZNVKjfIjYGpE7AL8ND//OiLinIjoi4i+SZMmNfDyLeIqIzNrI6lJ4VTgUrJ2gfWBC4ArgK8mvv8+sjmTSrbNtw2KiJUR8Uz+8jzgHxPP3ZlKayFsuml9A9NcZWRmBUrtkhpkCSA1CVRaCEyXNI0sGcwC9i8/QNJWZV1d9wNuH+G12l/5Wgj1TH3tKiMzK1hqQzOS3gbMBrYmW0fhkoj4ecp7I2K1pEOAa8hGQF8QEbdKOhHoj4gFwKGS9gNWA48AB9T1k3SSkayF4JlOzawJFAmrdkn6LPA54H+Ae4EpwIHAaRHxpUIjHEJfX1/09/e36vIjMzCQVRnVkxAmTYIlS5wQzKwhJN0UEX3V9qWWFA4H3hYRfyk76UVkDcItSwodp9So7CojM2tT9cxyurji9RKyCfEsRakdoZ5GZfcyMrMmS00KxwPnS5ou6cWSdgTOAY7zNNqJSu0IKdZbD17xCrchmFnTpX6Qn03WyLwIGADuAD5IlhieI2scfq6IADteqevpKafA6tVp7zniCLjxRicEM2s6T51dpPKupym8FoKZtVhdU2dbnertevryl3stBDNrKbcDFGVgAObNS68y2mMPVxmZWcs5KRSh3q6nPT1uVDaztpA8otkSldoRbr017fhSO4ITgpm1gbpKCpImS9q9qGDGhHq7nrodwczaSOpynFMk/ZqsK+rP8m3/R1L3rY42lHrbEdz11MzaTD3jFK4EJrBmPMJPgb2KCKpjzZuX1o7Q0wNz53pNZTNrO6ltCrsB74yIF8qW4/y7pJcUF1qHGRiA009PO9ZVRmbWplJLCg8BLyvfIGkG8LeGR9SJSo3LTz01/LHuempmbSw1KZwO/FjSgcB6kmaTrcR2amGRdZLUxmV3PTWzNpc6ovkCSSuBTwJLgQ8DX4yIHxYYW2coNS4P15bgrqdm1gGSkoKknoi4gmxdZiupZ5Ca2xHMrAOkVh89KOnrkt5QaDSdpJ71ETbayO0IZtYRUpPC3mRTZn9H0l8lnSzpVQXG1f7mzYO770479ogjnBDMrCMkJYWI+GNEHBURU4ADgM2AX0i6JfVCkvaRtEjSYklzhzjufZJCUtX1Q9vG/PmwatXwx5XaEszMOsBIJsS7A7idrDvq1JQ3SOoB5gP7AjOA2XmX1srjJgCHAb8bQVzNMzAAK1cOf5wbl82sw6ROc7GppI9J+jnZ2sxvIeuO+tLE6+wGLI6IJRHxLHAJMLPKcf+RnzfhK3iLlBqXU7hx2cw6TGpJ4X6y5TgvBraJiPdExGURkfrhvQ1ZV9aSZfm2QZJeA0yOiCuHOpGkOZL6JfUvX7488fINUk/jsgepmVkHSp3mYoeIeKCoICSNA84ga68YUkScQ7Y2NH19fVFUTFXV07jsQWpm1oFqJgVJe0RE6SvxzpJ2rnZcRPwi4Tr3AZPLXm+bbyuZALwSuF4SwJbAAkn7RUR/wvmbI7VxubfXCcHMOtJQJYWvk31QA5xf45gAtk+4zkJguqRpZMlgFrD/4Eki/g70ll5Luh44oq0SQmrj8oYbwsEHFx+PmVkBaiaFiHhl2fNpo7lIRKyWdAhwDdADXBARt0o6EeiPiAWjOX9TzJuXdtwOO7gLqpl1LEUMXy0v6YqIWKe3kKTvR8R7C4ksQV9fX/T3N6EwMTAAW2wx/Cyoe+zhtgQza3uSboqIqmPBUnsfvbXG9reMKKJOUs+02E4IZtbhhux9lFfvAGxQ9rxke+DeQqJqJ6k9jty4bGZjwHBdUks9hsaxdu+hIBt3cHwBMbWXlB5Hblw2szFiyKQQEQcCSPpNRJzbnJDaSGqPIzcum9kYkbrIzrkwODdRL6CyfUuKCa0NpPQ48rTYZjaGpC6yszPZFBf/QFZ1pPxfyLqYjj0DA3D66cMf52mxzWwMSe19dCZwHbA58DjZ1NlnAx8pKK7WqqfHkauNzGwMSZ376B+AvSLiOUmKiL9LOhL4C/Ct4sJrEfc4MrMulVpSWAWsnz9fIWlK/t6JhUTVSqVqI/c4MrMulJoU/hd4f/78cuBq4JdAymR4naOeaiP3ODKzMSi199H7y15+HrgVGA98s4igWia12sg9jsxsjEptUxgUES8AFxUQS+t9/etp1UbucWRmY9RQ6ylcxJpupzVFxIcbGlEreaCamXW5oUoKi5sWRbvYfPOhE4OrjcxsjBtqPYUTmhlIyw0MgFR7v6uNzKwLpI5oflutfYnLcba/efPgiSdq758wwdVGZjbmpTY0Vy7HOQnYAFhG2nKc7e/rX4dnnqm9P8KlBDMb81K7pK61HKekHuALwBBfrTvIwACsWDH0MY880pxYzMxaKHXw2loi4nngJOCo1PdI2kfSIkmLJc2tsv9Tkv4s6WZJv5I0YySx1a00YG04E8fe4G0zs0ojSgq5vYAXUg7MSxbzgX2BGcDsKh/6F0fEqyJiV+A04IxRxJYuZcDahhvCQQc1JRwzs1ZKbWheytpjFjYCNgRSJ//ZDVhcWntB0iXATOC20gER8XjZ8RuTMEaiIVIGrHlsgpl1idSG5n+teP0kcGfFB/lQtiFbvrNkGfC6yoMkfRo4nKwRu2qPJ0lzgDkAU6ZMSbz8EFIGrHlsgpl1idSG5l8WHUh+nfnAfEn7kzVkr7NeQ0ScA5wD0NfXN/rSxMSJQzcyT5rkhGBmXSO1+uglwKHAq8kmwhsUEXsnnOI+YHLZ623zbbVcQrawT/E+/vFsquzVq9fd57YEM+syqdVH3yVbdvMHwNMjuM5CYLqkaWTJYBawf/kBkqZHxF35y3cCd1G0gQH44Q/h+efX3SfBtGluSzCzrpKaFHYHeiPi2ZFcJCJWSzoEuIYsuVwQEbdKOhHoj4gFwCGS9gSeAx6lGUt9zpsH99yTDUyrNG4czJzpqiMz6yqKah+IlQdJVwFzI+KW4kNK19fXF/39/SM/waRJw7cnPPzwyM9vZtaGJN0UEX3V9qWWFA4ArpL0O+Ch8h0RceLowmuh4XoepfRMMjMbQ1KTwklkDcX3AJuUbW/OWIKiDNfzyKOYzazLpCaFWcCOEfFAkcE01cAAzJgBN9xQfb97HplZF0pNCkvIGoDHhtJ8R4trrCO04YYexWxmXSk1KVwELJD0/1i3TaHz1lMozXdUa6rs3XaDK690zyMz6zqpSeHT+b//VbE96MT1FIab7+j2250QzKwrjWg9hY7nXkdmZlWNZurszjVcryL3OjKzLpWUFCQtlfS3ao+iAyzEwQdnjcnVuNeRmXWxkU6dvRVwGNnEdZ3noIPgzDPXbVdwryMz63Ijnjpb0vXAT4CvNjimYg0MwJ57wuNVloIYPx5+9jM3MptZ1xpNm8IzQOc1QA/VHXVgICtBmJl1qdQJ8SrnN9oIeAdwS0TMKiKwFCOaEM+T4JlZl2vEhHiTK14/CZxBNqits7g7qplZTaltCgcWHUjTeBI8M7OahmxTkPQGSafW2HeKpN2LCatA7o5qZlbTcA3NnwdqTCPK9cAxDY2mGY48Mut2ul5FIcndUc3Mhk0Ku5J1O63mZ8A/NjSaZhg/Hm68EV772uz1uHFZ4/JRR2Xb3R3VzLrYcG0KmwAbAE9X2bc+MCH1QpL2IRvT0AOcFxGnVOw/HPg4sBpYDnw0Iu5NPX9dxo+HbbaBHXeERYsKuYSZWScarqRwB7B3jX175/uHJakHmA/sC8wAZkuaUXHYH4G+iNgFuBw4LeXcI3bvvTB1aqGXMDPrNMMlhS8DZ0t6r6RxAJLGSXovcBZZt9QUuwGLI2JJRDxLNj3GzPIDIuK6iHgqf3kjsG3qDzEi99zjpGBmVmHIpBARF5N9Y78QWCXpfmBV/npeRHwn8TrbAEvLXi/Lt9XyMeDqxHOnGxiA446D3l5Yvhy+9a3s9cBAwy9lZtaJhh2nEBFnSDoPeD0wEVgJ/DYiqkweNHqS/hXoA95cY/8cYA7AlClT0k9cWoLz7rvXTIT31FNw2mnwve+5kdnMjMS5jyLi8Yi4JiIuzv+tNyHcx9qjorfNt61F0p5k3Vz3i4iqa2VGxDkR0RcRfZMmTUqPoDTnUeXMqKtWZdvnzUs/l5nZGNWsRXYWAtMlTZO0ATALWFB+gKRXA2eTJYTGTz401BKcq1Z5IjwzM5qUFCJiNXAIcA1wO3BZRNwq6URJ++WHzQPGA9+VdLOkBTVONzKe88jMbFipE+KNWkRcBVxVse3Ysud7FhqA5zwyMxtW96zR7DmPzMyG1T1JoTTnUWVi8JxHZmaDuicplOY8OuqobK4jz3lkZraOpJXX2tWIVl4zM+tyQ6281j0lBTMzG5aTgpmZDXJSMDOzQU4KZmY2qKMbmiUtB0a6EE8vMMRotpZq19gcV30cV/3aNbaxFtd2EVF18riOTgqjIam/Vut7q7VrbI6rPo6rfu0aWzfF5eojMzMb5KRgZmaDujkpnNPqAIbQrrE5rvo4rvq1a2xdE1fXtimYmdm6urmkYGZmFZwUzMxsUFcmBUn7SFokabGkuS2MY7Kk6yTdJulWSYfl24+XdF++At3Nkt7RgtjukfTn/Pr9+bbNJf1U0l35v5s1Oaadyu7JzZIel/Rvrbpfki6Q9LCkv5Rtq3qPlPnv/G/uFkmvaXJc8yTdkV/7B5I2zbdPlfR02b07q8lx1fzdSTo6v1+LJP1zUXENEdulZXHdI+nmfHtT7tkQnw/F/o1FRFc9gB7gbmB7YAPgT8CMFsWyFfCa/PkE4E5gBnA8cESL79M9QG/FttOAufnzucCpLf49Pghs16r7BewBvAb4y3D3CHgHcDUgYHfgd02Oa29gvfz5qWVxTS0/rgX3q+rvLv9/8CfgRcC0/P9sTzNjq9j/JeDYZt6zIT4fCv0b68aSwm7A4ohYEhHPApcAM1sRSEQ8EBF/yJ8/QbZ+9TatiCXRTODC/PmFwL+0LhTeDtwdESMd0T5qEXED8EjF5lr3aCbwzcjcCGwqaatmxRUR10a2VjrAjcC2RVy73riGMBO4JCKeiYi/AovJ/u82PTZJAt4PfKeo69eIqdbnQ6F/Y92YFLYBlpa9XkYbfBBLmgq8GvhdvumQvAh4QbOraXIBXCvpJklz8m1bRMQD+fMHgS1aEFfJLNb+T9rq+1VS6x6109/dR8m+UZZMk/RHSb+U9KYWxFPtd9dO9+tNwEMRcVfZtqbes4rPh0L/xroxKbQdSeOB7wH/FhGPA2cCOwC7Ag+QFV2b7Y0R8RpgX+DTkvYo3xlZebUl/ZklbQDsB3w339QO92sdrbxHtUg6BlgNfDvf9AAwJSJeDRwOXCxpkyaG1Ja/uwqzWfsLSFPvWZXPh0FF/I11Y1K4D5hc9nrbfFtLSFqf7Bf+7Yj4PkBEPBQRz0fEC8C5FFhsriUi7sv/fRj4QR7DQ6XiaP7vw82OK7cv8IeIeCiPseX3q0yte9TyvztJBwDvAj6Yf5iQV8+szJ/fRFZ3v2OzYhrid9fy+wUgaT3gvcClpW3NvGfVPh8o+G+sG5PCQmC6pGn5N85ZwIJWBJLXVZ4P3B4RZ5RtL68HfA/wl8r3FhzXxpImlJ6TNVL+hew+fSQ/7CPAFc2Mq8xa39xafb8q1LpHC4AP5z1Edgf+XlYFUDhJ+wBHAftFxFNl2ydJ6smfbw9MB5Y0Ma5av7sFwCxJL5I0LY/r982Kq8yewB0Rsay0oVn3rNbnA0X/jRXdgt6OD7JW+jvJMvwxLYzjjWRFv1uAm/PHO4CLgD/n2xcAWzU5ru3Jen78Cbi1dI+AicDPgbuAnwGbt+CebQysBF5Stq0l94ssMT0APEdWf/uxWveIrEfI/Pxv7s9AX5PjWkxW31z6OzsrP/Z9+e/4ZuAPwLubHFfN3x1wTH6/FgH7Nvt3mW//BvCpimObcs+G+Hwo9G/M01yYmdmgbqw+MjOzGpwUzMxskJOCmZkNclIwM7NBTgpmZjbIScHqIumDkq5twnWmSop88FBLSbpe0sdbHYdZMzgp2DokvVHSbyT9XdIjkn4t6bUAEfHtiNi71TFaZ1M2Zfa3Wh2Hravl38KsveRzuPwYOAi4jGx68TcBz7QyrnpIWi/WzAhqZnVwScEq7QgQEd+JbE6apyObdvkWyObPkfSr0sF5Fc+n8gU/HpM0Px+ej6QeSV+StELSXyUdUl4lpGzhkj3LzlXz26OkAyXdLukJSUskfbJs31skLZP0OUkPAv9T8d4X5bG9smzbJGULpbxU0maSfixpuaRH8+dVp5aujLGymkvSSySdL+kBZYvH/GdpSoQq5+qR9HlJd+c/102SJuf7/knSwry0tlDSP5W97/r8vL+RNCDpR5ImSvq2soWHFiqbVbP8d3Roft9WKFtwZ1y+b5ykL0i6V9kiM9+U9JKKn+0jkv6Wv/eYsvOOkzQ3j3+lpMskbT7ce5VNufF54AN5/H/Ktx+Qx/hE/vfywWr3zYrlpGCV7gSel3ShpH2VNg31u4DXAruQzTtfWiXrE2ST1+1KtoDJv4wirofz62wCHAh8WWuvLLUlsDnZojtzyt8YEc8A3yebM6nk/cAvI5vwbxxZItkOmAI8DXxthHF+g2wW0peRTXW8N1CrPeLwPKZ35D/XR4Gn8g/WK4H/JpvS4AzgSkkTy947C/gQ2dTIOwC/zX+Gzcnm3T+u4lrvAfrIfg8z82sBHJA/3ko2vcl41v3Z3wjsRLaGxbGSds63f4bsd/pmYGvgUbJpFoZ8b0T8BPgv4NKIGB8R/6Bsjq3/JpvOYgLwT2TTOlizFTmfiB+d+QB2JvtwW0b2AbeAbA53yD5AflV2bJBNs116fRlrVoX6BfDJsn175seXVgC7B9izbP/xwLfy51PLj60S4w+Bw/LnbwGeBTYc4mfak2xRntLrXwMfrnHsrsCjZa+vBz5eGWNlnGTz2j8DvLhs/2zguhrXWQTMrLL9Q8DvK7b9FjigLJ5jyvZ9Cbi67PW7gZsrfkf7lL0+GPh5/vznwMFl+3Yim/9nvbKfbduy/b8HZuXPbwfeXrZvqzreW3kfNwYeI5tX6MWV98SP5j1cUrB1RMTtEXFARGwLvJLsW+BXhnjLg2XPnyL7tkn+vvJFP8qf1yUvtdyorOH7MbJv171lhyyPiFVDnOI6YCNJr8urVnYlmxIcSRtJOjuvQnkcuIFs1aqq1T5D2A5YH3ggr656DDgbeGmN4yeTTV5WaWugckW5e1l7wZSHyp4/XeX1eNZWfu/vza9R7Vr3sibBldT6/W4H/KDsZ70deD7xvWuJiCeBDwCfIrt/V0p6ebVjrVhOCjakiLiDrNTwymEOreYB1l72cXLF/ieBjcpeb1ntJJJeRDan/OlkJZZNgavIZoUcDHWoQCLiebJSzOz88ePIljgE+CzZN+TXRcQmZOv1UnH+lJiXkpUUeiNi0/yxSUS8okZYS8mqfirdT/aBW24Ko1tPoPzeT8mvUe1aU8hKh+VJppalZNU9m5Y9Nox8LY5hrPP7iohrImIvshLHHWTrK1iTOSnYWiS9XNJnSw2tecPnbLJ1fet1GXCYpG0kbQp8rmL/zWRz5q8vqQ/4PzXOswHZAu7LgdWS9iWrq6/XxWTfRj+YPy+ZQPbt+rG8Pr+yPr4y5j0kTckbZI8u7Yhs7vprgS9J2iRviN1B0ptrnOs84D8kTVdml7zd4CpgR0n7S1pP0gfIFmz/8Qh+5pIjlTWoTwYOY82iMd8B/l3Z+iLjWVPXn9J76yzgJEnbwWDjfep65w8BU8savLeQNDNvW3gGGABeSP7prGGcFKzSE8DrgN9JepIsGfyF7Nt0vc4l+5C8Bfgj2YfdarIqBoAvkn1TfhQ4gbU/qAfl3+gPJUsyjwL7M4KFkSLid2Tf9Ldm7TWKvwK8GFhB9vP+ZIhz/JTsA/UW4CbW/aD+MFkSuy2P9XKyb77VnEH2M10LPE62oMqLI1vV611k93wl2eI474qIFWk/aVVX5PHeTNaIfX6+/QKyNQ1uAP4KrCJrQE7xVbLfw7WSniC7d69LfG9pKdWVkv5A9ll0OFnJ5RGyxuuDEs9lDeT1FKxp8m/4Z0VEZdWIFUhSANMjYnGrY7H255KCFUbSiyW9I68C2YasWuYHrY7LzGpzUrAiiaxa6FGy6qPbgWNbGpGZDcnVR2ZmNsglBTMzG+SkYGZmg5wUzMxskJOCmZkNclIwM7NB/x+P+SJkchUFeAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#plot variance explained to see what latent dimensions to use\n",
    "explained = svd.explained_variance_ratio_.cumsum()\n",
    "plt.plot(explained, '.-', ms=16, color='red')\n",
    "plt.xlabel('Singular value components', fontsize= 12)\n",
    "plt.ylabel('Cumulative percent of variance', fontsize=12)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>190</th>\n",
       "      <th>191</th>\n",
       "      <th>192</th>\n",
       "      <th>193</th>\n",
       "      <th>194</th>\n",
       "      <th>195</th>\n",
       "      <th>196</th>\n",
       "      <th>197</th>\n",
       "      <th>198</th>\n",
       "      <th>199</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Toy Story (1995)</th>\n",
       "      <td>0.102600</td>\n",
       "      <td>0.082370</td>\n",
       "      <td>0.069459</td>\n",
       "      <td>0.091250</td>\n",
       "      <td>-0.151136</td>\n",
       "      <td>0.012090</td>\n",
       "      <td>0.234567</td>\n",
       "      <td>-0.019392</td>\n",
       "      <td>-0.124586</td>\n",
       "      <td>-0.008614</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.032497</td>\n",
       "      <td>-0.134418</td>\n",
       "      <td>-0.043143</td>\n",
       "      <td>0.145434</td>\n",
       "      <td>-0.088043</td>\n",
       "      <td>-0.058114</td>\n",
       "      <td>-0.011443</td>\n",
       "      <td>0.085556</td>\n",
       "      <td>0.026559</td>\n",
       "      <td>-0.005955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jumanji (1995)</th>\n",
       "      <td>0.032689</td>\n",
       "      <td>0.008539</td>\n",
       "      <td>0.059189</td>\n",
       "      <td>0.091842</td>\n",
       "      <td>-0.106015</td>\n",
       "      <td>0.007100</td>\n",
       "      <td>0.189676</td>\n",
       "      <td>-0.012325</td>\n",
       "      <td>-0.078054</td>\n",
       "      <td>0.001669</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001222</td>\n",
       "      <td>0.003288</td>\n",
       "      <td>-0.023079</td>\n",
       "      <td>0.010664</td>\n",
       "      <td>0.015974</td>\n",
       "      <td>0.001728</td>\n",
       "      <td>-0.006249</td>\n",
       "      <td>-0.012477</td>\n",
       "      <td>-0.000346</td>\n",
       "      <td>0.029158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Grumpier Old Men (1995)</th>\n",
       "      <td>0.148118</td>\n",
       "      <td>0.105052</td>\n",
       "      <td>-0.042318</td>\n",
       "      <td>0.121888</td>\n",
       "      <td>0.098552</td>\n",
       "      <td>-0.040117</td>\n",
       "      <td>-0.011973</td>\n",
       "      <td>0.000952</td>\n",
       "      <td>0.004727</td>\n",
       "      <td>-0.002093</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006399</td>\n",
       "      <td>-0.005632</td>\n",
       "      <td>0.001505</td>\n",
       "      <td>0.000543</td>\n",
       "      <td>0.000506</td>\n",
       "      <td>0.002536</td>\n",
       "      <td>0.000441</td>\n",
       "      <td>-0.004060</td>\n",
       "      <td>-0.004075</td>\n",
       "      <td>0.006066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Waiting to Exhale (1995)</th>\n",
       "      <td>0.824508</td>\n",
       "      <td>0.064191</td>\n",
       "      <td>-0.292164</td>\n",
       "      <td>0.377882</td>\n",
       "      <td>0.274419</td>\n",
       "      <td>-0.082577</td>\n",
       "      <td>-0.046063</td>\n",
       "      <td>0.001273</td>\n",
       "      <td>0.028399</td>\n",
       "      <td>-0.000377</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000135</td>\n",
       "      <td>-0.000139</td>\n",
       "      <td>0.000247</td>\n",
       "      <td>-0.000035</td>\n",
       "      <td>-0.000170</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000286</td>\n",
       "      <td>-0.000020</td>\n",
       "      <td>-0.000168</td>\n",
       "      <td>-0.000238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Father of the Bride Part II (1995)</th>\n",
       "      <td>0.113317</td>\n",
       "      <td>0.120747</td>\n",
       "      <td>-0.006328</td>\n",
       "      <td>-0.037081</td>\n",
       "      <td>-0.007424</td>\n",
       "      <td>0.010526</td>\n",
       "      <td>-0.009221</td>\n",
       "      <td>-0.001934</td>\n",
       "      <td>0.004463</td>\n",
       "      <td>-0.011439</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001208</td>\n",
       "      <td>-0.014851</td>\n",
       "      <td>0.023732</td>\n",
       "      <td>0.012979</td>\n",
       "      <td>0.008908</td>\n",
       "      <td>-0.000236</td>\n",
       "      <td>-0.004764</td>\n",
       "      <td>0.009549</td>\n",
       "      <td>0.009244</td>\n",
       "      <td>-0.012308</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 200 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         0         1         2         3    \\\n",
       "Toy Story (1995)                    0.102600  0.082370  0.069459  0.091250   \n",
       "Jumanji (1995)                      0.032689  0.008539  0.059189  0.091842   \n",
       "Grumpier Old Men (1995)             0.148118  0.105052 -0.042318  0.121888   \n",
       "Waiting to Exhale (1995)            0.824508  0.064191 -0.292164  0.377882   \n",
       "Father of the Bride Part II (1995)  0.113317  0.120747 -0.006328 -0.037081   \n",
       "\n",
       "                                         4         5         6         7    \\\n",
       "Toy Story (1995)                   -0.151136  0.012090  0.234567 -0.019392   \n",
       "Jumanji (1995)                     -0.106015  0.007100  0.189676 -0.012325   \n",
       "Grumpier Old Men (1995)             0.098552 -0.040117 -0.011973  0.000952   \n",
       "Waiting to Exhale (1995)            0.274419 -0.082577 -0.046063  0.001273   \n",
       "Father of the Bride Part II (1995) -0.007424  0.010526 -0.009221 -0.001934   \n",
       "\n",
       "                                         8         9    ...       190  \\\n",
       "Toy Story (1995)                   -0.124586 -0.008614  ... -0.032497   \n",
       "Jumanji (1995)                     -0.078054  0.001669  ...  0.001222   \n",
       "Grumpier Old Men (1995)             0.004727 -0.002093  ...  0.006399   \n",
       "Waiting to Exhale (1995)            0.028399 -0.000377  ...  0.000135   \n",
       "Father of the Bride Part II (1995)  0.004463 -0.011439  ...  0.001208   \n",
       "\n",
       "                                         191       192       193       194  \\\n",
       "Toy Story (1995)                   -0.134418 -0.043143  0.145434 -0.088043   \n",
       "Jumanji (1995)                      0.003288 -0.023079  0.010664  0.015974   \n",
       "Grumpier Old Men (1995)            -0.005632  0.001505  0.000543  0.000506   \n",
       "Waiting to Exhale (1995)           -0.000139  0.000247 -0.000035 -0.000170   \n",
       "Father of the Bride Part II (1995) -0.014851  0.023732  0.012979  0.008908   \n",
       "\n",
       "                                         195       196       197       198  \\\n",
       "Toy Story (1995)                   -0.058114 -0.011443  0.085556  0.026559   \n",
       "Jumanji (1995)                      0.001728 -0.006249 -0.012477 -0.000346   \n",
       "Grumpier Old Men (1995)             0.002536  0.000441 -0.004060 -0.004075   \n",
       "Waiting to Exhale (1995)            0.000005  0.000286 -0.000020 -0.000168   \n",
       "Father of the Bride Part II (1995) -0.000236 -0.004764  0.009549  0.009244   \n",
       "\n",
       "                                         199  \n",
       "Toy Story (1995)                   -0.005955  \n",
       "Jumanji (1995)                      0.029158  \n",
       "Grumpier Old Men (1995)             0.006066  \n",
       "Waiting to Exhale (1995)           -0.000238  \n",
       "Father of the Bride Part II (1995) -0.012308  \n",
       "\n",
       "[5 rows x 200 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#This is my content matrix.\n",
    "latent_matrix_1_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>190</th>\n",
       "      <th>191</th>\n",
       "      <th>192</th>\n",
       "      <th>193</th>\n",
       "      <th>194</th>\n",
       "      <th>195</th>\n",
       "      <th>196</th>\n",
       "      <th>197</th>\n",
       "      <th>198</th>\n",
       "      <th>199</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Toy Story (1995)</th>\n",
       "      <td>36.391240</td>\n",
       "      <td>-4.958788</td>\n",
       "      <td>14.331820</td>\n",
       "      <td>1.844157</td>\n",
       "      <td>-3.077921</td>\n",
       "      <td>-1.351171</td>\n",
       "      <td>8.341428</td>\n",
       "      <td>-0.561087</td>\n",
       "      <td>-1.898204</td>\n",
       "      <td>1.657854</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.284568</td>\n",
       "      <td>1.821130</td>\n",
       "      <td>-0.932767</td>\n",
       "      <td>-1.930437</td>\n",
       "      <td>1.394376</td>\n",
       "      <td>-0.751668</td>\n",
       "      <td>-2.149879</td>\n",
       "      <td>-0.465503</td>\n",
       "      <td>1.439896</td>\n",
       "      <td>-0.911504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jumanji (1995)</th>\n",
       "      <td>20.447499</td>\n",
       "      <td>0.675231</td>\n",
       "      <td>11.354479</td>\n",
       "      <td>-7.325453</td>\n",
       "      <td>-3.505243</td>\n",
       "      <td>3.585614</td>\n",
       "      <td>4.393397</td>\n",
       "      <td>-5.564563</td>\n",
       "      <td>0.434474</td>\n",
       "      <td>0.351435</td>\n",
       "      <td>...</td>\n",
       "      <td>0.341699</td>\n",
       "      <td>-0.508641</td>\n",
       "      <td>0.838471</td>\n",
       "      <td>1.799928</td>\n",
       "      <td>1.850423</td>\n",
       "      <td>-1.592821</td>\n",
       "      <td>0.738655</td>\n",
       "      <td>0.063008</td>\n",
       "      <td>-1.046210</td>\n",
       "      <td>-0.204062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Grumpier Old Men (1995)</th>\n",
       "      <td>8.407816</td>\n",
       "      <td>-5.190801</td>\n",
       "      <td>4.378314</td>\n",
       "      <td>-6.189039</td>\n",
       "      <td>-0.333121</td>\n",
       "      <td>1.924148</td>\n",
       "      <td>-1.548363</td>\n",
       "      <td>0.795947</td>\n",
       "      <td>-3.196045</td>\n",
       "      <td>2.121791</td>\n",
       "      <td>...</td>\n",
       "      <td>0.021261</td>\n",
       "      <td>-0.024558</td>\n",
       "      <td>0.077206</td>\n",
       "      <td>-0.225024</td>\n",
       "      <td>0.332259</td>\n",
       "      <td>0.404377</td>\n",
       "      <td>0.930755</td>\n",
       "      <td>-0.449721</td>\n",
       "      <td>1.278904</td>\n",
       "      <td>0.324372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Waiting to Exhale (1995)</th>\n",
       "      <td>0.408037</td>\n",
       "      <td>-0.664700</td>\n",
       "      <td>0.739903</td>\n",
       "      <td>-0.202676</td>\n",
       "      <td>-0.498374</td>\n",
       "      <td>1.055409</td>\n",
       "      <td>0.184081</td>\n",
       "      <td>-0.508294</td>\n",
       "      <td>-0.429506</td>\n",
       "      <td>0.086538</td>\n",
       "      <td>...</td>\n",
       "      <td>0.208043</td>\n",
       "      <td>-0.088896</td>\n",
       "      <td>0.000594</td>\n",
       "      <td>0.161013</td>\n",
       "      <td>0.056840</td>\n",
       "      <td>-0.015643</td>\n",
       "      <td>-0.308005</td>\n",
       "      <td>0.209546</td>\n",
       "      <td>-0.130141</td>\n",
       "      <td>-0.042825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Father of the Bride Part II (1995)</th>\n",
       "      <td>5.755079</td>\n",
       "      <td>-1.757369</td>\n",
       "      <td>3.961287</td>\n",
       "      <td>-3.673278</td>\n",
       "      <td>-3.688047</td>\n",
       "      <td>3.237859</td>\n",
       "      <td>1.140585</td>\n",
       "      <td>-0.690786</td>\n",
       "      <td>-2.171827</td>\n",
       "      <td>1.365963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.383106</td>\n",
       "      <td>-0.403908</td>\n",
       "      <td>-0.103732</td>\n",
       "      <td>-0.393780</td>\n",
       "      <td>-0.477676</td>\n",
       "      <td>0.144159</td>\n",
       "      <td>0.313400</td>\n",
       "      <td>0.148340</td>\n",
       "      <td>-0.701960</td>\n",
       "      <td>-0.707823</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 200 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          0         1          2         3    \\\n",
       "Toy Story (1995)                    36.391240 -4.958788  14.331820  1.844157   \n",
       "Jumanji (1995)                      20.447499  0.675231  11.354479 -7.325453   \n",
       "Grumpier Old Men (1995)              8.407816 -5.190801   4.378314 -6.189039   \n",
       "Waiting to Exhale (1995)             0.408037 -0.664700   0.739903 -0.202676   \n",
       "Father of the Bride Part II (1995)   5.755079 -1.757369   3.961287 -3.673278   \n",
       "\n",
       "                                         4         5         6         7    \\\n",
       "Toy Story (1995)                   -3.077921 -1.351171  8.341428 -0.561087   \n",
       "Jumanji (1995)                     -3.505243  3.585614  4.393397 -5.564563   \n",
       "Grumpier Old Men (1995)            -0.333121  1.924148 -1.548363  0.795947   \n",
       "Waiting to Exhale (1995)           -0.498374  1.055409  0.184081 -0.508294   \n",
       "Father of the Bride Part II (1995) -3.688047  3.237859  1.140585 -0.690786   \n",
       "\n",
       "                                         8         9    ...       190  \\\n",
       "Toy Story (1995)                   -1.898204  1.657854  ... -0.284568   \n",
       "Jumanji (1995)                      0.434474  0.351435  ...  0.341699   \n",
       "Grumpier Old Men (1995)            -3.196045  2.121791  ...  0.021261   \n",
       "Waiting to Exhale (1995)           -0.429506  0.086538  ...  0.208043   \n",
       "Father of the Bride Part II (1995) -2.171827  1.365963  ...  0.383106   \n",
       "\n",
       "                                         191       192       193       194  \\\n",
       "Toy Story (1995)                    1.821130 -0.932767 -1.930437  1.394376   \n",
       "Jumanji (1995)                     -0.508641  0.838471  1.799928  1.850423   \n",
       "Grumpier Old Men (1995)            -0.024558  0.077206 -0.225024  0.332259   \n",
       "Waiting to Exhale (1995)           -0.088896  0.000594  0.161013  0.056840   \n",
       "Father of the Bride Part II (1995) -0.403908 -0.103732 -0.393780 -0.477676   \n",
       "\n",
       "                                         195       196       197       198  \\\n",
       "Toy Story (1995)                   -0.751668 -2.149879 -0.465503  1.439896   \n",
       "Jumanji (1995)                     -1.592821  0.738655  0.063008 -1.046210   \n",
       "Grumpier Old Men (1995)             0.404377  0.930755 -0.449721  1.278904   \n",
       "Waiting to Exhale (1995)           -0.015643 -0.308005  0.209546 -0.130141   \n",
       "Father of the Bride Part II (1995)  0.144159  0.313400  0.148340 -0.701960   \n",
       "\n",
       "                                         199  \n",
       "Toy Story (1995)                   -0.911504  \n",
       "Jumanji (1995)                     -0.204062  \n",
       "Grumpier Old Men (1995)             0.324372  \n",
       "Waiting to Exhale (1995)           -0.042825  \n",
       "Father of the Bride Part II (1995) -0.707823  \n",
       "\n",
       "[5 rows x 200 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#This is my collaborative matrix.\n",
    "latent_matrix_2_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running a content/ collaborative and hybrid cosine Similarity:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>collaborative</th>\n",
       "      <th>hybrid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Bug's Life, A (1998)</th>\n",
       "      <td>0.901307</td>\n",
       "      <td>0.591832</td>\n",
       "      <td>0.746569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Toy Story 2 (1999)</th>\n",
       "      <td>0.778154</td>\n",
       "      <td>0.688699</td>\n",
       "      <td>0.733426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Shrek (2001)</th>\n",
       "      <td>0.423949</td>\n",
       "      <td>0.672226</td>\n",
       "      <td>0.548088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Monsters, Inc. (2001)</th>\n",
       "      <td>0.433832</td>\n",
       "      <td>0.610359</td>\n",
       "      <td>0.522096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Up (2009)</th>\n",
       "      <td>0.553247</td>\n",
       "      <td>0.426161</td>\n",
       "      <td>0.489704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Aladdin (1992)</th>\n",
       "      <td>0.235383</td>\n",
       "      <td>0.642712</td>\n",
       "      <td>0.439048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Antz (1998)</th>\n",
       "      <td>0.433832</td>\n",
       "      <td>0.441279</td>\n",
       "      <td>0.437556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Who Framed Roger Rabbit? (1988)</th>\n",
       "      <td>0.358991</td>\n",
       "      <td>0.507358</td>\n",
       "      <td>0.433174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Toy Story 3 (2010)</th>\n",
       "      <td>0.355778</td>\n",
       "      <td>0.503356</td>\n",
       "      <td>0.429567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ice Age (2002)</th>\n",
       "      <td>0.380024</td>\n",
       "      <td>0.477317</td>\n",
       "      <td>0.428671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Groundhog Day (1993)</th>\n",
       "      <td>0.196106</td>\n",
       "      <td>0.639584</td>\n",
       "      <td>0.417845</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  content  collaborative    hybrid\n",
       "Bug's Life, A (1998)             0.901307       0.591832  0.746569\n",
       "Toy Story 2 (1999)               0.778154       0.688699  0.733426\n",
       "Shrek (2001)                     0.423949       0.672226  0.548088\n",
       "Monsters, Inc. (2001)            0.433832       0.610359  0.522096\n",
       "Up (2009)                        0.553247       0.426161  0.489704\n",
       "Aladdin (1992)                   0.235383       0.642712  0.439048\n",
       "Antz (1998)                      0.433832       0.441279  0.437556\n",
       "Who Framed Roger Rabbit? (1988)  0.358991       0.507358  0.433174\n",
       "Toy Story 3 (2010)               0.355778       0.503356  0.429567\n",
       "Ice Age (2002)                   0.380024       0.477317  0.428671\n",
       "Groundhog Day (1993)             0.196106       0.639584  0.417845"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "#take the latent vectors for a selected movie from both content and collaborative matrixes\n",
    "a_1=np.array(latent_matrix_1_df.loc['Toy Story (1995)']).reshape(1,-1)\n",
    "a_2=np.array(latent_matrix_2_df.loc['Toy Story (1995)']).reshape(1,-1)\n",
    "\n",
    "#calculate the similarity of this movie with the others in the list\n",
    "score_1 = cosine_similarity(latent_matrix_1_df, a_1).reshape(-1)\n",
    "score_2 = cosine_similarity(latent_matrix_2_df, a_2).reshape(-1)\n",
    "\n",
    "#an avg measure of both content and collaborative\n",
    "hybrid = ((score_1 + score_2)/2.0)\n",
    "\n",
    "#form a data frame of similar movies\n",
    "dictDf = {'content': score_1, 'collaborative':score_2, 'hybrid':hybrid}\n",
    "similar = pd.DataFrame(dictDf, index = latent_matrix_1_df.index)\n",
    "\n",
    "#sort it on the basis of either: content, collaborative or hybrid,\n",
    "#here : content\n",
    "similar.sort_values('hybrid', ascending=False, inplace=True)\n",
    "\n",
    "similar[1:].head(11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}