1_Doc2Vec_Movies.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Word2Vec\n",
    "import gensim \n",
    "from gensim import corpora\n",
    "from gensim import models\n",
    "from gensim import similarities\n",
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
    "\n",
    "# For text preprocessing\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from string import digits\n",
    "\n",
    "# Generics\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.manifold import TSNE\n",
    "import json\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "stemmer = nltk.stem.PorterStemmer()\n",
    "w_tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
    "stop_words = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_word(word):\n",
    "    \"\"\"\n",
    "    INPUT : String\n",
    "    OUTPUT: String lemmatized\n",
    "    Apply standard cleaning operations of strings like:\n",
    "    - Lemmatization\n",
    "    - Stemming\n",
    "    \"\"\"\n",
    "    word = lemmatizer.lemmatize(word)\n",
    "    return stemmer.stem(word)\n",
    "\n",
    "def preprocess_doc(document):\n",
    "    \n",
    "    \"\"\"\n",
    "    INPUT : String\n",
    "    OUTPUT: List of strings \n",
    "    Apply standard cleaning operations to each of the words\n",
    "    and return a list of the cleaned words\n",
    "    \"\"\"\n",
    "    document = document.lower().translate(str.maketrans(' ', ' ', string.punctuation)).split()\n",
    "    return [preprocess_word(x)  for x in document if x not in stop_words]\n",
    "\n",
    "\n",
    "def tag(list_of_docs):\n",
    "    \"\"\"\n",
    "    INPUT : A dataframe row\n",
    "    OUTPUT: List of strings\n",
    "    Associate a tag/number with each document of the training corpus.\n",
    "    \"\"\"\n",
    "    for i, line in enumerate(list_of_docs):\n",
    "        yield  TaggedDocument(line, [i])\n",
    "        \n",
    "def getvector(query):\n",
    "    \"\"\"\n",
    "    INPUT: String\n",
    "    OUTUT: np.array containing the embeddings for query\n",
    "    \"\"\"\n",
    "    # Parse the query\n",
    "    return model.infer_vector(preprocess_doc(query))\n",
    "        \n",
    "def query(query, n_results=5):\n",
    "    \"\"\"\n",
    "    INPUT: String to query, numbers of docs to retireve\n",
    "    OUTUT: Dataframe containing the n most similar docs to the query\n",
    "    \"\"\"\n",
    "    \n",
    "    # Parse the query\n",
    "    query_preprocessed = getvector(query) \n",
    "    \n",
    "    # Get matches\n",
    "    results = pd.DataFrame( model.docvecs.most_similar([query_preprocessed], topn=n_results), columns=['doc_ID', 'distance']) \n",
    "    \n",
    "    # Match query results with original df\n",
    "    get_ids = df.loc[results['doc_ID'].tolist()]\n",
    "    # Add distances to dataframe\n",
    "    get_ids['distance'] = results['distance'].tolist()        \n",
    "    return get_ids[['Name','Plot','distance']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Read the preprocessed DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wiki_ID</th>\n",
       "      <th>Plot</th>\n",
       "      <th>Name</th>\n",
       "      <th>genres</th>\n",
       "      <th>len</th>\n",
       "      <th>Corpus</th>\n",
       "      <th>genres_all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>The nation of Panem consists of a wealthy Capi...</td>\n",
       "      <td>The Hunger Games</td>\n",
       "      <td>Action/Adventure</td>\n",
       "      <td>4559</td>\n",
       "      <td>[nation, panem, consist, wealthi, capitol, twe...</td>\n",
       "      <td>[Action/Adventure, Science Fiction, Action, Dr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Poovalli Induchoodan  is sentenced for six yea...</td>\n",
       "      <td>Narasimham</td>\n",
       "      <td>Musical</td>\n",
       "      <td>3099</td>\n",
       "      <td>[pooval, induchoodan, sentenc, six, year, pris...</td>\n",
       "      <td>[Musical, Action, Drama, Bollywood]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>The Lemon Drop Kid , a New York City swindler,...</td>\n",
       "      <td>The Lemon Drop Kid</td>\n",
       "      <td>Screwball comedy</td>\n",
       "      <td>4917</td>\n",
       "      <td>[lemon, drop, kid, new, york, citi, swindler, ...</td>\n",
       "      <td>[Screwball comedy, Comedy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Seventh-day Adventist Church pastor Michael Ch...</td>\n",
       "      <td>A Cry in the Dark</td>\n",
       "      <td>Crime Fiction</td>\n",
       "      <td>2425</td>\n",
       "      <td>[seventhday, adventist, church, pastor, michae...</td>\n",
       "      <td>[Crime Fiction, Drama, Docudrama, World cinema...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>The president is on his way to give a speech. ...</td>\n",
       "      <td>End Game</td>\n",
       "      <td>Thriller</td>\n",
       "      <td>1937</td>\n",
       "      <td>[presid, way, give, speech, travel, man, show,...</td>\n",
       "      <td>[Thriller, Action/Adventure, Action, Drama]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Wiki_ID                                               Plot  \\\n",
       "0        0  The nation of Panem consists of a wealthy Capi...   \n",
       "1        1  Poovalli Induchoodan  is sentenced for six yea...   \n",
       "2        2  The Lemon Drop Kid , a New York City swindler,...   \n",
       "3        3  Seventh-day Adventist Church pastor Michael Ch...   \n",
       "4        4  The president is on his way to give a speech. ...   \n",
       "\n",
       "                 Name            genres   len  \\\n",
       "0    The Hunger Games  Action/Adventure  4559   \n",
       "1          Narasimham           Musical  3099   \n",
       "2  The Lemon Drop Kid  Screwball comedy  4917   \n",
       "3   A Cry in the Dark     Crime Fiction  2425   \n",
       "4            End Game          Thriller  1937   \n",
       "\n",
       "                                              Corpus  \\\n",
       "0  [nation, panem, consist, wealthi, capitol, twe...   \n",
       "1  [pooval, induchoodan, sentenc, six, year, pris...   \n",
       "2  [lemon, drop, kid, new, york, citi, swindler, ...   \n",
       "3  [seventhday, adventist, church, pastor, michae...   \n",
       "4  [presid, way, give, speech, travel, man, show,...   \n",
       "\n",
       "                                          genres_all  \n",
       "0  [Action/Adventure, Science Fiction, Action, Dr...  \n",
       "1                [Musical, Action, Drama, Bollywood]  \n",
       "2                         [Screwball comedy, Comedy]  \n",
       "3  [Crime Fiction, Drama, Docudrama, World cinema...  \n",
       "4        [Thriller, Action/Adventure, Action, Drama]  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path   = \"./preprocessed/\"\n",
    "df     = pd.read_pickle(path+\"df_preprocessed.pkl\")\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create a generator that will be used to iterate over the movies during the training (row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "Corpus_Tagged = list(tag(df['Corpus'].tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train a Doc2Vec model\n",
    "----\n",
    "\n",
    "**Word2Vec** is a more recent model that embeds words in a lower-dimensional vector space using a shallow neural network\n",
    "The result is a set of word-vectors where vectors close together in vector space have similar meanings based on context, and word-vectors distant to each other have differing meanings. \n",
    "\n",
    "With the Word2Vec model, one can calculate the vectors for each word in a document. But what if we want to calculate a vector for the entire document?\n",
    "\n",
    "2 Possibilities: \n",
    "\n",
    "- Average the vectors for each word in the document\n",
    "- Doc2Vec (Mikolov,2014)\n",
    "\n",
    "\n",
    "There are two implementations:\n",
    "- **Distributed Memory (PV-DM)**:  task of predicting a center word based an average of both context word-vectors and the full document’s doc-vector\n",
    "- **Distributed Bag of Words (PV-DBOW)**: task of predicting a target word just from the full document’s doc-vector.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters for the training\n",
    "\n",
    "dm   = 1                               # Defines the training algorithm.  (dm=1) --> (PV-DM),  (dm=0) --> (PV-DBOW) \n",
    "size = 100                             # Dimensionality of the feature vectors\n",
    "window = 5                             # How many neighbours (to the left & to the right) will be analyzed for the current word\n",
    "min_count = 5                          # Ignore all words with total frequency lower than this.\n",
    "workers =multiprocessing.cpu_count()   # Faster training with multicore machines\n",
    "\n",
    "model = Doc2Vec(Corpus_Tagged,\n",
    "                vector_size=size ,\n",
    "                workers=workers,\n",
    "                min_count=min_count,\n",
    "                epochs=40 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "path   = \"./models/\"\n",
    "model.save(path+ \"doc2vec_100_Movies\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Doc2Vec.load(\"./models/doc2vec_100_Movies\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wiki_ID</th>\n",
       "      <th>Plot</th>\n",
       "      <th>Name</th>\n",
       "      <th>genres</th>\n",
       "      <th>len</th>\n",
       "      <th>Corpus</th>\n",
       "      <th>genres_all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>The nation of Panem consists of a wealthy Capi...</td>\n",
       "      <td>The Hunger Games</td>\n",
       "      <td>Action/Adventure</td>\n",
       "      <td>4559</td>\n",
       "      <td>[nation, panem, consist, wealthi, capitol, twe...</td>\n",
       "      <td>[Action/Adventure, Science Fiction, Action, Dr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Poovalli Induchoodan  is sentenced for six yea...</td>\n",
       "      <td>Narasimham</td>\n",
       "      <td>Musical</td>\n",
       "      <td>3099</td>\n",
       "      <td>[pooval, induchoodan, sentenc, six, year, pris...</td>\n",
       "      <td>[Musical, Action, Drama, Bollywood]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>The Lemon Drop Kid , a New York City swindler,...</td>\n",
       "      <td>The Lemon Drop Kid</td>\n",
       "      <td>Screwball comedy</td>\n",
       "      <td>4917</td>\n",
       "      <td>[lemon, drop, kid, new, york, citi, swindler, ...</td>\n",
       "      <td>[Screwball comedy, Comedy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Seventh-day Adventist Church pastor Michael Ch...</td>\n",
       "      <td>A Cry in the Dark</td>\n",
       "      <td>Crime Fiction</td>\n",
       "      <td>2425</td>\n",
       "      <td>[seventhday, adventist, church, pastor, michae...</td>\n",
       "      <td>[Crime Fiction, Drama, Docudrama, World cinema...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>The president is on his way to give a speech. ...</td>\n",
       "      <td>End Game</td>\n",
       "      <td>Thriller</td>\n",
       "      <td>1937</td>\n",
       "      <td>[presid, way, give, speech, travel, man, show,...</td>\n",
       "      <td>[Thriller, Action/Adventure, Action, Drama]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21101</th>\n",
       "      <td>21101</td>\n",
       "      <td>Lucy  is working as a dancer in a sleazy strip...</td>\n",
       "      <td>I Don't Want to Be Born</td>\n",
       "      <td>Horror</td>\n",
       "      <td>4801</td>\n",
       "      <td>[luci, work, dancer, sleazi, strip, joint, sta...</td>\n",
       "      <td>[Horror]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21102</th>\n",
       "      <td>21102</td>\n",
       "      <td>Twenty-something Eun-mo listens to a taxi driv...</td>\n",
       "      <td>Paju</td>\n",
       "      <td>Romantic drama</td>\n",
       "      <td>1406</td>\n",
       "      <td>[twentysometh, eunmo, listen, taxi, driver, dr...</td>\n",
       "      <td>[Romantic drama, Romance Film, Drama, World ci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21103</th>\n",
       "      <td>21103</td>\n",
       "      <td>In 1928 Hollywood, director Leo Andreyev  look...</td>\n",
       "      <td>The Last Command</td>\n",
       "      <td>Silent film</td>\n",
       "      <td>2971</td>\n",
       "      <td>[1928, hollywood, director, leo, andreyev, loo...</td>\n",
       "      <td>[Silent film, Indie, Black-and-white, Period p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21104</th>\n",
       "      <td>21104</td>\n",
       "      <td>Abdur Rehman Khan , a middle-aged dry fruit se...</td>\n",
       "      <td>Kabuliwala</td>\n",
       "      <td>Drama</td>\n",
       "      <td>1289</td>\n",
       "      <td>[abdur, rehman, khan, middleag, dri, fruit, se...</td>\n",
       "      <td>[Drama]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21105</th>\n",
       "      <td>21105</td>\n",
       "      <td>1940 - Operation Dynamo has just taken place. ...</td>\n",
       "      <td>Dad's Army</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>3489</td>\n",
       "      <td>[1940, oper, dynamo, taken, place, newli, conq...</td>\n",
       "      <td>[Comedy]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21106 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Wiki_ID                                               Plot  \\\n",
       "0            0  The nation of Panem consists of a wealthy Capi...   \n",
       "1            1  Poovalli Induchoodan  is sentenced for six yea...   \n",
       "2            2  The Lemon Drop Kid , a New York City swindler,...   \n",
       "3            3  Seventh-day Adventist Church pastor Michael Ch...   \n",
       "4            4  The president is on his way to give a speech. ...   \n",
       "...        ...                                                ...   \n",
       "21101    21101  Lucy  is working as a dancer in a sleazy strip...   \n",
       "21102    21102  Twenty-something Eun-mo listens to a taxi driv...   \n",
       "21103    21103  In 1928 Hollywood, director Leo Andreyev  look...   \n",
       "21104    21104  Abdur Rehman Khan , a middle-aged dry fruit se...   \n",
       "21105    21105  1940 - Operation Dynamo has just taken place. ...   \n",
       "\n",
       "                          Name            genres   len  \\\n",
       "0             The Hunger Games  Action/Adventure  4559   \n",
       "1                   Narasimham           Musical  3099   \n",
       "2           The Lemon Drop Kid  Screwball comedy  4917   \n",
       "3            A Cry in the Dark     Crime Fiction  2425   \n",
       "4                     End Game          Thriller  1937   \n",
       "...                        ...               ...   ...   \n",
       "21101  I Don't Want to Be Born            Horror  4801   \n",
       "21102                     Paju    Romantic drama  1406   \n",
       "21103         The Last Command       Silent film  2971   \n",
       "21104               Kabuliwala             Drama  1289   \n",
       "21105               Dad's Army            Comedy  3489   \n",
       "\n",
       "                                                  Corpus  \\\n",
       "0      [nation, panem, consist, wealthi, capitol, twe...   \n",
       "1      [pooval, induchoodan, sentenc, six, year, pris...   \n",
       "2      [lemon, drop, kid, new, york, citi, swindler, ...   \n",
       "3      [seventhday, adventist, church, pastor, michae...   \n",
       "4      [presid, way, give, speech, travel, man, show,...   \n",
       "...                                                  ...   \n",
       "21101  [luci, work, dancer, sleazi, strip, joint, sta...   \n",
       "21102  [twentysometh, eunmo, listen, taxi, driver, dr...   \n",
       "21103  [1928, hollywood, director, leo, andreyev, loo...   \n",
       "21104  [abdur, rehman, khan, middleag, dri, fruit, se...   \n",
       "21105  [1940, oper, dynamo, taken, place, newli, conq...   \n",
       "\n",
       "                                              genres_all  \n",
       "0      [Action/Adventure, Science Fiction, Action, Dr...  \n",
       "1                    [Musical, Action, Drama, Bollywood]  \n",
       "2                             [Screwball comedy, Comedy]  \n",
       "3      [Crime Fiction, Drama, Docudrama, World cinema...  \n",
       "4            [Thriller, Action/Adventure, Action, Drama]  \n",
       "...                                                  ...  \n",
       "21101                                           [Horror]  \n",
       "21102  [Romantic drama, Romance Film, Drama, World ci...  \n",
       "21103  [Silent film, Indie, Black-and-white, Period p...  \n",
       "21104                                            [Drama]  \n",
       "21105                                           [Comedy]  \n",
       "\n",
       "[21106 rows x 7 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.reset_index(drop=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sanity_check(query, id):\n",
    "    return model.docvecs.most_similar([getvector(query)], topn=1)[0][0] == id\n",
    " \n",
    "def test_coherence(df, nitem = 500, verbose=0, seed=1):\n",
    "    df_temp = df.sample(nitem)\n",
    "    \n",
    "    #print(df_temp)\n",
    "    \n",
    "    ids    =  df_temp['Wiki_ID'].tolist()\n",
    "    docs    = df_temp['Plot'].tolist()\n",
    "    titles  = df_temp['Name'].tolist()\n",
    "    \n",
    "    wrongs = 0\n",
    "    for i, doc,t in zip(ids,docs,titles):\n",
    "        if(sanity_check(doc,i) == False ):\n",
    "            if verbose == 1:\n",
    "                print(i)\n",
    "                print(\"WRONG: [\" + t + \"] \\n \")\n",
    "            wrongs += 1\n",
    "            \n",
    "    print(\"\\n\\n ***** Correctness: [ \" + str(((nitem-wrongs)/nitem * 100)) + \"% ] ***** with seed \"+ str(seed)+\" \\n \\t \\t \" )    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      " ***** Correctness: [ 100.0% ] ***** with seed 2 \n",
      " \t \t \n"
     ]
    }
   ],
   "source": [
    "test_coherence(df,300, verbose=1, seed=2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Deep Learning",
   "language": "python",
   "name": "deeplearning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}