CABA NLP presentation-May31.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "from nltk.stem import LancasterStemmer, PorterStemmer, WordNetLemmatizer\n",
    "from textblob import TextBlob, Word\n",
    "import spacy\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "note1 = 'WORKER slipped while carrying groceries. Worker fractured his elbow'\n",
    "note2 = 'worker developed carpal tunnel from repetitive typing'\n",
    "note3 = 'worker got traumatized from NLP presentation '"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = note1 + ' '+note2 + ' ' + note3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WORKER slipped while carrying groceries. Worker fractured his elbow worker developed carpal tunnel from repetitive typing worker got traumatized from NLP presentation '"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = defaultdict(int)\n",
    "for word in corpus.split():\n",
    "    vocab[word] = 1\n",
    "unique_words = vocab.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['NLP',\n",
       " 'WORKER',\n",
       " 'Worker',\n",
       " 'carpal',\n",
       " 'carrying',\n",
       " 'developed',\n",
       " 'elbow',\n",
       " 'fractured',\n",
       " 'from',\n",
       " 'got',\n",
       " 'groceries.',\n",
       " 'his',\n",
       " 'presentation',\n",
       " 'repetitive',\n",
       " 'slipped',\n",
       " 'traumatized',\n",
       " 'tunnel',\n",
       " 'typing',\n",
       " 'while',\n",
       " 'worker']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(unique_words) # our vocabulory-- list of distinct words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Preprocessing**    \n",
    "Lowering the case    \n",
    "stemming    \n",
    "lemmatization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Lowering the case**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lower(txt):\n",
    "    print(txt.lower())\n",
    "    return txt.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "worker slipped while carrying groceries. worker fractured his elbow\n",
      "worker developed carpal tunnel from repetitive typing\n",
      "worker got traumatized from nlp presentation \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'worker got traumatized from nlp presentation '"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lower(note1)\n",
    "lower(note2)\n",
    "lower(note3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Stemming**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We need to stem each word in the sentence as stemmer sees each word as a sentence\n",
    "def stemmer(sentence):\n",
    "    '''the function takes the sentence as an argument and returns the sentence with each word stemmed'''\n",
    "    stemmer = PorterStemmer()\n",
    "    return_str = []\n",
    "    for word in sentence.split():\n",
    "        word = word.lower()\n",
    "        return_str.append(stemmer.stem(word))\n",
    "    print(' '.join(return_str))\n",
    "    return ' '.join(return_str)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "worker slip while carri groceries. worker fractur hi elbow\n",
      "worker develop carpal tunnel from repetit type\n",
      "worker got traumat from nlp present\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'worker got traumat from nlp present'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stemmer(note1)\n",
    "stemmer(note2)\n",
    "stemmer(note3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Lemmatization with WordNetLemmatizer**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /Users/pradnya.n/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatizer(sentence):\n",
    "    '''the function takes the sentence as an argument and returns the sentence with each word lemmatized'''\n",
    "    lemma = WordNetLemmatizer()\n",
    "    return_str = []\n",
    "    for word in sentence.split():\n",
    "        word = word.lower()\n",
    "        return_str.append(lemma.lemmatize(word))\n",
    "    print(' '.join(return_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "worker slipped while carrying groceries. worker fractured his elbow\n",
      "worker developed carpal tunnel from repetitive typing\n",
      "worker got traumatized from nlp presentation\n"
     ]
    }
   ],
   "source": [
    "lemmatizer(note1)\n",
    "lemmatizer(note2)\n",
    "lemmatizer(note3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "worker slipped while carrying groceries. worker fractured his elbow\n"
     ]
    }
   ],
   "source": [
    "lemma = WordNetLemmatizer()\n",
    "doc1 = lemma.lemmatize(note1.lower())\n",
    "print(\" \".join([lemma.lemmatize(token) for token in doc1.split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "worker developed carpal tunnel from repetitive typing\n"
     ]
    }
   ],
   "source": [
    "doc2 = lemma.lemmatize(note2.lower())\n",
    "print(\" \".join([lemma.lemmatize(token) for token in doc2.split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "worker got traumatized from nlp presentation\n"
     ]
    }
   ],
   "source": [
    "doc3 = lemma.lemmatize(note3.lower())\n",
    "print(\" \".join([lemma.lemmatize(token) for token in doc3.split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatizer2(sentence):\n",
    "    '''the function takes the sentence as an argument and returns the sentence with each word lemmatized'''\n",
    "    lemma = WordNetLemmatizer()\n",
    "    return_str = []\n",
    "    for word in sentence.split():\n",
    "        word = word.lower()\n",
    "        w = Word(word)\n",
    "        return_str.append(w.lemmatize())\n",
    "    print(' '.join(return_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "worker slipped while carrying groceries. worker fractured his elbow\n",
      "worker developed carpal tunnel from repetitive typing\n",
      "worker got traumatized from nlp presentation\n"
     ]
    }
   ],
   "source": [
    "lemmatizer2(note1)\n",
    "lemmatizer2(note2)\n",
    "lemmatizer2(note3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(sentence):\n",
    "    tokens = lower(sentence)\n",
    "    tokens = stemmer(tokens)\n",
    "    tokens = lemmatizer2(tokens)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "worker slipped while carrying groceries. worker fractured his elbow\n",
      "worker slip while carri groceries. worker fractur hi elbow\n",
      "worker slip while carri groceries. worker fractur hi elbow\n"
     ]
    }
   ],
   "source": [
    "tokenize(note1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Spacy Lemmatization**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize spacy 'en' model, keeping only tagger component needed for lemmatization\n",
    "nlp_spacy = spacy.load('en', disable=['parser', 'ner'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'worker slip while carry grocery . worker fracture -PRON- elbow'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc1 = nlp_spacy(note1.lower())\n",
    "# Extract the lemma for each token and join\n",
    "\" \".join([token.lemma_ for token in doc1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'worker develop carpal tunnel from repetitive typing'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc2 = nlp_spacy(note2.lower())\n",
    "# Extract the lemma for each token and join\n",
    "\" \".join([token.lemma_ for token in doc2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'worker get traumatize from nlp presentation'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc3 = nlp_spacy(note3.lower())\n",
    "# Extract the lemma for each token and join\n",
    "\" \".join([token.lemma_ for token in doc3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = [note1, note2, note3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**CountVectorizer**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>carpal</th>\n",
       "      <th>carrying</th>\n",
       "      <th>developed</th>\n",
       "      <th>elbow</th>\n",
       "      <th>fractured</th>\n",
       "      <th>from</th>\n",
       "      <th>got</th>\n",
       "      <th>groceries</th>\n",
       "      <th>his</th>\n",
       "      <th>nlp</th>\n",
       "      <th>presentation</th>\n",
       "      <th>repetitive</th>\n",
       "      <th>slipped</th>\n",
       "      <th>traumatized</th>\n",
       "      <th>tunnel</th>\n",
       "      <th>typing</th>\n",
       "      <th>while</th>\n",
       "      <th>worker</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>doc_1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       carpal  carrying  developed  elbow  fractured  from  got  groceries  \\\n",
       "doc_1       0         1          0      1          1     0    0          1   \n",
       "doc_2       1         0          1      0          0     1    0          0   \n",
       "doc_3       0         0          0      0          0     1    1          0   \n",
       "\n",
       "       his  nlp  presentation  repetitive  slipped  traumatized  tunnel  \\\n",
       "doc_1    1    0             0           0        1            0       0   \n",
       "doc_2    0    0             0           1        0            0       1   \n",
       "doc_3    0    1             1           0        0            1       0   \n",
       "\n",
       "       typing  while  worker  \n",
       "doc_1       0      1       2  \n",
       "doc_2       1      0       1  \n",
       "doc_3       0      0       1  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countvec = CountVectorizer()\n",
    "X = countvec.fit_transform(corpus)\n",
    "header = countvec.get_feature_names()\n",
    "idx = ['doc_1', 'doc_2', 'doc_3']\n",
    "countvec = X.toarray()\n",
    "\n",
    "countvec = pd.DataFrame(countvec, columns = header, index = idx)\n",
    "countvec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "countvec.to_excel('Count_Vectorizer.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**TFIDF**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>carpal</th>\n",
       "      <th>carrying</th>\n",
       "      <th>developed</th>\n",
       "      <th>elbow</th>\n",
       "      <th>fractured</th>\n",
       "      <th>from</th>\n",
       "      <th>got</th>\n",
       "      <th>groceries</th>\n",
       "      <th>his</th>\n",
       "      <th>nlp</th>\n",
       "      <th>presentation</th>\n",
       "      <th>repetitive</th>\n",
       "      <th>slipped</th>\n",
       "      <th>traumatized</th>\n",
       "      <th>tunnel</th>\n",
       "      <th>typing</th>\n",
       "      <th>while</th>\n",
       "      <th>worker</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>doc_1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.345129</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.345129</td>\n",
       "      <td>0.345129</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.345129</td>\n",
       "      <td>0.345129</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.345129</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.345129</td>\n",
       "      <td>0.407678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_2</th>\n",
       "      <td>0.410747</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.410747</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.312384</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.410747</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.410747</td>\n",
       "      <td>0.410747</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.242594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.342620</td>\n",
       "      <td>0.450504</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.450504</td>\n",
       "      <td>0.450504</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.450504</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.266075</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         carpal  carrying  developed     elbow  fractured      from       got  \\\n",
       "doc_1  0.000000  0.345129   0.000000  0.345129   0.345129  0.000000  0.000000   \n",
       "doc_2  0.410747  0.000000   0.410747  0.000000   0.000000  0.312384  0.000000   \n",
       "doc_3  0.000000  0.000000   0.000000  0.000000   0.000000  0.342620  0.450504   \n",
       "\n",
       "       groceries       his       nlp  presentation  repetitive   slipped  \\\n",
       "doc_1   0.345129  0.345129  0.000000      0.000000    0.000000  0.345129   \n",
       "doc_2   0.000000  0.000000  0.000000      0.000000    0.410747  0.000000   \n",
       "doc_3   0.000000  0.000000  0.450504      0.450504    0.000000  0.000000   \n",
       "\n",
       "       traumatized    tunnel    typing     while    worker  \n",
       "doc_1     0.000000  0.000000  0.000000  0.345129  0.407678  \n",
       "doc_2     0.000000  0.410747  0.410747  0.000000  0.242594  \n",
       "doc_3     0.450504  0.000000  0.000000  0.000000  0.266075  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_vec = TfidfVectorizer()\n",
    "Y = tfidf_vec.fit_transform(corpus)\n",
    "header = tfidf_vec.get_feature_names()\n",
    "idx = ['doc_1', 'doc_2', 'doc_3']\n",
    "tfidf_vec = Y.toarray()\n",
    "\n",
    "tfidf_vec = pd.DataFrame(tfidf_vec, columns = header, index = idx)\n",
    "tfidf_vec\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vec.to_excel('TFIDF_Vectorizer.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "calculating for word \"worker\" in note_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>carpal</th>\n",
       "      <th>carrying</th>\n",
       "      <th>developed</th>\n",
       "      <th>elbow</th>\n",
       "      <th>fractured</th>\n",
       "      <th>from</th>\n",
       "      <th>got</th>\n",
       "      <th>groceries</th>\n",
       "      <th>his</th>\n",
       "      <th>nlp</th>\n",
       "      <th>presentation</th>\n",
       "      <th>repetitive</th>\n",
       "      <th>slipped</th>\n",
       "      <th>traumatized</th>\n",
       "      <th>tunnel</th>\n",
       "      <th>typing</th>\n",
       "      <th>while</th>\n",
       "      <th>worker</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>doc_1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_2</th>\n",
       "      <td>1.693147</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.287682</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.287682</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.693147</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         carpal  carrying  developed     elbow  fractured      from       got  \\\n",
       "doc_1  0.000000  1.693147   0.000000  1.693147   1.693147  0.000000  0.000000   \n",
       "doc_2  1.693147  0.000000   1.693147  0.000000   0.000000  1.287682  0.000000   \n",
       "doc_3  0.000000  0.000000   0.000000  0.000000   0.000000  1.287682  1.693147   \n",
       "\n",
       "       groceries       his       nlp  presentation  repetitive   slipped  \\\n",
       "doc_1   1.693147  1.693147  0.000000      0.000000    0.000000  1.693147   \n",
       "doc_2   0.000000  0.000000  0.000000      0.000000    1.693147  0.000000   \n",
       "doc_3   0.000000  0.000000  1.693147      1.693147    0.000000  0.000000   \n",
       "\n",
       "       traumatized    tunnel    typing     while  worker  \n",
       "doc_1     0.000000  0.000000  0.000000  1.693147     2.0  \n",
       "doc_2     0.000000  1.693147  1.693147  0.000000     1.0  \n",
       "doc_3     1.693147  0.000000  0.000000  0.000000     1.0  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_vec2 = TfidfVectorizer(norm= None)\n",
    "Y2 = tfidf_vec2.fit_transform(corpus)\n",
    "header = tfidf_vec2.get_feature_names()\n",
    "idx = ['doc_1', 'doc_2', 'doc_3']\n",
    "tfidf_vec2 = Y2.toarray()\n",
    "\n",
    "tfidf_vec2 = pd.DataFrame(tfidf_vec2, columns = header, index = idx)\n",
    "tfidf_vec2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4076776724611254\n"
     ]
    }
   ],
   "source": [
    "tf = 2\n",
    "N = 3\n",
    "df = 3\n",
    "idf = (np.log((N+1)/(1+df)))+1\n",
    "doc1 = tfidf_vec2.iloc[0,:]\n",
    "norm = np.linalg.norm(doc1)\n",
    "print(tf*idf/norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "carpal          0.000000\n",
       "carrying        0.345129\n",
       "developed       0.000000\n",
       "elbow           0.345129\n",
       "fractured       0.345129\n",
       "from            0.000000\n",
       "got             0.000000\n",
       "groceries       0.345129\n",
       "his             0.345129\n",
       "nlp             0.000000\n",
       "presentation    0.000000\n",
       "repetitive      0.000000\n",
       "slipped         0.345129\n",
       "traumatized     0.000000\n",
       "tunnel          0.000000\n",
       "typing          0.000000\n",
       "while           0.345129\n",
       "worker          0.407678\n",
       "Name: doc_1, dtype: float64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Cosine similarity**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#vec1 = np.array([2.3,0,1.5]).reshape(1,3)\n",
    "vec1 = np.array([[2.3,0,1.5]]) \n",
    "vec2 = np.array([5.4,2,0]).reshape(1,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.7854683]])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity(vec1, vec2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**LSA (Topic Modeling)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18, 3)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.T.shape # check shape of the term-document matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "svd_model = TruncatedSVD(n_components=2, algorithm='randomized', n_iter=100, random_state=42) ## n_componemts is the k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.12058087 0.95723259]\n"
     ]
    }
   ],
   "source": [
    "svd_model.fit(Y.T)\n",
    "print(svd_model.singular_values_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.49760883,  0.60774595,  0.61889443],\n",
       "       [ 0.86520243, -0.3985414 , -0.30428525]])"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VT = svd_model.components_\n",
    "VT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils.extmath import randomized_svd\n",
    "n = 2\n",
    "U, Sigma, VT = randomized_svd(Y.T, \n",
    "                              n_components=n,\n",
    "                              n_iter=100,\n",
    "                              random_state=42) # sklearn's implementation only shows the k largest singular values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.22276815, -0.17101342],\n",
       "       [ 0.15325919,  0.31194778],\n",
       "       [ 0.22276815, -0.17101342],\n",
       "       [ 0.15325919,  0.31194778],\n",
       "       [ 0.15325919,  0.31194778],\n",
       "       [ 0.35864919, -0.2389722 ],\n",
       "       [ 0.24881244, -0.14320631],\n",
       "       [ 0.15325919,  0.31194778],\n",
       "       [ 0.15325919,  0.31194778],\n",
       "       [ 0.24881244, -0.14320631],\n",
       "       [ 0.24881244, -0.14320631],\n",
       "       [ 0.22276815, -0.17101342],\n",
       "       [ 0.15325919,  0.31194778],\n",
       "       [ 0.24881244, -0.14320631],\n",
       "       [ 0.22276815, -0.17101342],\n",
       "       [ 0.22276815, -0.17101342],\n",
       "       [ 0.15325919,  0.31194778],\n",
       "       [ 0.45955778,  0.18289954]])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "U"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.12058087, 0.95723259])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.49760883,  0.60774595,  0.61889443],\n",
       "       [ 0.86520243, -0.3985414 , -0.30428525]])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VT"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}