TFIDF_Sklearn_Scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c3f65ee4",
   "metadata": {},
   "source": [
    "# Text Feature Extraction : TF-IDF Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "db65ee92",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import pandas as pd\n",
    "import math\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "62c148de",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = \"She is a wanderlust\", \"She is lovely\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a87db56f",
   "metadata": {},
   "source": [
    "## Sklearn TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "726a5980",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "13e88fdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit: creates dict of count\n",
    "# transform: tfidf calculated or features created\n",
    "vectors = vectorizer.fit_transform(corpus).todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b2ba0f1c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.        , 1.40546511, 1.        , 1.40546511])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.idf_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "199a6561",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['is', 'lovely', 'she', 'wanderlust']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9f65ba15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0.50154891, 0.        , 0.50154891, 0.70490949],\n",
       "        [0.50154891, 0.70490949, 0.50154891, 0.        ]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f6179777",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is</th>\n",
       "      <th>lovely</th>\n",
       "      <th>she</th>\n",
       "      <th>wanderlust</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.501549</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.501549</td>\n",
       "      <td>0.704909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.501549</td>\n",
       "      <td>0.704909</td>\n",
       "      <td>0.501549</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         is    lovely       she  wanderlust\n",
       "0  0.501549  0.000000  0.501549    0.704909\n",
       "1  0.501549  0.704909  0.501549    0.000000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data = vectors, columns=vectorizer.get_feature_names())\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7eb4685a",
   "metadata": {},
   "source": [
    "## Standard TFIDF"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9273cb7e",
   "metadata": {},
   "source": [
    "#### Term Frequency (TF): Word count in a doc / Words in a doc\n",
    "Creates a dictionary of word count via CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "584d08f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = list(map(str.lower, corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d6eea0ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer()\n",
    "count_occurrences = cv.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "eaf9fcc0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 1, 1],\n",
       "       [1, 1, 1, 0]], dtype=int64)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_occurrences.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b29586be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'is': 1, 'lovely': 0, 'she': 1, 'wanderlust': 1}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bagOfWords_1 = dict.fromkeys(cv.get_feature_names())\n",
    "for ind,key in enumerate(bagOfWords_1):\n",
    "    bagOfWords_1[key] = count_occurrences.toarray()[0][ind]\n",
    "bagOfWords_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "566f745c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'is': 1, 'lovely': 1, 'she': 1, 'wanderlust': 0}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bagOfWords_2 = dict.fromkeys(cv.get_feature_names())\n",
    "for ind,key in enumerate(bagOfWords_2):\n",
    "    bagOfWords_2[key] = count_occurrences.toarray()[1][ind]\n",
    "bagOfWords_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ee60027d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is</th>\n",
       "      <th>lovely</th>\n",
       "      <th>she</th>\n",
       "      <th>wanderlust</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   is  lovely  she  wanderlust\n",
       "0   1       0    1           1\n",
       "1   1       1    1           0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(data = count_occurrences.toarray(), columns= cv.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ad5cae21",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_tf(bow, doc):\n",
    "    tf_dict ={}\n",
    "    doc_count = len(doc)\n",
    "    for word, count in bow.items():\n",
    "        tf_dict[word] = count/doc_count\n",
    "    return tf_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "15feb8c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_doc1 = compute_tf(bagOfWords_1, corpus[0].split(' '))\n",
    "tf_doc2 = compute_tf(bagOfWords_2, corpus[1].split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "439d4e18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'is': 0.25, 'lovely': 0.0, 'she': 0.25, 'wanderlust': 0.25}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_doc1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "651829ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is</th>\n",
       "      <th>lovely</th>\n",
       "      <th>she</th>\n",
       "      <th>wanderlust</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         is    lovely       she  wanderlust\n",
       "0  0.250000  0.000000  0.250000        0.25\n",
       "1  0.333333  0.333333  0.333333        0.00"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf = pd.DataFrame([tf_doc1, tf_doc2])\n",
    "tf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ad8f18f",
   "metadata": {},
   "source": [
    "#### Inverse Document Frequency (IDF): log (Total docs / Docs with the word present)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6de1f423",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_idf(docs):    \n",
    "    N = len(docs)\n",
    "    idfDict = dict.fromkeys(docs[0].keys(),0)\n",
    "    for doc in docs:\n",
    "        for word, val in doc.items():\n",
    "            if val > 0:\n",
    "                idfDict[word] +=1 \n",
    "    \n",
    "    for word, val in idfDict.items():\n",
    "        # standard notation\n",
    "        idfDict[word] = math.log(N / float(val))\n",
    "        #sklearn notation\n",
    "        #idfDict[word] = (math.log((N+1) / (val+1))) + 1\n",
    "    return idfDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f5ff02c1",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'is': 0.0,\n",
       " 'lovely': 0.6931471805599453,\n",
       " 'she': 0.0,\n",
       " 'wanderlust': 0.6931471805599453}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idfs = compute_idf([bagOfWords_1,bagOfWords_2])\n",
    "idfs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "867c6e72",
   "metadata": {},
   "source": [
    "#### Dot product of both: TF * IDF "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c8056cac",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_tfidf(tf,idf):\n",
    "    tfidf = {}\n",
    "    for word, tfVal in tf.items():\n",
    "        tfidf[word] = tfVal * idf[word]\n",
    "    return tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "71a5f8d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_doc1 = compute_tfidf(tf_doc1, idfs)\n",
    "tfidf_doc2 = compute_tfidf(tf_doc2, idfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c80f978c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'is': 0.0, 'lovely': 0.0, 'she': 0.0, 'wanderlust': 0.17328679513998632}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_doc1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "75f0c724",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is</th>\n",
       "      <th>lovely</th>\n",
       "      <th>she</th>\n",
       "      <th>wanderlust</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.173287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.231049</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    is    lovely  she  wanderlust\n",
       "0  0.0  0.000000  0.0    0.173287\n",
       "1  0.0  0.231049  0.0    0.000000"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame([tfidf_doc1, tfidf_doc2])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f006e202",
   "metadata": {},
   "source": [
    "- Clearly, standard TF-IDF is different than the Sklearn's. \n",
    "- Though the TF term remains the same while IDF term differs. \n",
    "- Sklearn adds smoothing and L2-normalization to the result\n",
    "- Thus, after smoothing the idf term, the dot product of TF & IDF value is L2-normalized (Eucledian)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f58e9ad2",
   "metadata": {},
   "source": [
    "# END"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}