(KDT)_TF_IDF_이해와_유사도_계산.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "(KDT) TF-IDF 이해와 유사도 계산",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/learn-programmers/programmers_kdt_II/blob/main/(KDT)_TF_IDF_%EC%9D%B4%ED%95%B4%EC%99%80_%EC%9C%A0%EC%82%AC%EB%8F%84_%EA%B3%84%EC%82%B0.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NqmkCypbPYj-"
      },
      "source": [
        "## 카운트 방식의 텍스트 유사도 계산"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uf6gpe7HCoDq"
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MMK46g_zC2VU"
      },
      "source": [
        "text = [\n",
        "   'The sky is blue.',             # ‘sky’, ‘blue’       \n",
        "   'The sun is bright.',          # ‘sun’, ‘bright’     \n",
        "   'The sun in the sky is bright',    # ‘sun’, ‘sky’, ‘bright’\n",
        "   'We can see the shining sun, the bright sun.'  # ‘ see’, ‘shining’, ‘sun’, ‘bright’ \n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZtMjZ-LSDmU4"
      },
      "source": [
        "countvectorizer = CountVectorizer(analyzer='word', stop_words='english')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sLMkEsj4C4yO",
        "outputId": "e48c9674-a8f2-42f8-99de-1af5c11c62f3"
      },
      "source": [
        "count_wm = countvectorizer.fit_transform(text)\n",
        "print(count_wm.shape)\n",
        "print(count_wm.todense())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(4, 5)\n",
            "[[1 0 0 1 0]\n",
            " [0 1 0 0 1]\n",
            " [0 1 0 1 1]\n",
            " [0 1 1 0 2]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 182
        },
        "id": "v8Cy3exlhc61",
        "outputId": "6e396ea9-3c1a-4722-cde5-41c06f8d2064"
      },
      "source": [
        "count_tokens = countvectorizer.get_feature_names()\n",
        "print(count_tokens)\n",
        "df_countvect = pd.DataFrame(data = count_wm.toarray(), index = ['Doc1','Doc2', 'Doc3', 'Doc4'], columns = count_tokens)\n",
        "df_countvect.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['blue', 'bright', 'shining', 'sky', 'sun']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>blue</th>\n",
              "      <th>bright</th>\n",
              "      <th>shining</th>\n",
              "      <th>sky</th>\n",
              "      <th>sun</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Doc1</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc2</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc3</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc4</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      blue  bright  shining  sky  sun\n",
              "Doc1     1       0        0    1    0\n",
              "Doc2     0       1        0    0    1\n",
              "Doc3     0       1        0    1    1\n",
              "Doc4     0       1        1    0    2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VJn-k9qqQVDN",
        "outputId": "69e4e290-7e10-4feb-dcb2-aff6dae8b58b"
      },
      "source": [
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "\n",
        "cosine_similarities = cosine_similarity(count_wm)\n",
        "print(cosine_similarities)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1.         0.         0.40824829 0.        ]\n",
            " [0.         1.         0.81649658 0.8660254 ]\n",
            " [0.40824829 0.81649658 1.         0.70710678]\n",
            " [0.         0.8660254  0.70710678 1.        ]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 166
        },
        "id": "awixZRU0QbQO",
        "outputId": "9dece9f1-5267-40f3-da61-a60523d94a2f"
      },
      "source": [
        "pd.DataFrame(data = cosine_similarities, index = ['Doc1','Doc2', 'Doc3', 'Doc4'], columns = ['Doc1','Doc2', 'Doc3', 'Doc4'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Doc1</th>\n",
              "      <th>Doc2</th>\n",
              "      <th>Doc3</th>\n",
              "      <th>Doc4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Doc1</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.408248</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc2</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.816497</td>\n",
              "      <td>0.866025</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc3</th>\n",
              "      <td>0.408248</td>\n",
              "      <td>0.816497</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.707107</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc4</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.866025</td>\n",
              "      <td>0.707107</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          Doc1      Doc2      Doc3      Doc4\n",
              "Doc1  1.000000  0.000000  0.408248  0.000000\n",
              "Doc2  0.000000  1.000000  0.816497  0.866025\n",
              "Doc3  0.408248  0.816497  1.000000  0.707107\n",
              "Doc4  0.000000  0.866025  0.707107  1.000000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ODwM3LQVQEwg"
      },
      "source": [
        "## TF-IDF 방식의 텍스트 유사도 계산"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CdK9ZQzHDNvM"
      },
      "source": [
        "TF-IDF = TF(t, d) * IDF(t)\n",
        "\n",
        "*   TF(t, d): 단어 t가 문서 d에서 몇번 나왔나?\n",
        "*   IDF(t): 단어가 t가 전체 문서들중 몇개의 문서에서 나왔는지? Ln(N/DF)\n",
        "    * N은 총 문서수를 나타내고 DF는 단어가 나온 문서를 말한다\n",
        "\n",
        "기본적인 아이디어는 전체 문서에서 빈도수가 낮은 단어가 특정 문서에서 많이 나오는 경우 해당 문서에서 해당 단어의 점수가 더 높게 계산됨\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WfBpE1-CPDtH"
      },
      "source": [
        "tfidfvectorizer = TfidfVectorizer(analyzer='word', stop_words='english')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jP_e0fYMTZBd",
        "outputId": "f6dd87f5-9915-42ce-b882-4a52bd3598c0"
      },
      "source": [
        "tfidf_wm = tfidfvectorizer.fit_transform(text)\n",
        "print(tfidf_wm.shape)\n",
        "print(tfidf_wm.todense())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(4, 5)\n",
            "[[0.78528828 0.         0.         0.6191303  0.        ]\n",
            " [0.         0.70710678 0.         0.         0.70710678]\n",
            " [0.         0.53256952 0.         0.65782931 0.53256952]\n",
            " [0.         0.36626037 0.57381765 0.         0.73252075]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 182
        },
        "id": "fnhEUPZvhmR-",
        "outputId": "1cb5a381-35be-4c4e-caf2-fe09b74a44e7"
      },
      "source": [
        "tfidf_tokens = tfidfvectorizer.get_feature_names()\n",
        "print(tfidf_tokens)\n",
        "df_tfidfvect = pd.DataFrame(data = tfidf_wm.toarray(), index = ['Doc1','Doc2', 'Doc3', 'Doc4'], columns = tfidf_tokens)\n",
        "df_tfidfvect.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['blue', 'bright', 'shining', 'sky', 'sun']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>blue</th>\n",
              "      <th>bright</th>\n",
              "      <th>shining</th>\n",
              "      <th>sky</th>\n",
              "      <th>sun</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Doc1</th>\n",
              "      <td>0.785288</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.619130</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc2</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.707107</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.707107</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc3</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.532570</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.657829</td>\n",
              "      <td>0.532570</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc4</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.366260</td>\n",
              "      <td>0.573818</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.732521</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          blue    bright   shining       sky       sun\n",
              "Doc1  0.785288  0.000000  0.000000  0.619130  0.000000\n",
              "Doc2  0.000000  0.707107  0.000000  0.000000  0.707107\n",
              "Doc3  0.000000  0.532570  0.000000  0.657829  0.532570\n",
              "Doc4  0.000000  0.366260  0.573818  0.000000  0.732521"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cgi1QKmhIaV5",
        "outputId": "8338a61f-dda5-4a08-f357-ab47154f3193"
      },
      "source": [
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "\n",
        "cosine_similarities = cosine_similarity(tfidf_wm)\n",
        "print(cosine_similarities)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1.         0.         0.40728206 0.        ]\n",
            " [0.         1.         0.75316704 0.77695558]\n",
            " [0.40728206 0.75316704 1.         0.58517734]\n",
            " [0.         0.77695558 0.58517734 1.        ]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nZ5NXXP2RgiJ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 166
        },
        "outputId": "606b154b-e1a0-4e7f-b6e7-7a3c07f78efd"
      },
      "source": [
        "pd.DataFrame(data = cosine_similarities, index = ['Doc1','Doc2', 'Doc3', 'Doc4'], columns = ['Doc1','Doc2', 'Doc3', 'Doc4'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Doc1</th>\n",
              "      <th>Doc2</th>\n",
              "      <th>Doc3</th>\n",
              "      <th>Doc4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Doc1</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.407282</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc2</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.753167</td>\n",
              "      <td>0.776956</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc3</th>\n",
              "      <td>0.407282</td>\n",
              "      <td>0.753167</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.585177</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Doc4</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.776956</td>\n",
              "      <td>0.585177</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          Doc1      Doc2      Doc3      Doc4\n",
              "Doc1  1.000000  0.000000  0.407282  0.000000\n",
              "Doc2  0.000000  1.000000  0.753167  0.776956\n",
              "Doc3  0.407282  0.753167  1.000000  0.585177\n",
              "Doc4  0.000000  0.776956  0.585177  1.000000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    }
  ]
}