new_work_times_project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "new work times project.ipynb",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/threegenie/knn_project/blob/main/new_work_times_project.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aM1RotxEYdrT",
        "outputId": "990b24ef-6f8f-4759-8b30-16cdb32a154e"
      },
      "source": [
        "import pandas as pd\n",
        "import os, sys\n",
        "import numpy as np\n",
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/drive')\n",
        "# df1 = pd.read_csv('/content/drive/My Drive/df_2016.csv',encoding= 'unicode_escape')\n",
        "# df2 = pd.read_csv('/content/drive/My Drive/df_2017.csv',encoding= 'unicode_escape')\n",
        "# df3 = pd.read_csv('/content/drive/My Drive/df_2018.csv',encoding= 'unicode_escape')\n",
        "# df4 = pd.read_csv('/content/drive/My Drive/df_2019.csv',encoding= 'unicode_escape')\n",
        "df = pd.read_csv('/content/drive/My Drive/df_2020.csv',encoding= 'unicode_escape')\n",
        "\n",
        "#원래는 2016~2020 New York Times 기사를 전부 쓰고 싶었으나..... 시간적 한계로 2020년 데이터만 사용 ㅠㅠ "
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "Xa8KHt28Zdgn",
        "outputId": "669dd24e-480a-4d1e-cfba-bfaa2ff66c65"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>year</th>\n",
              "      <th>sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>2020</td>\n",
              "      <td>The fugitive from Japan has put Lebanon in an ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>2020</td>\n",
              "      <td>Mixed reaction to the news that Maj. Gen. Qass...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>2020</td>\n",
              "      <td>In bronze, silver or aluminum, a statue is an ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>2020</td>\n",
              "      <td>Also this week, holiday fireworks in Iceland a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>2020</td>\n",
              "      <td>The teenager, accused in the death of the Barn...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0  year                                           sentence\n",
              "0           0  2020  The fugitive from Japan has put Lebanon in an ...\n",
              "1           1  2020  Mixed reaction to the news that Maj. Gen. Qass...\n",
              "2           2  2020  In bronze, silver or aluminum, a statue is an ...\n",
              "3           3  2020  Also this week, holiday fireworks in Iceland a...\n",
              "4           4  2020  The teenager, accused in the death of the Barn..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AOssle3nZXKv",
        "outputId": "2ce0a65d-6dc6-495a-ea03-6af07d9e20d4"
      },
      "source": [
        "df.info()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 69119 entries, 0 to 69118\n",
            "Data columns (total 3 columns):\n",
            " #   Column      Non-Null Count  Dtype \n",
            "---  ------      --------------  ----- \n",
            " 0   Unnamed: 0  69119 non-null  int64 \n",
            " 1   year        69119 non-null  int64 \n",
            " 2   sentence    69119 non-null  object\n",
            "dtypes: int64(2), object(1)\n",
            "memory usage: 1.6+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6klntVdoREjo"
      },
      "source": [
        "# 분석에 필요없는 열 제거\n",
        "df = df.drop(['Unnamed: 0','year'],axis=1)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CFBB_Q5VyvlN"
      },
      "source": [
        "# 중복 기사 제거\n",
        "df = df.drop_duplicates('sentence',keep='first')"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "a0NH4h-yRV7M",
        "outputId": "354b97b7-1e5a-48f8-f6f5-90e060553ace"
      },
      "source": [
        "# 제거 결과 확인\n",
        "df.head()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The fugitive from Japan has put Lebanon in an ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Mixed reaction to the news that Maj. Gen. Qass...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>In bronze, silver or aluminum, a statue is an ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Also this week, holiday fireworks in Iceland a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The teenager, accused in the death of the Barn...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            sentence\n",
              "0  The fugitive from Japan has put Lebanon in an ...\n",
              "1  Mixed reaction to the news that Maj. Gen. Qass...\n",
              "2  In bronze, silver or aluminum, a statue is an ...\n",
              "3  Also this week, holiday fireworks in Iceland a...\n",
              "4  The teenager, accused in the death of the Barn..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MA_5HU8ayxNw",
        "outputId": "3afb43c3-fad0-446f-a94a-672e094881b0"
      },
      "source": [
        "df.info()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 42420 entries, 0 to 69113\n",
            "Data columns (total 1 columns):\n",
            " #   Column    Non-Null Count  Dtype \n",
            "---  ------    --------------  ----- \n",
            " 0   sentence  42420 non-null  object\n",
            "dtypes: object(1)\n",
            "memory usage: 662.8+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lnmJiEcbQdHj"
      },
      "source": [
        "#### 토큰 정제"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZU-UAgJyZdBP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "34435633-dd2e-4788-9022-d75c0eff98c1"
      },
      "source": [
        "!pip install squarify"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting squarify\n",
            "  Downloading https://files.pythonhosted.org/packages/0b/2b/2e77c35326efec19819cd1d729540d4d235e6c2a3f37658288a363a67da5/squarify-0.4.3-py3-none-any.whl\n",
            "Installing collected packages: squarify\n",
            "Successfully installed squarify-0.4.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X0jKPfscQfNE"
      },
      "source": [
        "import re\n",
        "import squarify\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import spacy\n",
        "from spacy.tokenizer import Tokenizer"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tCjjxvRmQo56",
        "outputId": "6b578509-06e6-42c2-e10f-717c09205aa5"
      },
      "source": [
        "#기본 불용어 불러오기\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "tokenizer = Tokenizer(nlp.vocab)\n",
        "print(nlp.Defaults.stop_words)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'yourself', 'five', 'down', '‘s', 'always', 'someone', 'whole', 'unless', 'upon', 'used', 'almost', 'much', 'least', 'yet', 'us', 'else', 'for', 'part', 'afterwards', 'them', 'make', 'can', 'per', 'now', 'four', 'became', 'below', 'whose', 'must', 'is', 'be', 'same', 'being', 'amount', 'front', 'after', 'only', 'various', 'how', 'did', 're', 'nowhere', 'hence', 'could', 'any', 'throughout', 'by', '’ll', 'again', 'through', 'why', 'sixty', 'indeed', 'too', 'his', 'within', 'or', 'and', 'myself', \"'m\", 'this', 'hereafter', 'because', 'regarding', 'over', 'anyway', 'thence', 'never', 'she', 'call', 'there', '‘ll', \"'ll\", 'all', 'as', 'eight', 'although', 'full', \"'d\", 'elsewhere', '’m', 'itself', 'two', 'without', '’s', 'who', 'at', 'meanwhile', 'others', 'before', 'more', 'via', 'ten', 'take', 'ourselves', 'back', 'toward', 'to', 'nine', 'whether', 'whereas', 'together', 'bottom', 'therefore', 'third', 'go', 'amongst', 'empty', 'another', 'eleven', 'until', 'whence', 'still', 'except', 'using', 'however', 'first', 'such', 'own', 'onto', 'seeming', 'keep', 'done', 'rather', 'during', 'besides', 'has', 'perhaps', 'would', \"'re\", 'will', 'these', 'when', 'latter', 'show', 'really', 'anyhow', 'so', 'whereupon', 'beside', 'then', 'name', 'somewhere', 'side', 'no', 'are', \"n't\", 'well', 'thereupon', 'against', 'than', 'few', 'forty', 'from', 'above', 'otherwise', 'hereupon', 'last', 'alone', 'fifteen', 'get', 'where', 'under', 'made', 'sometime', 'into', 'had', 'some', 'nobody', 'around', 'a', 'but', 'between', 'also', 'an', 'our', 'fifty', 'that', 'latterly', 'on', 'nor', 'does', 'what', '‘re', 'moreover', 'herein', 'very', 'whoever', 'six', 'ever', 'he', 'thus', 'please', 'their', 'if', 'your', 'seems', 'becoming', 'whatever', 'mine', 'see', 'yourselves', 'further', 'twelve', 'three', 'formerly', 'here', 'becomes', 'one', 'former', 'namely', 'you', '‘d', 'everything', '’re', 'less', 'towards', 'should', 'already', 'across', 'out', 'they', 'we', 'every', 'often', 'anything', 'doing', 'those', 'my', 'do', 'herself', 'anyone', 'with', 'were', 'of', 'once', 'something', 'say', 'nevertheless', 'her', 'give', 'not', 'whereby', 'among', 'have', \"'ve\", 'mostly', 'was', 'twenty', 'each', 'n’t', 'n‘t', 'due', 'its', 'move', 'off', 'just', 'whither', 'nothing', 'in', 'top', 'many', 'other', 'neither', '‘ve', 'ca', 'whereafter', 'everywhere', 'sometimes', 'the', 'put', 'yours', 'hers', 'behind', 'thereby', 'thereafter', 'several', 'about', 'up', 'none', 'since', '‘m', 'while', 'enough', 'cannot', 'whom', 'even', 'himself', 'most', '’ve', 'serious', 'may', 'me', 'hereby', 'wherein', 'been', 'both', 'i', 'anywhere', 'though', 'become', 'seemed', 'thru', 'which', 'next', 'noone', 'everyone', 'themselves', 'either', 'therein', 'am', \"'s\", 'somehow', '’d', 'wherever', 'along', 'hundred', 'quite', 'whenever', 'beyond', 'him', 'beforehand', 'might', 'ours', 'seem', 'it'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ukKWGqVMQqKZ"
      },
      "source": [
        "#불용어 추가하기\n",
        "st_words = nlp.Defaults.stop_words.union(['this', 'i', 'it', 'is', 'in', 'and', 'a', 'the', 'but', 'to', 'the',\n",
        "                                          'said','york','u.s.','not','day','date','year','time'])"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1K1uKAmHPBU7"
      },
      "source": [
        "def tokenize(text):\n",
        "  text = text.lower()\n",
        "  text = text.replace(\"\\n\",\" \")\n",
        "\n",
        "  subst = \"\"\n",
        "  regex = \"[^a-zA-Z0-9 ]\"\n",
        "  doc = re.sub(regex, subst, text)\n",
        "\n",
        "  doc = nlp(doc)\n",
        "  tokens = []\n",
        "\n",
        "  for token in doc :\n",
        "    if (token.text.lower() not in st_words) & (token.is_stop == False) & (token.is_punct == False) & (token.is_alpha):\n",
        "      tokens.append(token.lemma_)\n",
        "      \n",
        "  return tokens"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BwRoDK59PNhO",
        "outputId": "9595ba74-dafd-4e45-a16e-86339eb7714a"
      },
      "source": [
        "df['Tokens']=df['sentence'].apply(tokenize)\n",
        "df['Tokens'].head(3)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    [fugitive, japan, lebanon, awkward, position, ...\n",
              "1    [mixed, reaction, news, maj, gen, qassim, sule...\n",
              "2    [bronze, silver, aluminum, statue, idea, solid...\n",
              "Name: Tokens, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5mvZh1GpRAIT"
      },
      "source": [
        "#### 단어 빈도에 따라 시각화"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VeToevWmRCpk"
      },
      "source": [
        "def word_count(docs):\n",
        "   \n",
        "    # 전체 코퍼스에서 단어 빈도 카운트\n",
        "    word_counts = Counter()\n",
        "\n",
        "    # 단어가 존재하는 문서의 빈도 카운트, 단어가 한 번 이상 존재하면 +1\n",
        "    word_in_docs = Counter()\n",
        "\n",
        "    # 전체 문서의 갯수\n",
        "    total_docs = len(docs)\n",
        "\n",
        "    for doc in docs:\n",
        "        word_counts.update(doc)\n",
        "        word_in_docs.update(set(doc))\n",
        "\n",
        "    temp = zip(word_counts.keys(), word_counts.values())\n",
        "\n",
        "    wc = pd.DataFrame(temp, columns = ['word', 'count'])\n",
        "\n",
        "    # 단어의 순위\n",
        "    # method='first': 같은 값의 경우 먼저나온 요소를 우선\n",
        "    wc['rank'] = wc['count'].rank(method='first', ascending=False)\n",
        "    total = wc['count'].sum()\n",
        "\n",
        "    # 코퍼스 내 단어의 비율\n",
        "    wc['percent'] = wc['count'].apply(lambda x: x / total)\n",
        "\n",
        "    wc = wc.sort_values(by='rank')\n",
        "\n",
        "    # 누적 비율\n",
        "    # cumsum() : cumulative sum\n",
        "    wc['cul_percent'] = wc['percent'].cumsum()\n",
        "\n",
        "    temp2 = zip(word_in_docs.keys(), word_in_docs.values())\n",
        "    ac = pd.DataFrame(temp2, columns=['word', 'word_in_docs'])\n",
        "    wc = ac.merge(wc, on='word')\n",
        "    \n",
        "    # 전체 문서 중 존재하는 비율\n",
        "    wc['word_in_docs_percent'] = wc['word_in_docs'].apply(lambda x: x / total_docs)\n",
        "\n",
        "    return wc.sort_values(by='rank')"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IflUi3YTQ-1j",
        "outputId": "4e077859-2f35-42ca-c26b-340ca04ca6db"
      },
      "source": [
        "from collections import Counter\n",
        "word_counts = Counter()\n",
        "df['Tokens'].apply(lambda x: word_counts.update(x))\n",
        "word_counts.most_common(10)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('new', 7017),\n",
              " ('coronavirus', 6146),\n",
              " ('trump', 5318),\n",
              " ('president', 4088),\n",
              " ('pandemic', 3023),\n",
              " ('not', 3016),\n",
              " ('state', 2111),\n",
              " ('people', 2090),\n",
              " ('know', 2031),\n",
              " ('need', 2008)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "GK0GLE8nREaB",
        "outputId": "5fa13538-26ea-4dcd-c7e4-7f008520dd1f"
      },
      "source": [
        "wc = word_count(df['Tokens'])\n",
        "wc.head(5)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>word_in_docs</th>\n",
              "      <th>count</th>\n",
              "      <th>rank</th>\n",
              "      <th>percent</th>\n",
              "      <th>cul_percent</th>\n",
              "      <th>word_in_docs_percent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>new</td>\n",
              "      <td>6208</td>\n",
              "      <td>7017</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.009849</td>\n",
              "      <td>0.009849</td>\n",
              "      <td>0.146346</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5481</th>\n",
              "      <td>coronavirus</td>\n",
              "      <td>5754</td>\n",
              "      <td>6146</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.008626</td>\n",
              "      <td>0.018475</td>\n",
              "      <td>0.135644</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>496</th>\n",
              "      <td>trump</td>\n",
              "      <td>4288</td>\n",
              "      <td>5318</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.007464</td>\n",
              "      <td>0.025939</td>\n",
              "      <td>0.101084</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>818</th>\n",
              "      <td>president</td>\n",
              "      <td>3711</td>\n",
              "      <td>4088</td>\n",
              "      <td>4.0</td>\n",
              "      <td>0.005738</td>\n",
              "      <td>0.031677</td>\n",
              "      <td>0.087482</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10599</th>\n",
              "      <td>pandemic</td>\n",
              "      <td>2937</td>\n",
              "      <td>3023</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.004243</td>\n",
              "      <td>0.035920</td>\n",
              "      <td>0.069236</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              word  word_in_docs  ...  cul_percent  word_in_docs_percent\n",
              "35             new          6208  ...     0.009849              0.146346\n",
              "5481   coronavirus          5754  ...     0.018475              0.135644\n",
              "496          trump          4288  ...     0.025939              0.101084\n",
              "818      president          3711  ...     0.031677              0.087482\n",
              "10599     pandemic          2937  ...     0.035920              0.069236\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 264
        },
        "id": "hr4WIVLiRKZx",
        "outputId": "538b66be-90fb-479f-f8d8-02ff56e3fc99"
      },
      "source": [
        "import squarify\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "colors=['springgreen','violet','dodgerblue','gold','wheat','lightcoral',\n",
        "        'mediumslateblue','greenyellow','aquamarine','deeppink']\n",
        "\n",
        "wc_top5 = wc[wc['rank'] <= 10]\n",
        "squarify.plot(sizes=wc_top5['percent'], label=wc_top5['word'], alpha=0.6, color=colors)\n",
        "plt.axis('off')\n",
        "plt.title('Token Top 10')\n",
        "plt.show()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD3CAYAAAC+eIeLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUVeLG8fdMS0gmvVdCCDUBpIhKDYLSQZEmUVdERV0VRBF/ijR3kXWRshZEmnRpAREQQU2kBARMgITQS0iZ9IT0TLu/P0jYbKQIwpkA7+d58jzOvWfuPXfA73NyJ0yEoiggIiI5VLaeABHR/YTRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0aXbTggRKYRIs/U8iOoiRpeuSwhRUuPLKoQor/E4yobzOlZjHhYhREWNx+/fpnN0E0LECCEuCSEuXGV/SNX+MiHECSFEj9txXrq3Mbp0XYqi6Ku/AFwE0L/GtpU2nFd4jXntBvB6jXlNv02nKQWwGMD4a+xfDSABgAeADwCsF0J43aZz0z2K0aVbIoSwE0LMEUJkVH3NEULYXWPsm0KIZCFEYNXzZgohLgohsoQQXwkh6lWNixRCpAkh3hZCZAshDEKIkTc5L5UQYqIQIqXqGMuEEC5V+0KEEIoQ4uWqORuEEO9c61iKohxQFGU5gHNXOU9jAG0ATFYUpVxRlA0AEgE8dTPzpfsPo0u36gMADwN4AEArAO0BTKw9SAgxCcDzALoqipIGYAaAxlXPCwMQAGBSjaf4AnCp2j4KwBdCCLebmNfzVV/dAIQC0AP4vNaYbgAaAXgcwIRbvC0QDuCcoijFNbYdqdpOdE2MLt2qKADTFEXJVhQlB8BUAM/W2C+EELNwOWzdFEXJEUIIAC8DeEtRlPyqYE0HMLzG80xVxzUpirINQAmAJjc5r1mKopxTFKUEwP8BGC6E0NQYM1VRlFJFURIBLAHw9E1d+WV6AJdqbbsEwOkWjkX3Ec2NhxBdlT+AlBqPU6q2VXPF5cAOUxSlOk5eABwA/H65vwAAAUBd43l5iqKYazwuw+XA/ZV5aQD41NiWWmt/i5s4frUSAM61tjkDKL7KWKIruNKlW5UBoH6Nx8FV26oVAOgHYIkQomPVtlwA5QDCFUVxrfpyqXoz7E7Oywwgq8a2oOvM+886BiBUCFFzZduqajvRNTG6dKtWA5gohPASQnji8n3ZFTUHKIoSi8vf7kcLIdorimIFsADAbCGENwAIIQKEED1v87zeEkI0EELocfn2xZpaq+cPhRAOQohwACMBrLnagarelLMHoL38UNgLIXRV13YKwGEAk6u2PwmgJYANt/Fa6B7E6NKt+geAQwCO4vK79vFV2/6Hoig7AbwA4HshRBsAEwCcAbBfCFEE4Cfc3D3bG1kMYDmAXQDOA6gA8EatMb9WzeFnADMVRdlxjWN1weWV+TZcXhGXA6g5djiAdri8qp8BYHDV/W2iaxL8EHO6XwghQnA5xNpaK18iabjSJSKSiNElIpKItxeIiCTiSpeISKLr/uOIgo8K5suaCMmnsv7H1lO4b8yePNjWU7CJKV4xtp6CbeS8Pvpau7jSJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJbrLGOKP62cHdR9i63ncS7zy3htxwZJnL+NcjC7d9T6OiWl8MifHwdbzIPozNLaeANFftSk5uUlrf//8Jl5eZbaei2wZB5Oc1g4Z93ijPl3OGBKSfSwVRk2xIce5ftd254es+fQ3ANjz8cKGB79c3RqACOrY+uLgb2f+FjdzSWjaviM+QzfM2bd97IyIxFVbW4zP3r0642CSU/Qz7z36+skt39n40q4q3pyq7180v08LtX9mkiXDx0M4lu5yGfvjSUu24+iS1Z0uKRX2dkJj/lr/9K7O2oaF5yy59lHFSztnW4v1APCx44C4oXZtslIs+XYDiuZ3z1NKHcPVflmKxGtgdKnOOWIw6IetWtUn3Ns7Mzk728fDwaF028iRPx5MS3Mdv21b50qzWePn5FS0bNiw2OikpIDz+fleY77/vrtu2zbz3ldf3eRsb2+x9TXIcHFvvEt01Hs9+n89JSY1LsGz4Gyqx6tHNm7QOtazzA3tOSwr8VSSSqNR4mYueeilg2uinQK8K79uO7Tvb3NXhDQZ0M1waN6aVgCQfiDRz87JsSLn2FmHU9t2+fm1aWaw9bVdT6a1yGWRPurnXrpmuzoVzu7xWcWvoWsq45t8rX961yPaBkUbK494v1qyplOS2/tbRhav6DC2XrfEYXZtMo+Y0/V9iub1GWrXZu2Y0vVt22qCMxc7RcXPK98dvMN0oqms+TO6VCdll5S4fDFw4M/dw8J29Vy0qMf8AwdCvzl0qNWUHj32DmnZ0vBKdHS7d7dta7ti+PB9yxMSIqb26LGvV5MmubaetyyVRSX264a83fOpVZ/sCIl8sDA1LsHTr03zdL2fpxEAXIJ9C3KSzzmVZuXZ+bRqYnALDawAgKZPdj+dsuuQ30NjnrlgqqjUlubka0uz8x0b9e165uTmGL+0uMO+TZ/scd62V3d9XkJf3EvXLA8AIjR+ORcs+fqTlmyf4cVLHqseY4ZVDQAJlrTAd0s3ub1bugkAUKYYdbnWEk2COc1vvdOLOwDg1XqdL44v3VQpa/6MLtVJng4Oxd3DwvIAoLmPT875/HznMpNJN6RlSwMAvPTQQ6dGrlvXw7aztB2NQz2jo5dbydkf9/qFRD5YCABqnfbKCl+oVIrVZBbXO4Z3eKOsPdMXNHEJ9rvU8LFHDIfmr22afeyMzxPfTN9/p+f/V2iF+sp1qqFScpQSewehrUxxn7ah9lhFARLd3t/krKo73/3wjTSqkzTqGv9jCaEUVVbqbDmfukatUVv/FrNkx/GNPzXaPX1B2LXGNXy8Q07W0VN+hRfS7S0mkzj53S9h9bs+mAEAQR1aGY4s29wyqMMDhoa9OuYZEo77q7UaS/Vq+W7hJOyM3sKpeGbZz6EAYFUU7DAedweANprAtPfKNkdUj91uPO4BAK01gYb5FXvCAGB++d6gUhjtZM2X0aW7gpNOZ3TU6YwbkpJ8AWDhgQONWvr5GQDAQas1FVZU3HdRrufuYv7bL4u3Jyza0KLyUslVr9+zaYOyDm8//9uSzs/1+09or8FezRvmPvRmVAoANOrTJbM8/5K+UZ8uBrVWqzh6upX4tGqSKfcqbo/Vzs//srLyYNP6+ZMGBxV8OPTbyvgQAFjq9GzcUXO6Z3D+pMEB+ROHflYe2xwA5joO/v2gOcUvMH/ikI3GIw08hGOJrLkKRbn2+3YFHxXMlzURkk9l/Y+tp3BVRwwG/dOrV/dOHjduHQC8s3Vry1KjUTu4RYsL1W+k+To5FS0fNizW39nZ+MW+fQ3+s3dve51aXWffSJs9ebCtp2ATU7xibD0F28h5ffS1djG697G6Gt17EaN7n7lOdHl7gYhIIkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISCJG9xZsP7ndM2pVVAdbz4OI7j733Ec7VporhZ3G7o5+EHyvJr1yezXp9YfPbpVxbiK6u9Xp6H4S80mj5fHLWwkIhLiH5P2j5z8OvhL9SmRxZbG9s51zxfzB82MjfCNKnvjmiUg7tZ3lTN4ZjwjfiKzn2j53asLWCZ0rLZUaXyffoqXDl8b6O/sbH/nskf7NfZpnJ6Qn+JeZynTTe0//dVCLQZlHM47qX1r/0qOV5koNAEztOXXvwPCBWX0W9uk+uOXg0y+0f+EiADzxzRORPRr1SPHWe1fM2zevVcwrMdv/Hv33tqmXUp0zizKdvfReJZ0bdE49ajjqtSpq1V4A6PZVt16vPvLqkScjnswcsnxI1zO5Z7wAKP2a9Ts5o++MRBu+vERkA3X29sLu87vdFh9c3GbrC1u3HH376Pp5g+bFjd08ttPA8IGnEt9OXN+3Wd/TY78be+Vb/OzSbMf9b+z/bunwpfvGbh7bbcKjE35LfDtxfZhHWP6ErRPaVo8zW80i/q34jRO6TYibtWtWWwAIdguu+Gn0T1sPjzscvWDIgp+m7JjSEQAGhA84u/nY5lAAKDOWqY4ajgY81/a5i7XnerHgottPo3/asnXU1p+vdT0xZ2I88krzHJPeSVqX9E7S+rGdx568na8XEd0d6mx0fzz5o3+X0C7ngt2CKwAgwCWg8nTuae8xncacAYBxXcadPplz0rd6fK8mvc5p1VolqzhLV2Ys0w1pOaTqNwy8dCrRkOhXPe7JiCfPA0DX0K65OSU5TgBQaa5URa2K6hIxM2LwqLWjHssoynAFgKg2UamJmYn+pcZS1Yr4FUHhPuEGZ3vnP3xsYMcGHVOutr2mCL+IoqySLKeolVEdlx5aGuipv7s+KJqIbo86G92b5ahzNP2Zcfbay5+1qlFpFItiUQHAtJ3TWrg7uJcfHnd4/cExB6MtVosaAJzsnCwRvhGGVQmrgjYf29xwQPiAs1c7poPW4cq5NWqNYlWsV35NisliUgOAv7O/cd8b+9Z3bNAxY/nvy5sPXT60661fLRHdrepsdHs26Zmx69yu0NTCVDsASL+UbtfYs3HWZ3s/awgAc3bPCWvq3fQPn3Lv4+RjdNQ5GqMTo30BYNGBRY1a+rW87m83La4s1nk7epdpVBrM2jWrcc1oDgwfeHbtkbVNkrOT/aJaR6XeaN4N3RsWn80762G2mpGclex4Nu+sNwBcLLhob7FaxGsdXjs/6bFJB8/knvG8uVeEiO4FdfaNtM4NOheMfHBkQu+FvQeohEoJ9QjNnTNwzt7RG0ZHLvt9WavqN9Ku9tzZA2bHTNg6ofPkHZM1PnqfomVPL7vquGpvdHrj2AtrX3h8+8ntjR8MejBVp9aZq/eNaD0ibdKPk7o9GPTgBQedg/VG8x4YMTBz8cHFxS1mthga6BpYGOIWkgsAZ/LOOIzbPC6yOujjuow7cDOvBxHdG/ibI+5j/M0R8vA3R9xn+JsjiIjqBkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISKI6+8+Aie4l5lf9bT0Fm4jOWWjrKdjEILx+zX2M7n1M+1zbGw+i2+MTW0+A6greXiAikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0SUikojRJSKSiNElIpKI0aW7RtvHX+u/7ecDnn/1OCfPpDo8Onj8Y7djTnXFjr3/16KislBzu8bdD9ZMyWz88YDzHWWfl9Gl+06TsKCyX9b/e6et53E7JZ1e16LCWHTDmP7ZcXTn8MWn2+7wsbP6p0ZN7RPWICD37PkMz+BA74LoRZNj3vvnopZ7fkuqbzSZNM0b18+MXjx5t0qlQtvHX+sf3iQkOz7xtH9ZWYXuXx++9OuQ/l0yLxWXqgePmhZ5/mKmR6C/Z6HRZFZXn2Pp2h2Bn361oZ3ZbFb5ebsXrV3wYayHm7M5uG3UiEc7PXBm/+/Hg9RqlTJryiu7Js9c1j4zO9/52cE9jnw04fnjh4+d1Q996aPep+K+WWcymcXIsTMfOpBwIkgIoTzZp+PxGR+8eMyWr9+NlFcUaFZtGdSjrCJPryhW0SCw67mKyksOyzb16Wenc64YPSxuy5ptwzrlFpzytliN6pCArucGPPrl7zvjPoioPe7w8RWBcYfntLNazSonB9+iIb1XxjrYe5htfY0AcC6+TD+93/k+fo3tcg2nKz296+sK3t8aGnMyrtR12fiMR4wVitbBWVUx7tv6sQFN7csSthd5LHwjvbO5UtG4+WuLxq8PifUI1BrHhJ/o79/YLu9cfLmfYoXqxc8DYtsPdMmpea6sc5X2c6Iudr6UbdYDwDMf+8V1GOqadSeuiytduiMyswtcRz/T99iZ/UvXOjrYG6fOWtF84lsjjiX9umDjqbhv1lUaTZoFK38Irh5vsVhEUuyCje+PHRH37y/XtgWAaZ+uaG5vrzOf2b907aRxzxy6kJrpBQApaVn2cxdsbBMbPXNL8q5F0eFNQ3I++HhJy+pjBfp5lpzYs3hDy+ahhjcnfhH5/bKPdsZGz9y0YOW2drXnOfXT5c0ysvKcEmMXrD+5d8n6118YeEbG6/NXHD25OsihnkfZG88cXf/ms0nruj00KdHezqXsuSe2bRk9LG4LAPTuMuvg36MSov8+4vB6Q06C//m0WPfHOvwzqea4wuKL9vuPfN5m5JM7t7wedTja2yM85+d9k1ve6PwyFRjMrr1f8zy2MD18rZ2jyrjqA0P4krcyOr6/pcHOeeeaRXd62u3kwjfSHwSAr15O6zZsiu9v8y82X+/f2C5/4RtpbauPYyxXNPNTmm949hO/3QtfT4+sfZ7PR6Z26DfWK/HLs802vhsdsnPxW+ld79Q1caVLd4S7q1PJoL6dsgBg+BPdTn+9fGtE9La9xQtWbGtVaTRpSkvL7cMaBBQAuAgAT/XtfB4AunVolfuP2SudAODQkVN+Lz/bNwkAHu3UOj/I3ysPAH6MPeSdZsh17dBvzEAAMFss6uaNgq+sSqKe6n4BAJo3Cs4vK6/Uenm4mLw8XExajcZiyMrX1Zzn3oPHAkYO75lsp9MqABDo51V5Z1+Zvy7Ap13+3oRZj2zcOeqhxg36poSHDcqsPeZQ0oLQ5LObmimKRVVRecnBkHPYrUFgZH7NMWdTdnoXlaa7LlzfdSAAWK1mtZd7szuyurtVend1SfWKs+uzbqe/m5nTOjfV6D6p29m+AKBYIfTu6rICg0lXUWrVdYlyMwBA79c9T3067EKP6uN0HuF6BgA6P+2WufD1dG1+hul//h6cTygPXPZuhtuydzMAAMYyRVeUa9Y4e2pu+6qf0SUphACmz13V6ae1n0RHNA0pfXn87LaVRuOV2wX29joLAGg0asVqVa77HZiiAA+EN0zfsWbGz1fb71DPzgoAKpVK0Wk1lv/OQcBoMt31390F+ra/9OLgXzccPbk6eM+hTx48l/pLes39hpzDTkdOrm416qmYaGe9v3Hl909Ems0V6trHUaDA17Nl+nMDt171dawLhPjfx3YOKpNnkLbg81PNNtXcXmD434je6Di1HysKMCexyaZ6zmoL7rC7/i8g1U35hcX6TdvjvAFgzXexYW1aNMoEgKAAr4q8giJN7N4joTc6RrtWjQ1rvosNA4DYuCNuqRk5HgDQM7Jt9vHTF30OHj7pXHUuzf7fj7vcyjw7tY9I+2bNjuaVRpMAgDRDjt2tHEem3IJTDnY6Z3Ontu+cbhfx0pHcghOeGrW9sbwiXwsA5RX5Wo3azqx38DbmXzpXLyM7Iaj6uTXHhQU/lp2bf8InPeuQ8+XnFWjSMg/c0ut4pxTnWfT7NhR6A8CulYVhDR6ol1V2yWpfvc1YblUlxZS4uflpjfZ6lXHPtwW+ALD9y9xGoa0dDNXH2fNtYUMA2Lum0NfOQWV089Maa54ntE29tOXvGSKqHydsL/K4U9fElS7dEb7eboVfLf0+/J0p8yODArwKJo17JvlSUaldq0dHD3F11pc3bhiYfaNjTHr7meTBo6ZFhj38t6GB/l6FIUG+OQAQEuRbMWPii7Ejx8zsbjJffnPtrdFPHXy4bbNLNzvPiW9FnRg5ZqZreNcXh2jUauuTvTse//iDUXX6jbTUzP3ue37/98OAUFQqjfXxDtN3X8jY47Pmh6f71LNzKx09LG6Lu0to7n+WRwxzqOdR6uXW5Motg2YNnzhec1yPDv+M3fTzy92tVpMaAB5+4M2Dgb7tb/p1vFPc/DSFP3yeG77ozfRIr2BdQdTHfsfaP+GctuStjI7fjMvQWS0Qkc+5JUZ00xe8Mj8wZuEb6Z2Xv2vQuPppi97dEBJbfRyNTphfCUl+ymq5/EZa7fO8sTQ4bu4zFzuODk4ebLUoqpAH6hla93LefSeuSSiKcu2dmDL/TpyU6obS8394X+m2qPnTAXfkBHeh6Z9I/3HQOqHNvG63/Nxz8WX6GQMv9P46tflf+ns0JvxE/2dn+O1r198l968c52YMwuHR19rH2wtERBLx9gLddg+ENyzhKpf+qtA2DiV/dZULAHOPNf3+dsznduFKl4hIIkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISCJGl4hIIkaXiEgija0nQLbTbHM/W0/h/tHUNqdN6Slsc+JqJ2x7epu5zp83V7pERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEt0/0Y036BHw6VD0WtEFgbOGoNVXfXCpQo0D6c5oPb8PQucOQrMvBmB3iitMFgGvfz8NqwJkFOugmvoSVif6AgCafTEA+9KcbXw1RHSXun+iCwCZpS4Y+/AxpI1bByedEZ8dCMWozV3wZd89ODcmGtO778erWztBq1YQ6HwJv15ww6YTvmjglotfzvuhxKhCbpkjHgkssvWlENHdSWPrCUjl5VCMXmF5AIAI7xxcKNTjZK4Phq9/7MoYs0UNAGjnZ8APZ/yQUuiE19snYPmRZth6yoAmHjk2mTsR3RPur+hq1ZYr/61WKcgps4eDthIpYzf8YWz3UAPmHWqO3DJHLBhwCPMOPoCd5/zQPsAgc8pEdG+5v24v1OakM8LbsRgz40IBAFYF2HHWHQDQr3E2krN9IaDA2c6CRh652HiiOXqFMbpEdMvu7+gCwOqnfsHKxKaoP2cwgmYPxbdJIQAAvc4KT8cStPTJAgB0DMpEuUmLRxvk23C2RHSXE4qiXHsnpsyXOBeSLHjuFFtPge6wlJ7C1lO4PzVVRl9rF1e6REQSMbpERBIxukREEjG6REQSMbpERBIxunTXMV/K0BV8N765redBd86ET9Fs2pdoVHt7fDL0QZEYcqvHHTMdLQqKbPuPwhhduutYizN1ZfHfhtferpgr+fNRdZTJhJv6s/nX2zg+6TWcvt3zWPU9Wlwqtm10769/Bkz3hILoMQ9ZSrKc06fWf0qoNFaodRaVnb7SXJjm6j1669acBQN7B0xNXQcA+ev/3lKpLNV6RH3ze8bH4f21Pk1yjemH/WCq1LgNnRdTtPPj1ub88+72jXuc9Xx2xUFjWrw+++t+fbRejXPNOac91e71C7xf3hqjcnAz2/q666r4ZOj7vYI+jUOQe/oCPOsHoGDrfMQ07omh3R/B2d+OImD0MBzxckPlvxainckMlb83ir77ArGebjA//Tba7zqEELUK1odaIW3dHOx//v/Q1tEBpi8+xNGtsfB8bRoiAeChlkitPq/JBBE1Hu1/T4a/yQT1iP44NmMcjq/4Hn4fz0c7FydUXEiHW+MQ5P7yDX55+xNEFBTDofMz6OeiR0XS99hii9eLK12667gNmvubWu9TFDA5ZYNL76n7zXnnPN2HfhUX+JFhzY2eK9Q6a8CH56IdWg9Jzlv5t54eI5bs8Z94Zm1F8rbG5vwUOwCwFhlc9Z1eOxYwLX2tSudoLNw2kbcybsCQA9fXRuBY+i6sdXSAceJcNAcANxdUnP8J0cP7IH3mErTZ/y22nP8J0S0aI+fdmWiZkgG72ANokBqDtRdjsH7u+4ivfexXpyJyxtvYk/IL1tfcPukzNHXWw3h2BzYmb0X02h/Q7PdjcAKA82nw+Hoa4i7GYG1GNpzX/Qjf2e8hyc0JZbtXYIutggtwpUv3AK1Xo2xdUNviPzO2XstBFwBAG9gmX3M6pkDr07QMAFTOPsWmrON6tZN3pcrBvcSx9dAsAHB48NnTJbu/iABw9E7N/17g7oKSob2QBQDP9sfpL1YhAgBGD8VZANj2K7zTMuHa+ikMBACzGerwMGT5esKo08LS8yV07dMFKaOH4mLN42ZkQ1daDt3TfZEJAC8Nxel9hxEMAL8eROD5NLjX745QACgrh+7wcbjY2cHSqD5yIhqhFAAaXV6BOwGXj2FrjC7d9YTW/r/f+qt1CmC9cv9QMVWqa45VaetZAEAIlSLUWkuNPQospqrn1br9KHir+EZqv0TVj12dYQYABUDrZkjftQI/137usS2IXrEZARt2InTRekT82VWoogAfvYm9Lw5BWs3tK76Hn1aLK3+2ahUUs+Xm7infSby9QHcdlYOHSTGVa6+2T+vZsNxaXmRvzk+xs1aWqCrP/lr/Zo9vLcvTlx3Z4A0AZYdWhumC2tWJFVJdllcI/YYd8AaAlVsQ1i7if1eVvbsg+9gZ+Px2BM4AkF8ITVwCXHILoMnKhe6V4UhdNgP7LqTDo+bz/L1hdKwH45of4AsAi9b/9ycaItsjbcE6NC+vvNyxuAS45BdefyFpbw9jXiGu+ndHFq506a6jca9fqfULz0yfHDREaHRmlYNbefU+oa1n1T/yYnzmrPZPqhzdy9Tu9Qtv9vgqZ7/C4t2fhxdseDNS7R5c4Nrno+TbewX3Hj8vFH6+AuFv/gORwf4o+GgMktdsu3yLAQBCA1Ex+z3EjhiP7iYT1ADw7os46O4KU7/R6Gk0Qa0AGD8K+2ofe95kxL42DZHvzoTycMv/rmo/GoPjKRnQN+6FQVAgXJxQvnMxdlxvnkN64ni/V9DHwxWltrqvy08Zu4/xU8b+yJgWr6/50w93OxmfMhafDP3A19A7NRb3xGt2WydUSbwAAAECSURBVPBTxoiI6gZGl6gGXWCbkntllStLm+Yo4Sr3z2N0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkEoqi2HoORET3Da50iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJPp/gqCF13NPHOYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "NUHmxIeXQQr9",
        "outputId": "30227011-4518-4136-f342-8e104b99e7eb"
      },
      "source": [
        "sns.set_palette('GnBu')\n",
        "sns.barplot(x=wc['count'][:10],y=wc['word'][:10],orient='h')\n",
        "plt.title('Token Top 10')\n",
        "plt.show()"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAawAAAEWCAYAAAA6maO/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de7xd07338c9XQoiQhKgHxXZXUSLCSUpSUVX6eNqjaF1al/actKoUjx6cOsV59ULrpap1aahLUXUSPBztcSmJRFqXnftNUNJSl7jk5haS/J4/5tjMLGsnK8lae6251vf9eq3XGnPMscb8zW3Jb48x555DEYGZmVmjW6feAZiZmVXCCcvMzArBCcvMzArBCcvMzArBCcvMzArBCcvMzArBCcusgUg6QNIL9Y7DrBE5YZnViKQ3c6/lkt7JbR9Xx7hm5uJYJund3Pa/V+kYwyWNkbRQ0twy+9vS/rclPSnpoGoc15qbE5ZZjUREr44X8Hfg/+TqbqljXP1zcY0HvpOL68dVOsxbwHXA9zrZfyswGdgU+D4wWtJmVTq2NSknLLMuJqmHpMskvZhel0nq0Unb0yTNkvTx9LlLJP1d0iuSrpa0QWp3gKQXJP1fSfMkvSTppNWMax1J50n6W+rjt5J6p31tkkLSiBTzS5LO6qyviHg8Im4Cni1znJ2BgcD5EfFORNwOTAeOWJ14rfU4YZl1ve8Dg4EBwJ7AvsB5pY0k/QA4Efh0RLwAXATsnD63I7AV8IPcR/4X0DvVfwO4QlLf1YjrxPQaDmwP9AJ+VdJmOLATcDBw9hpO5fUHno2Ixbm6qanerFNOWGZd7zjgPyNiXkS8ClwIfC23X5IuJUsKwyPiVUkCRgBnRMQb6R/7HwNH5z73fur3/Yj4I/AmsMtqxnVpRDwbEW8C5wJHS+qea3NhRLwVEdOB64FjVuvMM72AhSV1C4GN1qAvayHdV93EzKpsS+Bvue2/pboOfciS01ciouMf9s2AnsDELHcBIKBb7nOvR8TS3PbbZMlhbeLqDmyeq3u+ZP8nV6P/Dm8CG5fUbQwsLtPW7AMeYZl1vReBbXPb26S6DvOBw4DrJe2X6l4D3gH6R0Sf9OqdbpyoZVxLgVdydVuvJO5KzQS2l5QfUe2Z6s065YRl1vVuBc6TtJmkfmTXoW7ON4iIsWRTdHdI2jcilgPXAD+X9DEASVtJ+lyV4zpD0naSepFNOd5WMmr7D0k9JfUHTgJuK9dRuoFjfWDdbFPrS1ovndtTwBTg/FR/OLAHcHsVz8WakBOWWdf7IdAOTCO7O25SqltBRDwAfB34b0kDgbOBZ4BHJS0C/sTqXaNaleuAm4BxwHPAu8CpJW0eTjE8CFwSEfd30tcwshHhH8lGYu8A+bZHA4PIRpMXAUem63lmnZIXcDSzVZHURpbE1i0ZcZl1GY+wzMysEJywzMysEDwlaGZmheARlpmZFYL/cLiG+vXrF21tbfUOw8ysMCZOnPhaRJR9ELITVg21tbXR3t5e7zDMzApD0t862+cpQTMzKwSPsGpo6atv8OpVN6+6oZlZk9js5K/WrG+PsMzMrBCcsMzMrBCcsMzMrBCcsMzMrBCcsMzMrBCcsMzMrBBaLmFJapM0W9I1kmZKul/SBpJ2kHSvpImSxkvaVVI3Sc8p00fSMknDUj/jJO1U7/MxM2sVLZewkp2AKyKiP7AAOAIYCZwaEXsDZwFXRsQyYA6wG7A/2UJ7QyX1ALaOiKdLO5Y0QlK7pPbX31zURadjZtb8WvUPh5+LiCmpPBFoAz4FjJLU0aZHeh9PtnrqdsBPgH8lW3X1iXIdR8RIsuTHgG2396PwzcyqpFVHWEty5WXAJsCCiBiQe30i7R8HDAX2JVvuuw9wAFkiMzOzLtKqCavUIuA5SUcBpGtWe6Z9j5ONvpZHxLvAFOCbZInMzMy6iBPWh44DviFpKjAT+CJARCwBngceTe3GAxsB0+sRpJlZq2q5a1gRMRfYPbd9SW73IZ18Zmiu/Dvgd7WKz8zMyvMIy8zMCsEJy8zMCsEJy8zMCqHlrmF1pe6bbVLTxczMzFqJR1hmZlYITlhmZlYITlhmZlYIvoZVQ+/Oe4Ynr/hivcMwsyaz6yl31TuEuvAIy8zMCsEJy8zMCsEJy8zMCsEJy8zMCsEJy8zMCqElE5akQZIur3ccZmZWuYa7rV1S94hYWstjREQ70F6PY5uZ2Zqp6QhL0vGSpkmaKukmSW2SHkp1D0raJrW7QdLVkh4DfippgKRHU7s7JfVN7cZKuljS45KekjQ01bdJGi9pUnp9KtX/XtL/zsVzg6QjJR0g6Z5Ud0GKbQJwk6QTJf0q95l7Uvtu6fMzJE2XdEYtf3ZmZraimiUsSf2B84ADI2JP4LvAL4EbI2IP4BYgPy33ceBTEXEm8Fvg7NRuOnB+rl33iNgXOD1XPw/4bEQMBL6S6/c24MspnvWAzwB/KBPubsBBEXHMSk5pALBVROweEZ8Eru/kvEdIapfUPv/N91bSnZmZrY5ajrAOBEZFxGsAEfEGMIQPV+u9Cdg/135URCyT1BvoExEPp/obgWG5dnek94lAWyqvC1wjaTowiiwBAfwPMFxSD+BQYFxEvFMm1rs7qc97Fthe0i8lHQIsKtcoIkZGxKCIGNS313qr6NLMzCrVSDddvFVhuyXpfRkfXoM7A3gF2BMYBKwHEBHvAmOBz5GNvG6r4NhLWfHnsn7qa37qfyzwLeDaCuM1M7MqqGXCegg4StKmAJI2Af4MHJ32HweML/1QRCwE5ndcnwK+Bjxc2q5Eb+CliFie2nfL7bsNOAkYCtxbQdxzgQGS1pG0NbBvir8fsE5E3E421Tmwgr7MzKxKanaXYETMlPQj4GFJy4DJwKnA9ZK+B7xKlkjKOQG4WlJPsqm4ztp1uBK4XdLxZEkpP2K6n2z68a6IqOSi0gTgOWAWMBuYlOq3SrF3JPlzK+jLzMyqRBFR7xia1u7b9InRZ3+63mGYWZNp5qe1S5oYEYPK7Wuka1hmZmadcsIyM7NCcMIyM7NCaLhHMzWT9T+2Y1PPNZuZdSWPsMzMrBCcsMzMrBCcsMzMrBB8DauG5r/2NKOvP6TeYZjZGjrypEoejmNdxSMsMzMrBCcsMzMrBCcsMzMrBCcsMzMrBCcsMzMrhKZJWJL6SPp2veMwM7PaaJqEBfQBPpKwJPnWfTOzJtBMCesiYAdJUyQ9IWm8pLuBWZLaJM3oaCjpLEkXpPJYST+X1C5ptqR9JN0h6WlJP0xt2iQ9KemW1GZ0WlzSzMy6SDMlrHOAv0bEAOB7ZEvYfzcidq7gs++lBcOuBu4CTgF2B06UtGlqswtwZUR8AlhEmdEcgKQRKfm1L3qzkgWOzcysEs2UsEo9HhHPVdj27vQ+HZgZES9FxBLgWWDrtO/5iJiQyjcD+5frKCJGRsSgiBi0ca/11jR2MzMr0cwJ661ceSkrnuv6JW2XpPfluXLHdsc1sCj5TOm2mZnVUDMlrMXARp3sewX4mKRNJfUADluD/reRNCSVjwUeWYM+zMxsDTXNHXQR8bqkCenminfIklTHvvcl/SfwOPAP4Mk1OMQc4BRJ1wGzgKuqELaZmVWoaRIWQEQcu5J9lwOXl6k/IFceC4wt3SepDVgaEV+tUqhmZraammlK0MzMmlhTjbBqJSLmkt3mbmZmdeKEVUN9++3kBeDMzKrEU4JmZlYITlhmZlYITlhmZlYITlhmZlYIvumihl6c/zQX/Nfn6h2GWc1c8OX76h2CtRCPsMzMrBCcsMzMrBCcsMzMrBCcsMzMrBCaNmFJ+pak48vUt6Unuq9pv6dL6rl20ZmZ2eoqTMKS1G112kfE1RHx2xqEcjrghGVm1sUaImGlUc+Tkm6RNFvSaEk9Jc2VdLGkScBRkg6W9BdJkySNktQrff4iSbMkTZN0Saq7QNJZqby3pKmSpgKn5I7bTdLPJD2RPvvNVH+ApLEpjo64JOk0YEtgjKQxXf1zMjNrZQ2RsJJdgCsj4hPAIuDbqf71iBgI/Ak4DzgobbcDZ0raFDgc6B8RewA/LNP39cCpEbFnSf03gIURsQ+wD/CvkrZL+/YiG03tBmwP7JfW1HoRGB4Rw8udhKQRktoltb+96L01+DGYmVk5jZSwno+ICal8M7B/Kt+W3geTJY8JkqYAJwDbAguBd4HfSPoS8Ha+U0l9gD4RMS5V3ZTbfTBwfOrvMWBTYKe07/GIeCEilgNTgLZKTiIiRkbEoIgY1HPj9Sr5iJmZVaCRnnQRnWy/ld4FPBARx5R+UNK+wGeAI4HvAAdWeEyRjbxW+HN9SQcAS3JVy2isn5WZWctppBHWNpKGpPKxwCMl+x8F9pO0I4CkDSXtnK5j9Y6IPwJnACtM+0XEAmCBpI4R23G53fcBJ0taN/W5s6QNVxHnYmCj1Tw3MzNbS42UsOYAp0iaDfQFrsrvjIhXgROBWyVNA/4C7EqWPO5JdY8AZ5bp+yTgijT1p1z9tcAsYFK61f3XrHokNRK41zddmJl1LUWUzsTVIQipDbgnIppqGfotd+gdI34yuN5hmNWMH35r1SZpYkQMKrevkUZYZmZmnWqIGwkiYi7QVKMrMzOrLo+wzMysEBpihNWstuy7k+f4zcyqxCMsMzMrBCcsMzMrBCcsMzMrBCcsMzMrBN90UUNPL3iJz99Z7uHxZpk/Hn5evUMwKwyPsMzMrBCcsMzMrBCcsMzMrBCcsMzMrBCaMmFJGiup7NN+V7OfLSWNrkZMZma2dnyX4EpExItkqxibmVmd1XWEJalN0pOSbpE0W9JoST0l/UDSE5JmSBopSan9WEkXS3pc0lOShqb6DST9PvVxJ7BB7hgHS/qLpEmSRqUVipE0V9JPJE2R1C5poKT7JP1V0rdy8c1I5W6SLkkxTZN0apf/wMzMWlgjTAnuAlwZEZ8AFgHfBn4VEfukBR03AA7Lte8eEfsCpwPnp7qTgbdTH+cDewNI6gecBxwUEQOBdlZckfjvETEAGA/cQDaaGgxcWCbOEUAbMCAi9gBuKXcykkakBNj+3qK3VusHYWZmnWuEKcHnI2JCKt8MnAY8J+nfgJ7AJsBM4L9TmzvS+0SyBAIwDLgcICKmSZqW6gcDuwET0iBtPeAvuWPfnd6nA70iYjGwWNISSX1K4jwIuDoilqbjvFHuZCJiJDASoPeOW9V/OWczsyax0oQl6ZdAp//oRsRpVYihtP8ArgQGRcTzki4A1s/tX5Lel7HqhCvggYg4ppP9HX0tz5U7thshmZuZWbKqKcF2spHM+sBA4On0GkA2WqmGbSQNSeVjgUdS+bV0vamSmx7Gpc8iaXdgj1T/KLCfpB3Tvg0l7byGcT4AfFNS99TXJmvYj5mZrYGVjiIi4kYASScD+3dMh0m6muy6TzXMAU6RdB0wC7gK6AvMAF4Gnqigj6uA6yXNBmaTJVki4lVJJwK3SuqR2p4HPLUGcV4L7AxMk/Q+cA3wqzXox8zM1oAiVn2ZRdIcYEjHdRtJfYFHI2KXtTq41Abck26uaDq9d9wq9vvZyfUOwxqYH35rtiJJEyOi7N/RVnqd5iJgsqQxZNeFhgEXVCc8MzOzVVtlwpK0Dtm03T+lF8DZEfHy2h48IuYCTTm6MjOz6lplwoqI5ZKuiIi9gLu6ICYzM7OPqHRK8EFJRwB3RCUXvQyAnfps4WsUZmZVUumTLr4JjALek7Q4vRbVMC4zM7MVVDTCioiNah2ImZnZylT8NAdJXyC7OxBgbETcU5uQzMzMPqqihCXpImAfPnzg63cl7RcR59YssibwzPw3OGx02WfkmgFwz5HH1TsEs8KodIT1ebKnlC8HkHQjMBlwwjIzsy6xOsuL5J9e3rvagZiZma1MpSOsHwOTJI3lwyddnFOroMzMzEpVmrAOA64D5gNzqdKTLszMzCpVacL6DTAU+AKwA9lzBcdFxC9qFpmZmVlORdewImIM8CPgP8iW1RhEtix9y5F0oqQt6x2HmVmrqfS29geBDcmWlx8P7BMR82oZWAM7kWytrhfrHIeZWUup9C7BacB7ZE9W3wPYXdIGNYuqC0lqkzRb0jWSZkq6X9IGkgZIelTSNEl3Suor6Uiy0eUtkqY0y8/AzKwIKp0SPCMihgFfAl4HrgcW1DKwLrYTcEVE9Cc7ryOA35LdXLIHMB04PyJGA+3AcRExICLeKe1I0ghJ7ZLa31vkxy2amVVLpVOC3yG76WJvsrsEryObGmwWz0XElFSeSHZjSZ+IeDjV3Uj28N9VioiRwEiAPjts7yfbm5lVSaV3Ca4PXApMjIilNYynXpbkystY8Y+kzcysAVT6tPZLah1Ig1kIzJc0NCLGA18DOkZbiwE/vd7MrItV/LT2FnQCcLWknsCzwEmp/oZU/w4wpNx1LDMzq76WT1gRMZfs7seO7fxocnCZ9rcDt9c+MjMzy1udh9+amZnVjROWmZkVghOWmZkVQstfw6qlHftu4hVlzcyqxCMsMzMrBCcsMzMrBCcsMzMrBF/DqqG/zn+Tw29/pN5htKw7j9i/3iGYWRV5hGVmZoXghGVmZoXghGVmZoXghGVmZoXghGVmZoXQcglL0ulpyZCqtDMzs67RcgkLOB2oJBFV2s7MzLpAUycsSRtK+oOkqZJmSDof2BIYI2lManOVpHZJMyVdmOpOK9PuYEl/kTRJ0ihJvep1XmZmraipExZwCPBiROwZEbsDlwEvAsMjYnhq8/2IGATsAXxa0h4RcXm+naR+wHnAQRExEGgHzix3QEkjUgJsX7JoQY1Pz8ysdTR7wpoOfFbSxZKGRsTCMm2+LGkSMBnoD+xWps3gVD9B0hTgBGDbcgeMiJERMSgiBvXYuE91zsLMzJr70UwR8ZSkgcDngR9KejC/X9J2wFnAPhExX9INwPpluhLwQEQcU+uYzcysvKYeYUnaEng7Im4GfgYMBBYDG6UmGwNvAQslbQ4cmvt4vt2jwH6Sdkz9bihp5y44BTMzS5p6hAV8EviZpOXA+8DJwBDgXkkvputTk4EngeeBCbnPjixpdyJwq6Qeaf95wFNddSJmZq1OEVHvGJpW3x12jQN+em29w2hZflq7WfFImphuhPuIpp4SNDOz5uGEZWZmhdDs17Dqaoe+vTwtZWZWJR5hmZlZIThhmZlZIThhmZlZIThhmZlZIfimixqat+B9rrjzlXqH0ZJOOXzzeodgZlXmEZaZmRWCE5aZmRWCE5aZmRWCE5aZmRWCE9ZKSDpR0q/qHYeZmTlhmZlZQRQ6YUlqk/SkpFskzZY0WlJPSXtLeljSREn3SdoitR8g6VFJ0yTdKalvqh8r6ReSpkiaIWnfMsfaTNLtkp5Ir/26+nzNzFpZoRNWsgtwZUR8AlgEnAL8EjgyIvYGrgN+lNr+Fjg7IvYApgPn5/rpGREDgG+nz5T6BfDziNgHOAIou9CVpBGS2iW1v7nojbU/OzMzA5rjD4efj4iOlYJvBv4d2B14QBJAN+AlSb2BPhHxcGp7IzAq18+tABExTtLGkvqUHOcgYLfUJ8DGknpFxJv5RhExkmy1YrbZcU+vjmlmViXNkLBKk8JiYGZEDMlXpoS1Ov2Ubq8DDI6Id1c/RDMzW1vNMCW4jaSO5HQs8CiwWUedpHUl9Y+IhcB8SUNT268BD+f6+Upqvz+wMLXPux84tWND0oDqn4qZmXWmGUZYc4BTJF0HzCK7fnUfcHkaVXUHLgNmAicAV0vqCTwLnJTr511Jk4F1ga+XOc5pwBWSpqU+xwHfqs0pmZlZqWZIWEsj4qsldVOAYaUNI2IKMLiTfm6OiNNL2t8A3JDKr5FGYWZm1vWaYUrQzMxaQKFHWBExl+yOwLXt54C1DsbMzGrKIywzMyuEQo+wGt3H+qzrhQTNzKrEIywzMysEJywzMysEJywzMysEX8OqobdfW8rka+fVO4yWsde/fKzeIZhZDXmEZWZmheCEZWZmheCEZWZmheCEZWZmheCEZWZmhdC0CUtSm6QZ9Y7DzMyqo2kTlpmZNZeWSFiStpc0WdL3JN0h6V5JT0v6aa7NMZKmS5oh6eJUd5SkS1P5u5KezfU3oT5nY2bWmpo+YUnaBbgdOBF4FRhAthDjJ4GvSNpa0pbAxcCBaf8+kv4ZGA8MTV0NBV6XtFUqj+vkeCMktUtqn7/49dqdmJlZi2n2hLUZcBdwXERMTXUPRsTCiHgXmAVsC+wDjI2IVyNiKXALMCwiXgZ6SdoI2Br4HdlKxkPJktlHRMTIiBgUEYP6brRpTU/OzKyVNHvCWgj8Hdg/V7ckV17Gqh9P9WfgJGAOH464hgCeEjQz60LNnrDeAw4Hjpd07EraPQ58WlI/Sd2AY4CH077xwFlkU4CTgeHAkohYWLuwzcysVLMnLCLiLeAw4Axg407avAScA4wBpgITI+KutHs82XTguIhYBjwPPFLruM3MbEVN+7T2iJgL7J7KC8iuU5W2OSxXvhW4tUybvwLKbR9cg3DNzGwVmn6EZWZmzcEJy8zMCsEJy8zMCqFpr2E1gp79unsVXDOzKvEIy8zMCsEJy8zMCsEJy8zMCsHXsGro/Vfe5pXLJtY7jJaw+el71zsEM6sxj7DMzKwQnLDMzKwQnLDMzKwQnLDMzKwQnLDMzKwQnLDWgKS5kvrVOw4zs1bihGVmZoXQ1AlLUpuk2ZKukTRT0v2SNpC0g6R7JU2UNF7Srqn9ZpJul/REeu2X6jdNn50p6Vpy62OZmVnXaOqElewEXBER/YEFwBHASODUiNgbOAu4MrX9BfDziNgntbs21Z8PPJL6uBPYprODSRohqV1S+xtvza/JCZmZtaJWeNLFcxExJZUnAm3Ap4BR0gcDpR7p/SBgt1z9xpJ6AcOALwFExB8kdZqJImIkWUJkz613i+qdhplZa2uFhLUkV14GbA4siIgBZdquAwyOiHfzlbkEZmZmddIKU4KlFgHPSToKQJk90777gVM7GkrqSGrjgGNT3aFA364L18zMoDUTFsBxwDckTQVmAl9M9acBgyRNkzQL+FaqvxAYJmkm2dTg37s6YDOzVtfUU4IRMRfYPbd9SW73IWXavwZ8pUz968DBNQjRzMwq1KojLDMzKxgnLDMzKwQnLDMzK4SmvoZVb+tu3tMr4ZqZVYlHWGZmVgiK8MMYakXSYmBOveNYTf2A1+odxGoqYsxQzLiLGDMUM+4ixgxrH/e2EbFZuR2eEqytORExqN5BrA5J7Y65axQx7iLGDMWMu4gxQ23j9pSgmZkVghOWmZkVghNWbY2sdwBrwDF3nSLGXcSYoZhxFzFmqGHcvunCzMwKwSMsMzMrBCcsMzMrBCesGpB0iKQ5kp6RdE4DxHOdpHmSZuTqNpH0gKSn03vfVC9Jl6fYp0kamPvMCan905JOqHHMW0saI2mWpJmSvtvocUtaX9LjkqammC9M9dtJeizFdpuk9VJ9j7T9TNrfluvr3FQ/R9LnahVz7njdJE2WdE+BYp4rabqkKZLaU13Dfj/SsfpIGi3pSUmzJQ0pQMy7pJ9xx2uRpNPrEndE+FXFF9AN+CuwPbAeMBXYrc4xDQMGAjNydT8Fzknlc4CLU/nzwP8AAgYDj6X6TYBn03vfVO5bw5i3AAam8kbAU8BujRx3OnavVF4XeCzF8l/A0an+auDkVP42cHUqHw3clsq7pe9ND2C79H3qVuPvyJnA74B70nYRYp4L9Cupa9jvRzrejcC/pPJ6QJ9Gj7kk/m7Ay8C29Yi75ifYai9gCHBfbvtc4NwGiKuNFRPWHGCLVN6C7I+cAX4NHFPaDjgG+HWufoV2XRD/XcBnixI30BOYBPwT2V/9dy/9fgD3AUNSuXtqp9LvTL5djWL9OPAgcCBwT4qhoWNOx5jLRxNWw34/gN7Ac6Sb3YoQc5lzOBiYUK+4PSVYfVsBz+e2X0h1jWbziHgplV8GNk/lzuKv23mlaae9yEYsDR13mlqbAswDHiAbaSyIiKVljv9BbGn/QmDTro4ZuAz4N2B52t60ADEDBHC/pImSRqS6Rv5+bAe8Clyfpl+vlbRhg8dc6mjg1lTu8ridsIzIft1pyL9vkNQLuB04PSIW5fc1YtwRsSwiBpCNWvYFdq1zSCsl6TBgXkRMrHcsa2D/iBgIHAqcImlYfmcDfj+6k03NXxURewFvkU2lfaABY/5Auo75BWBU6b6uitsJq/r+AWyd2/54qms0r0jaAiC9z0v1ncXf5eclaV2yZHVLRNxRlLgBImIBMIZsOq2PpI7nduaP/0FsaX9v4PUujnk/4AuS5gK/J5sW/EWDxwxARPwjvc8D7iT7BaGRvx8vAC9ExGNpezRZAmvkmPMOBSZFxCtpu8vjdsKqvieAndJdVuuRDaHvrnNM5dwNdNylcwLZNaKO+uPTnT6DgYVp2H8fcLCkvuluoINTXU1IEvAbYHZEXFqEuCVtJqlPKm9Ads1tNlniOrKTmDvO5UjgofSb6t3A0emOvO2AnYDHaxFzRJwbER+PiDay7+pDEXFcI8cMIGlDSRt1lMn+u86ggb8fEfEy8LykXVLVZ4BZjRxziWP4cDqwI76ujbsrLtS12ovsLpmnyK5ffL8B4rkVeAl4n+y3vG+QXXd4EHga+BOwSWor4IoU+3RgUK6frwPPpNdJNY55f7IphmnAlPT6fCPHDewBTE4xzwB+kOq3J/vH+xmy6ZQeqX79tP1M2r99rq/vp3OZAxzaRd+TA/jwLsGGjjnFNzW9Znb8f9bI3490rAFAe/qO/D+yu+UaOuZ0vA3JRtK9c3VdHrcfzWRmZoXgKUEzMysEJywzMysEJywzMysEJywzMysEJywzMysEJywz61R6KnfPesdhBl5x2MxWIj0BY1BEvFbvWMw8wjIrOEnHp3WHpkq6SVKbpIdS3YOStkntbpB0ZO5zb6b3AySN1YfrNN2SnlJwGrAlMEbSmPqcndmHuq+6iZk1Kkn9gfOAT0XEa5I2IVtz6caIuFHS14HLgX9eRVd7Af2BF4EJwH4RcbmkM4HhHmFZI/AIy6zYDgRGdSSUiHiD7IG7v0v7byJ7zNWqPB4RL0TEcrLHYLXVIFazteKEZdY6lpL+n5e0DtmKtx2W5HFG89UAAAChSURBVMrL8OyLNSAnLLNiewg4StKmAGlK8M9kT14HOA4Yn8pzgb1T+QvAuhX0vxjYqFrBmq0N/xZlVmARMVPSj4CHJS0je1r8qWSr2n6PbIXbk1Lza4C7JE0F7iVbQHBVRgL3SnoxIoZX/wzMKufb2s3MrBA8JWhmZoXghGVmZoXghGVmZoXghGVmZoXghGVmZoXghGVmZoXghGVmZoXw/wEPftO3aKC6bQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rlz1p6DTm1ul"
      },
      "source": [
        "#### KNN + TfidVectorizer "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hVJUNp-knIQK",
        "outputId": "b779ce53-3d9f-4abd-c54d-a090aa0c683d"
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "tfidf = TfidfVectorizer(stop_words=st_words\n",
        "                        ,tokenizer=tokenize\n",
        "                        ,ngram_range = (1,2)\n",
        "                        ,max_features = 5000\n",
        "                        )\n",
        "dtm = tfidf.fit_transform(df['sentence'])\n",
        "dtm = pd.DataFrame(dtm.todense(), columns=tfidf.get_feature_names())"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/feature_extraction/text.py:385: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['d', 'm', 'nt', 's', 've'] not in stop_words.\n",
            "  'stop_words.' % sorted(inconsistent))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 270
        },
        "id": "-xI6L090qoci",
        "outputId": "687b41c8-9eae-4583-f68c-a603dc4c05e5"
      },
      "source": [
        "dtm.head()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>aaron</th>\n",
              "      <th>abandon</th>\n",
              "      <th>abc</th>\n",
              "      <th>ability</th>\n",
              "      <th>able</th>\n",
              "      <th>aboard</th>\n",
              "      <th>abortion</th>\n",
              "      <th>abroad</th>\n",
              "      <th>abruptly</th>\n",
              "      <th>absence</th>\n",
              "      <th>absentee</th>\n",
              "      <th>absentee ballot</th>\n",
              "      <th>abuse</th>\n",
              "      <th>academic</th>\n",
              "      <th>academy</th>\n",
              "      <th>accelerate</th>\n",
              "      <th>accept</th>\n",
              "      <th>acceptance</th>\n",
              "      <th>access</th>\n",
              "      <th>accident</th>\n",
              "      <th>acclaim</th>\n",
              "      <th>accord</th>\n",
              "      <th>accord new</th>\n",
              "      <th>account</th>\n",
              "      <th>accountability</th>\n",
              "      <th>accurate</th>\n",
              "      <th>accusation</th>\n",
              "      <th>accuse</th>\n",
              "      <th>accuser</th>\n",
              "      <th>ace</th>\n",
              "      <th>achieve</th>\n",
              "      <th>achievement</th>\n",
              "      <th>acknowledge</th>\n",
              "      <th>acquire</th>\n",
              "      <th>acquit</th>\n",
              "      <th>acquittal</th>\n",
              "      <th>acre</th>\n",
              "      <th>act</th>\n",
              "      <th>action</th>\n",
              "      <th>active</th>\n",
              "      <th>...</th>\n",
              "      <th>world war</th>\n",
              "      <th>worldwide</th>\n",
              "      <th>worried</th>\n",
              "      <th>worry</th>\n",
              "      <th>worsen</th>\n",
              "      <th>worth</th>\n",
              "      <th>wound</th>\n",
              "      <th>wrap</th>\n",
              "      <th>wrestle</th>\n",
              "      <th>write</th>\n",
              "      <th>write book</th>\n",
              "      <th>writer</th>\n",
              "      <th>writing</th>\n",
              "      <th>writing prompt</th>\n",
              "      <th>wrong</th>\n",
              "      <th>wuhan</th>\n",
              "      <th>x</th>\n",
              "      <th>xi</th>\n",
              "      <th>yang</th>\n",
              "      <th>yankees</th>\n",
              "      <th>yard</th>\n",
              "      <th>yemen</th>\n",
              "      <th>yes</th>\n",
              "      <th>yield</th>\n",
              "      <th>yorker</th>\n",
              "      <th>yorkers</th>\n",
              "      <th>yorks</th>\n",
              "      <th>young</th>\n",
              "      <th>young adult</th>\n",
              "      <th>young child</th>\n",
              "      <th>young people</th>\n",
              "      <th>young woman</th>\n",
              "      <th>youth</th>\n",
              "      <th>youtube</th>\n",
              "      <th>zealand</th>\n",
              "      <th>zero</th>\n",
              "      <th>zone</th>\n",
              "      <th>zoo</th>\n",
              "      <th>zoom</th>\n",
              "      <th>zuckerberg</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.215219</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 5000 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   aaron  abandon  abc  ability  able  ...  zero  zone  zoo  zoom  zuckerberg\n",
              "0    0.0      0.0  0.0      0.0   0.0  ...   0.0   0.0  0.0   0.0         0.0\n",
              "1    0.0      0.0  0.0      0.0   0.0  ...   0.0   0.0  0.0   0.0         0.0\n",
              "2    0.0      0.0  0.0      0.0   0.0  ...   0.0   0.0  0.0   0.0         0.0\n",
              "3    0.0      0.0  0.0      0.0   0.0  ...   0.0   0.0  0.0   0.0         0.0\n",
              "4    0.0      0.0  0.0      0.0   0.0  ...   0.0   0.0  0.0   0.0         0.0\n",
              "\n",
              "[5 rows x 5000 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rei9OJWdnEgl",
        "outputId": "9e0016b8-3222-4f09-ab7f-ebcbca145181"
      },
      "source": [
        "from sklearn.neighbors import NearestNeighbors\n",
        "nn = NearestNeighbors(n_neighbors=5, algorithm='kd_tree') #최근접 5이웃\n",
        "nn.fit(dtm)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "NearestNeighbors(algorithm='kd_tree', leaf_size=30, metric='minkowski',\n",
              "                 metric_params=None, n_jobs=None, n_neighbors=5, p=2,\n",
              "                 radius=1.0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iTYYrqpjniSE",
        "outputId": "7d29ae97-0789-4564-fe95-a11b2625fa66"
      },
      "source": [
        "result = nn.kneighbors([dtm.iloc[10000]])\n",
        "result"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[0.        , 1.        , 1.        , 1.        , 1.16302513]]),\n",
              " array([[10000,  5144, 10509, 35412, 15591]]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "zfwugSbHnx-Y",
        "outputId": "8a670dec-2497-4340-8651-bf9210f99a33"
      },
      "source": [
        "df.iloc[result[1][0]]"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>Tokens</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>16469</th>\n",
              "      <td>Neighboring Iran, badly hit by the virus, cont...</td>\n",
              "      <td>[neighbor, iran, badly, hit, virus, continue, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7717</th>\n",
              "      <td>What is this illustration saying? Maze</td>\n",
              "      <td>[illustration, say, maze]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17301</th>\n",
              "      <td>No. Should I Still Be Going Out?</td>\n",
              "      <td>[go]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58484</th>\n",
              "      <td>Dani Raymon bamboozles us. Puns and Anagrams</td>\n",
              "      <td>[dani, raymon, bamboozle, pun, anagram]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26540</th>\n",
              "      <td>For this issue, a look at Afghanistan as the v...</td>\n",
              "      <td>[issue, look, afghanistan, virus, threaten, ex...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                sentence                                             Tokens\n",
              "16469  Neighboring Iran, badly hit by the virus, cont...  [neighbor, iran, badly, hit, virus, continue, ...\n",
              "7717              What is this illustration saying? Maze                          [illustration, say, maze]\n",
              "17301                   No. Should I Still Be Going Out?                                               [go]\n",
              "58484       Dani Raymon bamboozles us. Puns and Anagrams            [dani, raymon, bamboozle, pun, anagram]\n",
              "26540  For this issue, a look at Afghanistan as the v...  [issue, look, afghanistan, virus, threaten, ex..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IeB9nDmelLVr",
        "outputId": "53989bbe-af29-45b8-e377-0991924fff66"
      },
      "source": [
        "pd.set_option('display.max_rows', None)\n",
        "pd.set_option('display.max_columns', None)\n",
        "pd.set_option('display.width', None)\n",
        "pd.set_option('display.max_colwidth', -1)\n",
        "print(df['sentence'].iloc[result[1][0]])"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "16469    Neighboring Iran, badly hit by the virus, continues to allow thousands of people to cross into Afghanistan daily despite requests to close the border. In Afghanistan, Coronavirus Complicates War and Peace\n",
            "7717     What is this illustration saying? Maze                                                                                                                                                                      \n",
            "17301    No. Should I Still Be Going Out?                                                                                                                                                                            \n",
            "58484    Dani Raymon bamboozles us. Puns and Anagrams                                                                                                                                                                \n",
            "26540    For this issue, a look at Afghanistan as the virus threatens to explode with few resources to contain it. Behind the Cover: Afghanistanâs Next War                                                        \n",
            "Name: sentence, dtype: object\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:4: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
            "  after removing the cwd from sys.path.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NJuunYnOpoOS"
      },
      "source": [
        "10000번째 기사를 읽었다면 16469, 77117, 17301, 58484, 26540 번째의 기사가 추천된다.\n",
        "주로 전쟁에 관한 기사들이 추천되는 것을 볼 수 있다. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bZ0d7vj5kOC-"
      },
      "source": [
        "#### 특정 주제에 관한 기사들을 분류하는 모델 만들기 - coronavirus에 대한 기사"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lzaX5Pu69A-h"
      },
      "source": [
        "**타겟 라벨 만들기**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "czc-DGAaE8NI"
      },
      "source": [
        "for row in df['sentence']:\n",
        "  df['sentence'] = df['sentence'].str.lower()"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RLyxSaLP6aiW"
      },
      "source": [
        "corona = df['sentence'].str.contains('coronavirus')\n",
        "df['label'] = corona"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UiTmfbdQ8nMW"
      },
      "source": [
        "df['label'] = df['label'].astype(int)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JPwELOts8Hrv",
        "outputId": "4dd5a021-6fb9-453d-f69d-cff05c046623"
      },
      "source": [
        "df.info()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 42420 entries, 0 to 69113\n",
            "Data columns (total 3 columns):\n",
            " #   Column    Non-Null Count  Dtype \n",
            "---  ------    --------------  ----- \n",
            " 0   sentence  42420 non-null  object\n",
            " 1   Tokens    42420 non-null  object\n",
            " 2   label     42420 non-null  int64 \n",
            "dtypes: int64(1), object(2)\n",
            "memory usage: 1.3+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "44BIZED750Nj",
        "outputId": "351dedf7-e363-4898-85ca-7eca44a38821"
      },
      "source": [
        "df['label'].value_counts()"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    36616\n",
              "1    5804 \n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qg27AFQf9tft"
      },
      "source": [
        "**훈련, 테스트셋 분리**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5AVYTeV09MyG",
        "outputId": "54bd885a-5a74-44c4-cfe8-ec5147c504a1"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "train = df['sentence']\n",
        "target = df['label']\n",
        "\n",
        "x_train, x_test, y_train, y_test = train_test_split(train, target, test_size = 0.3, random_state = 11)\n",
        "\n",
        "x_train.shape, x_test.shape, y_train.shape, y_test.shape"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((29694,), (12726,), (29694,), (12726,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MWY5EsLBRkJv"
      },
      "source": [
        "**Baseline**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D_oWRQeqQ0KH"
      },
      "source": [
        "#Baseline - logistic regression\n",
        "import numpy as np\n",
        "import scipy.stats as stats\n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "vect = TfidfVectorizer(stop_words=st_words\n",
        "                       , ngram_range=(1,2)\n",
        "                       , min_df=2\n",
        "                       , max_df=0.7\n",
        "                        ,tokenizer=tokenize\n",
        "                       , max_features=10000\n",
        "                      )\n",
        "\n",
        "svd = TruncatedSVD(algorithm='randomized'\n",
        "                   , n_iter=5\n",
        "                   , random_state=11)\n",
        "\n",
        "\n",
        "lgs = LogisticRegression()\n",
        "\n",
        "pipe_baseline = Pipeline([\n",
        "  ('vect', vect)\n",
        "  , ('svd', svd)\n",
        "  , ('lgs', lgs)\n",
        "])"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-bIsvwE4SHek",
        "outputId": "321145e5-1645-4996-d23c-9db41f35cd8c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "pipe_baseline.fit(x_train,y_train)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/feature_extraction/text.py:385: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['d', 'm', 'nt', 's', 've'] not in stop_words.\n",
            "  'stop_words.' % sorted(inconsistent))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(memory=None,\n",
              "         steps=[('vect',\n",
              "                 TfidfVectorizer(analyzer='word', binary=False,\n",
              "                                 decode_error='strict',\n",
              "                                 dtype=<class 'numpy.float64'>,\n",
              "                                 encoding='utf-8', input='content',\n",
              "                                 lowercase=True, max_df=0.7, max_features=10000,\n",
              "                                 min_df=2, ngram_range=(1, 2), norm='l2',\n",
              "                                 preprocessor=None, smooth_idf=True,\n",
              "                                 stop_words={\"'d\", \"'ll\", \"'m\", \"'re\", \"'s\",\n",
              "                                             \"'ve\", 'a', 'about', 'above',\n",
              "                                             'acr...\n",
              "                ('svd',\n",
              "                 TruncatedSVD(algorithm='randomized', n_components=2, n_iter=5,\n",
              "                              random_state=11, tol=0.0)),\n",
              "                ('lgs',\n",
              "                 LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
              "                                    fit_intercept=True, intercept_scaling=1,\n",
              "                                    l1_ratio=None, max_iter=100,\n",
              "                                    multi_class='auto', n_jobs=None,\n",
              "                                    penalty='l2', random_state=None,\n",
              "                                    solver='lbfgs', tol=0.0001, verbose=0,\n",
              "                                    warm_start=False))],\n",
              "         verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RPmeI5mMSbkZ",
        "outputId": "fc341ab5-7c84-4be2-9873-83a5d96efe7f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "y_pred_base = pipe_baseline.predict(x_test)\n",
        "accuracy_score(y_test, y_pred_base)"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8615432971868615"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bdx-PhrhSnP_",
        "outputId": "2608c992-b13d-4f83-9fa4-25fc4fd55f1a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(classification_report(y_test, y_pred_base))"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.87      0.98      0.92     10976\n",
            "           1       0.48      0.10      0.16      1750\n",
            "\n",
            "    accuracy                           0.86     12726\n",
            "   macro avg       0.68      0.54      0.54     12726\n",
            "weighted avg       0.82      0.86      0.82     12726\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YihN8rMxOcEA"
      },
      "source": [
        "**Pipeline**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gPti9_xPJK6g"
      },
      "source": [
        "vect = TfidfVectorizer(stop_words=st_words\n",
        "                       , ngram_range=(1,2)\n",
        "                       , min_df=2\n",
        "                       , max_df=0.7\n",
        "                        ,tokenizer=tokenize\n",
        "                       , max_features=10000\n",
        "                      )\n",
        "\n",
        "svd = TruncatedSVD(algorithm='randomized'\n",
        "                   , n_iter=5\n",
        "                   , random_state=11)\n",
        "\n",
        "clf = RandomForestClassifier(n_estimators=500\n",
        "                             , random_state=11)\n",
        "\n",
        "pipe = Pipeline([\n",
        "  ('vect', vect)\n",
        "  , ('svd', svd)\n",
        "  , ('clf', clf)\n",
        "])"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kJP7dUOAJ8KE",
        "outputId": "4864d713-d217-4f9c-98ec-65756644a3e9"
      },
      "source": [
        "pipe.fit(x_train,y_train)"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/feature_extraction/text.py:385: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['d', 'm', 'nt', 's', 've'] not in stop_words.\n",
            "  'stop_words.' % sorted(inconsistent))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(memory=None,\n",
              "         steps=[('vect',\n",
              "                 TfidfVectorizer(analyzer='word', binary=False,\n",
              "                                 decode_error='strict',\n",
              "                                 dtype=<class 'numpy.float64'>,\n",
              "                                 encoding='utf-8', input='content',\n",
              "                                 lowercase=True, max_df=0.7, max_features=10000,\n",
              "                                 min_df=2, ngram_range=(1, 2), norm='l2',\n",
              "                                 preprocessor=None, smooth_idf=True,\n",
              "                                 stop_words={\"'d\", \"'ll\", \"'m\", \"'re\", \"'s\",\n",
              "                                             \"'ve\", 'a', 'about', 'above',\n",
              "                                             'acr...\n",
              "                 RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,\n",
              "                                        class_weight=None, criterion='gini',\n",
              "                                        max_depth=None, max_features='auto',\n",
              "                                        max_leaf_nodes=None, max_samples=None,\n",
              "                                        min_impurity_decrease=0.0,\n",
              "                                        min_impurity_split=None,\n",
              "                                        min_samples_leaf=1, min_samples_split=2,\n",
              "                                        min_weight_fraction_leaf=0.0,\n",
              "                                        n_estimators=500, n_jobs=None,\n",
              "                                        oob_score=False, random_state=11,\n",
              "                                        verbose=0, warm_start=False))],\n",
              "         verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "To6xF2FoOgxb"
      },
      "source": [
        "**Cross Validation Parameter Search**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D9fDxg4oKPhY",
        "outputId": "f8771ff2-d995-475a-8bda-1cd413f745d5"
      },
      "source": [
        "# 파라미터 셋팅\n",
        "parameters = {\n",
        "    'vect__max_df': (0.7, 1.0) \n",
        "    ,'vect__min_df': (2, 5, 10) \n",
        "    ,'vect__max_features': (5000, 20000) \n",
        "    ,'clf__n_estimators': (100, 500) \n",
        "    ,'clf__max_depth': (10, 20, None) \n",
        "}\n",
        "\n",
        "random_search = RandomizedSearchCV(pipe, parameters, cv=5, n_jobs=-1, verbose=1)\n",
        "random_search.fit(x_train,y_train)"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 40 concurrent workers.\n",
            "[Parallel(n_jobs=-1)]: Done  22 out of  50 | elapsed: 23.7min remaining: 30.2min\n",
            "[Parallel(n_jobs=-1)]: Done  50 out of  50 | elapsed: 34.1min finished\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/feature_extraction/text.py:385: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['d', 'm', 'nt', 's', 've'] not in stop_words.\n",
            "  'stop_words.' % sorted(inconsistent))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomizedSearchCV(cv=5, error_score=nan,\n",
              "                   estimator=Pipeline(memory=None,\n",
              "                                      steps=[('vect',\n",
              "                                              TfidfVectorizer(analyzer='word',\n",
              "                                                              binary=False,\n",
              "                                                              decode_error='strict',\n",
              "                                                              dtype=<class 'numpy.float64'>,\n",
              "                                                              encoding='utf-8',\n",
              "                                                              input='content',\n",
              "                                                              lowercase=True,\n",
              "                                                              max_df=0.7,\n",
              "                                                              max_features=10000,\n",
              "                                                              min_df=2,\n",
              "                                                              ngram_range=(1,\n",
              "                                                                           2),\n",
              "                                                              norm='l2',\n",
              "                                                              preprocessor=None,\n",
              "                                                              smooth_idf=True,\n",
              "                                                              stop_words={\"'d\",\n",
              "                                                                          \"'...\n",
              "                                                                     random_state=11,\n",
              "                                                                     verbose=0,\n",
              "                                                                     warm_start=False))],\n",
              "                                      verbose=False),\n",
              "                   iid='deprecated', n_iter=10, n_jobs=-1,\n",
              "                   param_distributions={'clf__max_depth': (10, 20, None),\n",
              "                                        'clf__n_estimators': (100, 500),\n",
              "                                        'vect__max_df': (0.7, 1.0),\n",
              "                                        'vect__max_features': (5000, 20000),\n",
              "                                        'vect__min_df': (2, 5, 10)},\n",
              "                   pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
              "                   return_train_score=False, scoring=None, verbose=1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DLFER2mtKWy2",
        "outputId": "629232f4-b255-49f2-f898-88e04d7a3ab8"
      },
      "source": [
        "random_search.best_score_"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9264158724737765"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X6OHqBpHKXZa",
        "outputId": "bbfa5061-2189-4f3e-e254-6525059d201a"
      },
      "source": [
        "random_search.best_params_"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'clf__max_depth': 10,\n",
              " 'clf__n_estimators': 100,\n",
              " 'vect__max_df': 0.7,\n",
              " 'vect__max_features': 20000,\n",
              " 'vect__min_df': 10}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LcRsWwxoOmxR"
      },
      "source": [
        "**Accuracy score**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HEpJ7HzRNu0h",
        "outputId": "a5160240-3ce6-4786-8c7e-c70d488095d4"
      },
      "source": [
        "y_pred = random_search.predict(x_test)\n",
        "accuracy_score(y_test, y_pred)"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9252710985384253"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kbzKDmk2b-WU",
        "outputId": "48717739-8e9c-4bc5-c688-d7dbb2671dea"
      },
      "source": [
        "print(classification_report(y_test, y_pred))"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.96      0.96     10976\n",
            "           1       0.75      0.68      0.72      1750\n",
            "\n",
            "    accuracy                           0.93     12726\n",
            "   macro avg       0.85      0.82      0.84     12726\n",
            "weighted avg       0.92      0.93      0.92     12726\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}