bbc_project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "bbc project.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/threegenie/knn_project/blob/main/bbc_project.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 242
        },
        "id": "oO9gk4XWsFVK",
        "outputId": "85cb436a-6656-4f5d-931b-e6e582225ea9"
      },
      "source": [
        "import pandas as pd\n",
        "import os, sys\n",
        "import numpy as np\n",
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/drive')\n",
        "df = pd.read_csv('/content/drive/My Drive/BBCarticles_csv.csv',encoding= 'unicode_escape')\n",
        "df.head()"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Summary</th>\n",
              "      <th>Text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>TimeWarner said fourth quarter sales rose 2% t...</td>\n",
              "      <td>Ad sales boost Time Warner profit\\n\\nQuarterly...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>The dollar has hit its highest level against t...</td>\n",
              "      <td>Dollar gains on Greenspan speech\\n\\nThe dollar...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Yukos' owner Menatep Group says it will ask Ro...</td>\n",
              "      <td>Yukos unit buyer faces loan claim\\n\\nThe owner...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Rod Eddington, BA's chief executive, said the ...</td>\n",
              "      <td>High fuel prices hit BA's profits\\n\\nBritish A...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Pernod has reduced the debt it took on to fund...</td>\n",
              "      <td>Pernod takeover talk lifts Domecq\\n\\nShares in...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                             Summary                                               Text\n",
              "0  TimeWarner said fourth quarter sales rose 2% t...  Ad sales boost Time Warner profit\\n\\nQuarterly...\n",
              "1  The dollar has hit its highest level against t...  Dollar gains on Greenspan speech\\n\\nThe dollar...\n",
              "2  Yukos' owner Menatep Group says it will ask Ro...  Yukos unit buyer faces loan claim\\n\\nThe owner...\n",
              "3  Rod Eddington, BA's chief executive, said the ...  High fuel prices hit BA's profits\\n\\nBritish A...\n",
              "4  Pernod has reduced the debt it took on to fund...  Pernod takeover talk lifts Domecq\\n\\nShares in..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a-iDeLWMx4Q_"
      },
      "source": [
        "#### 토큰 정제"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MkgdoCK6x8uY",
        "outputId": "8d947bd7-26f6-4fdd-fba6-f6d054233bda"
      },
      "source": [
        "!pip install squarify"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: squarify in /usr/local/lib/python3.7/dist-packages (0.4.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AiewFQ12uJze"
      },
      "source": [
        "import re\n",
        "import squarify\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import spacy\n",
        "from spacy.tokenizer import Tokenizer"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kg1SfPQoyCPG",
        "outputId": "14988a2d-af1a-42d3-cd96-41bd901e4800"
      },
      "source": [
        "#결측값 처리\n",
        "df.dropna().info()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 2225 entries, 0 to 2224\n",
            "Data columns (total 2 columns):\n",
            " #   Column   Non-Null Count  Dtype \n",
            "---  ------   --------------  ----- \n",
            " 0   Summary  2225 non-null   object\n",
            " 1   Text     2225 non-null   object\n",
            "dtypes: object(2)\n",
            "memory usage: 52.1+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GCmL5Ie-yltW",
        "outputId": "718f98a3-341a-4acc-9e3c-e26706a4f0a2"
      },
      "source": [
        "#문자를 소문자로 통일\n",
        "df['Text'] = df['Text'].apply(lambda x : x.lower())\n",
        "df['Text'].head(3)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    ad sales boost time warner profit\\n\\nquarterly...\n",
              "1    dollar gains on greenspan speech\\n\\nthe dollar...\n",
              "2    yukos unit buyer faces loan claim\\n\\nthe owner...\n",
              "Name: Text, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6xgWZBAoysKe",
        "outputId": "f0c6d963-e79a-4236-ba77-622f7c0edd51"
      },
      "source": [
        "#줄바꿈 문자 정리\n",
        "df['Text'] = df['Text'].str.replace('\\n',' ')\n",
        "df['Text'].head(3)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    ad sales boost time warner profit  quarterly p...\n",
              "1    dollar gains on greenspan speech  the dollar h...\n",
              "2    yukos unit buyer faces loan claim  the owners ...\n",
              "Name: Text, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DpNdVAiazZEG"
      },
      "source": [
        "#알파벳, 숫자만 받을 수 있도록 함수 작성(text):\n",
        "\n",
        "regex = r\"[^a-zA-Z0-9 ]\"\n",
        "subst = \"\"\n",
        "\n",
        "def tokenize(text):\n",
        "\n",
        "    # 정규식 적용\n",
        "    tokens = re.sub(regex, subst, text)\n",
        "\n",
        "    # split\n",
        "    tokens = tokens.split()\n",
        "    \n",
        "    return tokens\n",
        "\n",
        "df['Tokens'] = df['Text'].apply(tokenize)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mEY4WvQQyEb-",
        "outputId": "4b2581a0-abdb-41c5-f6d8-63c79e54d09e"
      },
      "source": [
        "#기본 불용어 불러오기\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "tokenizer = Tokenizer(nlp.vocab)\n",
        "print(nlp.Defaults.stop_words)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'by', 'very', 'otherwise', \"'ve\", 'have', 'top', '’ll', 'myself', 'how', 'am', 'why', 'ours', 'not', 'seeming', 'as', 'anyhow', \"'re\", 'much', 'under', 'during', 'everywhere', 'side', 'move', 'any', 'to', 'well', 'twelve', '’re', 'all', 'him', 'because', 'whole', 'down', 'together', 'did', 'twenty', 'rather', 'n’t', 'cannot', 'sometime', 'we', 'amount', 'must', 'are', \"'ll\", 'another', 'either', 'name', 'get', 'two', 'so', 'each', 'thence', 'else', 'around', 'which', 'also', 'although', 'amongst', 'per', 'from', 'latter', 'some', 'somewhere', 'hereafter', 'beforehand', 'same', 'these', '‘m', 'whence', 'becomes', 'at', 'within', 'before', 're', 'nowhere', 'might', 'wherever', 'whose', 'through', 'above', 'and', 'whether', 'though', 'take', 'formerly', 'one', 'thereby', 'has', 'ten', 'put', 'hence', 'behind', \"'d\", 'never', 'nothing', 'below', 'using', 'is', 'itself', '’d', 'fifteen', 'more', '‘d', 'hereupon', 'however', 'serious', 'thereupon', 'four', 'namely', 'for', 'i', 'towards', 'every', 'regarding', 'somehow', 'full', 'yet', 'whereupon', 'anything', 'back', 'own', 'sometimes', 'until', 'my', 'least', 'they', 'front', 'due', 'should', 'third', 'just', 'with', 'both', 'go', 'while', 'that', 'again', 'other', 'no', 'first', 'besides', 'noone', 'nor', 'up', 'seem', 'you', 'afterwards', 'call', 'used', 'eight', 'neither', 'the', 'whereas', 'hereby', 'whatever', 'herein', 'will', 'yourself', 'or', 'therefore', 'after', 'indeed', 'about', 'none', \"'s\", 'than', '’s', 'on', 'still', 'n‘t', 'thru', 'less', 'me', 'please', 'now', 'being', 'even', 'former', '‘ve', 'what', 'thus', \"'m\", 'was', 'whereby', 'here', 'therein', 'this', 'empty', 'since', 'everything', 'their', 'into', 'perhaps', 'something', 'already', 'it', '‘ll', 'anyway', 'yours', 'be', 'your', 'bottom', 'unless', 'against', 'others', 'alone', 'between', 'mostly', 'hers', 'her', 'become', 'anyone', 'becoming', 'six', 'those', 'a', 'along', 'there', 'made', 'over', 'fifty', 'nobody', 'five', 'in', 'done', 'she', 'out', 'had', 'who', 'many', 'when', 'our', '‘s', 'he', 'everyone', '’ve', 'could', 'quite', 'really', 'whenever', 'onto', 'whoever', 'eleven', 'keep', 'nine', 'often', 'always', 'ever', 'ourselves', 'make', 'its', 'across', 'latterly', 'yourselves', 'give', 'seemed', 'moreover', 'where', 'nevertheless', 'say', 'throughout', 'does', 'elsewhere', 'next', 'too', '‘re', 'mine', 'became', 'whom', 'hundred', 'via', 'himself', 'off', 'someone', 'only', 'themselves', 'seems', 'but', 'ca', 'once', 'if', 'anywhere', 'few', 'sixty', 'of', 'doing', 'several', 'almost', 'except', 'show', 'were', 'do', 'may', 'see', 'an', 'would', 'meanwhile', 'most', 'last', 'herself', 'thereafter', 'further', 'such', 'part', 'whereafter', 'toward', \"n't\", 'beyond', 'among', 'enough', 'upon', 'then', 'various', 'us', 'whither', 'wherein', 'forty', 'them', 'without', 'three', '’m', 'his', 'can', 'been', 'beside'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fq4pUhU_yPE2"
      },
      "source": [
        "#불용어 추가하기\n",
        "st_words = nlp.Defaults.stop_words.union(['this', 'i', 'it', 'is', 'in', 'and', 'a', 'the', 'but', 'to', \"it's\", \"i'm\",'it.',' ',\n",
        "                                          'said','-','mr','new','said.','\"i','told','\"the','uk','best','year','time',\n",
        "                                          'like','\"we','number','bbc','said:','years','m','not'])"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "02_LubCnyUB-"
      },
      "source": [
        "def tokenize(text):\n",
        "  text = text.lower()\n",
        "  text = text.replace(\"\\n\",\" \")\n",
        "\n",
        "  subst = \"\"\n",
        "  regex = \"[^a-zA-Z0-9 ]\"\n",
        "  doc = re.sub(regex, subst, text)\n",
        "\n",
        "  doc = nlp(doc)\n",
        "  tokens = []\n",
        "\n",
        "  for token in doc :\n",
        "    if (token.text.lower() not in st_words) & (token.is_stop == False) & (token.is_punct == False) & (token.is_alpha):\n",
        "      tokens.append(token.lemma_)\n",
        "      \n",
        "  return tokens"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Re7eU4hdINiI",
        "outputId": "2fd0ba6b-81f0-4a99-d01d-cbc17bd4bfa6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "df['Tokens']=df['Text'].apply(tokenize)\n",
        "df['Tokens'].head(3)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    [ad, sale, boost, warner, profit, quarterly, p...\n",
              "1    [dollar, gain, greenspan, speech, dollar, hit,...\n",
              "2    [yukos, unit, buyer, face, loan, claim, owner,...\n",
              "Name: Tokens, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2BZgZriVHCwl",
        "outputId": "dea71bd5-0b3d-494d-c17d-a6ceaf96e6e3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(df['Tokens'].loc[29])"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['soar', 'oil', 'hit', 'world', 'economy', 'soar', 'cost', 'oil', 'hit', 'global', 'economic', 'growth', 'world', 'major', 'economy', 'weather', 'storm', 'price', 'rise', 'accord', 'oecd', 'late', 'biannual', 'report', 'oecd', 'cut', 'growth', 'prediction', 'world', 'main', 'industrialise', 'region', 'growth', 'reach', 'fall', 'previous', 'estimate', 'oecd', 'parisbased', 'economic', 'think', 'tank', 'believe', 'global', 'economy', 'regain', 'momentum', 'forecast', 'japanese', 'growth', 'scale', 'outlook', 'bad', 'eurozone', 'bloc', 'sluggish', 'growth', 'forecast', 'slip', 'oecd', 'overall', 'report', 'forecast', 'total', 'growth', 'member', 'country', 'oecd', 'slip', 'recover', 'nonetheless', 'good', 'reason', 'believe', 'despite', 'recent', 'oil', 'price', 'turbulence', 'world', 'economy', 'regain', 'momentum', 'nottoodistant', 'future', 'jeanphilippe', 'cotis', 'oecds', 'chief', 'economist', 'price', 'crude', 'high', 'start', 'record', 'high', 'set', 'late', 'october', 'dip', 'oil', 'price', 'improve', 'job', 'prospect', 'improve', 'consumer', 'confidence', 'spend', 'oecd', 'oil', 'shock', 'enormous', 'historical', 'standard', 'see', 'bad', 'seventy', 'oil', 'price', 'rise', 'think', 'shock', 'absorb', 'quarter', 'vincent', 'koen', 'senior', 'economist', 'oecd', 'bbcs', 'world', 'business', 'report', 'recovery', 'underway', 'interrupt', 'bit', 'oil', 'shock', 'regain', 'momentum', 'course', 'china', 'boom', 'economy', 'spectacular', 'comeback', 'japan', 'albeit', 'falter', 'recent', 'month', 'help', 'world', 'economic', 'recovery', 'oecd', 'support', 'strong', 'balance', 'sheet', 'high', 'profit', 'recovery', 'business', 'investment', 'continue', 'north', 'america', 'start', 'earnest', 'europe', 'add', 'report', 'warn', 'remain', 'see', 'continental', 'europe', 'play', 'strong', 'supportive', 'role', 'marked', 'upswing', 'final', 'domestic', 'demand', 'oecd', 'highlight', 'current', 'depressed', 'household', 'expenditure', 'germany', 'eurozone', 'overreliance', 'exportle', 'growth']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kmo1HtyczIAo"
      },
      "source": [
        "#### 단어 빈도에 따라 시각화"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qw9wBnBgzC_O",
        "outputId": "1664c36a-a460-49cc-8e1f-de41223f671e"
      },
      "source": [
        "from collections import Counter\n",
        "word_counts = Counter()\n",
        "df['Tokens'].apply(lambda x: word_counts.update(x))\n",
        "word_counts.most_common(10)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('people', 2041),\n",
              " ('game', 1464),\n",
              " ('win', 1424),\n",
              " ('not', 1263),\n",
              " ('world', 1214),\n",
              " ('come', 1192),\n",
              " ('government', 1189),\n",
              " ('play', 1159),\n",
              " ('film', 1122),\n",
              " ('go', 1111)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ICWgWcZv8V1H"
      },
      "source": [
        "def word_count(docs):\n",
        "   \n",
        "    # 전체 코퍼스에서 단어 빈도 카운트\n",
        "    word_counts = Counter()\n",
        "\n",
        "    # 단어가 존재하는 문서의 빈도 카운트, 단어가 한 번 이상 존재하면 +1\n",
        "    word_in_docs = Counter()\n",
        "\n",
        "    # 전체 문서의 갯수\n",
        "    total_docs = len(docs)\n",
        "\n",
        "    for doc in docs:\n",
        "        word_counts.update(doc)\n",
        "        word_in_docs.update(set(doc))\n",
        "\n",
        "    temp = zip(word_counts.keys(), word_counts.values())\n",
        "\n",
        "    wc = pd.DataFrame(temp, columns = ['word', 'count'])\n",
        "\n",
        "    # 단어의 순위\n",
        "    # method='first': 같은 값의 경우 먼저나온 요소를 우선\n",
        "    wc['rank'] = wc['count'].rank(method='first', ascending=False)\n",
        "    total = wc['count'].sum()\n",
        "\n",
        "    # 코퍼스 내 단어의 비율\n",
        "    wc['percent'] = wc['count'].apply(lambda x: x / total)\n",
        "\n",
        "    wc = wc.sort_values(by='rank')\n",
        "\n",
        "    # 누적 비율\n",
        "    # cumsum() : cumulative sum\n",
        "    wc['cul_percent'] = wc['percent'].cumsum()\n",
        "\n",
        "    temp2 = zip(word_in_docs.keys(), word_in_docs.values())\n",
        "    ac = pd.DataFrame(temp2, columns=['word', 'word_in_docs'])\n",
        "    wc = ac.merge(wc, on='word')\n",
        "    \n",
        "    # 전체 문서 중 존재하는 비율\n",
        "    wc['word_in_docs_percent'] = wc['word_in_docs'].apply(lambda x: x / total_docs)\n",
        "\n",
        "    return wc.sort_values(by='rank')"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "uM-VGrJK8koe",
        "outputId": "d89e9b5e-7c88-47b1-d870-bfdc0afe3131"
      },
      "source": [
        "wc = word_count(df['Tokens'])\n",
        "wc.head(5)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>word_in_docs</th>\n",
              "      <th>count</th>\n",
              "      <th>rank</th>\n",
              "      <th>percent</th>\n",
              "      <th>cul_percent</th>\n",
              "      <th>word_in_docs_percent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>846</th>\n",
              "      <td>people</td>\n",
              "      <td>795</td>\n",
              "      <td>2041</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.004945</td>\n",
              "      <td>0.004945</td>\n",
              "      <td>0.357303</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1465</th>\n",
              "      <td>game</td>\n",
              "      <td>441</td>\n",
              "      <td>1464</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.003547</td>\n",
              "      <td>0.008492</td>\n",
              "      <td>0.198202</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1149</th>\n",
              "      <td>win</td>\n",
              "      <td>622</td>\n",
              "      <td>1424</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.003450</td>\n",
              "      <td>0.011942</td>\n",
              "      <td>0.279551</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2016</th>\n",
              "      <td>not</td>\n",
              "      <td>628</td>\n",
              "      <td>1263</td>\n",
              "      <td>4.0</td>\n",
              "      <td>0.003060</td>\n",
              "      <td>0.015002</td>\n",
              "      <td>0.282247</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>424</th>\n",
              "      <td>world</td>\n",
              "      <td>668</td>\n",
              "      <td>1214</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.002941</td>\n",
              "      <td>0.017943</td>\n",
              "      <td>0.300225</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        word  word_in_docs  count  ...   percent  cul_percent  word_in_docs_percent\n",
              "846   people           795   2041  ...  0.004945     0.004945              0.357303\n",
              "1465    game           441   1464  ...  0.003547     0.008492              0.198202\n",
              "1149     win           622   1424  ...  0.003450     0.011942              0.279551\n",
              "2016     not           628   1263  ...  0.003060     0.015002              0.282247\n",
              "424    world           668   1214  ...  0.002941     0.017943              0.300225\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 264
        },
        "id": "xKftOqtz8Qie",
        "outputId": "8c1e99f2-eb5b-4bc5-9a68-1e9b06167467"
      },
      "source": [
        "import squarify\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "colors=['springgreen','violet','dodgerblue','gold','wheat','lightcoral',\n",
        "        'mediumslateblue','greenyellow','aquamarine','deeppink']\n",
        "\n",
        "wc_top5 = wc[wc['rank'] <= 10]\n",
        "squarify.plot(sizes=wc_top5['percent'], label=wc_top5['word'], alpha=0.6, color=colors)\n",
        "plt.axis('off')\n",
        "plt.title('Token Top 10')\n",
        "plt.show()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD3CAYAAAC+eIeLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcjElEQVR4nO3deVzVdb7H8ff37CD7jqIsAoq4opaOIpKaSy6lTmXmraxsKnGpW82tpqysnOpOm5NZt2ayzEpEywV3ccncM3cB2ZV9EQQOZ/vdPwQHmbB08Hvg8H4+Hj4end92Pj99PF59+x05CUVRQEREcqjsPQARUXvC6BIRScToEhFJxOgSEUnE6BIRScToEhFJxOhSixNCDBdC5Nl7DqLWiNGlaxJCXGr0yyaEqG30erod5zrZaA6rEMLY6PXzLfQe8UKIHUKIi0KIrF/ZH1K/v0YIcUYIMbIl3pccG6NL16QoikvDLwA5ACY02rbcjnNFN5prN4DZjeZ6o4XephrA5wCeaWb/CgA/A/AG8AKARCGEbwu9NzkoRpduiBBCL4R4Twhxof7Xe0IIfTPHzhFCnBJCBNWf944QIkcIUSiE+FgI4VR/3HAhRJ4Q4mkhRJEQIl8I8dB1zqUSQrwohMiuv8YyIYR7/b4QIYQihJhVP3O+EOK/m7uWoigHFEX5EkDGr7xPJIAYAC8rilKrKMoqAMcBTLmeean9YXTpRr0AYBCAvgD6ALgFwItNDxJCvATgQQBxiqLkAVgEILL+vHAAnQC81OiUAADu9dsfBvB3IYTndcz1YP2veABhAFwALG5yTDyACAC3A3juBh8LRAPIUBSlqtG2X+q3EzWL0aUbNR3Aq4qiFCmKUgzgFQAzGu0XQoi/4XLY4hVFKRZCCACzAMxXFKWsPlhvALi30Xnm+uuaFUXZAOASgG7XOdffFEXJUBTlEoD/AXCvEELT6JhXFEWpVhTlOIB/AJh2XXd+mQuAi022XQTgegPXonZE89uHEP2qjgCyG73Ort/WwAOXA3uPoigNcfIF4Azg8OX+AgAEAHWj80oVRbE0el2Dy4H7T+bSAPBvtC23yf5e13H9BpcAuDXZ5gag6leOJbqCK126URcABDd63aV+W4NyAOMB/EMIMaR+WwmAWgDRiqJ41P9yr/8w7GbOZQFQ2Ghb52vM/XudBBAmhGi8su1Tv52oWYwu3agVAF4UQvgKIXxw+bnsV40PUBQlBZf/cz9JCHGLoig2AJ8CeFcI4QcAQohOQojRLTzXfCFEqBDCBZcfX3zbZPX8FyGEsxAiGsBDAL79tQvVfyhnAKC9/FIYhBC6+ntLBXAUwMv12+8C0BvAqha8F3JAjC7dqIUADgE4hsuf2h+p33YVRVG2AJgJYK0QIgbAcwDSAewTQlQC2Irre2b7Wz4H8CWAXQAyARgBJDQ5Zmf9DNsAvKMoyuZmrjUMl1fmG3B5RVwLoPGx9wIYgMur+kUAptY/3yZqluCXmFN7IYQIweUQa5usfImk4UqXiEgiRpeISCI+XiAikogrXSIiia75wxHB72OprEGodXh3bl97j9CqHYPdvuOn1Vrgu8PeI7Q+xbMfa24XV7pERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbpERBIxukREEjG6REQSMbrNyH8nZqzl4gWdvecgIseisfcArVXgfx9JtvcMROR42m10y1Y+0Udo9FbPu949Ufz55MGWolTvwD+fWHfp4LKO1fs+724uOBkQ8PTBJFtNmab4kwnjtB17FZjzT/irOnhX+yfs2qRycrfa+x6IqO1pt48XDN1G5tdlHwgAAHPBaV/FbNQq5lpVXdqOQH3IoPzGx1qrCtxd4+ad7PRK3kqhdzVV7f4wzD5TE1Fb126j6xQ1tsRSkuZrrSrUCrXWqu3Yq7D21AYfU+7hAEPUmKuiq3LxrXKKGlMKALrAnsWW0iwX+0xNRG1du328ILRONrWrf1Xl1kXddJ36Fmo79SmtPbOpk7Uy313fdVjFVceqtP96lKBSKzAb2+2/rIjoP9Ou46EL6p9ffXBZb31EfL5Tz4kFtb+sitL4dC0Rol3/thDRTdSu66KPiM+31V50duo5qVDrG1ELtc6q6zIw/7fPJCK6MUJRlGZ3Br+PpRJnoVbg3bl97T1Cq3YMy+09QquzwHeHvUdofYpnP9bcrna90iUiko3RJSKSiNElIpKI0SUikojRpVZnbvSZCYfWXvSx9xz2tGn+X3v+b6f4u9/oMODBdY+90hcA1jz4Qv8NTy7sbe/Z6D/Tbn84gqg1O/Htxh4zNn+y3q9nRLW9Z6GWxeiS3WQcqXF5Y3zmuMBIfUl+Wp2PX7Cu/Pn1YVf9/aPX78gYmne6zs9iUtS9R7hkJHzR5XDKsrKOyX8v7fnX/RGbAWDb52WdtnxSGr1o3+XXbd3yOx6PrS4qdVs+9k9ju9818mxF1nm3aT8s/rHxMR9FT5rg3S20pODomUBLXZ1m/JKXdux+89N+FZnnvcJGDj43+atFB+01vywPVy2P2WA6FeGuMtT6CdfqXpqOxRN0vc4nXFoZWweLpqPKrTLR9ZGUILWHyd6zNsbHC2RX5fkWj7FP+Jz8v/PR3+k7qEwrXszv0Xj/Yx8HHVySEZW0JDMqMf1QTcdftlR5Dbvf80JprsmjMKPOAAApX5R1i5vhecY+d9Dypq9fstvg6Vbz0O5l65y83OqaO06t09jmZmxMiv7j6FNrHnh+9J3/eH3PnPTk79I27IqsyL6glzmzbD/UHffdaj4betbzL4m73eclp1qLfAFg1qUV8Qucx+3P8Xo1MVLtV5ZQvbK/vWdtitElu3LxUl/6w90ehQAQN8MzLf1QbUDj/cmLS8L+FHpq8pNdT08pyTF7Zhyu9VSpBGLucEtb915JRNkFsy7vlNF/5CNeufa5A/uJmjwqCwACY3qUuXUJLPfpHlqjc3G2ufh7V5WcznDoL2XaZj4bMFwbkeWmMlh9Va7mIdqw7BrFpK1W6nTTDQPyAWC2IS71qCUv0N6zNsXHC2RXQjT/+tzhGtcd/yzv8/bhyCTvIK1pwchzw01GmxoAJj7le/b1OzLHaA3C2vM2lwytXtX8j1Y6KK2T3goAQqVS1FrNlS9lEiqVYjNbRPNnkj1xpUt2VVVqdflpVYUfAOxaXhEePsCpoNE+rdYgLB4BGlN+Wp1T+sGazg37gqIMNa7e6pqtn5bFjH3S56w9Zif7GaHtVrDLnB5caTOqS2yXNHvNGV2chc7sIvSmb+oOBwDAR8ZdEf00nVvdd6lwpUt25RmoqUheXBL92Zzzw3276MrvfS3w1PGhacEA0Pd217LACH3JrM6n7nHz0VR37mEobHzu4CkeaVs/KzX0GOZS8etXJ0c1Ud+rOMl0NLtb+atTPVTOtSFq7zJ34WRa6jJtR8KllbHPVn+vCVS5Va5yfSTF3rM2xS+8oavI/MKbjCM1LosmZY39JLfHyhs5/82JmUNC+hhKpr0WKG2lyy+8+Xf2+sKbEtsljY/KxVJuq9H0r3hr4mKXP+4ap4suscswTV3jC2+40qU26fGw05O1emGZ/3WXn+w9C9nHlMr/G5ZlK/M0K1b1BH3P1FYT3N/A6JLdhMU4X7rRVe6SjKiklp6H2padHvO223uGG8EP0oiIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkuuZ3LxytLJc1B7US9vm+KKL2gytdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIomEoijN7ix/rXypxFmoFVDZPrD3CPQb3i6YY+8R6DcsXOL5WHP7uNIlIpKI0SUikkhj7wFkSFiTELM1dWuEq8G11reDb3UP/x7FbgY30+rjq6MsNos60DXw4rczvt3h4eRhufOfdw7Xa/SWcyXnfC4aLzq9Pvb1lG+OfhN5puiMf6RvZNGaB9ekAMBXh78Kem/PewMsVosqwDWgcvn05Snezt4WO98qEbVyDr/STT6T7Lvz3M7QA3MPJG54eENyekm6LwA8OODBzCPzj6w+9vSxxBCvkIp3dr7TreGcKmOV/sDcA2ueGf7M3rnfzx0zd+jc48eePvZdVlmW17a0bd455TmGxT8ujtny6JZ1R586mhTtH1388qaXe9vvLomorXD4le6ujF0BQ0OHZrnqXa2uelfroOBB2QCwL2ef19RlUwdWm6p1dZY6bUynmLyGc0ZEjMhWCRUGdh5Y5qZ3q43rGlcGAMGeweXpJemuWeVZHc5XnveIWxI3CQAsNos6yi+q0D53SERticNHtznPb3h++NKpSzfdFn5b2aLtiyL35ezr2LBPr9FbAUAt1IpWrbU2bBdCKGabWahVavQO7H1+/cPrt9ljdiJquxz+8cKwsGEFe7P2BlfVValLa0o1+3P2dwEAo8WoDfUKrTGajar1p9dHXM81R0WOKjpTdMb/UN4hNwAorynXHMg54H4z5icix+LwK92x3ccWrzu1Lnvg+wOnuhvca4M9gsvcDG6mh295+NDtn9x+l6ve1RjlF1VUba7W/t5rBnsGG18f+3rKrJWzRphtZjUAzBk65+AtXW65ePPuhIgcQbv44YjSmlKNt7O3paK2QhP/cfzEt+54a9eoyFEl9p6rNeIPR7R+/OGI1u9aPxzh8CtdAHhgxQPDcipyPM1Ws3pM9zGpDC4R2Uu7iO66h9dtt/cMRERAO/ggjYioNWF0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImo3SgoPuby45G/hdtzBkaXiOzCYq0Tst+zuPyM65mMtXaNbrv4P0eQY/lrSkrEsiNH+gghEOrpWbpw9OiDf1q9enil0WhwMxiMn06enNIzIODSpC++GK7XaCzppaU+lUaj0xtjxqR8ffRo5JniYv9uPj5F3z/wQAoAfHnkSNB7e/YMMNtsqgBX18oV06aleDs7W+x8m3azLiUhJj1na4RB51rr7ORb7efVozgiZMz5Tbufi7Xa6jQuzgGVU27/IqWiKtt57Y4n45+878hq4PIq8ruN942ZM+NEYlr2Zp/tP7002Gw1avU6N+PkUZ+leHtE1Cz9dvAEb4/wksKSE4HhXUalZ13YHezn1aPoQvHPHc3mGt2oIW/sjA6fXLDr4KLIc7lbQyyWOm1VTb5778hpv1htJlVa1sZIlUprvX/i98luLp3qLhQdcduwc/5Qo+miQaPWW8bFvberS+DgiuVr7xyu03YwlZSn+hrrKpwH95u7b1Cf2Zm7D711a1X1BY8Pv+o9JTJkXOrooYuOy/795UqX2pTdmZmenx06FJM8c+a6E/PnJ3581117561dO3RSjx6pJ596KnF89+5pc9eu/UPD8VV1dfpDCQlrnomL2zvnhx/GzBs69PiJ+fO/yyov99qWnu6dU1Fh+HDv3pitjz667ti8eUnR/v7FL23e3Nue92hPqVnJvpl5O0Mfv/dA4oxJG5LLKtJ9AWDDznnxwwY+tz/h/uOJ3h7hZZt/fK5/l8DBFTabRZVffNQVAH4+/UXXsM63nbNYjKqte18Ycs+4b7fMnn40qWf4lLOb9jw3sOE9rFaz+snpPyeNjn3rGADYFIt48r4jq4cNeG7vj0f+1r/huIqqXK/7JqzePHPytqTDJz8bqNU4WWbf/8sqP+8ehft+WRwJAOtSEoaNGfbOntnTjybF3/rSvo27nh7acH5NbanzY/f89P3k2/+Z/NPRD28FgNgBz+738exWkHD/sVX2CC7AlS61MRtTUzvGhYZmdPHwMAJAJ3f3utSSEr/khx7aDABPx8amfbRv360Nx48ID89WCYFbgoLK3PT62uFhYWUA0MXDozytpMQ1s7y8w/nKSo9hH388CQAsNpu6u59foT3urTXIOr8rILjj0Cy9ztWqh6s1KGBQttlSozWba3Q9I/6YDwADej6amrRl5kgACOt8W8bPp5Z1DYzre/Rc7raud478dGte4QH3ykt5Xl/+MOEOAFAUm3AyeNY0vEd0xJRzjd8zqutdmQAQEhRXsvPQIteG7QE+vS50cPIxw8nHrNU4mXpF3psNAL5e3cuKy0571RhLNaUV6f6rtzw8quEcm82ibvjnrsGjslQqDYI7DqmoM1U63ZzfsevH6JJDM2g0VgBQqVSKRq22NmxXCaFYbDahVqnQOyDgfPLMmdvsN2Xb1S/qv84lbZk5Mq/gQCYABPkPrMzM2+nl6tKp/Ilph9b82jl6rau58Wut2lD/Z6RRFMV65b++1SrdlT8vCAGt1tkGAAIqxWazqhSbVWg1TnUJ9x9b9Wvvo1Hr/3W+okh/ftwcPl6gNmVMZOSFnZmZYbkVFXoAOH/xoj7Sx6fwg717uwLAu3v2hEf5+hb83uuNiogoOlNc7H8oL88NAMprazX7c3Pdb870rV9Ip2EFOfl7g+tMVeoaY6kmr2B/F63G2azVdjCdTE8KAIDDJz6LCPDpnQ8AHf1iKoVQKSkHFsZ07TziHAAE+Q+sqDNVGc5k/OAHABaLUZV1frdnS8/awdnP3MHJt2rf0Q/DAEBRbMjI3e51rXMMeg+zxVKrbelZrgdXutSmxIaGls8cMODn0Z9/PlElhNLVy6vkg4kTf5yVlDT8i8OH+zR8kPZ7rxfi6Wl8Y/TolEdWrRphtlrVADB3yJCDt3bufPGm3UQrFhkytvhsxrrsJSsGTjXo3Ws93ILL9Do307i4d3ds2v1c7PZ9L2tcnP0rp9y+LKXhnK6dR5w7fPKzQSMHL/waALRaZ9vE2z7asuXHF4Zs2fuiTlGsolfkvcdDOsWWt/S8d478dPv6lLmxB098GmOzWVRhnW9LD+t8W1lzx4d2GlYmhEr58KveUyNDxp21x3NdoShKszvLXytfKnEWagVUtg/sPQL9hrcL5tzU69cYSzXOBm+Lsa5C81li/MTRQ9/aFR48quSmvqmDWbjE87Hm9nGlS0RXWbXpgWEVVTmeNptZHRE8JpXBbVmMLhFdZcakddvtPYMju2Z0vf7yvqw5qJWoeKXVfMhL5JD4txeIiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgk0th7AGpd3O9ZYO8R6DcsxAIAQI1+rX0HoWsY3+wernSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnSJiCRidImIJGJ0iYgkYnRv1IKUSExcMcTeYxBR28LoEhFJ5Lj/54gj+S4Y//U4RHqXIK3UB8Ee5Vh/3w7szfXAM1sGw2jRwk1vxDdTU9DdpwYb072RkByLOosGHV0rkXh3CoLcTIj+aAIivUpxpCAQNkWFxWNTMKl78VXvlVFuwPSkWBRVuwAA3hyxF3dHF9rjtomodXPslW7+JQ88MfAkzj/9HTroTHhhezTmbxqCdfdtQcbcJEzrdRYJyQMBALPWxmPB8P3ImZ+ISO8yJCT3v3KdWosG2fNW4a2RuzE7efi/vc9D3/8B8249jnNzViPp7i2YvzFO1i0SUdviuCtdAPByunRlxTmjdxre2dsPuZVeiP/iDgCATRHwcqpB/iUdqs06TO+VDwCYfUsq7kkceeU69/VKBwBM61WA2claXKjSXfU+P+cH4dmtnnh26+XXNRYdSmo08HG23OxbJKK2xbGjK5q8dtaa0dmtHKkJa67ann9Jh2tpeh3RZIMC4Pjja+Cmt97QnETUbjj244XSWhesOuUHAFh+PBx9Awpxsc5wZVutWYUdmZ4IdDHBRWfCNycCAAAfHYxAv4D8K9f55kRXAMC3JwLgrDUh0MV01fvEBObhz1t7Xnm9Md37pt4XEbVZjr3SDXSpwOID0ZiTPBxdPMrx5oiTuLN7HuZvGoKnNutgtQn8V5/jiA8tx9LxO5CQHItnt2gQ6FqJVXenXLmOTmNByHtTYK3/IK2pL+7ci/uThqDLu1NhVVTo65+PMeG75d0oEbUVjh1dtUrBjgd3XLVtTHgpxoT/8G/HjgkvRVqTxw4NZvZLw4R7frpq24LhqQBSAQAhHkbsmbmtZYYmIkfm2I8XiIhaGcdd6cYEXkLu/JX/8XVOPrG2BaYhIgLAlS4RkVSMLrVbCxYj8vQ5ONt7Dvp1/W9/YsKGbQd8mm5//b2vI//46Gtt9ntPGF1qt1ZuRLf0HEa3NTKbLU3/drzDcNxnutTuHDkFlwl/wrhe3VBwIg3+3h6o3vUlNv30CzwSXkNsnQmajn6oTPwAKd9uQKdzufCd9RJG6HSwHFuDNe6u4A+3tIB5Ly3po9dprX998ZET0x5/Y3B65nnvgxv/vm550vaOy1Zu6T5qWEz2J19t6AdFEYP698hZ9uGz+wHAs/vkmeNuG3jq8LG0oNf//NCextdc+O7ybp+v2NjX2dlg6hrSsVSn1bTZPyuudMmhFJTCfd4DOJmXgpWuzjB9+BXCZv0F8QsSsD9nBxIjQ1CWsBD9n34ImV07o/iTV7EtextWMbgtJ35In/xDR88GAMCZtFzfWqNJazSaVLt+OhYYEuR/8f1PV9+6+Zs31x1P+TTxdFq270f//CEEAEwms2ZA38iiM3s+T5wyPrag4Xqp5/Kcl365vv/2VW9/fyB58ffZuYWedrq1FsHokkPx9UTVmKEoBYCekSg+lwu36lropo9HPgDMno7Uo6cRaN8pHdvouAEl6Vn5vkUlFVqtVm2N7hZSuDHlkM/PJ9ID3N061PWKCs0P6Rxg1Ou0yoTbB6ft2X8yEABUKqHMfmhSZtPrbd19xK/hHGcnvW10/IBz8u+q5TC65FC0mn+tWNVqKBercO3v1aAWZzDobH4+HlX/+3Fit949wgr/MLBH/tadhzsVFJW7h3YJqGruPI1GY9VqNYrMWe2B0SWH5uYCk4szTN9sQAAAfLQCEf2iLq96nQ0wV1QyyjdDv15d879O2t47bnDv/PEjby34ftNPUWHBASUjhvYrPn46MzA7r9BgNlvEui37wmMH9bxwrWuNjI0pOn46MzD3fJHeaDSpNqccDpN1HzcDP0gjh7f0VexIeA2xz74NTaAvKld9iBQAmDYeZ599B7EvfsAP0lrasEG981esTul3x8hbC708XC1ajdrav3dkfmTXoJq5j961f9Q9fx7f8EHa4w9MyL7WtSK7BtU8NuOOw3F3PX2ns7PBFB7SsUTWfdwMQlGaX80LLFgqcRZqBZQzr9h7BPqdavT8YcnWyjl0/GPN7ePjBSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiRhdIiKJGF0iIokYXSIiiYSiKPaegYio3eBKl4hIIkaXiEgiRpeISCJGl4hIIkaXiEgiRpeISKL/B/KpU0BZaenDAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "0f-zAsvxdSLg",
        "outputId": "7568e594-a2c6-4dd9-e360-6727c73a95d9"
      },
      "source": [
        "sns.set_palette('GnBu')\n",
        "sns.barplot(x=wc['count'][:10],y=wc['word'][:10],orient='h')\n",
        "plt.title('Token Top 10')\n",
        "plt.show()"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAAEWCAYAAAA+bHOCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAeKklEQVR4nO3de5QdZZ3u8e8DSYghhCQkchDF5iqTIATScchAMHA4KCy8ggpyS3RWjiOC4MABBxCYpS5wUBRFWMGBBIwMw01gRgUEchEJ2IFciOEawoDEECDkAiQhye/8UW9LsbP7Fvala+/ns9ZeXfvdb1X9dnX3fvqtqq5SRGBmZlYUW9W7ADMzs55wcJmZWaE4uMzMrFAcXGZmVigOLjMzKxQHl5mZFYqDy6yXkjRe0ov1rsOst3FwmdWApDW5xyZJb+Wen1DHuhbm6tgoaW3u+b9UaB2HSnpA0kpJS8q83pJef1PSE5IOr8R6rXE5uMxqICIGtj+A/wE+lWubVse6RubqmgV8I1fX9yu0mjeAa4GzO3j9RuAxYAfgPOAWScMrtG5rQA4uszqStI2kH0t6KT1+LGmbDvqeLunPkj6Y5rtM0v9IWibpaknvS/3GS3pR0j9LelnSUkkTe1jXVpLOl/R8Wsb1krZPr7VICkmTUs1LJZ3V0bIi4pGIuAFYXGY9ewEHABdGxFsRcSuwADimJ/Vac3FwmdXXecCBwChgP+BjwPmlnSR9B5gAfDwiXgQuAfZK8+0B7Ax8JzfL/wK2T+1fBa6UNKQHdU1Ij0OB3YCBwM9K+hwK7AkcAZyzhbv4RgKLI2J1rm1eajcry8FlVl8nAP8aES9HxHLgYuCk3OuS9COycDg0IpZLEjAJODMiXksf+t8HjsvN93Za7tsR8RtgDfCRHtb1o4hYHBFrgG8Dx0nqk+tzcUS8ERELgOuA43v0zjMDgZUlbSuB7bZgWdYk+nTdxcyq6APA87nnz6e2doPJQupLEdH+AT8cGADMyTIMAAFb5+Z7NSI25J6/SRYS76WuPsCOubYXSl7/aA+W324NMKikbRCwukxfM8AjLrN6ewn4cO75Lqmt3QrgaOA6SQeltleAt4CRETE4PbZPJ1hUs64NwLJc24c6qbu7FgK7ScqPsPZL7WZlObjM6utG4HxJwyUNIztO9ct8h4iYTrbr7jZJH4uITcA1wOWS3g8gaWdJn6hwXWdK2lXSQLJdkTeVjOIukDRA0khgInBTuQWlEz36A32zp+ovqV96b08Bc4ELU/vngH2BWyv4XqzBOLjM6uu7QBswn+xsukdT27tExL3AV4C7JB0AnAM8A8yWtAr4PT07htWVa4EbgJnAc8Ba4LSSPjNSDfcBl0XEPR0s6xCyEeJvyEZmbwH5vscBrWSjy0uAY9PxPrOy5BtJmllPSGohC7O+JSMws5rwiMvMzArFwWVmZoXiXYVmZlYoHnGZmVmh+B+Qa2DYsGHR0tJS7zLMzAplzpw5r0TEZhdcdnDVQEtLC21tbfUuw8ysUCQ9X67duwrNzKxQPOKqgQ3LX2P5Vb/suqOZWQMZ/k8nVmW5HnGZmVmhOLjMzKxQHFxmZlYoDi4zMysUB1c3SJogqfS25WZmVgcOLjMzK5SGCC5JLZKekDRN0iJJt6Qb3I2WNEPSHEl3S9op9R8labak+ZJulzQktU+X9BNJcyU9LuljZdY1XNKtkv6UHgeV9jEzs+ppiOBKPgL8PCL+DlgFnAr8lOymdKPJboz3vdT3euCciNiX7OZ9F+aWMyAiRgFfT/OU+glweUSMAY4BflGuGEmTJLVJant1zar3/u7MzAxorH9AfiEiHkzTvwT+BdgHuFcSwNbAUknbA4MjYkbqOxW4ObecGwEiYqakQZIGl6zncGBEWibAIEkDI2JNvlNETAYmA4z68G6+BL+ZWYU0UnCVhsNqYGFEjM03puDqyXJKn28FHBgRa3teopmZvVeNtKtwF0ntIfVlYDYwvL1NUl9JIyNiJbBC0rjU9yRgRm45X0r9DwZWpv559wCntT+RNKryb8XMzDrSSCOuJ4FTJV0L/Jns+NbdwBVplNUH+DGwEDgFuFrSAGAxMDG3nLWSHgP6Al8ps57TgSslzU/LnAl8rTpvyczMSjVScG2IiNIrOs4FDintGBFzgQM7WM4vI+KMkv5TgClp+hXSqMzMzGqvkXYVmplZE2iIEVdELCE7g/C9Lmf8ey7GzMyqyiMuMzMrlIYYcfV2fYYPrdoN1czMmo1HXGZmVigOLjMzKxQHl5mZFYqPcdXA2pef4YkrP1PvMsws2fvUO+pdgr0HHnGZmVmhOLjMzKxQHFxmZlYoDi4zMysUB5eZmRWKg8vMzArFwWVmZoXS0P/HJekC4ERgOfACMAdYCUwC+gHPACdFxJuSpgBvAfsD7ye7ieTJwFjg4YiYkJZ5BHAxsA3wLDAxItbU7l2ZmTW3hh1xSRoDHAPsBxwJtKaXbouIMRGxH7AI+GputiFkQXUmcCdwOTAS+KikUZKGAecDh0fEAUAb8K0O1j9JUpukthVr1lf+DZqZNalGHnEdBNwREWuBtZLuSu37SPouMBgYCNydm+euiAhJC4BlEbEAQNJCoAX4IDACeFASZKO2h8qtPCImA5MB9tllcFT4vZmZNa1GDq6OTAE+GxHzJE0AxudeW5e+bspNtz/vA2wE7o2I46tfppmZldOwuwqBB4FPSeovaSBwdGrfDlgqqS9wQg+XORs4SNIeAJK2lbRXxSo2M7MuNeyIKyL+JOlOYD6wDFhAdmLGBcDDZCdsPEwWZN1d5vI0SrtR0jap+XzgqQqWbmZmnWjY4Eoui4iLJA0AZgJzIuJR4KrSju1nDabpJcA+Hbx2PzCmeiWbmVlnGj24JksaAfQHpqbQMjOzAmvo4IqIL9e7BjMzq6xGPjnDzMwaUEOPuHqL/u/fw3dcNTOrEI+4zMysUBxcZmZWKA4uMzMrFB/jqoEVrzzNLdd9st5lmDWVYyf+rt4lWJV4xGVmZoXi4DIzs0JxcJmZWaE4uMzMrFAcXGZmVigOrm6Q9BtJg+tdh5mZ+XT4bomIo+pdg5mZZTziAiSdLen0NH25pPvT9GGSpklaImmYpBZJiyRdI2mhpHskva++1ZuZNRcHV2YWMC5NtwIDJfVNbTNL+u4JXBkRI4HXgWPKLVDSJEltktpWrVlfpbLNzJqPgyszBxgtaRCwDniILMDGkYVa3nMRMTc3X0u5BUbE5IhojYjWQQP7VadqM7Mm5GNcQES8Lek5YALwR2A+cCiwB7CopPu63PRGwLsKzcxqyCOud8wCziLbNTgL+BrwWEREXasyM7N3cXC9YxawE/BQRCwD1rL5bkIzM6sz7ypMIuI+oG/u+V656ZY0+QqwT679slrVZ2ZmGY+4zMysUBxcZmZWKN5VWANDhu3pm9qZmVWIR1xmZlYoDi4zMysUB5eZmRWKg8vMzArFJ2fUwEsrnuai//xEvcsw6zUu+uLd9S7BCswjLjMzKxQHl5mZFYqDy8zMCsXBZWZmheLgeg8kTZD0gXrXYWbWTBxc780EwMFlZlZDDq4cSS2SFkm6RtJCSfdIep+kUZJmS5ov6XZJQyQdC7QC0yTNleQ7IZuZ1YCDa3N7AldGxEjgdeAY4HrgnIjYF1gAXBgRtwBtwAkRMSoi3sovRNIkSW2S2t5ctb7Gb8HMrHE5uDb3XETMTdNzgN2BwRExI7VNBQ7paiERMTkiWiOidcCgflUq1cys+Ti4NrcuN70RGFyvQszMbHMOrq6tBFZIGpeenwS0j75WA9vVpSozsyblaxV2zynA1ZIGAIuBial9Smp/CxhbepzLzMwqz8GVExFLgH1yzy/LvXxgmf63ArdWvzIzM2vnXYVmZlYoDi4zMysUB5eZmRWKj3HVwAeG7Okb55mZVYhHXGZmVigOLjMzKxQHl5mZFYqDy8zMCsUnZ9TA068v5ajbv1vvMsxq6jefO7/eJViD8ojLzMwKxcFlZmaF4uAyM7NCcXCZmVmhOLi6QdJ0Sa1l2idI+lk9ajIza1YOri5I2rreNZiZ2TsaOrgknS3p9DR9uaT70/RhkqZJOl7SAkmPS7o0N98aST+UNA8YW7LMiZKekvQIcFAt34+ZmTV4cAGzgHFpuhUYKKlvansKuBQ4DBgFjJH02dR3W+DhiNgvIv7QvjBJOwEXkwXWwcCIjlYsaZKkNklt61e9UeG3ZWbWvBo9uOYAoyUNAtYBD5EF2DjgdWB6RCyPiA3ANOCQNN9Gyt/Z+O9z86wHbupoxRExOSJaI6K136BtK/eOzMyaXKdXzpD0UyA6ej0iTq94RRUUEW9Leg6YAPwRmA8cCuwBLAFGdzDr2ojYWIsazcysZ7oacbWRjVr6AwcAT6fHKKBfdUurmFnAWcDMNP014DHgEeDjkoalEzCOB2Z0sayH0zw7pF2OX6he2WZmVk6nI66ImAog6Z+Ag9MuNSRdTRYCRTALOA94KCLekLQWmBURSyWdCzwACPjviLijswWleS4i2+X4OjC3uqWbmVmp7l5kdwgwCHgtPR+Y2nq9iLgP6Jt7vldu+kbgxjLzDCx5Pj43fR1wXTVqNTOzrnU3uC4BHpPUPjo5BLioWkWZmZl1pMvgkrQV8CTZGXV/n5rPiYi/VrMwMzOzcroMrojYJOnKiNgf6PQYkJmZWbV1d1fhfZKOAW6LiA5Pj7fy9hy8k2+qZ2ZWId39B+T/C9wMrJe0Oj1WVbEuMzOzsro14oqI7apdiJmZWXd0d1chkj7NO5dEmh4R/1WdkszMzDrWreCSdAkwhux6fgDflHRQRHy7apU1kGdWvMbRt0zruqNZgf3XsSfUuwRrEt0dcR0FjIqITQCSppJdNsnBZWZmNdWTq8MPzk1vX+lCzMzMuqO7I67vA49Kms47V844t1pFmZmZdaS7wXU0cC2wgux2IL5yhpmZ1UV3g+vfyW6++Glgd7LrFs6MiJ9UrTIzM7Myuvt/XA9Imkl2ZuGhZPe0Ggk4uMzMrKa6ezr8fcC2ZPehmgWMiYiXq1mYmZlZOd09q3A+sB7YB9gX2EfS+6pWVQVJOlnSfEnzJN0gqUXS/antPkm7pH5TJF0labakxZLGS7pW0iJJU3LLO0LSQ5IelXSzpIEdrtzMzCquW8EVEWdGxCHA54FXyW6k+Ho1C6sESSOB84HDImI/4JvAT4GpEbEv2T9UX5GbZQgwFjgTuBO4nGyX6EcljZI0LC3v8Ig4AGgDvtXBuidJapPUtn6VL+toZlYp3d1V+A2ykzNGk51VeC3ZLsPe7jDg5oh4BSAiXpM0liyAAW4AfpDrf1dEhKQFwLKIWAAgaSHQAnwQGAE8KAmgH9nu081ExGRgMsDg3XfzFfXNzCqku2cV9gd+BMyJiA1VrKfe1qWvm3LT7c/7ABuBeyPi+FoXZmZmme7uKrwsIh4uYGjdD3xB0g4AkoYCfwSOS6+fQM9GjrOBgyTtkZa3raS9KlivmZl1odtXhy+iiFgo6XvADEkbya6veBpwnaSzgeXAxB4sb7mkCcCNkrZJzecDT1W2cjMz60hDBxdAREwFppY0H1am34Tc9BKyMyjLvXY/2f+zmZlZHfTkIrtmZmZ15+AyM7NCcXCZmVmhNPwxrt5gjyFDfXdYM7MK8YjLzMwKxcFlZmaF4uAyM7NC8TGuGnh2xRo+d+sf6l2GWVXdfszB9S7BmoRHXGZmVigOLjMzKxQHl5mZFYqDy8zMCsXBZWZmheLg2kKSWiR9ud51mJk1m4YILkn1OK2/BXBwmZnVWFWDS9IFkp6U9AdJN0o6S9IoSbMlzZd0u6QhkvaW9EhuvhZJC9L0aEkzJM2RdLeknVL7dEk/ltQGfDM9v1TSI5KekjQu9Zsg6deS7pW0RNI3JH1L0mOpjqGp3+6SfpfWM0vS3ql9iqQrJP1R0mJJx6YyLwHGSZor6cxqbkczM3tH1YJL0hjgGGA/4EigNb10PXBOROwLLAAujIgngH6Sdk19vgTcJKkv8FPg2IgYDVwLfC+3mn4R0RoRP0zP+0TEx4AzgAtz/fYBPk92A8jvAW9GxP7AQ8DJqc9k4LS0nrOAn+fm3wk4GDiaLLAAzgVmRcSoiLi8zPufJKlNUtu6Va93Z5OZmVk3VHMX20HAHRGxFlgr6S5gW2BwRMxIfaYCN6fp/yQLrEvS1y8BHyELnXslAWwNLM2t46aSdd6Wvs4h25XX7oGIWA2slrQSuCu1LwD2lTQQ+Afg5rQegG1y8/86IjYBf5a0Y3fefERMJgtDhuy+d3RnHjMz61pvuuTTTWTBcRsQEfG0pI8CCyNibAfzvFHyfF36upF3v7d1uelNueebUr+tgNcjYlQH68nPrw76mJlZDVTzGNeDwKck9U8jmqPJgmZF+/En4CRgBkBEPEsWOBfwzkjqSWC4pLEAkvpKGlnpQiNiFfCcpC+k9UjSfl3MthrYrtK1mJlZ56oWXBHxJ+BOYD7wW7LdciuBU4B/kzQfGAX8a262m4ATyXYbEhHrgWOBSyXNA+aS7dKrhhOAr6b1LAQ+00X/+cBGSfN8coaZWe0oonqHXyQNjIg1kgYAM4FJEfFo1VbYSw3Zfe8Y/4Nf1LsMs6ry1eGt0iTNiYjW0vZqH+OaLGkE0B+Y2oyhZWZmlVXV4IoI/4OumZlVVG86q7Bh7T5koHejmJlVSENc8snMzJqHg8vMzArFwWVmZoXi4DIzs0LxyRk18PLrb3Pl7cvqXYZZxZz6uW5dstOsKjziMjOzQnFwmZlZoTi4zMysUBxcZmZWKA6uEpKmS9rsoo5mZtY7OLjMzKxQmja4JLVIekLSNEmLJN2Sbr+S73OVpDZJCyVdnNoOk/TrXJ//I+n2WtdvZtasmja4ko8AP4+IvwNWAV8vef28dC+YfYGPS9oXeADYW9Lw1GcicG3pgiVNSqHXtmbVa9V7B2ZmTabZg+uFiHgwTf8SKL2E+xclPQo8BowERkR2580bgBMlDQbGkt3h+V0iYnJEtEZE68BBQ6v3DszMmkyzXzmj9PbPf3suaVfgLGBMRKyQNIXshpgA1wF3AWuBmyNiQw1qNTMzPOLaRdLYNP1l4A+51wYBbwArJe0IHNn+QkS8BLwEnE8WYmZmViPNHlxPAqdKWgQMAa5qfyEi5pHtInwC+BXwYMm808h2NS6qUa1mZoZ3FW6IiBNL2sa3T0TEhE7mPRi4pgo1mZlZJ5o9uLaIpDlkuxH/ud61mJk1m6YNrohYAuyzhfOOrmw1ZmbWXc1+jMvMzAqmaUdctfT+wX194z0zswrxiMvMzArFwWVmZoXi4DIzs0LxMa4aePOVDTz2i5frXYZZj+3/j++vdwlmm/GIy8zMCsXBZWZmheLgMjOzQnFwmZlZoTi4zMysUJo+uCSdLmmRpBWSzk1tF0k6q961mZnZ5nw6PHwdODwiXqx3IWZm1rWmHnFJuhrYDfitpDMl/axMn+mSLpfUlkZmYyTdJulpSd+tfdVmZs2tqYMrIr4GvAQcCqzopOv6iGgFrgbuAE4luyXKBEk7lJtB0qQUdm0rVr9a4crNzJpXUwdXD9yZvi4AFkbE0ohYBywGPlRuhoiYHBGtEdE6ZLuy2WZmZlvAwdU969LXTbnp9uc+TmhmVkMOLjMzKxQHl5mZFUrT7+aKiJY0OSU9iIiLcq+Pz01PB6aXe83MzGrDIy4zMysUB5eZmRWKg8vMzAql6Y9x1cKAYX18J1kzswrxiMvMzArFwWVmZoXi4DIzs0LxMa4aeHvZmyz78Zx6l2HWIzueMbreJZiV5RGXmZkVioPLzMwKxcFlZmaF4uAyM7NCcXCZmVmhOLjMzKxQHFxmZlYo/j+uTki6ADgRWA68AMwBfg9cDQwAngW+EhEr6lakmVmT8YirA5LGAMcA+wFHAq3ppeuBcyJiX2ABcGEH80+S1Cap7bU3nGtmZpXi4OrYQcAdEbE2IlYDdwHbAoMjYkbqMxU4pNzMETE5IlojonXotkNqU7GZWRNwcJmZWaE4uDr2IPApSf0lDQSOBt4AVkgal/qcBMzoaAFmZlZ5PjmjAxHxJ0l3AvOBZWTHs1YCpwBXSxoALAYm1q9KM7Pm4+Dq3GURcVEKqZnAnIiYCxxY57rMzJqWg6tzkyWNAPoDUyPi0XoXZGbW7BxcnYiIL9e7BjMzezefnGFmZoXiEVcN9N1xgO8ma2ZWIR5xmZlZoSgi6l1Dw5O0Gniy3nV0YBjwSr2L6IBr2zKubcu4ti1Tzdo+HBHDSxu9q7A2noyI1q671Z6kNtfWc65ty7i2LePa3s27Cs3MrFAcXGZmVigOrtqYXO8COuHatoxr2zKubcu4thyfnGFmZoXiEZeZmRWKg8vMzArFwVVFkj4p6UlJz0g6tw7r/5CkByT9WdJCSd9M7RdJ+oukuelxVG6eb6d6n5T0iSrXt0TSglRDW2obKuleSU+nr0NSuyRdkWqbL+mAKtb1kdy2mStplaQz6rndJF0r6WVJj+faerytJJ2S+j8t6ZQq1vZvkp5I679d0uDU3iLprdw2vDo3z+j08/BMql9Vqq3H38dq/C53UNtNubqWSJqb2mu93Tr67OgVP3NEhB9VeABbA88CuwH9gHnAiBrXsBNwQJreDngKGAFcBJxVpv+IVOc2wK6p/q2rWN8SYFhJ2w+Ac9P0ucClafoo4LeAyG4r83ANv49/BT5cz+0GHAIcADy+pdsKGEp2D7mhwJA0PaRKtR0B9EnTl+Zqa8n3K1nOI6lepfqPrFJtPfo+Vut3uVxtJa//EPhOnbZbR58dveJnziOu6vkY8ExELI6I9cB/AJ+pZQERsTTSrVgiYjWwCNi5k1k+A/xHRKyLiOeAZ8jeRy19BpiapqcCn821Xx+Z2cBgSTvVoJ7/DTwbEc930qfq2y0iZgKvlVlvT7bVJ4B7I+K1iFgB3At8shq1RcQ9EbEhPZ0NfLCzZaT6BkXE7Mg+8a7PvZ+K1taJjr6PVfld7qy2NGr6InBjZ8uo4nbr6LOjV/zMObiqZ2fghdzzF+k8NKpKUguwP/BwavpGGtJf2z7cp/Y1B3CPpDmSJqW2HSNiaZr+K7BjnWprdxzv/vDoDdutXU+3Vb3q/ArZX+PtdpX0mKQZksaltp1TPbWqrSffx3pst3HAsoh4OtdWl+1W8tnRK37mHFxNQNJA4FbgjIhYBVwF7A6MApaS7ZKoh4Mj4gDgSOBUSYfkX0x/Qdbt/zUk9QM+DdycmnrLdttMvbdVRySdB2wApqWmpcAuEbE/8C3gV5IG1bisXvt9zDmed//BVJftVuaz42/q+TPn4KqevwAfyj3/YGqrKUl9yX7wpkXEbQARsSwiNkbEJuAa3tmtVdOaI+Iv6evLwO2pjmXtuwDT15frUVtyJPBoRCxLdfaK7ZbT021V0zolTQCOBk5IH3Kk3XCvpuk5ZMeO9kp15HcnVq22Lfg+1nq79QE+D9yUq7nm263cZwe95GfOwVU9fwL2lLRr+sv9OODOWhaQ9pP/O7AoIn6Ua88fG/oc0H5W053AcZK2kbQrsCfZgd9q1LatpO3ap8kO5j+eamg/8+gU4I5cbSens5cOBFbmdllUy7v+6u0N261ET7fV3cARkoak3WNHpLaKk/RJ4P8Bn46IN3PtwyVtnaZ3I9tWi1N9qyQdmH5uT869n0rX1tPvY61/lw8HnoiIv+0CrPV26+izg97yM/dez+7wo9Mzc44iOxvnWeC8Oqz/YLKh/HxgbnocBdwALEjtdwI75eY5L9X7JBU4O6mT2nYjOztrHrCwffsAOwD3AU8DvweGpnYBV6baFgCtVd522wKvAtvn2uq23cgCdCnwNtlxgq9uybYiO970THpMrGJtz5Ad22j/ubs69T0mfb/nAo8Cn8otp5UsRJ4Ffka6sk8Vauvx97Eav8vlakvtU4CvlfSt9Xbr6LOjV/zM+ZJPZmZWKN5VaGZmheLgMjOzQnFwmZlZoTi4zMysUBxcZmZWKA4uM+sWZVfIH1DvOsx8OryZdYukJWT/n/NKvWux5uYRl1kDkXRyunjsPEk3KLuP0/2p7T5Ju6R+UyQdm5tvTfo6XtJ0Sbcou5/WtHQ1hNOBDwAPSHqgPu/OLNOn3gWYWWVIGgmcD/xDRLwiaSjZrSemRsRUSV8BrqDr217sD4wEXgIeBA6KiCskfQs41CMuqzePuMwax2HAze3BEhGvAWOBX6XXbyC7lE9XHomIFyO7CO1cspsYmvUaDi6z5rSB9PsvaSuyO/u2W5eb3oj3zFgv4+Ayaxz3A1+QtANA2lX4R7KrmQOcAMxK00uA0Wn600Dfbix/Ndlt3M3qyn9JmTWIiFgo6XvADEkbgceA04DrJJ0NLAcmpu7XAHdImgf8DnijG6uYDPxO0ksRcWjl34FZ9/h0eDMzKxTvKjQzs0JxcJmZWaE4uMzMrFAcXGZmVigOLjMzKxQHl5mZFYqDy8zMCuX/A2389nLCA8YeAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q3vkzeqC9wie"
      },
      "source": [
        "#### TfidVectorizer + KNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "id": "XvQBRavT90yX",
        "outputId": "20c41711-297a-44c8-98a0-f1cfc95a5c68"
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "tfidf = TfidfVectorizer(stop_words= st_words, max_features=5000)\n",
        "dtm = tfidf.fit_transform(df['Text'])\n",
        "dtm = pd.DataFrame(dtm.todense(), columns=tfidf.get_feature_names())\n",
        "\n",
        "dtm.head()"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/feature_extraction/text.py:385: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['ll', 've'] not in stop_words.\n",
            "  'stop_words.' % sorted(inconsistent))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>000</th>\n",
              "      <th>000m</th>\n",
              "      <th>05</th>\n",
              "      <th>10</th>\n",
              "      <th>100</th>\n",
              "      <th>100m</th>\n",
              "      <th>102</th>\n",
              "      <th>10bn</th>\n",
              "      <th>10th</th>\n",
              "      <th>11</th>\n",
              "      <th>110</th>\n",
              "      <th>11bn</th>\n",
              "      <th>12</th>\n",
              "      <th>120</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>150</th>\n",
              "      <th>1500m</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>1960s</th>\n",
              "      <th>1970</th>\n",
              "      <th>1970s</th>\n",
              "      <th>1973</th>\n",
              "      <th>1980s</th>\n",
              "      <th>1981</th>\n",
              "      <th>1983</th>\n",
              "      <th>1984</th>\n",
              "      <th>1985</th>\n",
              "      <th>1987</th>\n",
              "      <th>1989</th>\n",
              "      <th>1990s</th>\n",
              "      <th>1991</th>\n",
              "      <th>1992</th>\n",
              "      <th>1993</th>\n",
              "      <th>1994</th>\n",
              "      <th>1995</th>\n",
              "      <th>...</th>\n",
              "      <th>worried</th>\n",
              "      <th>worries</th>\n",
              "      <th>worry</th>\n",
              "      <th>worrying</th>\n",
              "      <th>worse</th>\n",
              "      <th>worst</th>\n",
              "      <th>worth</th>\n",
              "      <th>wouldn</th>\n",
              "      <th>wright</th>\n",
              "      <th>write</th>\n",
              "      <th>writer</th>\n",
              "      <th>writers</th>\n",
              "      <th>writing</th>\n",
              "      <th>written</th>\n",
              "      <th>wrong</th>\n",
              "      <th>wrote</th>\n",
              "      <th>wto</th>\n",
              "      <th>xbox</th>\n",
              "      <th>xp</th>\n",
              "      <th>yachvili</th>\n",
              "      <th>yahoo</th>\n",
              "      <th>yard</th>\n",
              "      <th>yards</th>\n",
              "      <th>yelling</th>\n",
              "      <th>yen</th>\n",
              "      <th>yes</th>\n",
              "      <th>york</th>\n",
              "      <th>yorkshire</th>\n",
              "      <th>young</th>\n",
              "      <th>younger</th>\n",
              "      <th>youngsters</th>\n",
              "      <th>youth</th>\n",
              "      <th>yuan</th>\n",
              "      <th>yugansk</th>\n",
              "      <th>yuganskneftegas</th>\n",
              "      <th>yukos</th>\n",
              "      <th>yushchenko</th>\n",
              "      <th>zealand</th>\n",
              "      <th>zero</th>\n",
              "      <th>zone</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.028497</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.029905</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.038506</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.065307</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.059999</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.095496</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.420712</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.372661</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.021711</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.022784</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.029337</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.032392</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.052154</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 5000 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        000  000m   05        10  ...  yushchenko  zealand  zero  zone\n",
              "0  0.028497   0.0  0.0  0.029905  ...         0.0      0.0   0.0   0.0\n",
              "1  0.000000   0.0  0.0  0.000000  ...         0.0      0.0   0.0   0.0\n",
              "2  0.000000   0.0  0.0  0.000000  ...         0.0      0.0   0.0   0.0\n",
              "3  0.021711   0.0  0.0  0.022784  ...         0.0      0.0   0.0   0.0\n",
              "4  0.000000   0.0  0.0  0.052154  ...         0.0      0.0   0.0   0.0\n",
              "\n",
              "[5 rows x 5000 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8UlOT9uz-HO7"
      },
      "source": [
        "def tokenize(document):\n",
        "    \n",
        "    doc = nlp(document)\n",
        "    \n",
        "    return [token.lemma_.strip() for token in doc if (token.is_stop != True) and (token.is_punct != True) and (token.is_alpha == True)]"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        },
        "id": "b3PtWpcc-LiG",
        "outputId": "28759053-11dd-455d-f1e1-3056db5874cd"
      },
      "source": [
        "tfidf = TfidfVectorizer(stop_words=st_words\n",
        "                        ,tokenizer=tokenize\n",
        "                        ,ngram_range=(1,2)\n",
        "                        ,max_df=.7\n",
        "                        ,min_df=3\n",
        "                       )\n",
        "\n",
        "dtm = tfidf.fit_transform(df['Text'])\n",
        "dtm = pd.DataFrame(dtm.todense(), columns=tfidf.get_feature_names())\n",
        "dtm.head()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/feature_extraction/text.py:385: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['d', 'good', 'tell', 've'] not in stop_words.\n",
            "  'stop_words.' % sorted(inconsistent))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>-PRON-</th>\n",
              "      <th>aaa</th>\n",
              "      <th>aaas</th>\n",
              "      <th>aaas title</th>\n",
              "      <th>aaron</th>\n",
              "      <th>abandon</th>\n",
              "      <th>abandon plan</th>\n",
              "      <th>abandonment</th>\n",
              "      <th>abate</th>\n",
              "      <th>abba</th>\n",
              "      <th>abbas</th>\n",
              "      <th>abbasi</th>\n",
              "      <th>abbasi mubanga</th>\n",
              "      <th>abbott</th>\n",
              "      <th>abc</th>\n",
              "      <th>abc network</th>\n",
              "      <th>aberdeen</th>\n",
              "      <th>abeyie</th>\n",
              "      <th>abide</th>\n",
              "      <th>abide member</th>\n",
              "      <th>ability</th>\n",
              "      <th>ability control</th>\n",
              "      <th>ability game</th>\n",
              "      <th>ability influence</th>\n",
              "      <th>ability record</th>\n",
              "      <th>able</th>\n",
              "      <th>able access</th>\n",
              "      <th>able buy</th>\n",
              "      <th>able choose</th>\n",
              "      <th>able control</th>\n",
              "      <th>able deliver</th>\n",
              "      <th>able download</th>\n",
              "      <th>able handle</th>\n",
              "      <th>able offer</th>\n",
              "      <th>able play</th>\n",
              "      <th>able sell</th>\n",
              "      <th>able spend</th>\n",
              "      <th>able start</th>\n",
              "      <th>able store</th>\n",
              "      <th>able tell</th>\n",
              "      <th>...</th>\n",
              "      <th>yulia tymoshenko</th>\n",
              "      <th>yushchenko</th>\n",
              "      <th>yusuf</th>\n",
              "      <th>z</th>\n",
              "      <th>zach</th>\n",
              "      <th>zach braff</th>\n",
              "      <th>zajec</th>\n",
              "      <th>zambia</th>\n",
              "      <th>zealand</th>\n",
              "      <th>zealand australia</th>\n",
              "      <th>zealand lion</th>\n",
              "      <th>zealand rugby</th>\n",
              "      <th>zealand summer</th>\n",
              "      <th>zealand world</th>\n",
              "      <th>zealander</th>\n",
              "      <th>zelda</th>\n",
              "      <th>zellweger</th>\n",
              "      <th>zen</th>\n",
              "      <th>zen micro</th>\n",
              "      <th>zeppelin</th>\n",
              "      <th>zero</th>\n",
              "      <th>zero tolerance</th>\n",
              "      <th>zeta</th>\n",
              "      <th>zeta jones</th>\n",
              "      <th>zhang</th>\n",
              "      <th>zheng</th>\n",
              "      <th>zidane</th>\n",
              "      <th>zimbabwe</th>\n",
              "      <th>zimbabwe burma</th>\n",
              "      <th>zinc</th>\n",
              "      <th>ziyi</th>\n",
              "      <th>ziyi zhang</th>\n",
              "      <th>zoe</th>\n",
              "      <th>zola</th>\n",
              "      <th>zombie</th>\n",
              "      <th>zombie bot</th>\n",
              "      <th>zone</th>\n",
              "      <th>zoom</th>\n",
              "      <th>zurich</th>\n",
              "      <th>zurich premiership</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 26993 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   -PRON-  aaa  aaas  aaas title  ...  zone  zoom  zurich  zurich premiership\n",
              "0     0.0  0.0   0.0         0.0  ...   0.0   0.0     0.0                 0.0\n",
              "1     0.0  0.0   0.0         0.0  ...   0.0   0.0     0.0                 0.0\n",
              "2     0.0  0.0   0.0         0.0  ...   0.0   0.0     0.0                 0.0\n",
              "3     0.0  0.0   0.0         0.0  ...   0.0   0.0     0.0                 0.0\n",
              "4     0.0  0.0   0.0         0.0  ...   0.0   0.0     0.0                 0.0\n",
              "\n",
              "[5 rows x 26993 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "njqGXsJc-VWW"
      },
      "source": [
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "distance_matrix  = cosine_similarity(dtm)\n",
        "data = pd.DataFrame(distance_matrix)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i2FwW_gw-WU_",
        "outputId": "414bba29-f7ba-484f-d677-910223367bdd"
      },
      "source": [
        "ind = data[data[100] < 1][100].sort_values(ascending=False)[:5]\n",
        "ind"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "120    0.336975\n",
              "150    0.307608\n",
              "136    0.302756\n",
              "289    0.302062\n",
              "437    0.295498\n",
              "Name: 100, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 179
        },
        "id": "WVNuEg7aDE8Z",
        "outputId": "01d52653-ad7b-4180-cedf-677b89560530"
      },
      "source": [
        "df['Text'].loc[10]"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"ask jeeves tips online ad revival  ask jeeves has become the third leading online search firm this week to thank a revival in internet advertising for improving fortunes.  the firm's revenue nearly tripled in the fourth quarter of 2004, exceeding $86m (â£46m). ask jeeves, once among the best-known names on the web, is now a relatively modest player. its $17m profit for the quarter was dwarfed by the $204m announced by rival google earlier in the week. during the same quarter, yahoo earned $187m, again tipping a resurgence in online advertising.  the trend has taken hold relatively quickly. late last year, marketing company doubleclick, one of the leading providers of online advertising, warned that some or all of its business would have to be put up for sale. but on thursday, it announced that a sharp turnaround had brought about an unexpected increase in profits. neither ask jeeves nor doubleclick thrilled investors with their profit news, however. in both cases, their shares fell by some 4%. analysts attributed the falls to excessive expectations in some quarters, fuelled by the dramatic outperformance of google on tuesday. \""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 179
        },
        "id": "7f8BD0LSDw6X",
        "outputId": "d01647ee-7982-4422-d826-3884403a6bad"
      },
      "source": [
        "df['Text'].loc[1885]"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'ask jeeves joins web log market  ask jeeves has bought the bloglines website to improve the way it handles content from web journals or blogs.  the bloglines site has become hugely popular as it gives users one place in which to read, search and share all the blogs they are interested in. ask jeeves said it was not planning to change bloglines but would use the 300 million articles it has archived to round out its index of the web. how much ask jeeves paid for bloglines was not revealed.  bloglines has become popular because it lets users build a list of the blogs they want to follow without having to visit each journal site individually. to do this it makes use of a technology known as really simple syndication (rss) that many blogs have adopted to let other sites know when new entries are made on their journals. the acquisition follows similar moves by other search sites. google acquired pyra labs, makers of the blogger software, in 2003. in 2004 msn introduced its own blog system and yahoo has tweaked its technology to do a better job of handling blog entries. jim lanzone, vice president of search properties at ask jeeves in the us, said it did not acquire bloglines just to get a foothold in the blog publishing world. he said ask jeeves was much more interested in helping people find information they were looking for rather than helping them write it.  \"the universe of readers is vastly larger than the universe of writers,\" he said. mr lanzone said the acquisition would sit well with ask\\'s my jeeves service which lets people customise their own web experience and build up a personal collection of useful links. \"search engines are about discovering information for the first time and rss is the ideal way to keep track of and monitor those sites,\" he said. it would also help drive information and entries from blogs to the portals that ask jeeves operates. there would be no instant sweeping changes to bloglines, said mr lanzone. \"our intent is to take our time to figure out the right business model not to try to monetise it right away,\" he said. though mr lanzone added that ask jeeves would be helping organise the database of 300m blog entries bloglines holds with its own net indexing technology. \"being able to search the blogosphere as one corpus of information will be very useful in its own right,\" said mr lanzone. rumours about the acquisition were broken by the napsterization weblog which said it got the hint from ask jeeves insiders. '"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qRNwq_UuekZB",
        "outputId": "a1614359-47bb-4486-c5df-4f6e5ed9984c"
      },
      "source": [
        "from sklearn.neighbors import NearestNeighbors\n",
        "nn = NearestNeighbors(n_neighbors=5, algorithm='kd_tree') #최근접 5이웃\n",
        "nn.fit(dtm)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "NearestNeighbors(algorithm='kd_tree', leaf_size=30, metric='minkowski',\n",
              "                 metric_params=None, n_jobs=None, n_neighbors=5, p=2,\n",
              "                 radius=1.0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9cSFsJJ_ezKA",
        "outputId": "d79c40c0-11a2-45ef-c670-e93c69f43b5e"
      },
      "source": [
        "result = nn.kneighbors([dtm.iloc[1000]])\n",
        "result"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[0.        , 1.24242228, 1.24489071, 1.25970784, 1.26560213]]),\n",
              " array([[1000,  951, 1129, 2090, 2036]]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "id": "pek9Bqche8Gs",
        "outputId": "3a9bf3ea-2874-484a-8d31-e01e0b2a9a0b"
      },
      "source": [
        "df.iloc[result[1][0]]"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Summary</th>\n",
              "      <th>Text</th>\n",
              "      <th>Tokens</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1000</th>\n",
              "      <td>UK Muslims should accept that people of Islami...</td>\n",
              "      <td>muslim police stops 'more likely'  uk muslims ...</td>\n",
              "      <td>[muslim, police, stop, likely, muslims, accept...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>951</th>\n",
              "      <td>Britain's young black and Asian people are to ...</td>\n",
              "      <td>citizenship event for 18s touted  citizenship ...</td>\n",
              "      <td>[citizenship, event, touted, citizenship, cere...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1129</th>\n",
              "      <td>Mr Blunkett said he wanted \"to go back to a ti...</td>\n",
              "      <td>blunkett unveils policing plans  people could ...</td>\n",
              "      <td>[blunkett, unveil, police, plan, people, give,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2090</th>\n",
              "      <td>Web users face a plethora of choices as each c...</td>\n",
              "      <td>the year search became personal  the odds are ...</td>\n",
              "      <td>[search, personal, odd, fire, browser, straigh...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2036</th>\n",
              "      <td>\"Blinkx users do not stop using other web sear...</td>\n",
              "      <td>search sites get closer to users  search sites...</td>\n",
              "      <td>[search, site, close, user, search, site, want...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                Summary  ...                                             Tokens\n",
              "1000  UK Muslims should accept that people of Islami...  ...  [muslim, police, stop, likely, muslims, accept...\n",
              "951   Britain's young black and Asian people are to ...  ...  [citizenship, event, touted, citizenship, cere...\n",
              "1129  Mr Blunkett said he wanted \"to go back to a ti...  ...  [blunkett, unveil, police, plan, people, give,...\n",
              "2090  Web users face a plethora of choices as each c...  ...  [search, personal, odd, fire, browser, straigh...\n",
              "2036  \"Blinkx users do not stop using other web sear...  ...  [search, site, close, user, search, site, want...\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NBJRWN1PfA0W",
        "outputId": "9d577f90-1319-4a66-ffc7-6fd626efc5ef"
      },
      "source": [
        "pd.set_option('display.max_rows', None)\n",
        "pd.set_option('display.max_columns', None)\n",
        "pd.set_option('display.width', None)\n",
        "pd.set_option('display.max_colwidth', -1)\n",
        "print(df['Summary'].iloc[result[1][0]])"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000    UK Muslims should accept that people of Islamic appearance are more likely to be stopped and searched by police, a Home Office minister has said.Statistics showed that of the 17 people found guilty of terrorist acts in the UK since the 11 September attacks, only four of the 12 whose ethnic backgrounds were known were Muslim, he added Figures published last week showed that people from ethnic minorities were increasingly likely to be targeted by police stop and search tactics.\"It is not aimed at a particular community, it is not police policy to aim these powers at a particular community,\" he added.The minister had been saying she understood there was a perception that stop and search was aimed at one community, but that was not what was happening, the spokesman said.Last July, the police were accused of Islamophobia by Muslim groups after stop and search figures showed the numbers of Asians targeted had risen by 300% since the introduction of anti-terror laws.\"She is demonising and alienating our community,\" he said.Hazel Blears said innocent Muslims would be targeted because of the search for Islamic extremists.Separate figures on police searches in England and Wales carried out under the Terrorism Act 2000 showed that ethnic minorities were more likely to be targeted.On ministers of religions, such as imams, she said faith groups would be asked what other qualifications and skills, such as civic knowledge and ability to engage the community, should be demanded.\"It means that some of our counter-terrorism powers will be disproportionately experienced by the Muslim community.\"                                      \n",
            "951     Britain's young black and Asian people are to be encouraged to learn about the nation's heritage as part of the government's new race strategy which will also target specific issues within different ethnic minority groups.Citizenship ceremonies could be introduced for people celebrating their 18th birthday, Charles Clarke has said.Mr Clarke said it would be a way of recognising young people reaching their voting age when they also gain greater independence from parents.\"It represents a move away from the one-size-fits-all approach to focus on specifics within cultural groups,\" said a Home Office spokesman.The Downing Street Strategy Unit in 2003 said people from Indian and Chinese backgrounds were doing well on average, often outperforming white people in education and earnings.The Home Office wants more initiatives which try to promote a sense of belonging by encouraging young people to take part in voluntary work.Officials say the home secretary wants young people to feel they belong and to understand their \"other cultural identities\" alongside being British.\"It is not right to say that if you are from a black or ethnic minority group you must be disadvantaged.\"Launching the initiative Mr Clarke said enormous progress had been made on race issues in recent years.The spokesman highlighted specific issues that affect particular communities - for example people of south Asian origin tend to suffer from a high incidence of heart disease.                                                                                                                                                                                      \n",
            "1129    Mr Blunkett said he wanted \"to go back to a time when I was very young, when you expected the police to be part of the community and the community to be part of policing and where people were joined together in partnership making it work\".The best performing police services would get more cash and extra freedoms, he said, but he would not shirk from stepping in where the public was being failed.Mr Blunkett said a new three digit number would be created for non-emergency phone calls to police.Police forces were already \"buried\" under existing government initiatives and there was little in the latest plans to reassure them,\" said Mr Davis.Earlier Lib Dem home affairs spokesman Mark Oaten said the government was right to want to increase the links between local people and the police.He has also started to recruit 25,000 Community Support Officers (CSOs) and the new plans would allow all police forces to give them the power to detain suspects.The home secretary said targets would be put in place to ensure that the public got a good response from police.Local people would also be able to \"trigger\" action on specific problems if they felt nothing was being done.The plans also include the idea of allowing people join police forces at different levels rather than the traditional way of making everybody spend specific amounts of time as a constable before being promoted.Opposition parties also want more local policing to tackle nuisance behaviour and other crime but they accuse ministers of tying the police up with paperwork.                                                                                                   \n",
            "2090    Web users face a plethora of choices as each company tries to outflank Google by rolling out new search products such as desktop search.Some may see this as the height of laziness, but in an era of information overload, search has become a vital tool in navigating the net.Microsoft is just one of the net giants muscling in on search.Search is not just about finding your way around the web.If Microsoft sees search as important, then nobody queries it,\" said Mr Holzle.Google has shown web commerce can work through its targeted small adverts, which appear at the top and down the right-hand side of a page and are related to the original search.\"Search was always important,\" said Urs Holzle, Google vice-president of operations.\"User experience has contributed to people searching more,\" said Yonca Brunini of Yahoo.\"People didn't realise that search was the future.But search is hardly a new phenomenon.In order to do a better job, search engines are trying to get to know you better, doing a better job of remembering, cataloguing and managing all the information you come across.Over the past 12 months, giants of the technology world such as Microsoft and Yahoo have sought to grab a slice of the search action.And as Google has shown, there is money in offering a service that people cannot live without.The odds are that when you fire up your browser, you go straight to your favourite search engine, rather than type in a web address.For all these advances, search is still a clumsy tool, often failing to come up with exactly what you had in mind.                                                                                  \n",
            "2036    \"Blinkx users do not stop using other web search systems,\" he said.Some of the latest to update their search systems are Ask Jeeves and Blinkx, which have both released a series of utilities that try to help people get more from the web.On average, said Mr Macklin, users conduct between five and 10 searches per day and the tools in My Ask Jeeves should stop them having to do searches twice and get to what they want much more easily.What users do with Blinkx and other desktop search engines shows that people tend to be very promiscuous in their use of search engines.By way of comparison Google's Desktop search tool catalogues search histories informally and lets people look through the sites they have visited.Mr Macklin said that too often when people use a search engine it was like the first time they ever used it, because there was no memory of what they had searched for before.Under My Ask Jeeves users can search the web or through the results they have already noted as interesting.Ask Jeeves has added the ability to \"save\" websites of interest so the next time a users visits the site they can search through the sites they have previously found.The variety of ways to search data was only helping users, said Mr Chandratillake and that it was likely that in the future people would use different ones for different tasks.Since Blinkx launched it has faced increased competition from firms such as Google, Copernic, Enfish, X1 and Apple all of whom now have programs that let people search their PC as well as the web.\"The future is all about developing your own personal web,\" said Tony Macklin, spokesman for Ask Jeeves.\n",
            "Name: Summary, dtype: object\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:4: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
            "  after removing the cwd from sys.path.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lsDv793ig0E7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6969f755-9d15-490f-cb70-1ce701d5cdc6"
      },
      "source": [
        "pd.set_option('display.max_rows', None)\n",
        "pd.set_option('display.max_columns', None)\n",
        "pd.set_option('display.width', None)\n",
        "pd.set_option('display.max_colwidth', -1)\n",
        "print(df['Tokens'].iloc[result[1][0]])"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000    [muslim, police, stop, likely, muslims, accept, people, islamic, appearance, likely, stop, search, police, home, office, minister, hazel, blear, innocent, muslims, target, search, islamic, extremist, qualification, religious, leader, enter, tough, common, inquiry, comment, describe, irresponsible, outrageous, islamic, human, rights, commission, ihrc, threat, likely, come, people, associate, extreme, form, islam, falsely, hide, islam, salford, mp, commons, home, affairs, committee, mean, counterterrorism, power, disproportionately, experience, muslim, community, reality, recognise, threat, particular, place, action, go, target, area, add, minister, religion, imam, faith, group, ask, qualification, skill, civic, knowledge, ability, engage, community, demand, minister, introduce, requirement, minister, speak, english, certain, level, ihrc, chairman, massoud, shadjareh, ...]\n",
            "951     [citizenship, event, touted, citizenship, ceremony, introduce, people, celebrate, birthday, charles, clarke, idea, try, overhaul, way, government, approach, inclusive, citizenship, particularly, ethnic, minority, pilot, scheme, base, ceremony, australia, start, october, clarke, way, recognise, young, people, reach, voting, age, gain, great, independence, parent, britain, young, black, asian, people, encourage, learn, nation, heritage, government, race, strategy, target, specific, issue, different, ethnic, minority, group, official, home, secretary, want, young, people, feel, belong, understand, cultural, identity, alongside, british, launch, follow, row, role, faith, school, britain, monday, school, inspection, chief, david, bell, accuse, islamic, school, fail, teach, pupil, obligation, british, society, muslim, council, britain, ofste, boss, ...]                        \n",
            "1129    [blunkett, unveil, police, plan, people, give, mobile, phone, local, bobby, overhaul, police, england, wales, unveil, david, blunkett, plan, include, dedicated, policing, team, neighbourhood, point, compulsory, customer, charter, home, secretary, target, place, ensure, public, get, good, response, police, local, people, able, trigger, action, specific, problem, feel, local, councillor, certain, condition, meet, invoke, power, police, refuse, request, complaint, frivolous, cause, annoyance, pose, heavy, burden, resource, blunkett, digit, create, nonemergency, phone, call, police, perform, police, service, cash, extra, freedom, shirk, step, public, fail, home, secretarys, power, suspend, sack, chief, constable, review, blunketts, battle, humberside, chief, david, westwood, intelligence, failure, soham, murderer, ian, huntley, ...]                                           \n",
            "2090    [search, personal, odd, fire, browser, straight, favourite, search, engine, type, web, address, height, laziness, era, information, overload, search, vital, tool, navigate, net, symptomatic, way, use, internet, change, google, show, money, offer, service, people, live, shortage, company, vie, loyalty, web, searcher, offer, wealth, different, service, tool, help, find, want, past, month, giant, technology, world, microsoft, yahoo, seek, grab, slice, search, action, user, experience, contribute, people, search, yonca, brunini, yahoo, people, familiar, internet, tend, spend, online, ask, query, second, thing, broadband, ms, brunini, news, website, internet, colour, tv, search, hardly, phenomenon, early, day, net, veteran, surfer, remember, oldtimer, hotbot, altavista, search, important, ...]                                                                                    \n",
            "2036    [search, site, close, user, search, site, want, know, better, content, provide, access, million, website, offer, way, well, job, remember, catalogue, manage, information, come, late, update, search, system, ask, jeeves, blinkx, release, series, utility, try, help, people, web, future, develop, personal, web, tony, macklin, spokesman, ask, jeeves, macklin, people, use, search, engine, memory, search, start, series, updates, service, collect, ask, jeeves, banner, help, people, remember, ask, jeeves, add, ability, save, website, interest, user, visit, site, search, site, previously, find, site, save, way, arrange, folder, note, attach, explain, save, macklin, people, want, save, site, see, want, add, bookmark, favourite, list, easily, search, ...]                                                                                                                                 \n",
            "Name: Tokens, dtype: object\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:4: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
            "  after removing the cwd from sys.path.\n"
          ],
          "name": "stderr"
        }
      ]
    }
  ]
}