Attempting_TITLE_clustering.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Attempting TITLE clustering.ipynb",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/JoanneBurke/Data-Analysis/blob/master/Attempting_TITLE_clustering.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e-n1H0VkbCwE",
        "colab_type": "text"
      },
      "source": [
        "# Attemtpting text clustering using doc2vec\n",
        "\n",
        "The idea behind this is to potentially find the 'odd texts' out. Hopefully once the elements have been clustered, a potential thing that could be done is topic modelling on each cluster and see what that gives - maybe by insepction this may give show the documents which do not tackle land conflcits"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BoRM4GZYcARy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install news-please"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oqpGXOpXcPdQ",
        "colab_type": "code",
        "outputId": "127a3874-1e5c-4760-9842-7e14c41f4520",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "%cd drive/My\\ Drive/Omdena/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n",
            "/content/drive/My Drive/Omdena\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FtOzYPsRhBzV",
        "colab_type": "code",
        "outputId": "9c6f80e2-f5ce-47c6-aef4-5cf70584f0d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "!tar -xvzf environmental-conflict-tracker/enwiki_dbow.tgz"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "enwiki_dbow/\n",
            "enwiki_dbow/doc2vec.bin.syn1neg.npy\n",
            "enwiki_dbow/doc2vec.bin.syn0.npy\n",
            "enwiki_dbow/doc2vec.bin\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k4oeR9SHcFqi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pickle\n",
        "import pandas as pd\n",
        "from newsplease import NewsPlease\n",
        "import gensim"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CPUAkxzmhpDf",
        "colab_type": "code",
        "outputId": "c08f5edc-64c5-4e08-a559-21b2d068bee3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!ls environmental-conflict-tracker/data/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "gold-standard  metadata  policy  reference  texts  urls\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cRMLoefYcbWD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gensim.models\n",
        "\n",
        "model = 'enwiki_dbow/doc2vec.bin'\n",
        "output_file = 'environmental-conflict-tracker/data/test_vectors.txt'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QoAvApukh517",
        "colab_type": "text"
      },
      "source": [
        "Now I need to extract the text documents"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PwVi5dJ-g80t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pickle\n",
        "import pandas as pd\n",
        "from newsplease import NewsPlease\n",
        "import numpy as np\n",
        "import os\n",
        "\n",
        "def load_obj(month):\n",
        "    month = str(month).zfill(2)\n",
        "    month_texts = []\n",
        "    path = 'environmental-conflict-tracker/data/texts/{}/'.format(month)\n",
        "    # print('bob')\n",
        "    for dirpath, dirnames, filenames in os.walk(path):\n",
        "      # print(filenames)\n",
        "      for file_ in filenames:\n",
        "        # print(file_)\n",
        "        with open(path + file_, \"rb\") as f:\n",
        "          # print(pickle.load(f).text)\n",
        "          try:\n",
        "            title_ = pickle.load(f).title\n",
        "            # print(title_)\n",
        "            if (title_ == 'Page Not Found'):\n",
        "              continue\n",
        "            month_texts.append(title_.split())\n",
        "            # print(month_texts)\n",
        "          except:\n",
        "            # print('bad')\n",
        "            continue\n",
        "    with open(month + \"_title.txt\", \"wb\") as fb:\n",
        "      pickle.dump(month_texts, fb)\n",
        "      print('saved ' + month)\n",
        "    return month_texts"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H-fN-gfTni9C",
        "colab_type": "code",
        "outputId": "8ff3495e-1695-4134-a449-fa18e3eb273c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "!ls"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " 01_title.txt   06_title.txt   all_texts.txt\n",
            " 01.txt         06.txt\t       cluster_topics.pkl\n",
            " 02_title.txt   07_title.txt   elbow_5.png\n",
            " 02.txt         07.txt\t       elbow.png\n",
            " 03_title.txt   08_title.txt   environmental-conflict-tracker\n",
            " 03.txt         08.txt\t       enwiki_dbow\n",
            " 04_title.txt   09.txt\t      'Getting Started.ipynb'\n",
            " 04.txt         10.txt\t       stanford-corenlp-full-2018-10-05\n",
            " 05_title.txt   11.txt\t       vectors.pkl\n",
            " 05.txt         12.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yaAy7H_yiYe3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "months = [str(el).zfill(2) for el in np.arange(13)[1:]]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wI2NyCA4ktSu",
        "colab_type": "code",
        "outputId": "8b60fe15-3a3c-46c1-a7aa-ab695db43822",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "for el in months:\n",
        "  load_obj(el)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "saved 01\n",
            "saved 02\n",
            "saved 03\n",
            "saved 04\n",
            "saved 05\n",
            "saved 06\n",
            "saved 07\n",
            "saved 08\n",
            "saved 09\n",
            "saved 10\n",
            "saved 11\n",
            "saved 12\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GZjYlpGemf49",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(\"01_title.txt\", \"rb\") as f:\n",
        "  month_1 = pickle.load(f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "97NGWN1GmpgB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "month_1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GdBfqbqLYpKW",
        "colab_type": "code",
        "outputId": "00978a20-13b5-414b-8543-a6c7f9b4db7f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "path = 'environmental-conflict-tracker/data/texts/01/'\n",
        "for dirpath, dirnames, filenames in os.walk(path):\n",
        "  print(len(filenames))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "6627\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S_kXDB10ZRTX",
        "colab_type": "text"
      },
      "source": [
        "What I had noticed previously was that for some reason some had text set to None. This should also ideally be flagged and removed. I find it weird"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KXk5zWKYsBAB",
        "colab_type": "code",
        "outputId": "32e64262-c522-4125-dd8a-76286949b95c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!pwd"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8YgkS7i9ZK15",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_docs = None\n",
        "for m in months:\n",
        "  with open(\"{}_title.txt\".format(m), \"rb\") as f:\n",
        "    month_texts = pickle.load(f)\n",
        "    if all_docs is None:\n",
        "      all_docs = month_texts\n",
        "    else:\n",
        "      all_docs += month_texts"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4HvMwSDBZjNO",
        "colab_type": "code",
        "outputId": "d1a19b47-a65c-443d-c265-dd2280065122",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(all_docs)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "64877"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pqlf_Ar3nKYm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "month_1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aq0CTi5ZZou3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(\"all_titles.txt\", \"wb\") as fb:\n",
        "      pickle.dump(all_docs, fb)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vylHQhCLn_jr",
        "colab_type": "text"
      },
      "source": [
        "# Attempt LDA on all the titles"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hAVvGzeqn_BD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_docs = [' '.join(x) for x in all_docs]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JrctNB_MobE6",
        "colab_type": "code",
        "outputId": "8a7399d8-d35a-4c4d-cf5b-8ce5032c5da4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "!pip install preprocessing\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: preprocessing in /usr/local/lib/python3.6/dist-packages (0.1.13)\n",
            "Requirement already satisfied: sphinx-rtd-theme==0.2.4 in /usr/local/lib/python3.6/dist-packages (from preprocessing) (0.2.4)\n",
            "Requirement already satisfied: nltk==3.2.4 in /usr/local/lib/python3.6/dist-packages (from preprocessing) (3.2.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk==3.2.4->preprocessing) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xQCestPEq8kY",
        "colab_type": "code",
        "outputId": "814862f0-8646-4235-9ca8-081b824012b0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "'a b c'.split()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['a', 'b', 'c']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A_MO2x6vrrQC",
        "colab_type": "code",
        "outputId": "b78f5cf3-56cd-4b09-f563-01ff887c1162",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8x6mmcRzoPES",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#remove stopwords, punctuation etc\n",
        "import string\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "\n",
        "stop_words = set(stopwords.words('english'))\n",
        "\n",
        "for i,title in enumerate(all_docs):\n",
        "  title = title.lower()\n",
        "  title = title.translate(str.maketrans('', '', string.punctuation))\n",
        "  title = ' '.join([w for w in title.split() if w not in stop_words])\n",
        "  all_docs[i] = title\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kwQX7CaVv5Mi",
        "colab_type": "code",
        "outputId": "0e53208f-7ecf-4247-a436-804c5c54f41a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "all_docs[i]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'rains fail chennai city prepared face another waterscarce summer'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tJcTP8Vnv-DB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(\"all_titles_preprocessed.txt\", \"wb\") as fb:\n",
        "      pickle.dump(all_docs, fb)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y2nQdk_Zwi8P",
        "colab_type": "code",
        "outputId": "236ed8b8-9b6a-4eae-c159-db451ad4c579",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 612
        }
      },
      "source": [
        "tf_vectorizer = LemmaCountVectorizer(max_df=0.95,\n",
        "                                   min_df=2,\n",
        "                                   stop_words='english',\n",
        "                                   decode_error='ignore')\n",
        "tf = tf_vectorizer.fit_transform(all_docs)\n",
        "lda = LatentDirichletAllocation(n_components=11, max_iter=5,\n",
        "                              learning_method = 'online',\n",
        "                              learning_offset = 50.,\n",
        "                              random_state = 0)\n",
        "lda.fit(tf)\n",
        "n_top_words = 5\n",
        "print(\"\\nTopics in LDA model: \")\n",
        "tf_feature_names = tf_vectorizer.get_feature_names()\n",
        "print_top_words(lda, tf_feature_names, n_top_words)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topic #0:protest violence kerala demand hold\n",
            "======================================================================\n",
            "\n",
            "Topic #1:india new development caste news\n",
            "======================================================================\n",
            "\n",
            "Topic #2:court case killed village bulandshahr\n",
            "======================================================================\n",
            "\n",
            "Topic #3:farmer killing modi govt pm\n",
            "======================================================================\n",
            "\n",
            "Topic #4:road r bjp minister project\n",
            "======================================================================\n",
            "\n",
            "Topic #5:people pakistan mp issue agustawestland\n",
            "======================================================================\n",
            "\n",
            "Topic #6:say man kill land dead\n",
            "======================================================================\n",
            "\n",
            "Topic #7:delhi farm loan year day\n",
            "======================================================================\n",
            "\n",
            "Topic #8:airport dead killed arrested held\n",
            "======================================================================\n",
            "\n",
            "Topic #9:protest chidambaram water cow chief\n",
            "======================================================================\n",
            "\n",
            "Topic #10:killed water indian state attack\n",
            "======================================================================\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4BIBflR0y3-9",
        "colab_type": "text"
      },
      "source": [
        "#Attempting to vectorize them with USE (UNIVERSAL SENTENCE ENCODER)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dx_FNaJty3bp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_hub as tf_hub"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KIZG1Yl22hm-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_url = 'https://tfhub.dev/google/universal-sentence-encoder/2'\n",
        "embed = tf_hub.Module(model_url)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lKfXJSv0_TKA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('all_titles.txt', 'rb') as f:\n",
        "  all_docs = pickle.load(f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-fWMGRHi_ak3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_docs = [' '.join(t) for t in all_docs]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5-fnaQi-7sM3",
        "colab_type": "code",
        "outputId": "b40f1cc9-c204-49c4-ac0a-e24cbaea4516",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "with tf.Session() as session:\n",
        "  session.run([tf.global_variables_initializer(), tf.tables_initializer()])\n",
        "  message_embeddings = session.run(embed(all_docs))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lvk5LTRk8qXm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " df = pd.DataFrame({'titles': all_docs, 'embed': message_embeddings.tolist()})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zAkLLtNe9vcp",
        "colab_type": "code",
        "outputId": "344c6089-720f-44b8-9e69-f674d2902256",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        }
      },
      "source": [
        "df.head(3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>titles</th>\n",
              "      <th>embed</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lawyer killed as truck hits car</td>\n",
              "      <td>[-0.020477019250392914, -0.06466207653284073, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Page not Found</td>\n",
              "      <td>[0.008759897202253342, -0.016028402373194695, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Telangana government makes harsh language a co...</td>\n",
              "      <td>[0.014865786768496037, -0.06467095762491226, -...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              titles                                              embed\n",
              "0                    Lawyer killed as truck hits car  [-0.020477019250392914, -0.06466207653284073, ...\n",
              "1                                     Page not Found  [0.008759897202253342, -0.016028402373194695, ...\n",
              "2  Telangana government makes harsh language a co...  [0.014865786768496037, -0.06467095762491226, -..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KFayE0dQ9xXI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.to_pickle('title_vectors.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GQF80AF8-AIu",
        "colab_type": "code",
        "outputId": "0bfbe23e-b13d-429c-9bd5-9be28b3e4a75",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "import numpy as np\n",
        "from sklearn.cluster import KMeans\n",
        "kmeans = KMeans(n_clusters=20,init='k-means++',max_iter=300,n_init=10,random_state=0)\n",
        "kmeans.fit(df['embed'].values.tolist())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "KMeans(algorithm='auto', copy_x=True, init='k-means++', max_iter=300,\n",
              "       n_clusters=20, n_init=10, n_jobs=None, precompute_distances='auto',\n",
              "       random_state=0, tol=0.0001, verbose=0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "10F2-r0l-RQr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "labels = kmeans.labels_"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "em_H7TDa-6ja",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['labels'] = labels\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uAgIcSRZ_AYm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df[df['labels'] == 3]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5BEJaQhZBoz5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.to_pickle('title_vectors.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ujDioHlCCXh3",
        "colab_type": "code",
        "outputId": "e470fb4f-f52b-481e-d81e-d93d1be109e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>titles</th>\n",
              "      <th>embed</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lawyer killed as truck hits car</td>\n",
              "      <td>[-0.020477019250392914, -0.06466207653284073, ...</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Page not Found</td>\n",
              "      <td>[0.008759897202253342, -0.016028402373194695, ...</td>\n",
              "      <td>19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Telangana government makes harsh language a co...</td>\n",
              "      <td>[0.014865786768496037, -0.06467095762491226, -...</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Nature provides artistic inspiration for retir...</td>\n",
              "      <td>[0.0431208610534668, -0.04448771849274635, -0....</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>18 arrested over Gurugram school bus attack du...</td>\n",
              "      <td>[-0.044236116111278534, -0.08111345022916794, ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              titles  ... labels\n",
              "0                    Lawyer killed as truck hits car  ...     11\n",
              "1                                     Page not Found  ...     19\n",
              "2  Telangana government makes harsh language a co...  ...      7\n",
              "3  Nature provides artistic inspiration for retir...  ...      6\n",
              "4  18 arrested over Gurugram school bus attack du...  ...      0\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mNd7YoxXEGKV",
        "colab_type": "code",
        "outputId": "d8af26ea-8f54-4d39-8fc8-165d3526bffb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        }
      },
      "source": [
        "df.groupby('labels')['titles'].count()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "labels\n",
              "0     1358\n",
              "1     5475\n",
              "2     3335\n",
              "3     2638\n",
              "4     2760\n",
              "5     2907\n",
              "6     2983\n",
              "7     5255\n",
              "8      485\n",
              "9     2890\n",
              "10     539\n",
              "11    4004\n",
              "12    5039\n",
              "13    6284\n",
              "14    3659\n",
              "15    3454\n",
              "16    3694\n",
              "17    4028\n",
              "18    3573\n",
              "19     517\n",
              "Name: titles, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Qw53-pFCCXi",
        "colab_type": "text"
      },
      "source": [
        "# Topics of Titles"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kLO2ccoyHodY",
        "colab_type": "code",
        "outputId": "eea5858d-9d23-44f1-f74c-b72b2cee32eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "df[df['labels'] == 8]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>titles</th>\n",
              "      <th>embed</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>81</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>93</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>127</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>128</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>243</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>436</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>718</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1115</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1166</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1167</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1321</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1338</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1719</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1857</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1858</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2027</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2056</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2269</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2270</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2271</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2482</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2511</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2512</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2671</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2672</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2688</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2713</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56890</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57027</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58311</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58312</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58708</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59576</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59620</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59695</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59767</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60188</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60239</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60658</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60862</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60984</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>61678</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>61731</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62277</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62280</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62369</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62580</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62582</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63535</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63550</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63781</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63984</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64116</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64118</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64214</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64285</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64778</th>\n",
              "      <td>Welcome To IANS Live</td>\n",
              "      <td>[-0.0005704158684238791, -0.043604832142591476...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>485 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                     titles  ... labels\n",
              "36     Welcome To IANS Live  ...      8\n",
              "39     Welcome To IANS Live  ...      8\n",
              "81     Welcome To IANS Live  ...      8\n",
              "91     Welcome To IANS Live  ...      8\n",
              "93     Welcome To IANS Live  ...      8\n",
              "127    Welcome To IANS Live  ...      8\n",
              "128    Welcome To IANS Live  ...      8\n",
              "243    Welcome To IANS Live  ...      8\n",
              "436    Welcome To IANS Live  ...      8\n",
              "718    Welcome To IANS Live  ...      8\n",
              "1115   Welcome To IANS Live  ...      8\n",
              "1166   Welcome To IANS Live  ...      8\n",
              "1167   Welcome To IANS Live  ...      8\n",
              "1321   Welcome To IANS Live  ...      8\n",
              "1338   Welcome To IANS Live  ...      8\n",
              "1719   Welcome To IANS Live  ...      8\n",
              "1857   Welcome To IANS Live  ...      8\n",
              "1858   Welcome To IANS Live  ...      8\n",
              "2027   Welcome To IANS Live  ...      8\n",
              "2056   Welcome To IANS Live  ...      8\n",
              "2269   Welcome To IANS Live  ...      8\n",
              "2270   Welcome To IANS Live  ...      8\n",
              "2271   Welcome To IANS Live  ...      8\n",
              "2482   Welcome To IANS Live  ...      8\n",
              "2511   Welcome To IANS Live  ...      8\n",
              "2512   Welcome To IANS Live  ...      8\n",
              "2671   Welcome To IANS Live  ...      8\n",
              "2672   Welcome To IANS Live  ...      8\n",
              "2688   Welcome To IANS Live  ...      8\n",
              "2713   Welcome To IANS Live  ...      8\n",
              "...                     ...  ...    ...\n",
              "56890  Welcome To IANS Live  ...      8\n",
              "57027  Welcome To IANS Live  ...      8\n",
              "58311  Welcome To IANS Live  ...      8\n",
              "58312  Welcome To IANS Live  ...      8\n",
              "58708  Welcome To IANS Live  ...      8\n",
              "59576  Welcome To IANS Live  ...      8\n",
              "59620  Welcome To IANS Live  ...      8\n",
              "59695  Welcome To IANS Live  ...      8\n",
              "59767  Welcome To IANS Live  ...      8\n",
              "60188  Welcome To IANS Live  ...      8\n",
              "60239  Welcome To IANS Live  ...      8\n",
              "60658  Welcome To IANS Live  ...      8\n",
              "60862  Welcome To IANS Live  ...      8\n",
              "60984  Welcome To IANS Live  ...      8\n",
              "61678  Welcome To IANS Live  ...      8\n",
              "61731  Welcome To IANS Live  ...      8\n",
              "62277  Welcome To IANS Live  ...      8\n",
              "62280  Welcome To IANS Live  ...      8\n",
              "62369  Welcome To IANS Live  ...      8\n",
              "62580  Welcome To IANS Live  ...      8\n",
              "62582  Welcome To IANS Live  ...      8\n",
              "63535  Welcome To IANS Live  ...      8\n",
              "63550  Welcome To IANS Live  ...      8\n",
              "63781  Welcome To IANS Live  ...      8\n",
              "63984  Welcome To IANS Live  ...      8\n",
              "64116  Welcome To IANS Live  ...      8\n",
              "64118  Welcome To IANS Live  ...      8\n",
              "64214  Welcome To IANS Live  ...      8\n",
              "64285  Welcome To IANS Live  ...      8\n",
              "64778  Welcome To IANS Live  ...      8\n",
              "\n",
              "[485 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZGU5EnAYCAav",
        "colab_type": "code",
        "outputId": "183e7930-27d6-4717-dceb-a0fdc0acfdf0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 663
        }
      },
      "source": [
        "from collections import defaultdict\n",
        "\n",
        "tf_vectorizer = LemmaCountVectorizer(max_df=0.95,\n",
        "                                   min_df=2,\n",
        "                                   stop_words='english',\n",
        "                                   decode_error='ignore')\n",
        "\n",
        "def get_top_words(model, feature_names, n_top_words):\n",
        "\n",
        "  d = defaultdict(list)\n",
        "\n",
        "  for index, topic in enumerate(model.components_):\n",
        "      message = \" \".join([feature_names[i] for i in topic.argsort()[:-n_top_words - 1 :-1]])\n",
        "      d[index].append(message)\n",
        "  return d\n",
        "\n",
        "all_topics = defaultdict(list)\n",
        "\n",
        "for i in range(20):\n",
        "  if (i==8):\n",
        "    continue\n",
        "  tf_ = tf_vectorizer.fit_transform(df[df['labels'] == i]['titles'].values)\n",
        "  lda = LatentDirichletAllocation(n_components=7, max_iter=5,\n",
        "                              learning_method = 'online',\n",
        "                              learning_offset = 50.,\n",
        "                              random_state = 0)\n",
        "  lda.fit(tf_)\n",
        "  n_top_words = 10\n",
        "  print(\"\\nTopics in LDA model: \")\n",
        "  tf_feature_names = tf_vectorizer.get_feature_names()\n",
        "  d = get_top_words(lda, tf_feature_names, n_top_words)\n",
        "\n",
        "  for k,v in d.items():\n",
        "    all_topics[k].append(v)\n",
        "\n",
        "df_topic = pd.DataFrame({})\n",
        "\n",
        "for k, v in all_topics.items():\n",
        "  df_topic['Topic ' + str(k)] = v"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n",
            "\n",
            "Topics in LDA model: \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t_DOgYfMIGV1",
        "colab_type": "code",
        "outputId": "cc62e5a3-e226-4d3a-aa25-a84545be32b4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "df_topic.iloc[8].values"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([list(['farmer farm loan waiver demand new ban hit congress r']),\n",
              "       list(['cow cattle land village help green farming held business industry']),\n",
              "       list(['farmer land crop govt farming india protest organic end support']),\n",
              "       list(['farmer price income march day milk villager 10 want change']),\n",
              "       list(['forest farmer mining agriculture plan govt minister say illegal coal']),\n",
              "       list(['farmer government indian conservation development rally wildlife dairy sugar pesticide']),\n",
              "       list(['agriculture farm water export sector need ministry policy stop president'])],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aQTvu0B0JzjY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_topic.to_pickle('title_topic.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vhJX8cc_C-h5",
        "colab_type": "code",
        "outputId": "b84a49b0-a516-4227-ac21-1006c7402bf3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "source": [
        "!ls"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " 01_title.txt   07.txt\t\t\t      all_titles.txt\n",
            " 01.txt         08_title.txt\t\t      cluster_topics.pkl\n",
            " 02_title.txt   08.txt\t\t\t      elbow_5.png\n",
            " 02.txt         09_title.txt\t\t      elbow.png\n",
            " 03_title.txt   09.txt\t\t\t      environmental-conflict-tracker\n",
            " 03.txt         10_title.txt\t\t      enwiki_dbow\n",
            " 04_title.txt   10.txt\t\t\t     'Getting Started.ipynb'\n",
            " 04.txt         11_title.txt\t\t      stanford-corenlp-full-2018-10-05\n",
            " 05_title.txt   11.txt\t\t\t      title_topic.pkl\n",
            " 05.txt         12_title.txt\t\t      title_vectors.pkl\n",
            " 06_title.txt   12.txt\t\t\t      vectors.pkl\n",
            " 06.txt         all_texts.txt\n",
            " 07_title.txt   all_titles_preprocessed.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VdaUh7dhCXGD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "vec = pd.read_pickle('title_vectors.pkl')\n",
        "topic = pd.read_pickle('title_topic.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gXuQBk-4DFhM",
        "colab_type": "code",
        "outputId": "641fd901-5548-4a15-fb04-fa0fc830be8f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        }
      },
      "source": [
        "topic.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Topic 0</th>\n",
              "      <th>Topic 1</th>\n",
              "      <th>Topic 2</th>\n",
              "      <th>Topic 3</th>\n",
              "      <th>Topic 4</th>\n",
              "      <th>Topic 5</th>\n",
              "      <th>Topic 6</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[nun bishop jalandhar link terror mulakkal pri...</td>\n",
              "      <td>[student university protest teacher delhi kill...</td>\n",
              "      <td>[student protest school delhi parent jnu class...</td>\n",
              "      <td>[senior nsui mu reinstatement aiims free fake ...</td>\n",
              "      <td>[student protest college caste school govt adm...</td>\n",
              "      <td>[school shut remain ghaziabad protest bandh fa...</td>\n",
              "      <td>[protest school village institute ministry emi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[killed r violence bulandshahr say killing lak...</td>\n",
              "      <td>[killing man mumbai old delhi year held woman ...</td>\n",
              "      <td>[kill man killed dead wife woman india mob ind...</td>\n",
              "      <td>[girl cow case old killer india year rape vill...</td>\n",
              "      <td>[dead shot delhi police killed pradesh officer...</td>\n",
              "      <td>[family delhi kerala case dead member alleged ...</td>\n",
              "      <td>[maharashtra killing farmer gauri hc cbi benga...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[deadline accused jalandhar bulandshahr set ma...</td>\n",
              "      <td>[dead babu verify damage sinha judiciary convi...</td>\n",
              "      <td>[michel agustawestland christian middleman ind...</td>\n",
              "      <td>[land r case court settlement dispute year cro...</td>\n",
              "      <td>[cbi case probe court land say scam chief deal...</td>\n",
              "      <td>[court sc supreme judge examine case say verdi...</td>\n",
              "      <td>[chidambaram case court karti medium inx cbi e...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[road gadkari safety say expressway bengal nit...</td>\n",
              "      <td>[road bridge india rail longest mumbai constru...</td>\n",
              "      <td>[india launch new lakh thailand land rover tat...</td>\n",
              "      <td>[road delhi mumbai traffic airport development...</td>\n",
              "      <td>[r infrastructure cr road development water tr...</td>\n",
              "      <td>[road land project mumbai village train bullet...</td>\n",
              "      <td>[vehicle pollution electric ashok leyland fore...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[airport air india pilot nod force crash near ...</td>\n",
              "      <td>[airport flight gold mumbai landing delhi r ma...</td>\n",
              "      <td>[airport flight international kerala kannur op...</td>\n",
              "      <td>[airport passenger detained airline plane cisf...</td>\n",
              "      <td>[navy indian island rescue seek vizag meghalay...</td>\n",
              "      <td>[land air india airport aai launch order expan...</td>\n",
              "      <td>[airport aircraft runway hour mumbai shut high...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                             Topic 0  ...                                            Topic 6\n",
              "0  [nun bishop jalandhar link terror mulakkal pri...  ...  [protest school village institute ministry emi...\n",
              "1  [killed r violence bulandshahr say killing lak...  ...  [maharashtra killing farmer gauri hc cbi benga...\n",
              "2  [deadline accused jalandhar bulandshahr set ma...  ...  [chidambaram case court karti medium inx cbi e...\n",
              "3  [road gadkari safety say expressway bengal nit...  ...  [vehicle pollution electric ashok leyland fore...\n",
              "4  [airport air india pilot nod force crash near ...  ...  [airport aircraft runway hour mumbai shut high...\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hiyJAjt-DNH3",
        "colab_type": "code",
        "outputId": "f2dd4133-6895-4403-d1a7-3e6b70a9cccd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "topic.iloc[15].values"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([list(['dead indian killed iraq shot 39 mosul body isi swaraj']),\n",
              "       list(['kashmir indian killed army kill soldier militant civilian force troop']),\n",
              "       list(['india kill rebel maoist near police killed anti 14 pakistan']),\n",
              "       list(['attack killed terror year killing grenade injured amritsar suspected dead']),\n",
              "       list(['pakistan killed india protest bsf killing jawan pak firing ceasefire']),\n",
              "       list(['killed terrorist kashmir encounter jammu security clash force militant blast']),\n",
              "       list(['killing kashmiri sikh condemns indian kashmir pakistan civilian assam massacre'])],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EQ8ylVMEFY8_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}