5.3 Topic Modeling.ipynb

{
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "codemirror_mode": {
        "version": 3,
        "name": "ipython"
      },
      "file_extension": ".py",
      "version": "3.6.5",
      "mimetype": "text/x-python",
      "name": "python"
    }
  },
  "cells": [
    {
      "execution_count": null,
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "import os\n",
        "print(os.listdir(\"../input\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ],
      "outputs": [],
      "metadata": {
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "outputs": [],
      "metadata": {
        "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
        "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
        "collapsed": true
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "import codecs\n",
        "input_file = codecs.open('../input/socialmedia-disaster-tweets-DFE.csv', 'r',encoding='utf-8', errors='replace')\n",
        "output_file = open('clean_socialmedia-disaster.csv', 'w')\n",
        "\n",
        "def sanitize_characters(raw, clean):    \n",
        "    for line in input_file:\n",
        "        out = line\n",
        "        output_file.write(line)\n",
        "sanitize_characters(input_file, output_file)"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "7eac873f76a74804d51ca98c9b04ad64dd907031"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "df = pd.read_csv('clean_socialmedia-disaster.csv')"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "09478f9dd057dbacd136415955a5ae31a6b70b09"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "df.head()"
      ],
      "outputs": [],
      "metadata": {
        "_uuid": "7ffabaa812cdfac13d0ef745db1ef72103370b69"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "df = df[df.choose_one != \"Can't Decide\"]"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "a7de4767557c653d3c75079956b0522fd6757a5b"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "df = df[['text','choose_one']]"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "7b43f6f6a8978e89195a0c301cdcf51a0ac95eec"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "df['relevant'] = df.choose_one.map({'Relevant':1,'Not Relevant':0})"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "58b7cb5fbcf66b2e2f6ba7c3f571f691b5be6400"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "import spacy"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "8f80c2bc1eba7afa3b58cefa8f577d11fe8084e4"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "nlp = spacy.load('en',disable=['tagger','parser','ner'])"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "b24a86fb756c130c2a5cc87e2bba829c0465fb8d"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "from tqdm import tqdm, tqdm_notebook\n",
        "tqdm.pandas(tqdm_notebook)\n"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "038455f8a8f246f8e9424c2845ab9fd2140b1b28"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "df['lemmas'] = df[\"text\"].progress_apply(lambda row: \n",
        "                                         [w.lemma_ for w in nlp(row)])"
      ],
      "outputs": [],
      "metadata": {
        "_uuid": "98784f318b759f58c060f248a9db82bbf7f36d00"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "df['joint_lemmas'] = df['lemmas'].progress_apply(lambda row: ' '.join(row))"
      ],
      "outputs": [],
      "metadata": {
        "_uuid": "9281d3bae79fd221eccce8b492af1c17fe9dba2a"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "df.head()"
      ],
      "outputs": [],
      "metadata": {
        "_uuid": "a312f6a5154374a53885871f466ae9d1b373fcd3"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "from sklearn.decomposition import LatentDirichletAllocation"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "fb9d5e6fc99d7ae13af770513f32fa5479b2ca66"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "lda = LatentDirichletAllocation(n_components=2,verbose=1)"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "08586ff09dd14f539b82c21f7cb77c3904b477c6"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "54d649f358fcc8cbad17fb2e7dddd85f93f2b65e"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "vectorizer = CountVectorizer(stop_words='english')"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "5798d6cab7754db8396a873be81a3e8d634e2d73"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "tf = vectorizer.fit_transform(df['joint_lemmas'])"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "b286a6c8880e3cd7287d2593d06c95eb532d627b"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "lda.fit(tf)"
      ],
      "outputs": [],
      "metadata": {
        "_uuid": "0a3c63476db8737d0b80b4900417be5c199cdccb"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "tf_feature_names = vectorizer.get_feature_names()"
      ],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "5321659c3c238283e8512575b24909df85f11ffc"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [
        "n_top_words = 5\n",
        "for topic_idx, topic in enumerate(lda.components_):\n",
        "        message = \"Topic #%d: \" % topic_idx\n",
        "        message += \" \".join([tf_feature_names[i]\n",
        "                             for i in topic.argsort()[:-n_top_words - 1:-1]])\n",
        "        print(message)"
      ],
      "outputs": [],
      "metadata": {
        "_uuid": "b9edaad4b5c5d3a31845a34e3a5d4f3483fa722a"
      },
      "cell_type": "code"
    },
    {
      "execution_count": null,
      "source": [],
      "outputs": [],
      "metadata": {
        "collapsed": true,
        "_uuid": "d06e63c72fedd4b40b3c93e6518e1217e144d787"
      },
      "cell_type": "code"
    }
  ],
  "nbformat_minor": 1,
  "nbformat": 4
}