notebooks/cis6930_week5a_nlp_basics_word_embeddings_student.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "cis6930-week5a-nlp-basics-word-embeddings-student.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oRgpqL0KVO43"
      },
      "source": [
        "# CIS6930 Week 5: NLP Basics & Word Embeddings (Student version)\n",
        "\n",
        "---\n",
        "\n",
        "This notebook does not use GPU. :)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z1wlP1kfWe8p"
      },
      "source": [
        "## spaCy Quick Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P9yzHwL53FDW"
      },
      "source": [
        "!pip install spacy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vrj9Au-OV0cN"
      },
      "source": [
        "# python -m spacy download en\n",
        "import spacy.cli\n",
        "spacy.cli.download(\"en_core_web_sm\") # en_core_web_lg"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vtmaV_D9V9yM"
      },
      "source": [
        "import spacy\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "doc = nlp(\"Apple is looking at buying U.K. startup for $1 billion\")\n",
        "for token in doc:\n",
        "    print([token.text, token.lemma_, token.pos_, token.dep_, token.ent_iob_, token.ent_type_])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_D08-5jRDhoj"
      },
      "source": [
        "token.lemma_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "unBmLxTSB72Y"
      },
      "source": [
        "# Named entity extraction (NER)\n",
        "for ent in doc.ents:\n",
        "    print([ent.text, ent.start_char, ent.end_char, ent.label_])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NLpFQ72cib1k"
      },
      "source": [
        "doc.vector"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "78abGQtksgYb"
      },
      "source": [
        "token.vector"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rxS3h8PSvG85"
      },
      "source": [
        "### Why not NLTK? NLTK's Known Pitfall"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HO7gLk-OvGAm"
      },
      "source": [
        "import nltk\n",
        "nltk.download(\"punkt\")  # \"all\" to download"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KknOV5NqvMy3"
      },
      "source": [
        "sentence = \"At eight o'clock on Thursday morning, Arthur didn't feel very good.\"\n",
        "nltk.word_tokenize(sentence)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uq8w8kLkvYS_"
      },
      "source": [
        "nltk.word_tokenize(\"It was eight o'clock on Thursday morning.Arthur didn't feel very good.\")\n",
        "#                                                          ^^^ No white space"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Oez1RtBO4sxu"
      },
      "source": [
        "## Pandas 101\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HjmJj-cA4sQM"
      },
      "source": [
        "# https://www.kaggle.com/crowdflower/twitter-airline-sentiment\n",
        "# License CC BY-NC-SA 4.0\n",
        "!gdown --id 1BS_TIqm7crkBRr8p6REZrMv4Uk9_-e6W"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wTNVvGdwKlrq"
      },
      "source": [
        "!ls"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pxuh78qW5Pyd"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_csv(\"Tweets.csv\")\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WYtpInK85cF0"
      },
      "source": [
        "df.columns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R8VE02Tx5dlf"
      },
      "source": [
        "df[[\"airline_sentiment\", \"airline\"]].head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NNrcpjUY_zAT"
      },
      "source": [
        "### Quick Analysis with Pandas DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rvBQ2ugG-rIq"
      },
      "source": [
        "agg_df = df.groupby([\"airline_sentiment\", \"airline\"]).size().unstack(0)\n",
        "agg_df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uzj0UPXs-9hl"
      },
      "source": [
        "agg_df.sum(axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4k5nbsLk_IZ1"
      },
      "source": [
        "agg_df.div(agg_df.sum(axis=1), axis=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3yLx9Gb5_p3-"
      },
      "source": [
        "agg_df.div(agg_df.sum(axis=1), axis=0).plot(kind=\"bar\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RRR_KP-W-2N_"
      },
      "source": [
        "### Data preparation for Text classification"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iUN6mgJe5g-J"
      },
      "source": [
        "subdf = df[[\"airline_sentiment\", \"text\"]]\n",
        "subdf.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uyECsUe5OxIj"
      },
      "source": [
        "subdf[\"text\"].head().iloc[1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XTMoHOsP5lcq"
      },
      "source": [
        "subdf[\"airline_sentiment\"].value_counts()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xcc0jHck5qEP"
      },
      "source": [
        "# Label encoding & Tokenization"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J4L0xkhD6LLm"
      },
      "source": [
        "# Non-NN solution\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.linear_model import LogisticRegression \n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "vectorizer = CountVectorizer(stop_words=\"english\")  # [\"text1\", \"text2\"] -> X\n",
        "X = vectorizer.fit_transform(subdf[\"text\"].tolist())\n",
        "\n",
        "le = LabelEncoder()\n",
        "y = le.fit_transform(subdf[\"airline_sentiment\"].values)\n",
        "y"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AWM7rU5ePBcZ"
      },
      "source": [
        "y.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dbi1DvQb7bpL"
      },
      "source": [
        "# Check the vocabulary\n",
        "len(vectorizer.vocabulary_)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kb_Q3iPs8H3q"
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y,\n",
        "                                                    test_size=0.20,\n",
        "                                                    random_state=42)\n",
        "\n",
        "# Dimensionality reduction using SVD: V -> Z (e.g., 500)\n",
        "#from sklearn.decomposition import TruncatedSVD\n",
        "#svd = TruncatedSVD(n_components=500)\n",
        "#X_train = svd.fit_transform(X_train)\n",
        "#X_test = svd.transform(X_test)\n",
        "\n",
        "clf = LogisticRegression()\n",
        "clf.fit(X_train, y_train)\n",
        "\n",
        "print(\"Train acc: {:.4f}\".format(clf.score(X_train, y_train)))\n",
        "print(\" Test acc: {:.4f}\".format(clf.score(X_test, y_test)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HYdumW79-gqY"
      },
      "source": [
        "### Creating confusion matrix using Pandas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2AKVXcQl9D-a"
      },
      "source": [
        "pred_df = pd.DataFrame({\"pred\": clf.predict(X_test),\n",
        "                        \"true\": y_test})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AHh_lu0Y9O0d"
      },
      "source": [
        "pred_df.groupby([\"pred\", \"true\"]).size().unstack(0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0W3rZRiaRDtu"
      },
      "source": [
        "le.classes_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ls3_A-TOv4Vr"
      },
      "source": [
        "## spaCy + Pandas: Clustering Tweets by Word Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jnLsl2WIwAWv"
      },
      "source": [
        "import numpy as np\n",
        "import seaborn as sns\n",
        "from sklearn.manifold import TSNE \n",
        "\n",
        "tsne = TSNE(n_components=2)\n",
        "\n",
        "sample_df = df.sample(n=2000, random_state=1) # Try increasing the number of samples and run the same analysis\n",
        "embs = np.array(sample_df[\"text\"].apply(lambda x: nlp(x).vector).tolist())\n",
        "embs2d = tsne.fit_transform(embs)\n",
        "\n",
        "emb_df = pd.DataFrame({\"x1\": embs2d[:, 0],\n",
        "                       \"x2\": embs2d[:, 1],\n",
        "                       \"sentiment\": sample_df[\"airline_sentiment\"],\n",
        "                       \"airline\": sample_df[\"airline\"]})\n",
        "\n",
        "emb_df.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8OV-gSvoyew_"
      },
      "source": [
        "sns.scatterplot(x=\"x1\",\n",
        "                y=\"x2\",\n",
        "                s=3,\n",
        "                hue=\"sentiment\",\n",
        "                data=emb_df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dKELbeDyyfaN"
      },
      "source": [
        "sns.scatterplot(x=\"x1\",\n",
        "                y=\"x2\",\n",
        "                s=3,\n",
        "                hue=\"airline\",\n",
        "                data=emb_df)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}