notebooks/statistics.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis - NLU\n",
    "\n",
    "## Statistics\n",
    "Student: Francesco Laiti\n",
    "\n",
    "---"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook contains the source code to extract statistics for ``subjectivity`` and ``movie_reviews`` datasets, available in the NLTK library."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import subjectivity, movie_reviews, stopwords\n",
    "import numpy as np\n",
    "\n",
    "nltk.download('movie_reviews')\n",
    "nltk.download('subjectivity')\n",
    "nltk.download('stopwords')\n",
    "\n",
    "stop_words = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_subj_data():\n",
    "    subj = subjectivity.sents(categories='subj')\n",
    "    obj = subjectivity.sents(categories='obj')\n",
    "    return subj + obj\n",
    "\n",
    "def get_pol_data():\n",
    "    neg = movie_reviews.paras(categories='neg')\n",
    "    pos = movie_reviews.paras(categories='pos')\n",
    "    return neg + pos\n",
    "\n",
    "subj_corpus = get_subj_data()\n",
    "movie_corpus  = get_pol_data()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Subjectivity dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Vocabulary size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subjectivity vocabulary size:  23906  words\n",
      "Subjectivity vocabulary size removing stop words:  23753  words\n"
     ]
    }
   ],
   "source": [
    "subj_sw_corpus = []\n",
    "for sent in subj_corpus:\n",
    "    tmp_seq = []\n",
    "    for x in sent:\n",
    "        if x not in stop_words:\n",
    "            tmp_seq.append(x)\n",
    "    subj_sw_corpus.append(tmp_seq)\n",
    "\n",
    "print('Subjectivity vocabulary size: ', len(set([w for d in subj_corpus for w in d])), ' words')\n",
    "print('Subjectivity vocabulary size removing stop words: ', len(set([w for sent in subj_sw_corpus for w in sent])), ' words')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Words per sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum words per sentence:  120\n",
      "Minimum words per sentence:  10\n",
      "Average words per sentence:  24\n"
     ]
    }
   ],
   "source": [
    "w_per_sents = []\n",
    "for d in subj_corpus:\n",
    "    w_per_sents.append(len(d))\n",
    "\n",
    "print('Maximum words per sentence: ', max(w_per_sents))\n",
    "print('Minimum words per sentence: ', min(w_per_sents))\n",
    "print('Average words per sentence: ', round(np.mean(w_per_sents)))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Movie reviews dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Vocabulary size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Movie review vocabulary size:  39768  words\n",
      "Movie review vocabulary size removing stop words:  39617  words\n"
     ]
    }
   ],
   "source": [
    "movie_sw_corpus = []\n",
    "for doc in movie_corpus:\n",
    "    tmp_sent = []\n",
    "    for sent in doc:\n",
    "        tmp_seq = []\n",
    "        for x in sent:\n",
    "            if x not in stop_words:\n",
    "                tmp_seq.append(x)\n",
    "        tmp_sent.append(tmp_seq)\n",
    "    movie_sw_corpus.append(tmp_sent)\n",
    "\n",
    "print('Movie review vocabulary size: ', len(set([w for list in movie_corpus for sent in list for w in sent])), ' words')\n",
    "print('Movie review vocabulary size removing stop words: ', len(set([w for list in movie_sw_corpus for sent in list for w in sent])), ' words')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Words per sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum words per sentence:  2879\n",
      "Minimum words per sentence:  19\n",
      "Average words per sentence:  792\n"
     ]
    }
   ],
   "source": [
    "w_per_sents = []\n",
    "for d in movie_corpus:\n",
    "    sum = 0\n",
    "    for w in d:\n",
    "        sum += len(w)\n",
    "    w_per_sents.append(sum)\n",
    "\n",
    "print('Maximum words per sentence: ', max(w_per_sents))\n",
    "print('Minimum words per sentence: ', min(w_per_sents))\n",
    "print('Average words per sentence: ', round(np.mean(w_per_sents)))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sents per document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum sents per document:  188\n",
      "Minimum sents per document:  1\n",
      "Average sents per document:  36\n"
     ]
    }
   ],
   "source": [
    "sent_per_doc = []\n",
    "for d in movie_corpus:\n",
    "    sent_per_doc.append(len(d))\n",
    "\n",
    "print('Maximum sents per document: ', max(sent_per_doc))\n",
    "print('Minimum sents per document: ', min(sent_per_doc))\n",
    "print('Average sents per document: ', round(np.mean(sent_per_doc)))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Words per sentence AFTER filtering out objective sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using .pkl with filtered sentences from  weights/transformer/filtered_polarity_sents.pkl\n",
      "Maximum words per sentence:  2348\n",
      "Minimum words per sentence:  19\n",
      "Average words per sentence:  498\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import os\n",
    "\n",
    "saved_path = 'weights/transformer/filtered_polarity_sents.pkl'\n",
    "dict_pols_filtered = {}\n",
    "\n",
    "if not os.path.exists(saved_path):\n",
    "    raise Exception('File .pkl not found. Run filter-task in transformer or GRU notebook and come back here!')\n",
    "else:\n",
    "    print('Using .pkl with filtered sentences from ', saved_path)\n",
    "    with open(saved_path, 'rb') as f:\n",
    "        dict_pols_filtered = pickle.load(f)\n",
    "\n",
    "w_per_sents = []\n",
    "for d in dict_pols_filtered['corpus']:\n",
    "    sum = 0\n",
    "    for w in d:\n",
    "        sum += len(w)\n",
    "    w_per_sents.append(sum)\n",
    "\n",
    "print('Maximum words per sentence: ', max(w_per_sents))\n",
    "print('Minimum words per sentence: ', min(w_per_sents))\n",
    "print('Average words per sentence: ', round(np.mean(w_per_sents)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "40d3a090f54c6569ab1632332b64b2c03c39dcf918b08424e98f38b5ae0af88f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}