Project51_UnSupervised_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load ../UnSupervised/utils.py\n",
    "class ItemsCount(object):\n",
    "    def __init__(self, value):\n",
    "        self._value = value\n",
    "\n",
    "    def __call__(self, sequence):\n",
    "        if isinstance(self._value, (bytes, str,)):\n",
    "            if self._value.endswith(\"%\"):\n",
    "                total_count = len(sequence)\n",
    "                percentage = int(self._value[:-1])\n",
    "                # at least one sentence should be chosen\n",
    "                count = max(1, total_count*percentage // 100)\n",
    "                return sequence[:count]\n",
    "            else:\n",
    "                return sequence[:int(self._value)]\n",
    "        elif isinstance(self._value, (int, float)):\n",
    "            return sequence[:int(self._value)]\n",
    "        else:\n",
    "            ValueError(\"Unsuported value of items count '%s'.\" % self._value)\n",
    "\n",
    "    def __repr__(self):\n",
    "        return to_string(\"<ItemsCount: %r>\" % self._value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load ../UnSupervised/base_summarizer.py\n",
    "from operator import attrgetter\n",
    "from collections import namedtuple\n",
    "\n",
    "SentenceInfo = namedtuple(\"SentenceInfo\", (\"sentence\", \"order\", \"rating\",))\n",
    "\n",
    "class BaseSummarizer(object):\n",
    "    \n",
    "    def __call__(self, document, sentences_count):\n",
    "        raise NotImplementedError(\"This method should be overriden in subclass\")\n",
    "\n",
    "    @staticmethod\n",
    "    def normalize_word(word):\n",
    "        return word.lower()\n",
    "\n",
    "    @staticmethod\n",
    "    def _get_best_sentences(sentences, count, rating, *args, **kwargs):\n",
    "        rate = rating\n",
    "        if isinstance(rating, dict):\n",
    "            assert not args and not kwargs\n",
    "            rate = lambda s: rating[s]\n",
    "\n",
    "        infos = (SentenceInfo(s, o, rate(s, *args, **kwargs))\n",
    "            for o, s in enumerate(sentences))\n",
    "\n",
    "        # sort sentences by rating in descending order\n",
    "        infos = sorted(infos, key=attrgetter(\"rating\"), reverse=True)\n",
    "        # get `count` first best rated sentences\n",
    "        if not isinstance(count, ItemsCount):\n",
    "            count = ItemsCount(count)\n",
    "        infos = count(infos)\n",
    "        # sort sentences by their order in document\n",
    "        infos = sorted(infos, key=attrgetter(\"order\"))\n",
    "\n",
    "        return tuple(i.sentence for i in infos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load ../UnSupervised/lsa_summarizer.py\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "import math\n",
    "import numpy\n",
    "import nltk\n",
    "\n",
    "from warnings import warn\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from numpy.linalg import svd as singular_value_decomposition\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "class LsaSummarizer(BaseSummarizer):\n",
    "    MIN_DIMENSIONS = 3\n",
    "    REDUCTION_RATIO = 1/1\n",
    "\n",
    "    _stop_words = list(stopwords.words('english'))\n",
    "\n",
    "    @property\n",
    "    def stop_words(self):\n",
    "        return self._stop_words\n",
    "\n",
    "    @stop_words.setter\n",
    "    def stop_words(self, words):\n",
    "        self._stop_words = words\n",
    "\n",
    "    def __call__(self, document, sentences_count):\n",
    "\n",
    "        dictionary = self._create_dictionary(document)\n",
    "        \n",
    "        if not dictionary:\n",
    "            return ()\n",
    "\n",
    "        sentences = sent_tokenize(document)\n",
    "\n",
    "        matrix = self._create_matrix(document, dictionary)\n",
    "        matrix = self._compute_term_frequency(matrix)\n",
    "        u, sigma, v = singular_value_decomposition(matrix, full_matrices=False)\n",
    "\n",
    "        ranks = iter(self._compute_ranks(sigma, v))\n",
    "        return self._get_best_sentences(sentences, sentences_count,\n",
    "            lambda s: next(ranks))\n",
    "\n",
    "    def _create_dictionary(self, document):\n",
    "        \"\"\"Creates mapping key = word, value = row index\"\"\"\n",
    "\n",
    "        words = word_tokenize(document)\n",
    "        words = tuple(words)\n",
    "\n",
    "        words = map(self.normalize_word, words)\n",
    "\n",
    "        unique_words = frozenset(w for w in words if w not in self._stop_words)\n",
    "\n",
    "        return dict((w, i) for i, w in enumerate(unique_words))\n",
    "\n",
    "    def _create_matrix(self, document, dictionary):\n",
    "        \"\"\"\n",
    "        Creates matrix of shape where cells\n",
    "        contains number of occurences of words (rows) in senteces (cols).\n",
    "        \"\"\"\n",
    "        sentences = sent_tokenize(document)\n",
    "        words_count = len(dictionary)\n",
    "        sentences_count = len(sentences)\n",
    "        if words_count < sentences_count:\n",
    "            message = (\n",
    "                \"Number of words (%d) is lower than number of sentences (%d). \"\n",
    "                \"LSA algorithm may not work properly.\"\n",
    "            )\n",
    "            warn(message % (words_count, sentences_count))\n",
    "\n",
    "        matrix = numpy.zeros((words_count, sentences_count))\n",
    "        for col, sentence in enumerate(sentences):\n",
    "            words = word_tokenize(sentence)\n",
    "            for word in words:\n",
    "                # only valid words is counted (not stop-words, ...)\n",
    "                if word in dictionary:\n",
    "                    row = dictionary[word]\n",
    "                    matrix[row, col] += 1\n",
    "\n",
    "        return matrix\n",
    "\n",
    "    def _compute_term_frequency(self, matrix, smooth=0.4):\n",
    "        \"\"\"\n",
    "        Computes TF metrics for each sentence (column) in the given matrix and  normalize \n",
    "        the tf weights of all terms occurring in a document by the maximum tf in that document \n",
    "        according to ntf_{t,d} = a + (1-a)\\frac{tf_{t,d}}{tf_{max}(d)^{'}}.\n",
    "        base_summarizer\n",
    "        The smoothing term $a$ damps the contribution of the second term - which may be viewed \n",
    "        as a scaling down of tf by the largest tf value in $d$\n",
    "        \"\"\"\n",
    "        assert 0.0 <= smooth < 1.0\n",
    "\n",
    "        max_word_frequencies = numpy.max(matrix, axis=0)\n",
    "        rows, cols = matrix.shape\n",
    "        for row in range(rows):\n",
    "            for col in range(cols):\n",
    "                max_word_frequency = max_word_frequencies[col]\n",
    "                if max_word_frequency != 0:\n",
    "                    frequency = matrix[row, col]/max_word_frequency\n",
    "                    matrix[row, col] = smooth + (1.0 - smooth)*frequency\n",
    "\n",
    "        return matrix\n",
    "\n",
    "    def _compute_ranks(self, sigma, v_matrix):\n",
    "        assert len(sigma) == v_matrix.shape[0]\n",
    "\n",
    "        dimensions = max(LsaSummarizer.MIN_DIMENSIONS,\n",
    "            int(len(sigma)*LsaSummarizer.REDUCTION_RATIO))\n",
    "        powered_sigma = tuple(s**2 if i < dimensions else 0.0\n",
    "            for i, s in enumerate(sigma))\n",
    "\n",
    "        ranks = []\n",
    "        \n",
    "        for column_vector in v_matrix.T:\n",
    "            rank = sum(s*v**2 for s, v in zip(powered_sigma, column_vector))\n",
    "            ranks.append(math.sqrt(rank))\n",
    "\n",
    "        return ranks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load ../UnSupervised/my_summarizer.py\n",
    "import pandas as pd\n",
    "import nltk\n",
    "nltk.download(\"punkt\", quiet=True)cases_final_v1\n",
    "nltk.download(\"stopwords\", quiet=True)\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from numba import jit, cuda\n",
    "\n",
    "\n",
    "SUMMARY_LINES =5\n",
    "\n",
    "# @jit(target =\"cuda\")\n",
    "def gen_summary(summarizer, text, lines):\n",
    "    try:\n",
    "        summary = summarizer(text, lines)\n",
    "        \n",
    "    except:\n",
    "#         print(\"*****************Exception*****************\")\n",
    "#         print(text)\n",
    "        summary = None\n",
    "        return summary\n",
    "    return \" \".join(summary)\n",
    "\n",
    "def get_df():\n",
    "    cases_df = pd.read_parquet(\"data/cases_final_v1.parquet.gzip\", engine=\"fastparquet\")\n",
    "    return cases_df\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    summarizer = LsaSummarizer()\n",
    "\n",
    "    stopwords = stopwords.words('english')\n",
    "    summarizer.stop_words = stopwords\n",
    "    print(\"Getting Datafrme\");\n",
    "    cases_df = get_df()\n",
    "    print(\"Applying summary\")\n",
    "    cases_df['opinion_text_summary'] = cases_df.opinion_text.apply(lambda x: gen_summary(summarizer, x, SUMMARY_LINES))\n",
    "\n",
    "    try:\n",
    "        cases_df.to_pickle(\"cases_summary.p\")\n",
    "    except:\n",
    "        print(\"Error writing to pickle\")\n",
    "\n",
    "    try:\n",
    "            cases_df.to_parquet(\"data/cases_summary_unsupervised.parquet.gzip\", engine=\"fastparquet\", row_group_offsets=100,\n",
    "                                compression=\"gzip\")\n",
    "    except:\n",
    "        print(\"Error writing to parquet\")\n",
    "\n",
    "    print(\"Complete \")\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}